Harden OpenCode compatibility CI and watcher
This commit is contained in:
@@ -26,4 +26,14 @@ class ExactLiveContractTest < Minitest::Test
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_accepts_only_one_authoritative_assistant_message
|
||||
assert_nil OpenCodeCompat::ExactLiveContract.assert_authoritative_assistant_count!(1)
|
||||
|
||||
[0, 2, "not-a-number"].each do |count|
|
||||
assert_raises(OpenCodeCompat::ExactLiveContract::ContractError) do
|
||||
OpenCodeCompat::ExactLiveContract.assert_authoritative_assistant_count!(count)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
153
test/image_contract_evidence_test.rb
Normal file
153
test/image_contract_evidence_test.rb
Normal file
@@ -0,0 +1,153 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "fileutils"
|
||||
require "json"
|
||||
require "minitest/autorun"
|
||||
require "open3"
|
||||
require "tmpdir"
|
||||
|
||||
class ImageContractEvidenceTest < Minitest::Test
|
||||
ROOT = File.expand_path("..", __dir__)
|
||||
RUNNER = File.join(ROOT, "scripts/run_image_contract.sh")
|
||||
VALID_IMAGE = "ghcr.io/anomalyco/opencode@sha256:#{'a' * 64}"
|
||||
|
||||
def setup
|
||||
@tmp = Dir.mktmpdir("opencode-image-evidence")
|
||||
@bin = File.join(@tmp, "bin")
|
||||
@docker_marker = File.join(@tmp, "docker-called")
|
||||
FileUtils.mkdir_p(@bin)
|
||||
docker = File.join(@bin, "docker")
|
||||
File.write(docker, <<~SH)
|
||||
#!/usr/bin/env bash
|
||||
: >"$DOCKER_CALLED_MARKER"
|
||||
exit 99
|
||||
SH
|
||||
FileUtils.chmod(0o755, docker)
|
||||
end
|
||||
|
||||
def teardown
|
||||
FileUtils.remove_entry(@tmp)
|
||||
end
|
||||
|
||||
def test_invalid_input_overwrites_stale_pass_without_touching_docker
|
||||
evidence_path = File.join(@tmp, "evidence.json")
|
||||
File.write(evidence_path, JSON.generate("status" => "pass"))
|
||||
|
||||
_output, _error, status = run_contract(
|
||||
"OPENCODE_IMAGE" => "latest",
|
||||
"OPENCODE_RUBY_PATH" => ROOT,
|
||||
"OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path
|
||||
)
|
||||
|
||||
assert_equal 2, status.exitstatus
|
||||
refute_path_exists @docker_marker
|
||||
evidence = JSON.parse(File.read(evidence_path))
|
||||
assert_equal "fail", evidence.fetch("status")
|
||||
assert_equal 2, evidence.dig("failure", "exit_status")
|
||||
assert_equal "latest", evidence.dig("image", "requested")
|
||||
assert_equal [], evidence.fetch("executed_profiles")
|
||||
end
|
||||
|
||||
def test_rejects_truncated_registry_and_local_image_ids_before_docker
|
||||
{
|
||||
"ghcr.io/anomalyco/opencode@sha256:#{'a' * 12}" => {},
|
||||
"sha256:#{'b' * 12}" => {"ALLOW_EXACT_IMAGE_ID" => "1"}
|
||||
}.each_with_index do |(image, extra_environment), index|
|
||||
evidence_path = File.join(@tmp, "truncated-#{index}.json")
|
||||
_output, error, status = run_contract(
|
||||
{
|
||||
"OPENCODE_IMAGE" => image,
|
||||
"OPENCODE_RUBY_PATH" => ROOT,
|
||||
"OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path
|
||||
}.merge(extra_environment)
|
||||
)
|
||||
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_match(/must be an OCI digest/, error)
|
||||
assert_equal image, JSON.parse(File.read(evidence_path)).dig("image", "requested")
|
||||
end
|
||||
refute_path_exists @docker_marker
|
||||
end
|
||||
|
||||
def test_rejects_a_checkout_at_a_different_commit_before_docker
|
||||
checkout, _commit = git_checkout
|
||||
evidence_path = File.join(@tmp, "mismatch.json")
|
||||
|
||||
_output, error, status = run_contract(
|
||||
base_environment(checkout, "f" * 40, evidence_path)
|
||||
)
|
||||
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_match(/does not match expected commit/, error)
|
||||
assert_failed_without_docker(evidence_path)
|
||||
end
|
||||
|
||||
def test_rejects_a_dirty_checkout_before_docker
|
||||
checkout, commit = git_checkout
|
||||
File.write(File.join(checkout, "dirty.txt"), "not certified\n")
|
||||
evidence_path = File.join(@tmp, "dirty.json")
|
||||
|
||||
_output, error, status = run_contract(base_environment(checkout, commit, evidence_path))
|
||||
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_match(/checkout must be clean/, error)
|
||||
assert_failed_without_docker(evidence_path)
|
||||
end
|
||||
|
||||
def test_rejects_a_non_git_path_before_docker
|
||||
checkout = File.join(@tmp, "not-a-checkout")
|
||||
FileUtils.mkdir_p(checkout)
|
||||
evidence_path = File.join(@tmp, "not-git.json")
|
||||
|
||||
_output, error, status = run_contract(
|
||||
base_environment(checkout, "f" * 40, evidence_path)
|
||||
)
|
||||
|
||||
assert_equal 2, status.exitstatus
|
||||
assert_match(/readable Git checkout/, error)
|
||||
assert_failed_without_docker(evidence_path)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def base_environment(checkout, commit, evidence_path)
|
||||
{
|
||||
"OPENCODE_IMAGE" => VALID_IMAGE,
|
||||
"OPENCODE_RUBY_PATH" => checkout,
|
||||
"OPENCODE_RUBY_COMMIT" => commit,
|
||||
"OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path
|
||||
}
|
||||
end
|
||||
|
||||
def run_contract(environment)
|
||||
env = {
|
||||
"PATH" => "#{@bin}:#{ENV.fetch('PATH')}",
|
||||
"DOCKER_CALLED_MARKER" => @docker_marker
|
||||
}.merge(environment)
|
||||
Open3.capture3(env, "bash", RUNNER)
|
||||
end
|
||||
|
||||
def git_checkout
|
||||
path = File.join(@tmp, "checkout-#{rand(1_000_000)}")
|
||||
FileUtils.mkdir_p(path)
|
||||
run_git!(path, "init", "--quiet")
|
||||
run_git!(path, "config", "user.name", "Compatibility Test")
|
||||
run_git!(path, "config", "user.email", "compat@example.test")
|
||||
File.write(File.join(path, "tracked.txt"), "clean\n")
|
||||
run_git!(path, "add", "tracked.txt")
|
||||
run_git!(path, "commit", "--quiet", "-m", "fixture")
|
||||
commit = run_git!(path, "rev-parse", "HEAD").strip
|
||||
[path, commit]
|
||||
end
|
||||
|
||||
def run_git!(path, *arguments)
|
||||
output, error, status = Open3.capture3("git", "-C", path, *arguments)
|
||||
assert status.success?, error
|
||||
output
|
||||
end
|
||||
|
||||
def assert_failed_without_docker(evidence_path)
|
||||
refute_path_exists @docker_marker
|
||||
assert_equal "fail", JSON.parse(File.read(evidence_path)).fetch("status")
|
||||
end
|
||||
end
|
||||
318
test/lockstep_client_contract_test.rb
Normal file
318
test/lockstep_client_contract_test.rb
Normal file
@@ -0,0 +1,318 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "minitest/autorun"
|
||||
require "rbconfig"
|
||||
require "stringio"
|
||||
require "tmpdir"
|
||||
require_relative "../ruby/lockstep_client_contract"
|
||||
|
||||
module LockstepClientContractTestSources
|
||||
class Git
|
||||
attr_reader :revision
|
||||
|
||||
def initialize(revision)
|
||||
@revision = revision
|
||||
end
|
||||
end
|
||||
|
||||
class Path; end
|
||||
end
|
||||
|
||||
class LockstepClientContractTest < Minitest::Test
|
||||
RUBY_COMMIT = "1" * 40
|
||||
RAILS_COMMIT = "2" * 40
|
||||
RUBY_TAG_OBJECT = "3" * 40
|
||||
RAILS_TAG_OBJECT = "4" * 40
|
||||
RUBY_TAG = "v0.0.1.alpha7"
|
||||
RAILS_TAG = "v0.0.1.alpha7"
|
||||
VERSION = "0.0.1.alpha7"
|
||||
|
||||
FakeSpec = Struct.new(:name, :version, :runtime_dependencies, :source, :full_gem_path, keyword_init: true)
|
||||
|
||||
def setup
|
||||
@root = Dir.mktmpdir("opencode-lockstep-contract")
|
||||
@ruby_path = File.join(@root, "opencode-ruby")
|
||||
@rails_path = File.join(@root, "opencode-rails")
|
||||
Dir.mkdir(@ruby_path)
|
||||
Dir.mkdir(@rails_path)
|
||||
@env = {
|
||||
"OPENCODE_RUBY_PATH" => @ruby_path,
|
||||
"OPENCODE_RUBY_COMMIT" => RUBY_COMMIT,
|
||||
"OPENCODE_RUBY_TAG" => RUBY_TAG,
|
||||
"OPENCODE_RUBY_TAG_OBJECT" => RUBY_TAG_OBJECT,
|
||||
"OPENCODE_RUBY_VERSION" => VERSION,
|
||||
"OPENCODE_RAILS_PATH" => @rails_path,
|
||||
"OPENCODE_RAILS_COMMIT" => RAILS_COMMIT,
|
||||
"OPENCODE_RAILS_TAG" => RAILS_TAG,
|
||||
"OPENCODE_RAILS_TAG_OBJECT" => RAILS_TAG_OBJECT,
|
||||
"OPENCODE_RAILS_VERSION" => VERSION
|
||||
}
|
||||
@heads = {@ruby_path => RUBY_COMMIT, @rails_path => RAILS_COMMIT}
|
||||
@git_results = {}
|
||||
seed_annotated_tag(@ruby_path, RUBY_TAG, RUBY_TAG_OBJECT, RUBY_COMMIT)
|
||||
seed_annotated_tag(@rails_path, RAILS_TAG, RAILS_TAG_OBJECT, RAILS_COMMIT)
|
||||
@ruby_spec = FakeSpec.new(name: "opencode-ruby", version: Gem::Version.new(VERSION))
|
||||
@rails_spec = FakeSpec.new(
|
||||
name: "opencode-rails",
|
||||
version: Gem::Version.new(VERSION),
|
||||
runtime_dependencies: [Gem::Dependency.new("opencode-ruby", "= #{VERSION}")],
|
||||
full_gem_path: @rails_path
|
||||
)
|
||||
@bundle_ruby_spec = FakeSpec.new(
|
||||
name: "opencode-ruby",
|
||||
source: LockstepClientContractTestSources::Git.new(RUBY_COMMIT)
|
||||
)
|
||||
end
|
||||
|
||||
def teardown
|
||||
FileUtils.remove_entry(@root)
|
||||
end
|
||||
|
||||
def test_verifies_and_emits_canonical_lockstep_evidence
|
||||
evidence_path = File.join(@root, "evidence", "lockstep.json")
|
||||
@env["OPENCODE_COMPAT_EVIDENCE_PATH"] = evidence_path
|
||||
output = StringIO.new
|
||||
|
||||
document = contract.emit(io: output)
|
||||
|
||||
expected_json = OpenCodeCompat::LockstepClientContract.canonical_json(document)
|
||||
assert_equal "#{expected_json}\n", output.string
|
||||
assert_equal "#{expected_json}\n", File.binread(evidence_path)
|
||||
assert_equal "pass", document.fetch("status")
|
||||
assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "checkout_commit")
|
||||
assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "bundler_git_revision")
|
||||
assert_equal RUBY_TAG_OBJECT, document.dig("opencode_ruby", "tag_provenance", "annotated_tag_object")
|
||||
assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "tag_provenance", "peeled_commit")
|
||||
assert_equal RUBY_TAG, document.dig("opencode_ruby", "tag_provenance", "tag")
|
||||
assert_equal RAILS_COMMIT, document.dig("opencode_rails", "checkout_commit")
|
||||
assert_equal RAILS_TAG_OBJECT, document.dig("opencode_rails", "tag_provenance", "annotated_tag_object")
|
||||
assert_equal RAILS_COMMIT, document.dig("opencode_rails", "tag_provenance", "peeled_commit")
|
||||
assert_equal "= #{VERSION}", document.dig("opencode_rails", "runtime_dependency", "requirement")
|
||||
assert_equal RUBY_VERSION, document.fetch("ruby_runtime_version")
|
||||
assert_nil document.dig("workflow", "run_id")
|
||||
assert_equal expected_json, JSON.generate(JSON.parse(expected_json).sort.to_h)
|
||||
end
|
||||
|
||||
def test_rejects_checkout_commit_mismatch_for_either_client
|
||||
{
|
||||
@ruby_path => "opencode-ruby checkout commit",
|
||||
@rails_path => "opencode-rails checkout commit"
|
||||
}.each do |path, message|
|
||||
original = @heads.fetch(path)
|
||||
@heads[path] = "f" * 40
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match message, error.message
|
||||
ensure
|
||||
@heads[path] = original
|
||||
end
|
||||
end
|
||||
|
||||
def test_rejects_non_annotated_tag_object_for_either_client
|
||||
{
|
||||
@ruby_path => RUBY_TAG_OBJECT,
|
||||
@rails_path => RAILS_TAG_OBJECT
|
||||
}.each do |path, object|
|
||||
set_git_result(path, ["cat-file", "-t", object], "commit")
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/annotated tag object type must equal "tag"/, error.message)
|
||||
ensure
|
||||
set_git_result(path, ["cat-file", "-t", object], "tag")
|
||||
end
|
||||
end
|
||||
|
||||
def test_rejects_tampered_annotated_tag_object_coordinate
|
||||
tampered_object = "f" * 40
|
||||
@env["OPENCODE_RUBY_TAG_OBJECT"] = tampered_object
|
||||
set_git_result(@ruby_path, ["cat-file", "-t", tampered_object], "tag")
|
||||
set_git_result(@ruby_path, ["rev-parse", "--verify", "#{tampered_object}^{commit}"], RUBY_COMMIT)
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
|
||||
assert_match(/opencode-ruby local tag ref object/, error.message)
|
||||
assert_match(/#{RUBY_TAG_OBJECT}/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_local_tag_ref_resolving_to_another_object
|
||||
set_git_result(
|
||||
@rails_path,
|
||||
["rev-parse", "--verify", "refs/tags/#{RAILS_TAG}"],
|
||||
"e" * 40
|
||||
)
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
|
||||
assert_match(/opencode-rails local tag ref object/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_wrong_object_or_ref_peel
|
||||
object_peel_key = ["rev-parse", "--verify", "#{RUBY_TAG_OBJECT}^{commit}"]
|
||||
set_git_result(@ruby_path, object_peel_key, "e" * 40)
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/opencode-ruby annotated tag peeled commit/, error.message)
|
||||
|
||||
set_git_result(@ruby_path, object_peel_key, RUBY_COMMIT)
|
||||
set_git_result(
|
||||
@ruby_path,
|
||||
["rev-parse", "--verify", "refs/tags/#{RUBY_TAG}^{commit}"],
|
||||
"e" * 40
|
||||
)
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/opencode-ruby local tag ref peeled commit/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_missing_local_tag
|
||||
@git_results.delete([@ruby_path, ["rev-parse", "--verify", "refs/tags/#{RUBY_TAG}"]])
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
|
||||
assert_match(/could not resolve opencode-ruby local tag ref/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_loaded_or_gem_version_drift
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) do
|
||||
contract(ruby_version: "0.0.1.alpha8").verify
|
||||
end
|
||||
assert_match(/loaded Opencode::VERSION/, error.message)
|
||||
|
||||
@rails_spec.version = Gem::Version.new("0.0.1.alpha8")
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/loaded opencode-rails gem version/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_non_lockstep_expected_versions
|
||||
@env["OPENCODE_RAILS_VERSION"] = "0.0.1.alpha8"
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) do
|
||||
contract(rails_version: "0.0.1.alpha8").verify
|
||||
end
|
||||
|
||||
assert_match(/lockstep client version/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_non_exact_or_wrong_rails_runtime_dependency
|
||||
["~> #{VERSION}", "= 0.0.1.alpha6"].each do |requirement|
|
||||
@rails_spec.runtime_dependencies = [Gem::Dependency.new("opencode-ruby", requirement)]
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/must require opencode-ruby = #{Regexp.escape(VERSION)}/, error.message)
|
||||
end
|
||||
end
|
||||
|
||||
def test_rejects_missing_or_duplicate_rails_runtime_dependency
|
||||
@rails_spec.runtime_dependencies = []
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/exactly one runtime dependency/, error.message)
|
||||
|
||||
@rails_spec.runtime_dependencies = [
|
||||
Gem::Dependency.new("opencode-ruby", "= #{VERSION}"),
|
||||
Gem::Dependency.new("opencode-ruby", "= #{VERSION}")
|
||||
]
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/exactly one runtime dependency/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_non_git_or_mismatched_bundler_source
|
||||
@bundle_ruby_spec.source = LockstepClientContractTestSources::Path.new
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/must be a Git source/, error.message)
|
||||
|
||||
@bundle_ruby_spec.source = LockstepClientContractTestSources::Git.new("f" * 40)
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/Bundler opencode-ruby revision/, error.message)
|
||||
end
|
||||
|
||||
def test_rejects_loaded_rails_gem_from_another_checkout
|
||||
other_path = File.join(@root, "other-opencode-rails")
|
||||
Dir.mkdir(other_path)
|
||||
@rails_spec.full_gem_path = other_path
|
||||
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
|
||||
assert_match(/loaded opencode-rails checkout/, error.message)
|
||||
end
|
||||
|
||||
def test_requires_full_exact_candidate_coordinates
|
||||
@env.delete("OPENCODE_RUBY_COMMIT")
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/missing required environment: OPENCODE_RUBY_COMMIT/, error.message)
|
||||
|
||||
@env["OPENCODE_RUBY_COMMIT"] = "main"
|
||||
error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify }
|
||||
assert_match(/full lowercase 40-character Git commit/, error.message)
|
||||
end
|
||||
|
||||
def test_cli_failure_writes_failure_evidence
|
||||
stub_path = File.join(@root, "cli-stubs")
|
||||
evidence_path = File.join(@root, "cli-evidence", "lockstep.json")
|
||||
FileUtils.mkdir_p(stub_path)
|
||||
File.binwrite(File.join(stub_path, "bundler.rb"), <<~RUBY)
|
||||
module Bundler
|
||||
LoadedDefinition = Struct.new(:specs)
|
||||
|
||||
def self.load
|
||||
LoadedDefinition.new([])
|
||||
end
|
||||
end
|
||||
RUBY
|
||||
File.binwrite(File.join(stub_path, "opencode-rails.rb"), <<~RUBY)
|
||||
module Opencode
|
||||
VERSION = #{VERSION.inspect}
|
||||
RAILS_VERSION = #{VERSION.inspect}
|
||||
end
|
||||
RUBY
|
||||
|
||||
_stdout, stderr, status = Open3.capture3(
|
||||
{"OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path},
|
||||
RbConfig.ruby,
|
||||
"-I",
|
||||
stub_path,
|
||||
File.expand_path("../ruby/lockstep_client_contract.rb", __dir__),
|
||||
unsetenv_others: true
|
||||
)
|
||||
|
||||
refute status.success?
|
||||
assert_equal 1, status.exitstatus
|
||||
failure = JSON.parse(File.binread(evidence_path))
|
||||
assert_equal "opencode-client-lockstep", failure.fetch("contract")
|
||||
assert_equal "fail", failure.fetch("status")
|
||||
assert_match(/missing required environment/, failure.fetch("error"))
|
||||
assert_equal failure, JSON.parse(stderr)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def contract(ruby_version: VERSION, rails_version: VERSION)
|
||||
OpenCodeCompat::LockstepClientContract.new(
|
||||
env: @env,
|
||||
git_resolver: method(:resolve_git),
|
||||
loaded_specs: {
|
||||
"opencode-ruby" => @ruby_spec,
|
||||
"opencode-rails" => @rails_spec
|
||||
},
|
||||
bundle_specs: [@bundle_ruby_spec],
|
||||
ruby_version: ruby_version,
|
||||
rails_version: rails_version
|
||||
)
|
||||
end
|
||||
|
||||
def resolve_git(path, *arguments)
|
||||
return @heads.fetch(path) if arguments == ["rev-parse", "--verify", "HEAD^{commit}"]
|
||||
|
||||
@git_results.fetch([path, arguments])
|
||||
end
|
||||
|
||||
def seed_annotated_tag(path, tag, object, commit)
|
||||
set_git_result(path, ["cat-file", "-t", object], "tag")
|
||||
set_git_result(path, ["rev-parse", "--verify", "refs/tags/#{tag}"], object)
|
||||
set_git_result(path, ["rev-parse", "--verify", "#{object}^{commit}"], commit)
|
||||
set_git_result(path, ["rev-parse", "--verify", "refs/tags/#{tag}^{commit}"], commit)
|
||||
end
|
||||
|
||||
def set_git_result(path, arguments, result)
|
||||
@git_results[[path, arguments]] = result
|
||||
end
|
||||
end
|
||||
62
test/matrix_json_test.rb
Normal file
62
test/matrix_json_test.rb
Normal file
@@ -0,0 +1,62 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "fileutils"
|
||||
require "json"
|
||||
require "minitest/autorun"
|
||||
require "open3"
|
||||
require "rbconfig"
|
||||
require "tmpdir"
|
||||
|
||||
class MatrixJsonTest < Minitest::Test
|
||||
ROOT = File.expand_path("..", __dir__)
|
||||
|
||||
def setup
|
||||
@tmp = Dir.mktmpdir("opencode-matrix-json")
|
||||
FileUtils.mkdir_p(File.join(@tmp, "scripts"))
|
||||
FileUtils.mkdir_p(File.join(@tmp, "manifests"))
|
||||
FileUtils.cp(File.join(ROOT, "scripts/matrix_json.rb"), File.join(@tmp, "scripts"))
|
||||
@manifest = JSON.parse(File.read(File.join(ROOT, "manifests/image-matrix.json")))
|
||||
end
|
||||
|
||||
def teardown
|
||||
FileUtils.remove_entry(@tmp)
|
||||
end
|
||||
|
||||
def test_emits_only_the_actual_shared_client_contract_profile
|
||||
output, error, status = run_matrix(@manifest)
|
||||
|
||||
assert status.success?, error
|
||||
JSON.parse(output).fetch("include").each do |target|
|
||||
assert_equal ["ruby-rest-sse"], target.fetch("profiles")
|
||||
assert_equal "shared-client-contract-only", target.fetch("certification_scope")
|
||||
end
|
||||
end
|
||||
|
||||
def test_rejects_a_manifest_that_overstates_the_executed_profile
|
||||
@manifest.fetch("public_ci").first["profiles"] = ["rails-persisted-turn"]
|
||||
|
||||
_output, error, status = run_matrix(@manifest)
|
||||
|
||||
refute status.success?
|
||||
assert_match(/executes exactly ruby-rest-sse/, error)
|
||||
end
|
||||
|
||||
def test_rejects_a_manifest_that_overstates_the_certification_scope
|
||||
@manifest.fetch("public_ci").first["certification_scope"] = "consumer-runtime"
|
||||
|
||||
_output, error, status = run_matrix(@manifest)
|
||||
|
||||
refute status.success?
|
||||
assert_match(/certification_scope must be shared-client-contract-only/, error)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def run_matrix(manifest)
|
||||
File.write(
|
||||
File.join(@tmp, "manifests/image-matrix.json"),
|
||||
JSON.pretty_generate(manifest) + "\n"
|
||||
)
|
||||
Open3.capture3(RbConfig.ruby, File.join(@tmp, "scripts/matrix_json.rb"))
|
||||
end
|
||||
end
|
||||
@@ -42,7 +42,9 @@ class RepositoryTest < Minitest::Test
|
||||
targets.each do |target|
|
||||
assert_match %r{\Aghcr\.io/anomalyco/opencode@sha256:[0-9a-f]{64}\z}, target.fetch("image")
|
||||
refute_includes target.fetch("image"), ":latest"
|
||||
refute_empty target.fetch("profiles")
|
||||
assert_equal ["ruby-rest-sse"], target.fetch("profiles")
|
||||
assert_equal "shared-client-contract-only",
|
||||
target.fetch("certification_scope", "shared-client-contract-only")
|
||||
next unless target.fetch("certification_status") == "certified"
|
||||
|
||||
assert_match(/\A[0-9a-f]{40}\z/, target.fetch("certified_client_commit"))
|
||||
@@ -51,14 +53,28 @@ class RepositoryTest < Minitest::Test
|
||||
end
|
||||
end
|
||||
|
||||
def test_candidate_is_bound_to_an_annotated_tag
|
||||
def test_candidate_client_train_is_lockstep_and_bound_to_annotated_tags
|
||||
candidate = json("manifests/client-candidate.json")
|
||||
provenance = candidate.fetch("tag_provenance")
|
||||
clients = candidate.fetch("clients")
|
||||
release_train = candidate.fetch("release_train")
|
||||
|
||||
assert_match(/\A[0-9a-f]{40}\z/, candidate.fetch("ref"))
|
||||
assert_match(/\Av\d+\.\d+\.\d+\.alpha\d+\z/, provenance.fetch("tag"))
|
||||
assert_match(/\A[0-9a-f]{40}\z/, provenance.fetch("annotated_tag_object"))
|
||||
assert_equal candidate.fetch("ref"), provenance.fetch("peeled_commit")
|
||||
assert_equal 2, candidate.fetch("schema_version")
|
||||
assert_equal "candidate", candidate.fetch("status")
|
||||
assert_equal %w[opencode-rails opencode-ruby], clients.keys.sort
|
||||
|
||||
clients.each do |name, client|
|
||||
provenance = client.fetch("tag_provenance")
|
||||
assert_equal "ajaynomics/#{name}", client.fetch("repository")
|
||||
assert_equal release_train, client.fetch("version")
|
||||
assert_match(/\A[0-9a-f]{40}\z/, client.fetch("ref"))
|
||||
assert_equal "v#{release_train}", provenance.fetch("tag")
|
||||
assert_match(/\A[0-9a-f]{40}\z/, provenance.fetch("annotated_tag_object"))
|
||||
assert_equal client.fetch("ref"), provenance.fetch("peeled_commit")
|
||||
end
|
||||
|
||||
dependency = clients.fetch("opencode-rails").fetch("runtime_dependencies").fetch("opencode-ruby")
|
||||
assert_equal "= #{release_train}", dependency.fetch("requirement")
|
||||
assert_equal clients.fetch("opencode-ruby").fetch("ref"), dependency.fetch("ref")
|
||||
end
|
||||
|
||||
def test_certified_migration_keeps_previous_tuple
|
||||
@@ -147,6 +163,41 @@ class RepositoryTest < Minitest::Test
|
||||
refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service)\b/i, workflow)
|
||||
end
|
||||
|
||||
def test_workflow_actions_are_pinned_to_exact_commits
|
||||
workflows = Dir.glob(File.join(ROOT, ".github/workflows/*.{yml,yaml}"))
|
||||
refute_empty workflows
|
||||
|
||||
workflows.each do |path|
|
||||
File.foreach(path).with_index(1) do |line, number|
|
||||
next unless (match = line.match(/\buses:\s+[^\s@]+@([^\s#]+)/))
|
||||
|
||||
assert_match(/\A[0-9a-f]{40}\z/, match[1], "#{path}:#{number} must pin an exact action commit")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_candidate_workflow_verifies_and_preserves_the_lockstep_client_tuple
|
||||
workflow = File.read(File.join(ROOT, ".github/workflows/candidate.yml"))
|
||||
|
||||
refute_includes workflow, "inputs.opencode_ruby_ref"
|
||||
refute_includes workflow, "inputs.opencode_rails_ref"
|
||||
assert_includes workflow, "repository: ajaynomics/opencode-ruby"
|
||||
assert_includes workflow, "repository: ajaynomics/opencode-rails"
|
||||
assert_includes workflow, "ruby: [\"3.2\", \"3.3\", \"3.4\"]"
|
||||
assert_includes workflow, "ruby/lockstep_client_contract.rb"
|
||||
assert_includes workflow, "OPENCODE_RUBY_TAG_OBJECT"
|
||||
assert_includes workflow, "OPENCODE_RAILS_TAG_OBJECT"
|
||||
assert_operator workflow.scan("fetch-tags: true").length, :>=, 2
|
||||
assert_operator workflow.scan("actions/upload-artifact@").length, :>=, 3
|
||||
assert_includes workflow, "OPENCODE_COMPAT_EVIDENCE_PATH"
|
||||
assert_includes workflow, "OPENCODE_REQUIRED_CONSUMER_PROFILES"
|
||||
assert_includes workflow, 'BUNDLE_PATH: ${{ runner.temp }}/opencode-ruby-bundle'
|
||||
assert_includes workflow, "Install candidate dependencies outside the checkout"
|
||||
refute_includes workflow, "OPENCODE_MATRIX_PROFILES"
|
||||
refute_includes workflow, "OPENCODE_CERTIFICATION_SCOPE"
|
||||
refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service|gh\s+pr\s+merge)\b/i, workflow)
|
||||
end
|
||||
|
||||
def test_tuple_promotion_has_no_command_execution_or_deployment_client
|
||||
paths = %w[
|
||||
lib/opencode_compat/runtime_tuple_promoter.rb
|
||||
@@ -164,8 +215,11 @@ class RepositoryTest < Minitest::Test
|
||||
runner = File.read(File.join(ROOT, "scripts/run_image_contract.sh"))
|
||||
|
||||
assert_includes probe, "ExactLiveContract.assert_final_text!"
|
||||
assert_includes probe, "ExactLiveContract.assert_authoritative_assistant_count!"
|
||||
refute_includes probe, "full_text.include?"
|
||||
assert_includes runner, "exact_live_contract.rb"
|
||||
assert_includes runner, "OPENCODE_COMPAT_EVIDENCE_PATH"
|
||||
assert_includes runner, "OPENCODE_EXPECTED_VERSION"
|
||||
refute_match(/request_count.*-lt\s+1/, runner)
|
||||
end
|
||||
end
|
||||
|
||||
234
test/watcher_test.rb
Normal file
234
test/watcher_test.rb
Normal file
@@ -0,0 +1,234 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "fileutils"
|
||||
require "json"
|
||||
require "minitest/autorun"
|
||||
require "open3"
|
||||
require "rbconfig"
|
||||
require "tmpdir"
|
||||
|
||||
class WatcherTest < Minitest::Test
|
||||
ROOT = File.expand_path("..", __dir__)
|
||||
RUBY = RbConfig.ruby
|
||||
|
||||
def setup
|
||||
@tmp = Dir.mktmpdir("opencode-watcher")
|
||||
FileUtils.mkdir_p(File.join(@tmp, "scripts"))
|
||||
FileUtils.mkdir_p(File.join(@tmp, "manifests"))
|
||||
FileUtils.cp(File.join(ROOT, "scripts/record_upstream_candidate.rb"), File.join(@tmp, "scripts"))
|
||||
%w[upstream.json image-matrix.json].each do |name|
|
||||
FileUtils.cp(File.join(ROOT, "manifests", name), File.join(@tmp, "manifests"))
|
||||
end
|
||||
end
|
||||
|
||||
def teardown
|
||||
FileUtils.remove_entry(@tmp)
|
||||
end
|
||||
|
||||
def test_same_tag_and_digest_is_an_exact_noop
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
before = watched_bytes
|
||||
|
||||
run_recorder!(
|
||||
upstream.fetch("release_tag"),
|
||||
upstream.fetch("published_at"),
|
||||
upstream.fetch("release_url"),
|
||||
upstream.fetch("image").split("@").last
|
||||
)
|
||||
|
||||
assert_equal before, watched_bytes
|
||||
end
|
||||
|
||||
def test_same_tag_and_digest_repairs_a_missing_public_target_then_noops
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
matrix = read_json("manifests/image-matrix.json")
|
||||
image = upstream.fetch("image")
|
||||
matrix.fetch("public_ci").reject! { |target| target.fetch("image") == image }
|
||||
write_json("manifests/image-matrix.json", matrix)
|
||||
upstream_before = File.binread(File.join(@tmp, "manifests/upstream.json"))
|
||||
|
||||
arguments = [
|
||||
upstream.fetch("release_tag"),
|
||||
upstream.fetch("published_at"),
|
||||
upstream.fetch("release_url"),
|
||||
image.split("@").last
|
||||
]
|
||||
run_recorder!(*arguments)
|
||||
|
||||
repaired = read_json("manifests/image-matrix.json")
|
||||
targets = repaired.fetch("public_ci").select { |target| target.fetch("image") == image }
|
||||
assert_equal 1, targets.length
|
||||
assert_equal "pending", targets.first.fetch("certification_status")
|
||||
assert_equal ["ruby-rest-sse"], targets.first.fetch("profiles")
|
||||
assert_equal upstream_before, File.binread(File.join(@tmp, "manifests/upstream.json"))
|
||||
|
||||
after = watched_bytes
|
||||
run_recorder!(*arguments)
|
||||
assert_equal after, watched_bytes
|
||||
end
|
||||
|
||||
def test_duplicate_public_targets_are_rejected_without_writing_either_manifest
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
matrix = read_json("manifests/image-matrix.json")
|
||||
target = matrix.fetch("public_ci").find { |entry| entry.fetch("image") == upstream.fetch("image") }
|
||||
duplicate = target.merge("id" => "#{target.fetch('id')}-duplicate")
|
||||
matrix.fetch("public_ci") << duplicate
|
||||
write_json("manifests/image-matrix.json", matrix)
|
||||
before = watched_bytes
|
||||
|
||||
_output, error, status = capture_recorder(
|
||||
upstream.fetch("release_tag"),
|
||||
upstream.fetch("published_at"),
|
||||
upstream.fetch("release_url"),
|
||||
upstream.fetch("image").split("@").last
|
||||
)
|
||||
|
||||
refute status.success?
|
||||
assert_match(/must appear exactly once in public_ci; found 2/, error)
|
||||
assert_equal before, watched_bytes
|
||||
end
|
||||
|
||||
def test_missing_public_target_with_conflicting_generated_id_is_rejected_without_writing
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
matrix = read_json("manifests/image-matrix.json")
|
||||
image = upstream.fetch("image")
|
||||
removed = matrix.fetch("public_ci").find { |target| target.fetch("image") == image }
|
||||
matrix.fetch("public_ci").reject! { |target| target.fetch("image") == image }
|
||||
matrix.fetch("public_ci") << removed.merge(
|
||||
"id" => "upstream-#{upstream.fetch('version')}-#{image.split('sha256:').last}",
|
||||
"image" => "ghcr.io/anomalyco/opencode@sha256:#{'e' * 64}"
|
||||
)
|
||||
write_json("manifests/image-matrix.json", matrix)
|
||||
before = watched_bytes
|
||||
|
||||
_output, error, status = capture_recorder(
|
||||
upstream.fetch("release_tag"),
|
||||
upstream.fetch("published_at"),
|
||||
upstream.fetch("release_url"),
|
||||
image.split("@").last
|
||||
)
|
||||
|
||||
refute status.success?
|
||||
assert_match(/public_ci id .* is already used by another image/, error)
|
||||
assert_equal before, watched_bytes
|
||||
end
|
||||
|
||||
def test_same_tag_with_new_digest_records_a_distinct_pending_target_once
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
replacement = "sha256:#{'f' * 64}"
|
||||
arguments = [
|
||||
upstream.fetch("release_tag"),
|
||||
upstream.fetch("published_at"),
|
||||
upstream.fetch("release_url"),
|
||||
replacement
|
||||
]
|
||||
|
||||
run_recorder!(*arguments)
|
||||
changed_upstream = read_json("manifests/upstream.json")
|
||||
matrix = read_json("manifests/image-matrix.json")
|
||||
target = matrix.fetch("public_ci").last
|
||||
|
||||
assert_equal "ghcr.io/anomalyco/opencode@#{replacement}", changed_upstream.fetch("image")
|
||||
expected_version = upstream.fetch("release_tag").delete_prefix("v")
|
||||
assert_equal "upstream-#{expected_version}-#{'f' * 64}", target.fetch("id")
|
||||
assert_equal "ghcr.io/anomalyco/opencode@#{replacement}", target.fetch("image")
|
||||
assert_equal "pending", target.fetch("certification_status")
|
||||
assert_equal "shared-client-contract-only", target.fetch("certification_scope")
|
||||
assert_empty target.fetch("consumers")
|
||||
assert_empty target.fetch("required_consumer_profiles")
|
||||
|
||||
after = watched_bytes
|
||||
run_recorder!(*arguments)
|
||||
assert_equal after, watched_bytes
|
||||
end
|
||||
|
||||
def test_recorder_rejects_untrusted_release_metadata
|
||||
upstream = read_json("manifests/upstream.json")
|
||||
digest = upstream.fetch("image").split("@").last
|
||||
|
||||
refute recorder_status("branch-name", upstream.fetch("published_at"), upstream.fetch("release_url"), digest).success?
|
||||
refute recorder_status(upstream.fetch("release_tag"), "yesterday", upstream.fetch("release_url"), digest).success?
|
||||
refute recorder_status(upstream.fetch("release_tag"), upstream.fetch("published_at"), "https://example.test/release", digest).success?
|
||||
refute recorder_status(upstream.fetch("release_tag"), upstream.fetch("published_at"), upstream.fetch("release_url"), "sha256:nope").success?
|
||||
end
|
||||
|
||||
def test_branch_names_bind_the_full_tag_and_digest_and_support_retries
|
||||
script = File.join(ROOT, "scripts/upstream_branch_name.rb")
|
||||
digest = "sha256:#{'a' * 64}"
|
||||
replacement = "sha256:#{'b' * 64}"
|
||||
|
||||
base = run_script!(script, "v1.18.3", digest).strip
|
||||
changed = run_script!(script, "v1.18.3", replacement).strip
|
||||
retry_branch = run_script!(script, "v1.18.3", digest, "123-2").strip
|
||||
|
||||
assert_equal "compat/upstream-1.18.3-#{'a' * 64}", base
|
||||
refute_equal base, changed
|
||||
assert_equal "#{base}-r123-2", retry_branch
|
||||
refute run_status(script, "v1.18.3", digest, "bad/value").success?
|
||||
end
|
||||
|
||||
def test_workflow_can_only_update_manifests_and_open_a_pr
|
||||
workflow = File.read(File.join(ROOT, ".github/workflows/watch-upstream.yml"))
|
||||
|
||||
assert_includes workflow, "contents: write"
|
||||
assert_includes workflow, "pull-requests: write"
|
||||
assert_includes workflow, "git add manifests/image-matrix.json manifests/upstream.json"
|
||||
assert_includes workflow, "gh pr create"
|
||||
assert_includes workflow, "scripts/upstream_branch_name.rb"
|
||||
assert_includes workflow, "git ls-remote --exit-code --heads"
|
||||
assert_includes workflow, "--base main"
|
||||
assert_includes workflow, ".baseRefName == \"main\""
|
||||
assert_includes workflow, "isCrossRepository == false"
|
||||
assert_includes workflow, "headRefOid"
|
||||
assert_includes workflow, "application/vnd.github.raw+json"
|
||||
assert_includes workflow, "current_image_count"
|
||||
assert_includes workflow, '[.public_ci[]? | select(.image == $image)] | length'
|
||||
assert_includes workflow, '[ "$current_image_count" -eq 1 ]'
|
||||
assert_includes workflow, ".release_tag == $tag and .image == $image"
|
||||
assert_includes workflow, "([.public_ci[]? | select(.image == $image)] | length) == 1"
|
||||
content_check = workflow.index(".release_tag == $tag and .image == $image")
|
||||
skip = workflow.index('echo "skip=true"')
|
||||
assert_operator content_check, :<, skip
|
||||
refute_match(/gh\s+pr\s+merge|repository_dispatch|workflow_dispatches|git\s+push\s+--force/i, workflow)
|
||||
refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service)\b/i, workflow)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def run_recorder!(*arguments)
|
||||
run_script!(File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments)
|
||||
end
|
||||
|
||||
def recorder_status(*arguments)
|
||||
run_status(File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments)
|
||||
end
|
||||
|
||||
def capture_recorder(*arguments)
|
||||
Open3.capture3(RUBY, File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments)
|
||||
end
|
||||
|
||||
def run_script!(script, *arguments)
|
||||
output, error, status = Open3.capture3(RUBY, script, *arguments)
|
||||
assert status.success?, error
|
||||
output
|
||||
end
|
||||
|
||||
def run_status(script, *arguments)
|
||||
_output, _error, status = Open3.capture3(RUBY, script, *arguments)
|
||||
status
|
||||
end
|
||||
|
||||
def read_json(relative_path)
|
||||
JSON.parse(File.read(File.join(@tmp, relative_path)))
|
||||
end
|
||||
|
||||
def write_json(relative_path, value)
|
||||
File.write(File.join(@tmp, relative_path), JSON.pretty_generate(value) + "\n")
|
||||
end
|
||||
|
||||
def watched_bytes
|
||||
%w[manifests/upstream.json manifests/image-matrix.json].map do |relative_path|
|
||||
File.binread(File.join(@tmp, relative_path))
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user