ajaynomics/opencode-compat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden runtime tuple recertification policy

Browse Source
This commit is contained in:
Ajay Krishnan
2026-07-18 16:46:56 -07:00
parent 84c0d80b76
commit 0d0ab1f23f
20 changed files with 1753 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
test/repository_test.rb
View File

@@ -12,7 +12,7 @@ class RepositoryTest < Minitest::Test
end
def test_every_json_document_parses
paths = Dir.glob(File.join(ROOT, "{evidence,fixtures,manifests,profiles}/**/*.json"))
paths = Dir.glob(File.join(ROOT, "{evidence,fixtures,manifests,profiles,schemas}/**/*.json"))
refute_empty paths
paths.each { |path| JSON.parse(File.read(path)) }
end
@@ -36,6 +36,19 @@ class RepositoryTest < Minitest::Test
end
end
def test_only_ruby_rest_sse_can_be_certified_by_the_shared_ruby_probe
Dir.glob(File.join(ROOT, "profiles/*.json")).each do |path|
profile = JSON.parse(File.read(path))
if profile.fetch("id") == "ruby-rest-sse"
assert_equal "shared-ruby-fixture-and-live-probe", profile.fetch("certification_mode")
assert_equal true, profile.fetch("shared_ruby_probe_sufficient")
else
assert_equal "executable-consumer-attestation", profile.fetch("certification_mode")
assert_equal false, profile.fetch("shared_ruby_probe_sufficient")
end
end
end
def test_public_matrix_uses_only_immutable_oci_digests
targets = json("manifests/image-matrix.json").fetch("public_ci")
refute_empty targets
@@ -78,6 +91,10 @@ class RepositoryTest < Minitest::Test
def test_runtime_tuple_profiles_exist
tuples = json("manifests/runtime-tuples.json")
schema = json("schemas/runtime-tuples.schema.json")
assert_equal 2, tuples.fetch("schema_version")
assert_equal 2, schema.dig("properties", "schema_version", "const")
tuples.fetch("consumers").each_value do |consumer|
profile = consumer.fetch("profile")
assert_path_exists File.join(ROOT, "profiles", "#{profile}.json")
@@ -120,6 +137,96 @@ class RepositoryTest < Minitest::Test
end
end
def test_pull_request_heads_are_explicitly_candidate_only
tuples = json("manifests/runtime-tuples.json")
tuples.fetch("consumers").each do |name, consumer|
candidate = consumer.fetch("candidate")
reference = candidate.fetch("consumer_ref")
assert_equal "pre-merge-pr-head-candidate-only", candidate.fetch("certification_scope"), name
assert_equal false, candidate.fetch("promotion_eligible"), name
assert_equal "pull-request-head", reference.fetch("kind"), name
assert_equal candidate.fetch("consumer_commit"), reference.fetch("commit"), name
assert_match(/\A[0-9a-f]{40}\z/, reference.fetch("tree"), name)
assert_match(%r{\Ahttps://.+/pulls/\d+\z}, reference.fetch("review_url"), name)
end
end
def test_candidate_gems_use_peeled_refs_and_loaded_source_proof
tuples = json("manifests/runtime-tuples.json")
tuples.fetch("consumers").each do |consumer_name, consumer|
candidate = consumer.fetch("candidate")
%w[opencode_ruby opencode_rails].each do |client_name|
client = candidate[client_name]
next unless client
commit = client.fetch("git_commit")
source = client.fetch("source")
proof = source.fetch("loaded_source_proof")
assert_equal "git", source.fetch("type"), "#{consumer_name} #{client_name}"
assert_equal commit, source.fetch("requested_ref"), "#{consumer_name} #{client_name}"
assert_equal commit, source.fetch("locked_revision"), "#{consumer_name} #{client_name}"
assert_equal "pass", proof.fetch("status"), "#{consumer_name} #{client_name}"
assert_equal "Bundler::Source::Git", proof.fetch("source_class"), "#{consumer_name} #{client_name}"
assert_equal client.fetch("version"), proof.fetch("loaded_version"), "#{consumer_name} #{client_name}"
assert_equal commit, proof.fetch("observed_ref"), "#{consumer_name} #{client_name}" if proof.key?("observed_ref")
assert_equal commit, proof.fetch("observed_revision"), "#{consumer_name} #{client_name}"
refute_empty proof.fetch("test"), "#{consumer_name} #{client_name}"
end
end
end
def test_ajent_records_current_selection_drift_and_exact_candidate_selection
ajent = json("manifests/runtime-tuples.json").fetch("consumers").fetch("ajent-rails")
expected_products = %w[aigl blackline raven]
current_runtime = ajent.fetch("current").fetch("runtime")
assert_match(/\Asha256:[0-9a-f]{64}\z/, current_runtime.fetch("docker_image_id"))
deployed = current_runtime.fetch("deployed_product_selection")
assert_equal "mutable-latest", deployed.fetch("strategy")
assert_equal "observed-configuration-drift-failure", deployed.fetch("status")
assert_equal expected_products, deployed.fetch("references").keys.sort
deployed.fetch("references").each_value { |reference| assert reference.end_with?(":latest") }
built = current_runtime.fetch("ci_built_product_images")
assert_equal expected_products, built.keys.sort
built.each_value do |coordinate|
assert_match(/\Asha256:[0-9a-f]{64}\z/, coordinate.fetch("docker_image_id"))
assert_equal ajent.dig("current", "consumer_commit"), coordinate.fetch("source_commit")
end
candidate_runtime = ajent.fetch("candidate").fetch("runtime")
selected = candidate_runtime.fetch("product_selection")
assert_equal expected_products, selected.fetch("references").keys.sort
selected.fetch("references").each_value do |reference|
assert reference.end_with?(":#{ajent.dig('candidate', 'consumer_commit')}")
refute reference.end_with?(":latest")
end
assert_equal ["blackline"], candidate_runtime.fetch("product_images").keys
assert_includes ajent.dig("candidate", "promotion_blockers"),
"aigl-and-raven-candidate-content-artifacts-have-not-been-built-and-canaried"
end
def test_manifest_candidates_cannot_be_promoted_from_pre_merge_evidence
promoter = OpenCodeCompat::RuntimeTuplePromoter.new(root: ROOT)
tuples = json("manifests/runtime-tuples.json")
tuples.fetch("consumers").each do |name, consumer|
error = assert_raises(OpenCodeCompat::PromotionError) do
promoter.promote(
consumer: name,
consumer_commit: consumer.dig("candidate", "consumer_commit"),
certification: {},
dry_run: true
)
end
assert_match(/pre-merge pull-request evidence is candidate-only/, error.message)
end
end
def test_candidate_evidence_is_bound_to_complete_tuple_fingerprints
promoter = OpenCodeCompat::RuntimeTuplePromoter.new(root: ROOT)
evidence_paths = {
@@ -138,6 +245,8 @@ class RepositoryTest < Minitest::Test
assert_equal consumer, evidence.fetch("consumer")
assert_equal fingerprints.fetch("profile"), evidence.fetch("profile")
assert_equal fingerprints.fetch("candidate_tuple_sha256"), evidence.fetch("tuple_sha256")
assert_equal "pre-merge-pr-head-candidate-only", evidence.fetch("certification_scope")
assert_equal false, evidence.fetch("promotion_eligible")
assert_equal "pass", evidence.fetch("status")
end
end

204
test/runtime_tuple_promoter_test.rb
View File

@@ -18,6 +18,7 @@ class RuntimeTuplePromoterTest < Minitest::Test
CANDIDATE_IMAGE = "ghcr.io/anomalyco/opencode@sha256:#{'b' * 64}"
CURRENT_TIME = "2026-07-17T12:00:00Z"
CANDIDATE_TIME = "2026-07-18T12:00:00Z"
CANDIDATE_TREE = "7" * 40
def setup
@root = Dir.mktmpdir("opencode-compat-promotion")
@@ -25,6 +26,7 @@ class RuntimeTuplePromoterTest < Minitest::Test
FileUtils.mkdir_p(File.join(@root, "evidence"))
FileUtils.mkdir_p(File.join(@root, "profiles"))
File.write(File.join(@root, "profiles", "rails-persisted-turn.json"), "{}\n")
write_post_merge_canary_evidence("post-merge-canary.json", CANDIDATE_COMMIT)
write_manifest(valid_manifest)
@promoter = OpenCodeCompat::RuntimeTuplePromoter.new(root: @root)
end
@@ -175,6 +177,154 @@ class RuntimeTuplePromoterTest < Minitest::Test
assert_equal before, File.binread(manifest_path)
end
def test_rejects_pre_merge_pull_request_evidence_as_candidate_only
manifest = valid_manifest
candidate = manifest.dig("consumers", CONSUMER, "candidate")
candidate["certification_scope"] = "pre-merge-pr-head-candidate-only"
candidate["promotion_eligible"] = false
candidate["consumer_ref"] = {
"kind" => "pull-request-head",
"repository" => "example/consumer",
"commit" => CANDIDATE_COMMIT,
"tree" => CANDIDATE_TREE,
"base_commit" => CURRENT_COMMIT,
"review_url" => "https://example.test/pulls/1"
}
candidate.delete("promotion_provenance")
write_manifest(manifest)
candidate_evidence, previous_evidence = write_matching_evidence
error = assert_raises(OpenCodeCompat::PromotionError) do
@promoter.promote(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT,
certification: certification(CANDIDATE_TIME, candidate_evidence),
previous_certification: certification(CURRENT_TIME, previous_evidence),
dry_run: true
)
end
assert_match(/pre-merge pull-request evidence is candidate-only/, error.message)
end
def test_accepts_explicit_identical_tree_attestation_with_post_merge_canary
manifest = valid_manifest
candidate = manifest.dig("consumers", CONSUMER, "candidate")
candidate["consumer_ref"] = {
"kind" => "pull-request-head",
"repository" => "example/consumer",
"commit" => CANDIDATE_COMMIT,
"tree" => CANDIDATE_TREE,
"base_commit" => CURRENT_COMMIT,
"review_url" => "https://example.test/pulls/1"
}
main_commit = "8" * 40
write_post_merge_canary_evidence("post-merge-attested-canary.json", main_commit)
candidate["promotion_provenance"] = {
"kind" => "identical-tree-attestation",
"pull_request_commit" => CANDIDATE_COMMIT,
"pull_request_tree" => CANDIDATE_TREE,
"main_commit" => main_commit,
"main_tree" => CANDIDATE_TREE,
"attested_at" => CANDIDATE_TIME,
"post_merge_canary" => {
"status" => "pass",
"checked_at" => CANDIDATE_TIME,
"evidence" => ["evidence/post-merge-attested-canary.json"]
}
}
write_manifest(manifest)
candidate_evidence, previous_evidence = write_matching_evidence
promoted = @promoter.promote(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT,
certification: certification(CANDIDATE_TIME, candidate_evidence),
previous_certification: certification(CURRENT_TIME, previous_evidence),
dry_run: true
)
assert_equal "certified", promoted.dig("consumers", CONSUMER, "current", "status")
end
def test_main_commit_promotion_still_requires_a_post_merge_canary
manifest = valid_manifest
manifest.dig("consumers", CONSUMER, "candidate", "promotion_provenance").delete("post_merge_canary")
write_manifest(manifest)
error = assert_raises(OpenCodeCompat::PromotionError) do
@promoter.promote(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT,
certification: {},
dry_run: true
)
end
assert_match(/passing post-merge canary/, error.message)
end
def test_rejects_candidate_without_loaded_exact_ref_source_proof
manifest = valid_manifest
manifest.dig("consumers", CONSUMER, "candidate", "opencode_ruby").delete("source")
write_manifest(manifest)
error = assert_raises(OpenCodeCompat::PromotionError) do
@promoter.fingerprints(consumer: CONSUMER, consumer_commit: CANDIDATE_COMMIT)
end
assert_match(/loaded exact-ref source proof/, error.message)
end
def test_explicit_degraded_bootstrap_certifies_current_without_faking_previous
manifest = valid_manifest
failed = manifest.dig("consumers", CONSUMER, "current")
failed["status"] = "observed-production-contract-failed"
write_manifest(manifest)
fingerprint = @promoter.fingerprints(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT
).fetch("candidate_tuple_sha256")
evidence = write_evidence(
"bootstrap-candidate.json",
commit: CANDIDATE_COMMIT,
timestamp: CANDIDATE_TIME,
fingerprint: fingerprint
)
bootstrapped = @promoter.bootstrap_current(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT,
certification: certification(CANDIDATE_TIME, evidence),
acknowledgement: OpenCodeCompat::RuntimeTuplePromoter::DEGRADED_BOOTSTRAP_ACKNOWLEDGEMENT,
dry_run: true
)
consumer = bootstrapped.dig("consumers", CONSUMER)
assert_equal "certified", consumer.dig("current", "status")
assert_equal CANDIDATE_COMMIT, consumer.dig("current", "consumer_commit")
assert_nil consumer["candidate"]
assert_nil consumer["previous"]
assert_equal "observed-production-contract-failed", consumer.dig("emergency_provenance", "status")
assert_nil consumer.dig("emergency_provenance", "certification")
assert_equal "degraded-no-certified-previous", consumer.dig("rollback_state", "status")
assert_equal "bootstrap-current-only", bootstrapped.fetch("migration_state")
end
def test_degraded_bootstrap_requires_exact_explicit_acknowledgement
error = assert_raises(OpenCodeCompat::PromotionError) do
@promoter.bootstrap_current(
consumer: CONSUMER,
consumer_commit: CANDIDATE_COMMIT,
certification: {},
acknowledgement: "yes",
dry_run: true
)
end
assert_match(/explicit acknowledgement/, error.message)
end
def test_rejects_evidence_that_does_not_match_the_complete_tuple
candidate, previous = write_matching_evidence
manifest = JSON.parse(File.read(manifest_path))
@@ -254,7 +404,7 @@ class RuntimeTuplePromoterTest < Minitest::Test
def valid_manifest
{
"schema_version" => 1,
"schema_version" => 2,
"migration_state" => "candidate",
"consumers" => {
CONSUMER => {
@@ -269,8 +419,25 @@ class RuntimeTuplePromoterTest < Minitest::Test
"candidate" => {
"status" => "compatibility-certified",
"certified_at" => "2026-07-18T10:00:00Z",
"opencode_ruby" => {"version" => "0.0.1.alpha7", "git_commit" => RUBY_CANDIDATE},
"opencode_rails" => {"version" => "0.0.1.alpha7", "git_commit" => RAILS_CANDIDATE},
"certification_scope" => "promotion-deployed",
"promotion_eligible" => true,
"consumer_ref" => {
"kind" => "main-commit",
"repository" => "example/consumer",
"commit" => CANDIDATE_COMMIT,
"tree" => CANDIDATE_TREE
},
"promotion_provenance" => {
"kind" => "main-commit",
"main_commit" => CANDIDATE_COMMIT,
"post_merge_canary" => {
"status" => "pass",
"checked_at" => CANDIDATE_TIME,
"evidence" => ["evidence/post-merge-canary.json"]
}
},
"opencode_ruby" => client("0.0.1.alpha7", RUBY_CANDIDATE),
"opencode_rails" => client("0.0.1.alpha7", RAILS_CANDIDATE),
"runtime" => {"image" => CANDIDATE_IMAGE, "reported_version" => "1.18.3"}
},
"previous" => nil
@@ -279,6 +446,27 @@ class RuntimeTuplePromoterTest < Minitest::Test
}
end
def client(version, commit)
{
"version" => version,
"git_commit" => commit,
"source" => {
"type" => "git",
"uri" => "https://example.test/client.git",
"requested_ref" => commit,
"locked_revision" => commit,
"loaded_source_proof" => {
"status" => "pass",
"source_class" => "Bundler::Source::Git",
"loaded_version" => version,
"observed_ref" => commit,
"observed_revision" => commit,
"test" => "test/dependency_provenance_test.rb"
}
}
}
end
def write_matching_evidence
fingerprints = @promoter.fingerprints(consumer: CONSUMER, consumer_commit: CANDIDATE_COMMIT)
candidate = write_evidence(
@@ -311,6 +499,16 @@ class RuntimeTuplePromoterTest < Minitest::Test
{"path" => path, "tuple_sha256" => fingerprint}
end
def write_post_merge_canary_evidence(name, commit)
document = {
"schema_version" => 1,
"status" => "pass",
"checked_at" => CANDIDATE_TIME,
"consumer_commit" => commit
}
File.write(File.join(@root, "evidence", name), JSON.pretty_generate(document) + "\n")
end
def certification(timestamp, evidence)
{
"status" => "pass",