Harden OpenCode compatibility CI and watcher

2026-07-18 23:07:50 -07:00
parent 84c0d80b76
commit 105550a8b4
19 changed files with 1906 additions and 139 deletions
--- a/scripts/matrix_json.rb
+++ b/scripts/matrix_json.rb
@@ -5,10 +5,25 @@ require "json"
 root = File.expand_path("..", __dir__)
 manifest = JSON.parse(File.read(File.join(root, "manifests/image-matrix.json")))
 matrix = manifest.fetch("public_ci").map do |target|
+  id = target.fetch("id")
+  profiles = target.fetch("profiles")
+  certification_scope = target.fetch("certification_scope", "shared-client-contract-only")
+  unless profiles == ["ruby-rest-sse"]
+    abort "#{id}: public CI executes exactly ruby-rest-sse; got profiles=#{profiles.inspect}"
+  end
+  unless certification_scope == "shared-client-contract-only"
+    abort "#{id}: certification_scope must be shared-client-contract-only; got #{certification_scope.inspect}"
+  end
+
  {
-    "id" => target.fetch("id"),
+    "id" => id,
    "version" => target.fetch("version"),
-    "image" => target.fetch("image")
+    "image" => target.fetch("image"),
+    "consumers" => target.fetch("consumers"),
+    "profiles" => profiles,
+    "required_consumer_profiles" => target.fetch("required_consumer_profiles"),
+    "certification_scope" => certification_scope,
+    "certification_status" => target.fetch("certification_status")
  }
 end

--- a/scripts/record_upstream_candidate.rb
+++ b/scripts/record_upstream_candidate.rb
@@ -8,6 +8,13 @@ tag, published_at, release_url, digest = ARGV
 abort "usage: record_upstream_candidate.rb TAG PUBLISHED_AT RELEASE_URL DIGEST" unless digest
 abort "invalid upstream release tag: #{tag.inspect}" unless tag.match?(/\Av\d+\.\d+\.\d+\z/)
 abort "invalid OCI digest: #{digest.inspect}" unless digest.match?(/\Asha256:[0-9a-f]{64}\z/)
+begin
+  Time.iso8601(published_at)
+rescue ArgumentError, TypeError
+  abort "invalid upstream publication timestamp: #{published_at.inspect}"
+end
+expected_release_url = "https://github.com/anomalyco/opencode/releases/tag/#{tag}"
+abort "invalid upstream release URL: #{release_url.inspect}" unless release_url == expected_release_url

 version = tag.delete_prefix("v")
 image = "ghcr.io/anomalyco/opencode@#{digest}"
@@ -15,30 +22,46 @@ upstream_path = File.join(root, "manifests/upstream.json")
 matrix_path = File.join(root, "manifests/image-matrix.json")
 upstream = JSON.parse(File.read(upstream_path))
 matrix = JSON.parse(File.read(matrix_path))
+public_ci = matrix.fetch("public_ci")
+abort "public_ci must be an array" unless public_ci.is_a?(Array)

-exit 0 if upstream.fetch("release_tag") == tag
+matching_targets = public_ci.select { |target| target.fetch("image") == image }
+if matching_targets.length > 1
+  abort "image #{image} must appear exactly once in public_ci; found #{matching_targets.length}"
+end

-upstream.merge!(
-  "release_tag" => tag,
-  "version" => version,
-  "published_at" => published_at,
-  "release_url" => release_url,
-  "image" => image,
-  "observed_at" => Time.now.utc.iso8601
-)
+matrix_changed = false
+if matching_targets.empty?
+  target_id = "upstream-#{version}-#{digest.delete_prefix('sha256:')}"
+  if public_ci.any? { |target| target.fetch("id") == target_id }
+    abort "cannot add #{image}: public_ci id #{target_id.inspect} is already used by another image"
+  end

-unless matrix.fetch("public_ci").any? { |target| target.fetch("image") == image }
-  matrix.fetch("public_ci") << {
-    "id" => "upstream-#{version}",
+  public_ci << {
+    "id" => target_id,
    "version" => version,
    "image" => image,
    "tag_provenance" => "ghcr.io/anomalyco/opencode:#{version}",
-    "consumers" => ["upstream-candidate"],
+    "consumers" => [],
    "profiles" => ["ruby-rest-sse"],
-    "required_consumer_profiles" => ["rails-persisted-turn", "voice-stream", "strict-v2", "plugin-ledger", "provider-hooks"],
+    "required_consumer_profiles" => [],
+    "certification_scope" => "shared-client-contract-only",
    "certification_status" => "pending"
  }
+  matrix_changed = true
 end

-File.write(upstream_path, JSON.pretty_generate(upstream) + "\n")
-File.write(matrix_path, JSON.pretty_generate(matrix) + "\n")
+upstream_changed = upstream.fetch("release_tag") != tag || upstream.fetch("image") != image
+if upstream_changed
+  upstream.merge!(
+    "release_tag" => tag,
+    "version" => version,
+    "published_at" => published_at,
+    "release_url" => release_url,
+    "image" => image,
+    "observed_at" => Time.now.utc.iso8601
+  )
+end
+
+File.write(upstream_path, JSON.pretty_generate(upstream) + "\n") if upstream_changed
+File.write(matrix_path, JSON.pretty_generate(matrix) + "\n") if matrix_changed
--- a/scripts/run_image_contract.sh
+++ b/scripts/run_image_contract.sh
@@ -2,38 +2,121 @@
 set -euo pipefail

 repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-image="${OPENCODE_IMAGE:?set OPENCODE_IMAGE to an immutable image digest}"
-gem_path="${OPENCODE_RUBY_PATH:?set OPENCODE_RUBY_PATH to the candidate checkout}"
+image="${OPENCODE_IMAGE:-}"
+gem_path="${OPENCODE_RUBY_PATH:-}"
+expected_gem_commit="${OPENCODE_RUBY_COMMIT:-}"
+evidence_path="${OPENCODE_COMPAT_EVIDENCE_PATH:-}"
+container_name="opencode-compat-${RANDOM}-$$"
+llm_container_name="opencode-compat-llm-${RANDOM}-$$"
+network_name="opencode-compat-net-${RANDOM}-$$"
+python_image="python@sha256:399babc8b49529dabfd9c922f2b5eea81d611e4512e3ed250d75bd2e7683f4b0"
+network_created=0
+llm_container_started=0
+opencode_container_started=0
+gem_commit=""

-if [[ "$image" != *@sha256:* ]]; then
-  if [[ "${ALLOW_EXACT_IMAGE_ID:-0}" != "1" || "$image" != sha256:* ]]; then
+write_failure_evidence() {
+  local exit_status="$1"
+  [[ -n "$evidence_path" ]] || return 0
+
+  mkdir -p "$(dirname "$evidence_path")"
+  jq -n \
+    --argjson schema_version 1 \
+    --arg kind shared-client-image-contract \
+    --arg status fail \
+    --arg checked_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    --arg matrix_target "${OPENCODE_MATRIX_ID:-local}" \
+    --arg image "$image" \
+    --arg ruby_commit "${gem_commit:-}" \
+    --arg rails_commit "${OPENCODE_RAILS_COMMIT:-}" \
+    --arg run_id "${OPENCODE_COMPAT_RUN_ID:-}" \
+    --arg run_attempt "${OPENCODE_COMPAT_RUN_ATTEMPT:-}" \
+    --arg head_sha "${OPENCODE_COMPAT_HEAD_SHA:-}" \
+    --arg repository "${OPENCODE_COMPAT_REPOSITORY:-}" \
+    --arg run_url "${OPENCODE_COMPAT_RUN_URL:-}" \
+    --argjson exit_status "$exit_status" \
+    '{
+      schema_version:$schema_version,
+      kind:$kind,
+      status:$status,
+      checked_at:$checked_at,
+      matrix_target:$matrix_target,
+      certification_scope:"shared-client-contract-only",
+      executed_profiles:[],
+      image:{requested:$image},
+      clients:{opencode_ruby:{commit:$ruby_commit},opencode_rails:{commit:$rails_commit,executed:false}},
+      failure:{exit_status:$exit_status},
+      workflow:{run_id:$run_id,run_attempt:$run_attempt,head_sha:$head_sha,repository:$repository,run_url:$run_url}
+    }' >"$evidence_path"
+}
+
+cleanup() {
+  local status=$?
+  trap - EXIT
+  if [[ "$status" != "0" ]]; then
+    echo "OpenCode compatibility probe failed; container log follows" >&2
+    if [[ "$opencode_container_started" == "1" ]]; then
+      docker logs "$container_name" >&2 2>/dev/null || true
+    fi
+    echo "Deterministic model request summary follows" >&2
+    if [[ "$llm_container_started" == "1" ]]; then
+      docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats >&2 2>/dev/null || true
+    fi
+    echo >&2
+    write_failure_evidence "$status" || true
+  fi
+  if [[ "$opencode_container_started" == "1" ]]; then
+    docker rm -f "$container_name" >/dev/null 2>&1 || true
+  fi
+  if [[ "$llm_container_started" == "1" ]]; then
+    docker rm -f "$llm_container_name" >/dev/null 2>&1 || true
+  fi
+  if [[ "$network_created" == "1" ]]; then
+    docker network rm "$network_name" >/dev/null 2>&1 || true
+  fi
+  exit "$status"
+}
+trap cleanup EXIT
+
+if [[ -n "$evidence_path" ]]; then
+  mkdir -p "$(dirname "$evidence_path")"
+  : >"$evidence_path"
+fi
+
+if [[ -z "$image" ]]; then
+  echo "set OPENCODE_IMAGE to an immutable image digest" >&2
+  exit 2
+fi
+if [[ -z "$gem_path" ]]; then
+  echo "set OPENCODE_RUBY_PATH to the candidate checkout" >&2
+  exit 2
+fi
+if [[ ! "$image" =~ ^[^[:space:]@]+@sha256:[0-9a-f]{64}$ ]]; then
+  if [[ "${ALLOW_EXACT_IMAGE_ID:-0}" != "1" || ! "$image" =~ ^sha256:[0-9a-f]{64}$ ]]; then
    echo "OPENCODE_IMAGE must be an OCI digest (or an exact local image ID with ALLOW_EXACT_IMAGE_ID=1)" >&2
    exit 2
  fi
 fi

-container_name="opencode-compat-${RANDOM}-$$"
-llm_container_name="opencode-compat-llm-${RANDOM}-$$"
-network_name="opencode-compat-net-${RANDOM}-$$"
-python_image="python@sha256:399babc8b49529dabfd9c922f2b5eea81d611e4512e3ed250d75bd2e7683f4b0"
-
-cleanup() {
-  status=$?
-  if [[ "$status" != "0" ]]; then
-    echo "OpenCode compatibility probe failed; container log follows" >&2
-    docker logs "$container_name" >&2 2>/dev/null || true
-    echo "Deterministic model request summary follows" >&2
-    docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats >&2 2>/dev/null || true
-    echo >&2
-  fi
-  docker rm -f "$container_name" >/dev/null 2>&1 || true
-  docker rm -f "$llm_container_name" >/dev/null 2>&1 || true
-  docker network rm "$network_name" >/dev/null 2>&1 || true
-  return "$status"
-}
-trap cleanup EXIT
+if [[ ! "$expected_gem_commit" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "OPENCODE_RUBY_COMMIT must be a full lowercase 40-character Git commit" >&2
+  exit 2
+fi
+if ! gem_commit="$(git -C "$gem_path" rev-parse --verify 'HEAD^{commit}' 2>/dev/null)"; then
+  echo "OPENCODE_RUBY_PATH must be a readable Git checkout" >&2
+  exit 2
+fi
+if [[ "$gem_commit" != "$expected_gem_commit" ]]; then
+  echo "opencode-ruby checkout HEAD $gem_commit does not match expected commit $expected_gem_commit" >&2
+  exit 2
+fi
+if [[ -n "$(git -C "$gem_path" status --porcelain --untracked-files=all)" ]]; then
+  echo "opencode-ruby checkout must be clean before certification" >&2
+  exit 2
+fi

 docker network create "$network_name" >/dev/null
+network_created=1
 docker run --detach \
  --name "$llm_container_name" \
  --network "$network_name" \
@@ -42,6 +125,7 @@ docker run --detach \
  "$python_image" \
  python /compat/fake_llm.py --port 8080 --port-file /tmp/compat-port \
  >/dev/null
+llm_container_started=1

 for _ in $(seq 1 100); do
  if docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/health >/dev/null 2>&1; then
@@ -93,6 +177,7 @@ docker run --detach \
  --env OPENCODE_PURE=1 \
  "$image" serve --hostname 0.0.0.0 --port 4096 \
  >/dev/null
+opencode_container_started=1

 host_port="$(docker port "$container_name" 4096/tcp | sed -E 's/.*:([0-9]+)$/\1/' | head -1)"
 base_url="http://127.0.0.1:${host_port}"
@@ -118,22 +203,75 @@ fi

 server_version="$(docker exec "$container_name" opencode --version)"
 image_id="$(docker inspect "$container_name" --format '{{.Image}}')"
-gem_commit="$(git -C "$gem_path" rev-parse HEAD 2>/dev/null || true)"
+image_os="$(docker image inspect "$image_id" --format '{{.Os}}')"
+image_architecture="$(docker image inspect "$image_id" --format '{{.Architecture}}')"
+image_platform="${image_os}/${image_architecture}"
+expected_version="${OPENCODE_EXPECTED_VERSION:-}"

-OPENCODE_BASE_URL="$base_url" \
-OPENCODE_RUBY_PATH="$gem_path" \
-OPENCODE_RUBY_COMMIT="$gem_commit" \
-  ruby "$repo_root/ruby/live_probe.rb"
+if [[ -n "$expected_version" && "$server_version" != "$expected_version" ]]; then
+  echo "OpenCode reported version $server_version; expected $expected_version" >&2
+  exit 1
+fi
+
+live_contract="$(
+  OPENCODE_BASE_URL="$base_url" \
+  OPENCODE_RUBY_PATH="$gem_path" \
+  OPENCODE_RUBY_COMMIT="$gem_commit" \
+    ruby "$repo_root/ruby/live_probe.rb"
+)"
+printf '%s\n' "$live_contract"

 llm_stats="$(docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats)"
 request_count="$(jq -r '.request_count' <<<"$llm_stats")"
 ruby "$repo_root/ruby/exact_live_contract.rb" "$request_count"

-jq -cn \
+evidence="$(jq -cn \
+  --argjson schema_version 1 \
  --arg status pass \
+  --arg kind shared-client-image-contract \
+  --arg matrix_target "${OPENCODE_MATRIX_ID:-local}" \
+  --arg certification_scope "shared-client-contract-only" \
+  --argjson executed_profiles '["ruby-rest-sse"]' \
+  --argjson required_consumer_profiles "${OPENCODE_REQUIRED_CONSUMER_PROFILES:-[]}" \
  --arg image "$image" \
  --arg image_id "$image_id" \
+  --arg image_platform "$image_platform" \
  --arg server_version "$server_version" \
  --arg gem_commit "$gem_commit" \
+  --arg companion_rails_commit "${OPENCODE_RAILS_COMMIT:-}" \
+  --arg run_id "${OPENCODE_COMPAT_RUN_ID:-}" \
+  --arg run_attempt "${OPENCODE_COMPAT_RUN_ATTEMPT:-}" \
+  --arg head_sha "${OPENCODE_COMPAT_HEAD_SHA:-}" \
+  --arg repository "${OPENCODE_COMPAT_REPOSITORY:-}" \
+  --arg run_url "${OPENCODE_COMPAT_RUN_URL:-}" \
+  --argjson live_contract "$live_contract" \
  --argjson llm_request_count "$request_count" \
-  '{status:$status,image:$image,image_id:$image_id,server_version:$server_version,opencode_ruby_commit:$gem_commit,llm_request_count:$llm_request_count}'
+  '{
+    schema_version:$schema_version,
+    kind:$kind,
+    status:$status,
+    checked_at:$live_contract.checked_at,
+    matrix_target:$matrix_target,
+    certification_scope:$certification_scope,
+    executed_profiles:$executed_profiles,
+    required_consumer_profiles:$required_consumer_profiles,
+    image:{requested:$image,docker_image_id:$image_id,platform:$image_platform,reported_version:$server_version},
+    clients:{
+      opencode_ruby:{version:$live_contract.opencode_ruby_version,commit:$gem_commit,executed:true},
+      opencode_rails:{commit:$companion_rails_commit,executed:false}
+    },
+    contract:{
+      expected_text:$live_contract.expected_text,
+      full_text:$live_contract.full_text,
+      observed_part_count:$live_contract.observed_part_count,
+      authoritative_assistant_message_count:$live_contract.authoritative_assistant_message_count,
+      llm_request_count:$llm_request_count
+    },
+    workflow:{run_id:$run_id,run_attempt:$run_attempt,head_sha:$head_sha,repository:$repository,run_url:$run_url}
+  }'
+)"
+printf '%s\n' "$evidence"
+
+if [[ -n "$evidence_path" ]]; then
+  printf '%s\n' "$evidence" >"$evidence_path"
+fi
--- a/scripts/upstream_branch_name.rb
+++ b/scripts/upstream_branch_name.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+tag = ARGV.fetch(0)
+digest = ARGV.fetch(1)
+disambiguator = ARGV[2]
+
+abort "release tag must be vMAJOR.MINOR.PATCH" unless tag.match?(/\Av\d+\.\d+\.\d+\z/)
+abort "digest must be an exact sha256" unless digest.match?(/\Asha256:[0-9a-f]{64}\z/)
+
+branch = "compat/upstream-#{tag.delete_prefix('v')}-#{digest.delete_prefix('sha256:')}"
+if disambiguator
+  unless disambiguator.match?(/\A[A-Za-z0-9][A-Za-z0-9-]*\z/)
+    abort "disambiguator must contain only letters, digits, and hyphens"
+  end
+
+  branch = "#{branch}-r#{disambiguator}"
+end
+
+abort "generated branch is longer than 128 characters" if branch.length > 128
+
+puts branch