diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index ec171ab..cc2867a 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -6,65 +6,197 @@ on: pull_request: branches: [main] workflow_dispatch: - inputs: - opencode_ruby_ref: - description: Candidate opencode-ruby ref; blank uses the manifest - required: false - type: string + +permissions: + contents: read jobs: repository: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1 with: ruby-version: "3.4" - - run: ruby test/repository_test.rb && ruby test/runtime_tuple_promoter_test.rb && ruby test/exact_live_contract_test.rb + - run: | + ruby test/repository_test.rb + ruby test/runtime_tuple_promoter_test.rb + ruby test/exact_live_contract_test.rb + ruby test/watcher_test.rb + ruby test/lockstep_client_contract_test.rb + ruby test/matrix_json_test.rb + ruby test/image_contract_evidence_test.rb prepare: runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.value }} - client_ref: ${{ steps.client.outputs.value }} + ruby_ref: ${{ steps.clients.outputs.ruby_ref }} + ruby_version: ${{ steps.clients.outputs.ruby_version }} + ruby_tag: ${{ steps.clients.outputs.ruby_tag }} + ruby_tag_object: ${{ steps.clients.outputs.ruby_tag_object }} + rails_ref: ${{ steps.clients.outputs.rails_ref }} + rails_version: ${{ steps.clients.outputs.rails_version }} + rails_tag: ${{ steps.clients.outputs.rails_tag }} + rails_tag_object: ${{ steps.clients.outputs.rails_tag_object }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - id: matrix run: echo "value=$(ruby scripts/matrix_json.rb)" >> "$GITHUB_OUTPUT" - - id: client - env: - INPUT_REF: ${{ inputs.opencode_ruby_ref }} + - id: clients run: | - if [ -n "$INPUT_REF" ]; then - echo "value=$INPUT_REF" >> "$GITHUB_OUTPUT" - else - echo "value=$(jq -r .ref manifests/client-candidate.json)" >> "$GITHUB_OUTPUT" + ruby_ref="$(jq -r '.clients["opencode-ruby"].ref' manifests/client-candidate.json)" + ruby_version="$(jq -r '.clients["opencode-ruby"].version' manifests/client-candidate.json)" + ruby_tag="$(jq -r '.clients["opencode-ruby"].tag_provenance.tag' manifests/client-candidate.json)" + ruby_tag_object="$(jq -r '.clients["opencode-ruby"].tag_provenance.annotated_tag_object' manifests/client-candidate.json)" + rails_ref="$(jq -r '.clients["opencode-rails"].ref' manifests/client-candidate.json)" + rails_version="$(jq -r '.clients["opencode-rails"].version' manifests/client-candidate.json)" + rails_tag="$(jq -r '.clients["opencode-rails"].tag_provenance.tag' manifests/client-candidate.json)" + rails_tag_object="$(jq -r '.clients["opencode-rails"].tag_provenance.annotated_tag_object' manifests/client-candidate.json)" + if ! [[ "$ruby_ref" =~ ^[0-9a-f]{40}$ ]]; then + echo "opencode-ruby candidate ref must be a full 40-hex commit" >&2 + exit 2 fi + if ! [[ "$rails_ref" =~ ^[0-9a-f]{40}$ ]]; then + echo "opencode-rails candidate ref must be a full 40-hex commit" >&2 + exit 2 + fi + if ! [[ "$ruby_tag_object" =~ ^[0-9a-f]{40}$ && "$rails_tag_object" =~ ^[0-9a-f]{40}$ ]]; then + echo "candidate annotated tag objects must be full 40-hex object IDs" >&2 + exit 2 + fi + test "$ruby_tag" = "v$ruby_version" + test "$rails_tag" = "v$rails_version" + test "$ruby_version" = "$rails_version" + echo "ruby_ref=$ruby_ref" >> "$GITHUB_OUTPUT" + echo "ruby_version=$ruby_version" >> "$GITHUB_OUTPUT" + echo "ruby_tag=$ruby_tag" >> "$GITHUB_OUTPUT" + echo "ruby_tag_object=$ruby_tag_object" >> "$GITHUB_OUTPUT" + echo "rails_ref=$rails_ref" >> "$GITHUB_OUTPUT" + echo "rails_version=$rails_version" >> "$GITHUB_OUTPUT" + echo "rails_tag=$rails_tag" >> "$GITHUB_OUTPUT" + echo "rails_tag_object=$rails_tag_object" >> "$GITHUB_OUTPUT" fixture-contract: needs: prepare runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ajaynomics/opencode-ruby - ref: ${{ needs.prepare.outputs.client_ref }} - path: client - - uses: ruby/setup-ruby@v1 + ref: ${{ needs.prepare.outputs.ruby_ref }} + path: ruby-client + - uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1 with: ruby-version: "3.4" bundler-cache: true - working-directory: client + working-directory: ruby-client + - id: client-sha + name: Verify immutable Ruby candidate checkout + env: + EXPECTED_CLIENT_SHA: ${{ needs.prepare.outputs.ruby_ref }} + run: | + actual_client_sha="$(git -C ruby-client rev-parse HEAD)" + test "$actual_client_sha" = "$EXPECTED_CLIENT_SHA" + echo "value=$actual_client_sha" >> "$GITHUB_OUTPUT" - name: Candidate unit tests - working-directory: client + working-directory: ruby-client run: bundle exec rake test - name: Shared fixture contract - working-directory: client + working-directory: ruby-client env: - OPENCODE_RUBY_PATH: ${{ github.workspace }}/client - OPENCODE_RUBY_COMMIT: ${{ needs.prepare.outputs.client_ref }} - run: bundle exec ruby ../ruby/opencode_ruby_fixture_contract.rb + OPENCODE_RUBY_PATH: ${{ github.workspace }}/ruby-client + OPENCODE_RUBY_COMMIT: ${{ steps.client-sha.outputs.value }} + OPENCODE_COMPAT_EVIDENCE_PATH: ${{ github.workspace }}/.compat-evidence/ruby-fixtures.json + OPENCODE_COMPAT_RUN_ID: ${{ github.run_id }} + OPENCODE_COMPAT_RUN_ATTEMPT: ${{ github.run_attempt }} + OPENCODE_COMPAT_HEAD_SHA: ${{ github.sha }} + OPENCODE_COMPAT_REPOSITORY: ${{ github.repository }} + OPENCODE_COMPAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p ../.compat-evidence + bundle exec ruby ../ruby/opencode_ruby_fixture_contract.rb + - name: Upload fixture evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ruby-fixtures-${{ github.run_id }}-${{ github.run_attempt }} + path: .compat-evidence/ruby-fixtures.json + include-hidden-files: true + if-no-files-found: warn + retention-days: 30 + + lockstep-client-contract: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ruby: ["3.2", "3.3", "3.4"] + name: lockstep clients Ruby ${{ matrix.ruby }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ajaynomics/opencode-ruby + ref: ${{ needs.prepare.outputs.ruby_ref }} + path: ruby-client + fetch-depth: 0 + fetch-tags: true + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ajaynomics/opencode-rails + ref: ${{ needs.prepare.outputs.rails_ref }} + path: rails-client + fetch-depth: 0 + fetch-tags: true + - uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + working-directory: rails-client + - name: Verify immutable candidate checkouts + env: + EXPECTED_RUBY_SHA: ${{ needs.prepare.outputs.ruby_ref }} + EXPECTED_RAILS_SHA: ${{ needs.prepare.outputs.rails_ref }} + run: | + test "$(git -C ruby-client rev-parse HEAD)" = "$EXPECTED_RUBY_SHA" + test "$(git -C rails-client rev-parse HEAD)" = "$EXPECTED_RAILS_SHA" + - name: Rails candidate unit tests + working-directory: rails-client + run: bundle exec rake test + - name: Lockstep client provenance contract + working-directory: rails-client + env: + OPENCODE_RUBY_PATH: ${{ github.workspace }}/ruby-client + OPENCODE_RAILS_PATH: ${{ github.workspace }}/rails-client + OPENCODE_RUBY_COMMIT: ${{ needs.prepare.outputs.ruby_ref }} + OPENCODE_RUBY_TAG: ${{ needs.prepare.outputs.ruby_tag }} + OPENCODE_RUBY_TAG_OBJECT: ${{ needs.prepare.outputs.ruby_tag_object }} + OPENCODE_RAILS_COMMIT: ${{ needs.prepare.outputs.rails_ref }} + OPENCODE_RAILS_TAG: ${{ needs.prepare.outputs.rails_tag }} + OPENCODE_RAILS_TAG_OBJECT: ${{ needs.prepare.outputs.rails_tag_object }} + OPENCODE_RUBY_VERSION: ${{ needs.prepare.outputs.ruby_version }} + OPENCODE_RAILS_VERSION: ${{ needs.prepare.outputs.rails_version }} + OPENCODE_COMPAT_EVIDENCE_PATH: ${{ github.workspace }}/.compat-evidence/lockstep-ruby-${{ matrix.ruby }}.json + OPENCODE_COMPAT_RUN_ID: ${{ github.run_id }} + OPENCODE_COMPAT_RUN_ATTEMPT: ${{ github.run_attempt }} + OPENCODE_COMPAT_HEAD_SHA: ${{ github.sha }} + OPENCODE_COMPAT_REPOSITORY: ${{ github.repository }} + OPENCODE_COMPAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p ../.compat-evidence + bundle exec ruby ../ruby/lockstep_client_contract.rb + - name: Upload lockstep evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: lockstep-ruby-${{ matrix.ruby }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .compat-evidence/lockstep-ruby-${{ matrix.ruby }}.json + include-hidden-files: true + if-no-files-found: warn + retention-days: 30 exact-image-contract: needs: prepare @@ -74,20 +206,56 @@ jobs: matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} name: image ${{ matrix.id }} steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ajaynomics/opencode-ruby - ref: ${{ needs.prepare.outputs.client_ref }} - path: client - - uses: ruby/setup-ruby@v1 + ref: ${{ needs.prepare.outputs.ruby_ref }} + path: ruby-client + - uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1 with: ruby-version: "3.4" - bundler-cache: true - working-directory: client + - name: Install candidate dependencies outside the checkout + working-directory: ruby-client + env: + BUNDLE_APP_CONFIG: ${{ runner.temp }}/opencode-ruby-bundle-config + BUNDLE_PATH: ${{ runner.temp }}/opencode-ruby-bundle + run: bundle install --jobs 4 --retry 3 + - id: client-sha + name: Verify immutable Ruby candidate checkout + env: + EXPECTED_CLIENT_SHA: ${{ needs.prepare.outputs.ruby_ref }} + run: | + actual_client_sha="$(git -C ruby-client rev-parse HEAD)" + test "$actual_client_sha" = "$EXPECTED_CLIENT_SHA" + echo "value=$actual_client_sha" >> "$GITHUB_OUTPUT" - name: Exercise complete REST and SSE turn env: - BUNDLE_GEMFILE: ${{ github.workspace }}/client/Gemfile + BUNDLE_GEMFILE: ${{ github.workspace }}/ruby-client/Gemfile + BUNDLE_APP_CONFIG: ${{ runner.temp }}/opencode-ruby-bundle-config + BUNDLE_PATH: ${{ runner.temp }}/opencode-ruby-bundle OPENCODE_IMAGE: ${{ matrix.image }} - OPENCODE_RUBY_PATH: ${{ github.workspace }}/client - run: bundle exec scripts/run_image_contract.sh + OPENCODE_EXPECTED_VERSION: ${{ matrix.version }} + OPENCODE_MATRIX_ID: ${{ matrix.id }} + OPENCODE_REQUIRED_CONSUMER_PROFILES: ${{ toJSON(matrix.required_consumer_profiles) }} + OPENCODE_RUBY_PATH: ${{ github.workspace }}/ruby-client + OPENCODE_RUBY_COMMIT: ${{ steps.client-sha.outputs.value }} + OPENCODE_RAILS_COMMIT: ${{ needs.prepare.outputs.rails_ref }} + OPENCODE_COMPAT_EVIDENCE_PATH: ${{ github.workspace }}/.compat-evidence/${{ matrix.id }}.json + OPENCODE_COMPAT_RUN_ID: ${{ github.run_id }} + OPENCODE_COMPAT_RUN_ATTEMPT: ${{ github.run_attempt }} + OPENCODE_COMPAT_HEAD_SHA: ${{ github.sha }} + OPENCODE_COMPAT_REPOSITORY: ${{ github.repository }} + OPENCODE_COMPAT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p .compat-evidence + bundle exec scripts/run_image_contract.sh + - name: Upload exact-image evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: image-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .compat-evidence/${{ matrix.id }}.json + include-hidden-files: true + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/watch-upstream.yml b/.github/workflows/watch-upstream.yml index 18a4424..47b91e6 100644 --- a/.github/workflows/watch-upstream.yml +++ b/.github/workflows/watch-upstream.yml @@ -17,7 +17,7 @@ jobs: open-compatibility-pr: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Resolve latest release @@ -27,55 +27,153 @@ jobs: run: | release="$(gh api repos/anomalyco/opencode/releases/latest)" tag="$(jq -r .tag_name <<<"$release")" + digest="$(docker buildx imagetools inspect "ghcr.io/anomalyco/opencode:${tag#v}" --format '{{json .Manifest}}' | jq -r .digest)" + image="ghcr.io/anomalyco/opencode@$digest" + published_at="$(jq -r .published_at <<<"$release")" + release_url="$(jq -r .html_url <<<"$release")" current="$(jq -r .release_tag manifests/upstream.json)" + current_image="$(jq -r .image manifests/upstream.json)" + current_image_count="$( + jq \ + --arg image "$image" \ + '[.public_ci[]? | select(.image == $image)] | length' \ + manifests/image-matrix.json + )" + ruby scripts/upstream_branch_name.rb "$tag" "$digest" >/dev/null + ruby -rtime -e 'Time.iso8601(ARGV.fetch(0))' "$published_at" + test "$release_url" = "https://github.com/anomalyco/opencode/releases/tag/$tag" echo "tag=$tag" >> "$GITHUB_OUTPUT" - echo "published_at=$(jq -r .published_at <<<"$release")" >> "$GITHUB_OUTPUT" - echo "release_url=$(jq -r .html_url <<<"$release")" >> "$GITHUB_OUTPUT" - if [ "$tag" = "$current" ]; then + echo "digest=$digest" >> "$GITHUB_OUTPUT" + echo "published_at=$published_at" >> "$GITHUB_OUTPUT" + echo "release_url=$release_url" >> "$GITHUB_OUTPUT" + if [ "$tag" = "$current" ] && [ "$image" != "$current_image" ]; then + echo "same_tag_digest_changed=true" >> "$GITHUB_OUTPUT" + else + echo "same_tag_digest_changed=false" >> "$GITHUB_OUTPUT" + fi + if [ "$tag" = "$current" ] && [ "$image" = "$current_image" ] && [ "$current_image_count" -eq 1 ]; then echo "changed=false" >> "$GITHUB_OUTPUT" else echo "changed=true" >> "$GITHUB_OUTPUT" fi - - name: Stop when a PR already exists + - name: Resolve collision-safe compatibility branch if: steps.release.outputs.changed == 'true' - id: existing + id: branch env: GH_TOKEN: ${{ github.token }} TAG: ${{ steps.release.outputs.tag }} + DIGEST: ${{ steps.release.outputs.digest }} run: | - branch="compat/upstream-${TAG#v}" - count="$(gh pr list --state open --head "$branch" --json number --jq length)" - echo "branch=$branch" >> "$GITHUB_OUTPUT" - if [ "$count" -gt 0 ]; then + base_branch="$(ruby scripts/upstream_branch_name.rb "$TAG" "$DIGEST")" + expected_image="ghcr.io/anomalyco/opencode@$DIGEST" + open_branch="" + open_prs="$( + gh pr list --state open --base main --limit 1000 \ + --json baseRefName,headRefName,headRefOid,headRepository,headRepositoryOwner,isCrossRepository + )" + while IFS=$'\t' read -r candidate_branch head_oid; do + [ -n "$candidate_branch" ] || continue + if ! candidate_upstream="$( + gh api \ + -H "Accept: application/vnd.github.raw+json" \ + "repos/$GITHUB_REPOSITORY/contents/manifests/upstream.json?ref=$head_oid" 2>/dev/null + )"; then + continue + fi + if ! candidate_matrix="$( + gh api \ + -H "Accept: application/vnd.github.raw+json" \ + "repos/$GITHUB_REPOSITORY/contents/manifests/image-matrix.json?ref=$head_oid" 2>/dev/null + )"; then + continue + fi + if jq -e \ + --arg tag "$TAG" \ + --arg image "$expected_image" \ + '.release_tag == $tag and .image == $image' \ + >/dev/null <<<"$candidate_upstream" && + jq -e \ + --arg image "$expected_image" \ + '([.public_ci[]? | select(.image == $image)] | length) == 1' \ + >/dev/null <<<"$candidate_matrix"; then + open_branch="$candidate_branch" + break + fi + done < <( + jq -r \ + --arg base "$base_branch" \ + --arg owner "${GITHUB_REPOSITORY%%/*}" \ + --arg repository "${GITHUB_REPOSITORY#*/}" \ + '.[] | select( + .baseRefName == "main" and + .isCrossRepository == false and + .headRepositoryOwner.login == $owner and + .headRepository.name == $repository and + (.headRefName == $base or (.headRefName | startswith($base + "-r"))) + ) | [.headRefName, .headRefOid] | @tsv' \ + <<<"$open_prs" + ) + if [ -n "$open_branch" ]; then + echo "branch=$open_branch" >> "$GITHUB_OUTPUT" echo "skip=true" >> "$GITHUB_OUTPUT" - else - echo "skip=false" >> "$GITHUB_OUTPUT" + exit 0 fi + + branch="$base_branch" + existing_count="$( + gh pr list --state all --base main --head "$base_branch" --limit 1000 \ + --json baseRefName,headRepository,headRepositoryOwner,isCrossRepository | + jq \ + --arg owner "${GITHUB_REPOSITORY%%/*}" \ + --arg repository "${GITHUB_REPOSITORY#*/}" \ + '[.[] | select( + .baseRefName == "main" and + .isCrossRepository == false and + .headRepositoryOwner.login == $owner and + .headRepository.name == $repository + )] | length' + )" + if [ "$existing_count" -gt 0 ] || git ls-remote --exit-code --heads origin "$base_branch" >/dev/null 2>&1; then + branch="$(ruby scripts/upstream_branch_name.rb "$TAG" "$DIGEST" "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")" + fi + echo "branch=$branch" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" - name: Resolve immutable image and update manifests - if: steps.release.outputs.changed == 'true' && steps.existing.outputs.skip == 'false' + if: steps.release.outputs.changed == 'true' && steps.branch.outputs.skip == 'false' env: TAG: ${{ steps.release.outputs.tag }} PUBLISHED_AT: ${{ steps.release.outputs.published_at }} RELEASE_URL: ${{ steps.release.outputs.release_url }} + DIGEST: ${{ steps.release.outputs.digest }} run: | - digest="$(docker buildx imagetools inspect "ghcr.io/anomalyco/opencode:${TAG#v}" --format '{{json .Manifest}}' | jq -r .digest)" - ruby scripts/record_upstream_candidate.rb "$TAG" "$PUBLISHED_AT" "$RELEASE_URL" "$digest" + ruby scripts/record_upstream_candidate.rb "$TAG" "$PUBLISHED_AT" "$RELEASE_URL" "$DIGEST" ruby test/repository_test.rb + ruby test/watcher_test.rb - name: Open compatibility PR - if: steps.release.outputs.changed == 'true' && steps.existing.outputs.skip == 'false' + if: steps.release.outputs.changed == 'true' && steps.branch.outputs.skip == 'false' env: GH_TOKEN: ${{ github.token }} TAG: ${{ steps.release.outputs.tag }} - BRANCH: ${{ steps.existing.outputs.branch }} + DIGEST: ${{ steps.release.outputs.digest }} + BRANCH: ${{ steps.branch.outputs.branch }} + SAME_TAG_DIGEST_CHANGED: ${{ steps.release.outputs.same_tag_digest_changed }} run: | git config user.name "opencode-compat-bot" git config user.email "opencode-compat-bot@users.noreply.github.com" + short_digest="${DIGEST#sha256:}" + short_digest="${short_digest:0:12}" git switch -c "$BRANCH" git add manifests/image-matrix.json manifests/upstream.json - git commit -m "Test OpenCode ${TAG} compatibility" + git commit -m "Test OpenCode ${TAG} compatibility (${short_digest})" git push --set-upstream origin "$BRANCH" + title="Test OpenCode ${TAG} compatibility (${short_digest})" + notice="Upstream watcher resolved ${TAG} to immutable OCI digest ${DIGEST}." + if [ "$SAME_TAG_DIGEST_CHANGED" = "true" ]; then + title="Review changed OCI digest for existing OpenCode ${TAG} (${short_digest})" + notice="Security-sensitive change: existing release ${TAG} now resolves to a different OCI digest, ${DIGEST}." + fi gh pr create \ --base main \ --head "$BRANCH" \ - --title "Test OpenCode ${TAG} compatibility" \ - --body "Upstream watcher detected ${TAG}, resolved its immutable OCI digest, and added it as a pending compatibility target. This PR only runs certification checks; promotion and consumer rollout remain manual." + --title "$title" \ + --body "${notice} The watcher added it as a pending compatibility target. This PR only runs certification checks; promotion and consumer rollout remain manual." diff --git a/README.md b/README.md index 84e12c1..a5d8d26 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ prompts, credentials, sessions, or user data belong here. - A gem release is a candidate until the fixture suite and every required image profile pass. +- `opencode-ruby` and `opencode-rails` are one candidate release train. The + manifest records both peeled commits, and CI rejects a Rails candidate whose + exact runtime dependency resolves to a different Ruby version or commit. - Image references are immutable OCI index digests. A tag may be recorded as human-readable provenance, but it is never an execution coordinate. - Custom consumer images are certified with an isolated canary on the host that @@ -46,11 +49,26 @@ Prerequisites are Ruby 3.2+, Python 3, `jq`, and Docker. ruby test/repository_test.rb ruby test/runtime_tuple_promoter_test.rb ruby test/exact_live_contract_test.rb +ruby test/watcher_test.rb OPENCODE_RUBY_PATH=/path/to/opencode-ruby-at-78b6f9c9e9c7d58b699af1c3c17764acd33de798 \ ruby ruby/opencode_ruby_fixture_contract.rb +BUNDLE_GEMFILE=/path/to/opencode-rails/Gemfile \ +OPENCODE_RUBY_PATH=/path/to/opencode-ruby \ +OPENCODE_RAILS_PATH=/path/to/opencode-rails \ +OPENCODE_RUBY_COMMIT=FULL_40_HEX_COMMIT \ +OPENCODE_RUBY_TAG=v0.0.1.alpha7 \ +OPENCODE_RUBY_TAG_OBJECT=FULL_40_HEX_ANNOTATED_TAG_OBJECT \ +OPENCODE_RAILS_COMMIT=FULL_40_HEX_COMMIT \ +OPENCODE_RAILS_TAG=v0.0.1.alpha7 \ +OPENCODE_RAILS_TAG_OBJECT=FULL_40_HEX_ANNOTATED_TAG_OBJECT \ +OPENCODE_RUBY_VERSION=0.0.1.alpha7 \ +OPENCODE_RAILS_VERSION=0.0.1.alpha7 \ + bundle exec ruby ruby/lockstep_client_contract.rb +BUNDLE_GEMFILE=/path/to/opencode-ruby-at-78b6f9c9e9c7d58b699af1c3c17764acd33de798/Gemfile \ OPENCODE_RUBY_PATH=/path/to/opencode-ruby-at-78b6f9c9e9c7d58b699af1c3c17764acd33de798 \ +OPENCODE_RUBY_COMMIT=78b6f9c9e9c7d58b699af1c3c17764acd33de798 \ OPENCODE_IMAGE='ghcr.io/anomalyco/opencode@sha256:e975a0647576016dfdf77d54b979ca30d32b4750472c10263e9894aad6628c2a' \ - scripts/run_image_contract.sh + bundle exec scripts/run_image_contract.sh ``` The live contract starts a deterministic local OpenAI-compatible model stub and @@ -58,15 +76,31 @@ an isolated OpenCode container. It creates a session, subscribes, submits an async prompt, observes terminal SSE, fetches the authoritative exchange, and deletes the session. Passing requires the authoritative final text to equal the expected text byte-for-byte and the deterministic model stub to observe exactly -one request. It never calls an external model provider. +one request and exactly one authoritative assistant message. It never calls an +external model provider. + +Candidate CI uploads machine-readable fixture, lockstep-client, and exact-image +evidence for 30 days. These artifacts are review inputs, not certification by +themselves. A person must review a passing artifact and commit the durable +certification document under `evidence/`; no workflow commits or promotes its +own result. ## Adding an upstream release -The scheduled watcher compares the latest upstream GitHub release with the -manifest. When a new release appears, it resolves the tag to an OCI digest and -opens a compatibility PR. The PR adds a pending image-matrix entry, which runs -the candidate suite. A person still decides whether to promote it and update -consumers. +The scheduled watcher compares both the latest upstream GitHub release tag and +its resolved OCI digest with the manifest. A new release or a changed digest +behind an existing tag gets a digest-specific branch and compatibility PR. The +PR adds a pending image-matrix entry, which runs the candidate suite after the +normal GitHub workflow approval boundary. A person still decides whether to +promote it and update consumers. The watcher never merges, dispatches consumer +workflows, deploys, or force-pushes. + +`public_ci` is the active compatibility set, not an append-only release log. +Keep images still referenced by a consumer's current, candidate, or previous +runtime tuple plus the newest pending upstream candidate. When a watcher PR +supersedes an unreferenced pending target, remove that older row during review; +durable certification documents and Git history are the archive. This keeps a +fast-moving upstream from making every future run retest every old release. See [docs/certification.md](docs/certification.md) for promotion, canary, and rollback evidence requirements. diff --git a/docs/certification.md b/docs/certification.md index 21c2c55..4e14ee7 100644 --- a/docs/certification.md +++ b/docs/certification.md @@ -9,11 +9,14 @@ commit. Changing any member invalidates the certification. The gate requires: 1. repository validation and the shared fixture corpus; -2. the public exact-image matrix against the deterministic model stub; -3. an isolated custom-image canary for Ajent Rails and Mushu; -4. the consumer's own focused and full required tests in its devcontainer; -5. a user-visible canary turn for stream consumers; -6. captured evidence including image ID/digest, server version, source commit, +2. exact `opencode-ruby` and `opencode-rails` candidate checkouts, all Rails + candidate tests on Ruby 3.2 through 3.4, an exact Rails-to-Ruby runtime + dependency, and loaded Bundler provenance for the Ruby commit; +3. the public exact-image matrix against the deterministic model stub; +4. an isolated custom-image canary for Ajent Rails and Mushu; +5. the consumer's own focused and full required tests in its devcontainer; +6. a user-visible canary turn for stream consumers; +7. captured evidence including image ID/digest, server version, source commit, gem commits, timestamps, and probe outcome. Health-only probes do not certify a tuple. @@ -94,6 +97,19 @@ certify Rails persistence, plugin hooks, voice streaming, strict route gates, or provider hooks. Those appear as `required_consumer_profiles` in the image matrix and need their own consumer evidence before a full tuple can pass. +The companion lockstep job proves that the exact Rails candidate loads and +tests against the exact Ruby candidate on every supported Ruby version. That +still does not certify a consumer's ActiveRecord schema, persistence callbacks, +container adapter, or application canary; those remain consumer-owned profile +evidence. + +CI writes canonical JSON artifacts for the shared fixture, lockstep client, and +each exact-image target. Artifact retention is 30 days and supplies reviewable +workflow provenance; it is not the long-term ledger. After review, copy the +relevant facts into a repository evidence document bound to the complete tuple +fingerprint. Automated workflows never update certified evidence or promote a +tuple. + The command clears `candidate` after promotion. It sets the repository-wide `migration_state` to `certified` only after every consumer has certified `current` and `previous` tuples and no pending candidate. @@ -112,6 +128,15 @@ have real passing evidence can the manifest honestly contain certified `current` and `previous` tuples. Until then, `promotion_readiness` remains blocked and the candidate PR must not be treated as a deploy authorization. +The schema-v1 promotion command deliberately cannot perform the first degraded +bootstrap transition when `current` is a known-failing baseline. Do not bypass +that guard by hand-editing the manifest or relabeling alpha2 evidence. A +separate reviewed state-machine change must first add an explicit +`bootstrap-current-only` state that preserves the failed baseline as +uncertified emergency provenance and leaves `previous` null. Itemized rollback +certification is complete only after a later, materially distinct passing tuple +can move the first certified current into `previous`. + ## Rollback Rollback restores the whole `previous` tuple. Do not roll back only the gem or @@ -125,11 +150,17 @@ the source commit used to build it. A source tag alone is insufficient. Run the shared live contract on a host that can pull the exact image: ```sh +BUNDLE_GEMFILE=/path/to/opencode-ruby/Gemfile \ OPENCODE_RUBY_PATH=/path/to/opencode-ruby \ +OPENCODE_RUBY_COMMIT=FULL_40_HEX_COMMIT \ OPENCODE_IMAGE='registry.example/image@sha256:...' \ -scripts/run_image_contract.sh +bundle exec scripts/run_image_contract.sh ``` +The Ruby checkout must be clean and its `HEAD` must equal +`OPENCODE_RUBY_COMMIT`. Install Bundler dependencies outside that checkout when +the normal cache path would create untracked files. + If an older private registry cannot expose an OCI repository digest, set `ALLOW_EXACT_IMAGE_ID=1` and pass the locally present `sha256:...` image ID. Record the registry tag, image ID, source commit, and reason a repository diff --git a/manifests/client-candidate.json b/manifests/client-candidate.json index a5caca0..b764ef2 100644 --- a/manifests/client-candidate.json +++ b/manifests/client-candidate.json @@ -1,11 +1,33 @@ { - "repository": "ajaynomics/opencode-ruby", - "ref": "78b6f9c9e9c7d58b699af1c3c17764acd33de798", - "version": "0.0.1.alpha7", - "tag_provenance": { - "tag": "v0.0.1.alpha7", - "annotated_tag_object": "abfce3ec8cb1ee468107d0845521769502251645", - "peeled_commit": "78b6f9c9e9c7d58b699af1c3c17764acd33de798" + "schema_version": 2, + "release_train": "0.0.1.alpha7", + "clients": { + "opencode-ruby": { + "repository": "ajaynomics/opencode-ruby", + "ref": "78b6f9c9e9c7d58b699af1c3c17764acd33de798", + "version": "0.0.1.alpha7", + "tag_provenance": { + "tag": "v0.0.1.alpha7", + "annotated_tag_object": "abfce3ec8cb1ee468107d0845521769502251645", + "peeled_commit": "78b6f9c9e9c7d58b699af1c3c17764acd33de798" + } + }, + "opencode-rails": { + "repository": "ajaynomics/opencode-rails", + "ref": "2a391ccad1d098e4bb51eb19b6ca52adaa79e5cb", + "version": "0.0.1.alpha7", + "tag_provenance": { + "tag": "v0.0.1.alpha7", + "annotated_tag_object": "61e128084b2b665c0ba20b332987e41794948d7c", + "peeled_commit": "2a391ccad1d098e4bb51eb19b6ca52adaa79e5cb" + }, + "runtime_dependencies": { + "opencode-ruby": { + "requirement": "= 0.0.1.alpha7", + "ref": "78b6f9c9e9c7d58b699af1c3c17764acd33de798" + } + } + } }, "status": "candidate" } diff --git a/ruby/exact_live_contract.rb b/ruby/exact_live_contract.rb index 4469130..7c8d0d1 100644 --- a/ruby/exact_live_contract.rb +++ b/ruby/exact_live_contract.rb @@ -13,13 +13,22 @@ module OpenCodeCompat end def assert_model_request_count!(value) - count = Integer(value, 10) + count = Integer(value.to_s, 10) return if count == 1 raise ContractError, "Expected exactly one deterministic model request, got #{count}" rescue ArgumentError, TypeError raise ContractError, "Deterministic model request count is not an integer: #{value.inspect}" end + + def assert_authoritative_assistant_count!(value) + count = Integer(value.to_s, 10) + return if count == 1 + + raise ContractError, "Expected exactly one authoritative assistant message, got #{count}" + rescue ArgumentError, TypeError + raise ContractError, "Authoritative assistant message count is not an integer: #{value.inspect}" + end end end diff --git a/ruby/live_probe.rb b/ruby/live_probe.rb index 5e5ed03..3c52361 100644 --- a/ruby/live_probe.rb +++ b/ruby/live_probe.rb @@ -37,7 +37,7 @@ begin messages = client.get_messages(session_id) assistant_messages = Array(messages).select { |message| message.dig(:info, :role) == "assistant" } - raise "No authoritative assistant exchange returned" if assistant_messages.empty? + OpenCodeCompat::ExactLiveContract.assert_authoritative_assistant_count!(assistant_messages.length) puts JSON.generate( status: "pass", diff --git a/ruby/lockstep_client_contract.rb b/ruby/lockstep_client_contract.rb new file mode 100644 index 0000000..92f0176 --- /dev/null +++ b/ruby/lockstep_client_contract.rb @@ -0,0 +1,338 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "open3" +require "rubygems" + +module OpenCodeCompat + class LockstepClientContract + class ContractError < StandardError; end + + CONTRACT_NAME = "opencode-client-lockstep" + REQUIRED_ENV = %w[ + OPENCODE_RUBY_PATH + OPENCODE_RUBY_COMMIT + OPENCODE_RUBY_TAG + OPENCODE_RUBY_TAG_OBJECT + OPENCODE_RUBY_VERSION + OPENCODE_RAILS_PATH + OPENCODE_RAILS_COMMIT + OPENCODE_RAILS_TAG + OPENCODE_RAILS_TAG_OBJECT + OPENCODE_RAILS_VERSION + ].freeze + + def self.from_environment(env: ENV, git_resolver: nil, git_head_resolver: nil) + require "bundler" + require "opencode-rails" + + new( + env: env, + git_resolver: git_resolver, + git_head_resolver: git_head_resolver, + loaded_specs: Gem.loaded_specs, + bundle_specs: Bundler.load.specs, + runtime_ruby_version: RUBY_VERSION, + ruby_version: Opencode::VERSION, + rails_version: Opencode::RAILS_VERSION + ) + end + + def self.canonical_json(value) + JSON.generate(canonicalize(value)) + end + + def self.canonicalize(value) + case value + when Hash + value.keys.map(&:to_s).sort.to_h do |key| + source_key = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key } + [key, canonicalize(value.fetch(source_key))] + end + when Array + value.map { |entry| canonicalize(entry) } + else + value + end + end + private_class_method :canonicalize + + def initialize( + env:, + loaded_specs:, + bundle_specs:, + ruby_version:, + rails_version:, + runtime_ruby_version: RUBY_VERSION, + git_resolver: nil, + git_head_resolver: nil + ) + @env = env + @loaded_specs = loaded_specs + @bundle_specs = bundle_specs + @ruby_version = ruby_version.to_s + @rails_version = rails_version.to_s + @runtime_ruby_version = runtime_ruby_version.to_s + @git_resolver = git_resolver || method(:resolve_git) + @git_head_resolver = git_head_resolver + end + + def verify + ensure_required_environment! + + ruby_path = realpath("OPENCODE_RUBY_PATH") + rails_path = realpath("OPENCODE_RAILS_PATH") + ruby_commit = exact_commit("OPENCODE_RUBY_COMMIT") + rails_commit = exact_commit("OPENCODE_RAILS_COMMIT") + expected_ruby_version = exact_version("OPENCODE_RUBY_VERSION") + expected_rails_version = exact_version("OPENCODE_RAILS_VERSION") + + assert_equal!(ruby_commit, checkout_head(ruby_path, "opencode-ruby"), "opencode-ruby checkout commit") + assert_equal!(rails_commit, checkout_head(rails_path, "opencode-rails"), "opencode-rails checkout commit") + ruby_tag_provenance = annotated_tag_provenance( + path: ruby_path, + label: "opencode-ruby", + tag_name: "OPENCODE_RUBY_TAG", + tag_object: "OPENCODE_RUBY_TAG_OBJECT", + expected_commit: ruby_commit + ) + rails_tag_provenance = annotated_tag_provenance( + path: rails_path, + label: "opencode-rails", + tag_name: "OPENCODE_RAILS_TAG", + tag_object: "OPENCODE_RAILS_TAG_OBJECT", + expected_commit: rails_commit + ) + assert_equal!(expected_ruby_version, @ruby_version, "loaded Opencode::VERSION") + assert_equal!(expected_rails_version, @rails_version, "loaded Opencode::RAILS_VERSION") + assert_equal!(expected_ruby_version, expected_rails_version, "lockstep client version") + + ruby_spec = loaded_spec!("opencode-ruby") + rails_spec = loaded_spec!("opencode-rails") + assert_equal!(expected_ruby_version, ruby_spec.version.to_s, "loaded opencode-ruby gem version") + assert_equal!(expected_rails_version, rails_spec.version.to_s, "loaded opencode-rails gem version") + assert_equal!(rails_path, File.realpath(rails_spec.full_gem_path), "loaded opencode-rails checkout") + + dependency = exact_runtime_dependency!(rails_spec, expected_ruby_version) + bundle_spec = bundled_ruby_spec! + source = bundle_spec.source + unless source && source.respond_to?(:revision) && source.class.name.match?(/(?:\A|::)Git\z/) + raise ContractError, "Bundler opencode-ruby source must be a Git source" + end + + bundler_revision = source.revision.to_s + assert_exact_commit!(bundler_revision, "Bundler opencode-ruby revision") + assert_equal!(ruby_commit, bundler_revision, "Bundler opencode-ruby revision") + + { + "contract" => CONTRACT_NAME, + "opencode_rails" => { + "checkout_commit" => rails_commit, + "gem_version" => rails_spec.version.to_s, + "loaded_version" => @rails_version, + "tag_provenance" => rails_tag_provenance, + "runtime_dependency" => { + "name" => dependency.name, + "requirement" => dependency.requirement.to_s + } + }, + "opencode_ruby" => { + "bundler_git_revision" => bundler_revision, + "bundler_source" => source.class.name, + "checkout_commit" => ruby_commit, + "gem_version" => ruby_spec.version.to_s, + "loaded_version" => @ruby_version, + "tag_provenance" => ruby_tag_provenance + }, + "ruby_runtime_version" => @runtime_ruby_version, + "schema_version" => 1, + "status" => "pass", + "workflow" => { + "head_sha" => @env["OPENCODE_COMPAT_HEAD_SHA"], + "repository" => @env["OPENCODE_COMPAT_REPOSITORY"], + "run_attempt" => @env["OPENCODE_COMPAT_RUN_ATTEMPT"], + "run_id" => @env["OPENCODE_COMPAT_RUN_ID"], + "run_url" => @env["OPENCODE_COMPAT_RUN_URL"] + } + } + rescue Errno::ENOENT, Errno::EACCES => error + raise ContractError, error.message + end + + def emit(io: $stdout) + document = verify + json = self.class.canonical_json(document) + evidence_path = @env["OPENCODE_COMPAT_EVIDENCE_PATH"].to_s + + unless evidence_path.empty? + expanded_path = File.expand_path(evidence_path) + FileUtils.mkdir_p(File.dirname(expanded_path)) + File.binwrite(expanded_path, "#{json}\n") + end + + io.puts(json) + document + end + + private + + def ensure_required_environment! + missing = REQUIRED_ENV.select { |name| @env[name].to_s.empty? } + return if missing.empty? + + raise ContractError, "missing required environment: #{missing.join(', ')}" + end + + def realpath(name) + File.realpath(@env.fetch(name)) + end + + def exact_commit(name) + value = @env.fetch(name) + assert_exact_commit!(value, name) + value + end + + def exact_tag(name) + value = @env.fetch(name) + if value.match?(/\A[A-Za-z0-9][A-Za-z0-9._-]*\z/) && !value.include?("..") && !value.end_with?(".") + return value + end + + raise ContractError, "#{name} must be a safe exact Git tag name, got #{value.inspect}" + end + + def exact_tag_object(name) + value = @env.fetch(name) + return value if value.match?(/\A[0-9a-f]{40}\z/) + + raise ContractError, + "#{name} must be a full lowercase 40-character Git object ID, got #{value.inspect}" + end + + def assert_exact_commit!(value, label) + return if value.match?(/\A[0-9a-f]{40}\z/) + + raise ContractError, "#{label} must be a full lowercase 40-character Git commit, got #{value.inspect}" + end + + def exact_version(name) + value = @env.fetch(name) + return value if Gem::Version.correct?(value) + + raise ContractError, "#{name} must be an exact RubyGems version, got #{value.inspect}" + end + + def checkout_head(path, label) + head = if @git_head_resolver + @git_head_resolver.call(path).to_s.strip + else + git_value(path, "#{label} checkout HEAD", "rev-parse", "--verify", "HEAD^{commit}") + end + assert_exact_commit!(head, "#{label} checkout HEAD") + head + rescue ContractError + raise + rescue StandardError => error + raise ContractError, "could not resolve #{label} checkout HEAD: #{error.message}" + end + + def annotated_tag_provenance(path:, label:, tag_name:, tag_object:, expected_commit:) + tag = exact_tag(tag_name) + object = exact_tag_object(tag_object) + tag_ref = "refs/tags/#{tag}" + + object_type = git_value(path, "#{label} annotated tag object type", "cat-file", "-t", object) + assert_equal!("tag", object_type, "#{label} annotated tag object type") + + resolved_object = git_value(path, "#{label} local tag ref", "rev-parse", "--verify", tag_ref) + assert_exact_commit!(resolved_object, "#{label} local tag ref object") + assert_equal!(object, resolved_object, "#{label} local tag ref object") + + object_commit = git_value(path, "#{label} annotated tag peel", "rev-parse", "--verify", "#{object}^{commit}") + assert_exact_commit!(object_commit, "#{label} annotated tag peeled commit") + assert_equal!(expected_commit, object_commit, "#{label} annotated tag peeled commit") + + ref_commit = git_value(path, "#{label} local tag ref peel", "rev-parse", "--verify", "#{tag_ref}^{commit}") + assert_exact_commit!(ref_commit, "#{label} local tag ref peeled commit") + assert_equal!(expected_commit, ref_commit, "#{label} local tag ref peeled commit") + + { + "annotated_tag_object" => object, + "peeled_commit" => object_commit, + "tag" => tag + } + end + + def git_value(path, label, *arguments) + @git_resolver.call(path, *arguments).to_s.strip + rescue StandardError => error + raise ContractError, "could not resolve #{label}: #{error.message}" + end + + def resolve_git(path, *arguments) + stdout, stderr, status = Open3.capture3("git", "-C", path, *arguments) + return stdout.strip if status.success? + + detail = stderr.strip + detail = "git exited #{status.exitstatus}" if detail.empty? + raise ContractError, detail + end + + def loaded_spec!(name) + @loaded_specs.fetch(name) + rescue KeyError + raise ContractError, "#{name} is not present in Gem.loaded_specs" + end + + def exact_runtime_dependency!(rails_spec, ruby_version) + dependencies = rails_spec.runtime_dependencies.select { |dependency| dependency.name == "opencode-ruby" } + unless dependencies.length == 1 + raise ContractError, "opencode-rails must declare exactly one runtime dependency on opencode-ruby" + end + + dependency = dependencies.first + expected = [["=", Gem::Version.new(ruby_version)]] + unless dependency.requirement.requirements == expected + raise ContractError, + "opencode-rails must require opencode-ruby = #{ruby_version}, got #{dependency.requirement}" + end + + dependency + end + + def bundled_ruby_spec! + specs = @bundle_specs.select { |spec| spec.name == "opencode-ruby" } + return specs.first if specs.length == 1 + + raise ContractError, "Bundler must resolve exactly one opencode-ruby spec, found #{specs.length}" + end + + def assert_equal!(expected, actual, label) + return if expected == actual + + raise ContractError, "#{label} must equal #{expected.inspect}, got #{actual.inspect}" + end + end +end + +if $PROGRAM_NAME == __FILE__ + begin + OpenCodeCompat::LockstepClientContract.from_environment.emit + rescue StandardError => error + failure = OpenCodeCompat::LockstepClientContract.canonical_json( + "contract" => OpenCodeCompat::LockstepClientContract::CONTRACT_NAME, + "error" => "#{error.class}: #{error.message}", + "schema_version" => 1, + "status" => "fail" + ) + if (evidence_path = ENV["OPENCODE_COMPAT_EVIDENCE_PATH"]) && !evidence_path.empty? + expanded_path = File.expand_path(evidence_path) + FileUtils.mkdir_p(File.dirname(expanded_path)) + File.binwrite(expanded_path, "#{failure}\n") + end + warn failure + exit 1 + end +end diff --git a/ruby/opencode_ruby_fixture_contract.rb b/ruby/opencode_ruby_fixture_contract.rb index 1bf8a51..70b5b2b 100644 --- a/ruby/opencode_ruby_fixture_contract.rb +++ b/ruby/opencode_ruby_fixture_contract.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true require "json" +require "fileutils" +require "time" root = File.expand_path("..", __dir__) gem_path = File.expand_path(ENV.fetch("OPENCODE_RUBY_PATH")) @@ -9,6 +11,12 @@ require "opencode-ruby" manifest = JSON.parse(File.read(File.join(root, "fixtures/manifest.json"))) failures = [] +write_evidence = lambda do |document| + next unless (evidence_path = ENV["OPENCODE_COMPAT_EVIDENCE_PATH"]) + + FileUtils.mkdir_p(File.dirname(evidence_path)) + File.write(evidence_path, JSON.pretty_generate(document) + "\n") +end manifest.fetch("fixtures").each do |entry| reply = Opencode::Reply.new @@ -56,14 +64,45 @@ manifest.fetch("fixtures").each do |entry| end unless failures.empty? + evidence = { + schema_version: 1, + kind: "shared-ruby-fixture-contract", + status: "fail", + checked_at: Time.now.utc.iso8601, + adapter: "opencode-ruby", + adapter_version: Opencode::VERSION, + adapter_commit: ENV["OPENCODE_RUBY_COMMIT"], + fixture_count: manifest.fetch("fixtures").length, + failures: failures, + workflow: { + run_id: ENV["OPENCODE_COMPAT_RUN_ID"], + run_attempt: ENV["OPENCODE_COMPAT_RUN_ATTEMPT"], + head_sha: ENV["OPENCODE_COMPAT_HEAD_SHA"], + repository: ENV["OPENCODE_COMPAT_REPOSITORY"], + run_url: ENV["OPENCODE_COMPAT_RUN_URL"] + } + } + write_evidence.call(evidence) warn failures.join("\n") exit 1 end -puts JSON.generate( +evidence = { + schema_version: 1, + kind: "shared-ruby-fixture-contract", status: "pass", + checked_at: Time.now.utc.iso8601, adapter: "opencode-ruby", adapter_version: Opencode::VERSION, adapter_commit: ENV["OPENCODE_RUBY_COMMIT"], - fixture_count: manifest.fetch("fixtures").length -) + fixture_count: manifest.fetch("fixtures").length, + workflow: { + run_id: ENV["OPENCODE_COMPAT_RUN_ID"], + run_attempt: ENV["OPENCODE_COMPAT_RUN_ATTEMPT"], + head_sha: ENV["OPENCODE_COMPAT_HEAD_SHA"], + repository: ENV["OPENCODE_COMPAT_REPOSITORY"], + run_url: ENV["OPENCODE_COMPAT_RUN_URL"] + } +} +puts JSON.generate(evidence) +write_evidence.call(evidence) diff --git a/scripts/matrix_json.rb b/scripts/matrix_json.rb index 6ac0e37..c4f6ba6 100755 --- a/scripts/matrix_json.rb +++ b/scripts/matrix_json.rb @@ -5,10 +5,25 @@ require "json" root = File.expand_path("..", __dir__) manifest = JSON.parse(File.read(File.join(root, "manifests/image-matrix.json"))) matrix = manifest.fetch("public_ci").map do |target| + id = target.fetch("id") + profiles = target.fetch("profiles") + certification_scope = target.fetch("certification_scope", "shared-client-contract-only") + unless profiles == ["ruby-rest-sse"] + abort "#{id}: public CI executes exactly ruby-rest-sse; got profiles=#{profiles.inspect}" + end + unless certification_scope == "shared-client-contract-only" + abort "#{id}: certification_scope must be shared-client-contract-only; got #{certification_scope.inspect}" + end + { - "id" => target.fetch("id"), + "id" => id, "version" => target.fetch("version"), - "image" => target.fetch("image") + "image" => target.fetch("image"), + "consumers" => target.fetch("consumers"), + "profiles" => profiles, + "required_consumer_profiles" => target.fetch("required_consumer_profiles"), + "certification_scope" => certification_scope, + "certification_status" => target.fetch("certification_status") } end diff --git a/scripts/record_upstream_candidate.rb b/scripts/record_upstream_candidate.rb index 4065c0f..dd8c024 100755 --- a/scripts/record_upstream_candidate.rb +++ b/scripts/record_upstream_candidate.rb @@ -8,6 +8,13 @@ tag, published_at, release_url, digest = ARGV abort "usage: record_upstream_candidate.rb TAG PUBLISHED_AT RELEASE_URL DIGEST" unless digest abort "invalid upstream release tag: #{tag.inspect}" unless tag.match?(/\Av\d+\.\d+\.\d+\z/) abort "invalid OCI digest: #{digest.inspect}" unless digest.match?(/\Asha256:[0-9a-f]{64}\z/) +begin + Time.iso8601(published_at) +rescue ArgumentError, TypeError + abort "invalid upstream publication timestamp: #{published_at.inspect}" +end +expected_release_url = "https://github.com/anomalyco/opencode/releases/tag/#{tag}" +abort "invalid upstream release URL: #{release_url.inspect}" unless release_url == expected_release_url version = tag.delete_prefix("v") image = "ghcr.io/anomalyco/opencode@#{digest}" @@ -15,30 +22,46 @@ upstream_path = File.join(root, "manifests/upstream.json") matrix_path = File.join(root, "manifests/image-matrix.json") upstream = JSON.parse(File.read(upstream_path)) matrix = JSON.parse(File.read(matrix_path)) +public_ci = matrix.fetch("public_ci") +abort "public_ci must be an array" unless public_ci.is_a?(Array) -exit 0 if upstream.fetch("release_tag") == tag +matching_targets = public_ci.select { |target| target.fetch("image") == image } +if matching_targets.length > 1 + abort "image #{image} must appear exactly once in public_ci; found #{matching_targets.length}" +end -upstream.merge!( - "release_tag" => tag, - "version" => version, - "published_at" => published_at, - "release_url" => release_url, - "image" => image, - "observed_at" => Time.now.utc.iso8601 -) +matrix_changed = false +if matching_targets.empty? + target_id = "upstream-#{version}-#{digest.delete_prefix('sha256:')}" + if public_ci.any? { |target| target.fetch("id") == target_id } + abort "cannot add #{image}: public_ci id #{target_id.inspect} is already used by another image" + end -unless matrix.fetch("public_ci").any? { |target| target.fetch("image") == image } - matrix.fetch("public_ci") << { - "id" => "upstream-#{version}", + public_ci << { + "id" => target_id, "version" => version, "image" => image, "tag_provenance" => "ghcr.io/anomalyco/opencode:#{version}", - "consumers" => ["upstream-candidate"], + "consumers" => [], "profiles" => ["ruby-rest-sse"], - "required_consumer_profiles" => ["rails-persisted-turn", "voice-stream", "strict-v2", "plugin-ledger", "provider-hooks"], + "required_consumer_profiles" => [], + "certification_scope" => "shared-client-contract-only", "certification_status" => "pending" } + matrix_changed = true end -File.write(upstream_path, JSON.pretty_generate(upstream) + "\n") -File.write(matrix_path, JSON.pretty_generate(matrix) + "\n") +upstream_changed = upstream.fetch("release_tag") != tag || upstream.fetch("image") != image +if upstream_changed + upstream.merge!( + "release_tag" => tag, + "version" => version, + "published_at" => published_at, + "release_url" => release_url, + "image" => image, + "observed_at" => Time.now.utc.iso8601 + ) +end + +File.write(upstream_path, JSON.pretty_generate(upstream) + "\n") if upstream_changed +File.write(matrix_path, JSON.pretty_generate(matrix) + "\n") if matrix_changed diff --git a/scripts/run_image_contract.sh b/scripts/run_image_contract.sh index 76d78b4..ba51c54 100755 --- a/scripts/run_image_contract.sh +++ b/scripts/run_image_contract.sh @@ -2,38 +2,121 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -image="${OPENCODE_IMAGE:?set OPENCODE_IMAGE to an immutable image digest}" -gem_path="${OPENCODE_RUBY_PATH:?set OPENCODE_RUBY_PATH to the candidate checkout}" +image="${OPENCODE_IMAGE:-}" +gem_path="${OPENCODE_RUBY_PATH:-}" +expected_gem_commit="${OPENCODE_RUBY_COMMIT:-}" +evidence_path="${OPENCODE_COMPAT_EVIDENCE_PATH:-}" +container_name="opencode-compat-${RANDOM}-$$" +llm_container_name="opencode-compat-llm-${RANDOM}-$$" +network_name="opencode-compat-net-${RANDOM}-$$" +python_image="python@sha256:399babc8b49529dabfd9c922f2b5eea81d611e4512e3ed250d75bd2e7683f4b0" +network_created=0 +llm_container_started=0 +opencode_container_started=0 +gem_commit="" -if [[ "$image" != *@sha256:* ]]; then - if [[ "${ALLOW_EXACT_IMAGE_ID:-0}" != "1" || "$image" != sha256:* ]]; then +write_failure_evidence() { + local exit_status="$1" + [[ -n "$evidence_path" ]] || return 0 + + mkdir -p "$(dirname "$evidence_path")" + jq -n \ + --argjson schema_version 1 \ + --arg kind shared-client-image-contract \ + --arg status fail \ + --arg checked_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg matrix_target "${OPENCODE_MATRIX_ID:-local}" \ + --arg image "$image" \ + --arg ruby_commit "${gem_commit:-}" \ + --arg rails_commit "${OPENCODE_RAILS_COMMIT:-}" \ + --arg run_id "${OPENCODE_COMPAT_RUN_ID:-}" \ + --arg run_attempt "${OPENCODE_COMPAT_RUN_ATTEMPT:-}" \ + --arg head_sha "${OPENCODE_COMPAT_HEAD_SHA:-}" \ + --arg repository "${OPENCODE_COMPAT_REPOSITORY:-}" \ + --arg run_url "${OPENCODE_COMPAT_RUN_URL:-}" \ + --argjson exit_status "$exit_status" \ + '{ + schema_version:$schema_version, + kind:$kind, + status:$status, + checked_at:$checked_at, + matrix_target:$matrix_target, + certification_scope:"shared-client-contract-only", + executed_profiles:[], + image:{requested:$image}, + clients:{opencode_ruby:{commit:$ruby_commit},opencode_rails:{commit:$rails_commit,executed:false}}, + failure:{exit_status:$exit_status}, + workflow:{run_id:$run_id,run_attempt:$run_attempt,head_sha:$head_sha,repository:$repository,run_url:$run_url} + }' >"$evidence_path" +} + +cleanup() { + local status=$? + trap - EXIT + if [[ "$status" != "0" ]]; then + echo "OpenCode compatibility probe failed; container log follows" >&2 + if [[ "$opencode_container_started" == "1" ]]; then + docker logs "$container_name" >&2 2>/dev/null || true + fi + echo "Deterministic model request summary follows" >&2 + if [[ "$llm_container_started" == "1" ]]; then + docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats >&2 2>/dev/null || true + fi + echo >&2 + write_failure_evidence "$status" || true + fi + if [[ "$opencode_container_started" == "1" ]]; then + docker rm -f "$container_name" >/dev/null 2>&1 || true + fi + if [[ "$llm_container_started" == "1" ]]; then + docker rm -f "$llm_container_name" >/dev/null 2>&1 || true + fi + if [[ "$network_created" == "1" ]]; then + docker network rm "$network_name" >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT + +if [[ -n "$evidence_path" ]]; then + mkdir -p "$(dirname "$evidence_path")" + : >"$evidence_path" +fi + +if [[ -z "$image" ]]; then + echo "set OPENCODE_IMAGE to an immutable image digest" >&2 + exit 2 +fi +if [[ -z "$gem_path" ]]; then + echo "set OPENCODE_RUBY_PATH to the candidate checkout" >&2 + exit 2 +fi +if [[ ! "$image" =~ ^[^[:space:]@]+@sha256:[0-9a-f]{64}$ ]]; then + if [[ "${ALLOW_EXACT_IMAGE_ID:-0}" != "1" || ! "$image" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "OPENCODE_IMAGE must be an OCI digest (or an exact local image ID with ALLOW_EXACT_IMAGE_ID=1)" >&2 exit 2 fi fi -container_name="opencode-compat-${RANDOM}-$$" -llm_container_name="opencode-compat-llm-${RANDOM}-$$" -network_name="opencode-compat-net-${RANDOM}-$$" -python_image="python@sha256:399babc8b49529dabfd9c922f2b5eea81d611e4512e3ed250d75bd2e7683f4b0" - -cleanup() { - status=$? - if [[ "$status" != "0" ]]; then - echo "OpenCode compatibility probe failed; container log follows" >&2 - docker logs "$container_name" >&2 2>/dev/null || true - echo "Deterministic model request summary follows" >&2 - docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats >&2 2>/dev/null || true - echo >&2 - fi - docker rm -f "$container_name" >/dev/null 2>&1 || true - docker rm -f "$llm_container_name" >/dev/null 2>&1 || true - docker network rm "$network_name" >/dev/null 2>&1 || true - return "$status" -} -trap cleanup EXIT +if [[ ! "$expected_gem_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "OPENCODE_RUBY_COMMIT must be a full lowercase 40-character Git commit" >&2 + exit 2 +fi +if ! gem_commit="$(git -C "$gem_path" rev-parse --verify 'HEAD^{commit}' 2>/dev/null)"; then + echo "OPENCODE_RUBY_PATH must be a readable Git checkout" >&2 + exit 2 +fi +if [[ "$gem_commit" != "$expected_gem_commit" ]]; then + echo "opencode-ruby checkout HEAD $gem_commit does not match expected commit $expected_gem_commit" >&2 + exit 2 +fi +if [[ -n "$(git -C "$gem_path" status --porcelain --untracked-files=all)" ]]; then + echo "opencode-ruby checkout must be clean before certification" >&2 + exit 2 +fi docker network create "$network_name" >/dev/null +network_created=1 docker run --detach \ --name "$llm_container_name" \ --network "$network_name" \ @@ -42,6 +125,7 @@ docker run --detach \ "$python_image" \ python /compat/fake_llm.py --port 8080 --port-file /tmp/compat-port \ >/dev/null +llm_container_started=1 for _ in $(seq 1 100); do if docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/health >/dev/null 2>&1; then @@ -93,6 +177,7 @@ docker run --detach \ --env OPENCODE_PURE=1 \ "$image" serve --hostname 0.0.0.0 --port 4096 \ >/dev/null +opencode_container_started=1 host_port="$(docker port "$container_name" 4096/tcp | sed -E 's/.*:([0-9]+)$/\1/' | head -1)" base_url="http://127.0.0.1:${host_port}" @@ -118,22 +203,75 @@ fi server_version="$(docker exec "$container_name" opencode --version)" image_id="$(docker inspect "$container_name" --format '{{.Image}}')" -gem_commit="$(git -C "$gem_path" rev-parse HEAD 2>/dev/null || true)" +image_os="$(docker image inspect "$image_id" --format '{{.Os}}')" +image_architecture="$(docker image inspect "$image_id" --format '{{.Architecture}}')" +image_platform="${image_os}/${image_architecture}" +expected_version="${OPENCODE_EXPECTED_VERSION:-}" -OPENCODE_BASE_URL="$base_url" \ -OPENCODE_RUBY_PATH="$gem_path" \ -OPENCODE_RUBY_COMMIT="$gem_commit" \ - ruby "$repo_root/ruby/live_probe.rb" +if [[ -n "$expected_version" && "$server_version" != "$expected_version" ]]; then + echo "OpenCode reported version $server_version; expected $expected_version" >&2 + exit 1 +fi + +live_contract="$( + OPENCODE_BASE_URL="$base_url" \ + OPENCODE_RUBY_PATH="$gem_path" \ + OPENCODE_RUBY_COMMIT="$gem_commit" \ + ruby "$repo_root/ruby/live_probe.rb" +)" +printf '%s\n' "$live_contract" llm_stats="$(docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats)" request_count="$(jq -r '.request_count' <<<"$llm_stats")" ruby "$repo_root/ruby/exact_live_contract.rb" "$request_count" -jq -cn \ +evidence="$(jq -cn \ + --argjson schema_version 1 \ --arg status pass \ + --arg kind shared-client-image-contract \ + --arg matrix_target "${OPENCODE_MATRIX_ID:-local}" \ + --arg certification_scope "shared-client-contract-only" \ + --argjson executed_profiles '["ruby-rest-sse"]' \ + --argjson required_consumer_profiles "${OPENCODE_REQUIRED_CONSUMER_PROFILES:-[]}" \ --arg image "$image" \ --arg image_id "$image_id" \ + --arg image_platform "$image_platform" \ --arg server_version "$server_version" \ --arg gem_commit "$gem_commit" \ + --arg companion_rails_commit "${OPENCODE_RAILS_COMMIT:-}" \ + --arg run_id "${OPENCODE_COMPAT_RUN_ID:-}" \ + --arg run_attempt "${OPENCODE_COMPAT_RUN_ATTEMPT:-}" \ + --arg head_sha "${OPENCODE_COMPAT_HEAD_SHA:-}" \ + --arg repository "${OPENCODE_COMPAT_REPOSITORY:-}" \ + --arg run_url "${OPENCODE_COMPAT_RUN_URL:-}" \ + --argjson live_contract "$live_contract" \ --argjson llm_request_count "$request_count" \ - '{status:$status,image:$image,image_id:$image_id,server_version:$server_version,opencode_ruby_commit:$gem_commit,llm_request_count:$llm_request_count}' + '{ + schema_version:$schema_version, + kind:$kind, + status:$status, + checked_at:$live_contract.checked_at, + matrix_target:$matrix_target, + certification_scope:$certification_scope, + executed_profiles:$executed_profiles, + required_consumer_profiles:$required_consumer_profiles, + image:{requested:$image,docker_image_id:$image_id,platform:$image_platform,reported_version:$server_version}, + clients:{ + opencode_ruby:{version:$live_contract.opencode_ruby_version,commit:$gem_commit,executed:true}, + opencode_rails:{commit:$companion_rails_commit,executed:false} + }, + contract:{ + expected_text:$live_contract.expected_text, + full_text:$live_contract.full_text, + observed_part_count:$live_contract.observed_part_count, + authoritative_assistant_message_count:$live_contract.authoritative_assistant_message_count, + llm_request_count:$llm_request_count + }, + workflow:{run_id:$run_id,run_attempt:$run_attempt,head_sha:$head_sha,repository:$repository,run_url:$run_url} + }' +)" +printf '%s\n' "$evidence" + +if [[ -n "$evidence_path" ]]; then + printf '%s\n' "$evidence" >"$evidence_path" +fi diff --git a/scripts/upstream_branch_name.rb b/scripts/upstream_branch_name.rb new file mode 100644 index 0000000..45aab1c --- /dev/null +++ b/scripts/upstream_branch_name.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +tag = ARGV.fetch(0) +digest = ARGV.fetch(1) +disambiguator = ARGV[2] + +abort "release tag must be vMAJOR.MINOR.PATCH" unless tag.match?(/\Av\d+\.\d+\.\d+\z/) +abort "digest must be an exact sha256" unless digest.match?(/\Asha256:[0-9a-f]{64}\z/) + +branch = "compat/upstream-#{tag.delete_prefix('v')}-#{digest.delete_prefix('sha256:')}" +if disambiguator + unless disambiguator.match?(/\A[A-Za-z0-9][A-Za-z0-9-]*\z/) + abort "disambiguator must contain only letters, digits, and hyphens" + end + + branch = "#{branch}-r#{disambiguator}" +end + +abort "generated branch is longer than 128 characters" if branch.length > 128 + +puts branch diff --git a/test/exact_live_contract_test.rb b/test/exact_live_contract_test.rb index 4f5d189..0568c1a 100644 --- a/test/exact_live_contract_test.rb +++ b/test/exact_live_contract_test.rb @@ -26,4 +26,14 @@ class ExactLiveContractTest < Minitest::Test end end end + + def test_accepts_only_one_authoritative_assistant_message + assert_nil OpenCodeCompat::ExactLiveContract.assert_authoritative_assistant_count!(1) + + [0, 2, "not-a-number"].each do |count| + assert_raises(OpenCodeCompat::ExactLiveContract::ContractError) do + OpenCodeCompat::ExactLiveContract.assert_authoritative_assistant_count!(count) + end + end + end end diff --git a/test/image_contract_evidence_test.rb b/test/image_contract_evidence_test.rb new file mode 100644 index 0000000..6ba5fce --- /dev/null +++ b/test/image_contract_evidence_test.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "minitest/autorun" +require "open3" +require "tmpdir" + +class ImageContractEvidenceTest < Minitest::Test + ROOT = File.expand_path("..", __dir__) + RUNNER = File.join(ROOT, "scripts/run_image_contract.sh") + VALID_IMAGE = "ghcr.io/anomalyco/opencode@sha256:#{'a' * 64}" + + def setup + @tmp = Dir.mktmpdir("opencode-image-evidence") + @bin = File.join(@tmp, "bin") + @docker_marker = File.join(@tmp, "docker-called") + FileUtils.mkdir_p(@bin) + docker = File.join(@bin, "docker") + File.write(docker, <<~SH) + #!/usr/bin/env bash + : >"$DOCKER_CALLED_MARKER" + exit 99 + SH + FileUtils.chmod(0o755, docker) + end + + def teardown + FileUtils.remove_entry(@tmp) + end + + def test_invalid_input_overwrites_stale_pass_without_touching_docker + evidence_path = File.join(@tmp, "evidence.json") + File.write(evidence_path, JSON.generate("status" => "pass")) + + _output, _error, status = run_contract( + "OPENCODE_IMAGE" => "latest", + "OPENCODE_RUBY_PATH" => ROOT, + "OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path + ) + + assert_equal 2, status.exitstatus + refute_path_exists @docker_marker + evidence = JSON.parse(File.read(evidence_path)) + assert_equal "fail", evidence.fetch("status") + assert_equal 2, evidence.dig("failure", "exit_status") + assert_equal "latest", evidence.dig("image", "requested") + assert_equal [], evidence.fetch("executed_profiles") + end + + def test_rejects_truncated_registry_and_local_image_ids_before_docker + { + "ghcr.io/anomalyco/opencode@sha256:#{'a' * 12}" => {}, + "sha256:#{'b' * 12}" => {"ALLOW_EXACT_IMAGE_ID" => "1"} + }.each_with_index do |(image, extra_environment), index| + evidence_path = File.join(@tmp, "truncated-#{index}.json") + _output, error, status = run_contract( + { + "OPENCODE_IMAGE" => image, + "OPENCODE_RUBY_PATH" => ROOT, + "OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path + }.merge(extra_environment) + ) + + assert_equal 2, status.exitstatus + assert_match(/must be an OCI digest/, error) + assert_equal image, JSON.parse(File.read(evidence_path)).dig("image", "requested") + end + refute_path_exists @docker_marker + end + + def test_rejects_a_checkout_at_a_different_commit_before_docker + checkout, _commit = git_checkout + evidence_path = File.join(@tmp, "mismatch.json") + + _output, error, status = run_contract( + base_environment(checkout, "f" * 40, evidence_path) + ) + + assert_equal 2, status.exitstatus + assert_match(/does not match expected commit/, error) + assert_failed_without_docker(evidence_path) + end + + def test_rejects_a_dirty_checkout_before_docker + checkout, commit = git_checkout + File.write(File.join(checkout, "dirty.txt"), "not certified\n") + evidence_path = File.join(@tmp, "dirty.json") + + _output, error, status = run_contract(base_environment(checkout, commit, evidence_path)) + + assert_equal 2, status.exitstatus + assert_match(/checkout must be clean/, error) + assert_failed_without_docker(evidence_path) + end + + def test_rejects_a_non_git_path_before_docker + checkout = File.join(@tmp, "not-a-checkout") + FileUtils.mkdir_p(checkout) + evidence_path = File.join(@tmp, "not-git.json") + + _output, error, status = run_contract( + base_environment(checkout, "f" * 40, evidence_path) + ) + + assert_equal 2, status.exitstatus + assert_match(/readable Git checkout/, error) + assert_failed_without_docker(evidence_path) + end + + private + + def base_environment(checkout, commit, evidence_path) + { + "OPENCODE_IMAGE" => VALID_IMAGE, + "OPENCODE_RUBY_PATH" => checkout, + "OPENCODE_RUBY_COMMIT" => commit, + "OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path + } + end + + def run_contract(environment) + env = { + "PATH" => "#{@bin}:#{ENV.fetch('PATH')}", + "DOCKER_CALLED_MARKER" => @docker_marker + }.merge(environment) + Open3.capture3(env, "bash", RUNNER) + end + + def git_checkout + path = File.join(@tmp, "checkout-#{rand(1_000_000)}") + FileUtils.mkdir_p(path) + run_git!(path, "init", "--quiet") + run_git!(path, "config", "user.name", "Compatibility Test") + run_git!(path, "config", "user.email", "compat@example.test") + File.write(File.join(path, "tracked.txt"), "clean\n") + run_git!(path, "add", "tracked.txt") + run_git!(path, "commit", "--quiet", "-m", "fixture") + commit = run_git!(path, "rev-parse", "HEAD").strip + [path, commit] + end + + def run_git!(path, *arguments) + output, error, status = Open3.capture3("git", "-C", path, *arguments) + assert status.success?, error + output + end + + def assert_failed_without_docker(evidence_path) + refute_path_exists @docker_marker + assert_equal "fail", JSON.parse(File.read(evidence_path)).fetch("status") + end +end diff --git a/test/lockstep_client_contract_test.rb b/test/lockstep_client_contract_test.rb new file mode 100644 index 0000000..a6a29a7 --- /dev/null +++ b/test/lockstep_client_contract_test.rb @@ -0,0 +1,318 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "rbconfig" +require "stringio" +require "tmpdir" +require_relative "../ruby/lockstep_client_contract" + +module LockstepClientContractTestSources + class Git + attr_reader :revision + + def initialize(revision) + @revision = revision + end + end + + class Path; end +end + +class LockstepClientContractTest < Minitest::Test + RUBY_COMMIT = "1" * 40 + RAILS_COMMIT = "2" * 40 + RUBY_TAG_OBJECT = "3" * 40 + RAILS_TAG_OBJECT = "4" * 40 + RUBY_TAG = "v0.0.1.alpha7" + RAILS_TAG = "v0.0.1.alpha7" + VERSION = "0.0.1.alpha7" + + FakeSpec = Struct.new(:name, :version, :runtime_dependencies, :source, :full_gem_path, keyword_init: true) + + def setup + @root = Dir.mktmpdir("opencode-lockstep-contract") + @ruby_path = File.join(@root, "opencode-ruby") + @rails_path = File.join(@root, "opencode-rails") + Dir.mkdir(@ruby_path) + Dir.mkdir(@rails_path) + @env = { + "OPENCODE_RUBY_PATH" => @ruby_path, + "OPENCODE_RUBY_COMMIT" => RUBY_COMMIT, + "OPENCODE_RUBY_TAG" => RUBY_TAG, + "OPENCODE_RUBY_TAG_OBJECT" => RUBY_TAG_OBJECT, + "OPENCODE_RUBY_VERSION" => VERSION, + "OPENCODE_RAILS_PATH" => @rails_path, + "OPENCODE_RAILS_COMMIT" => RAILS_COMMIT, + "OPENCODE_RAILS_TAG" => RAILS_TAG, + "OPENCODE_RAILS_TAG_OBJECT" => RAILS_TAG_OBJECT, + "OPENCODE_RAILS_VERSION" => VERSION + } + @heads = {@ruby_path => RUBY_COMMIT, @rails_path => RAILS_COMMIT} + @git_results = {} + seed_annotated_tag(@ruby_path, RUBY_TAG, RUBY_TAG_OBJECT, RUBY_COMMIT) + seed_annotated_tag(@rails_path, RAILS_TAG, RAILS_TAG_OBJECT, RAILS_COMMIT) + @ruby_spec = FakeSpec.new(name: "opencode-ruby", version: Gem::Version.new(VERSION)) + @rails_spec = FakeSpec.new( + name: "opencode-rails", + version: Gem::Version.new(VERSION), + runtime_dependencies: [Gem::Dependency.new("opencode-ruby", "= #{VERSION}")], + full_gem_path: @rails_path + ) + @bundle_ruby_spec = FakeSpec.new( + name: "opencode-ruby", + source: LockstepClientContractTestSources::Git.new(RUBY_COMMIT) + ) + end + + def teardown + FileUtils.remove_entry(@root) + end + + def test_verifies_and_emits_canonical_lockstep_evidence + evidence_path = File.join(@root, "evidence", "lockstep.json") + @env["OPENCODE_COMPAT_EVIDENCE_PATH"] = evidence_path + output = StringIO.new + + document = contract.emit(io: output) + + expected_json = OpenCodeCompat::LockstepClientContract.canonical_json(document) + assert_equal "#{expected_json}\n", output.string + assert_equal "#{expected_json}\n", File.binread(evidence_path) + assert_equal "pass", document.fetch("status") + assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "checkout_commit") + assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "bundler_git_revision") + assert_equal RUBY_TAG_OBJECT, document.dig("opencode_ruby", "tag_provenance", "annotated_tag_object") + assert_equal RUBY_COMMIT, document.dig("opencode_ruby", "tag_provenance", "peeled_commit") + assert_equal RUBY_TAG, document.dig("opencode_ruby", "tag_provenance", "tag") + assert_equal RAILS_COMMIT, document.dig("opencode_rails", "checkout_commit") + assert_equal RAILS_TAG_OBJECT, document.dig("opencode_rails", "tag_provenance", "annotated_tag_object") + assert_equal RAILS_COMMIT, document.dig("opencode_rails", "tag_provenance", "peeled_commit") + assert_equal "= #{VERSION}", document.dig("opencode_rails", "runtime_dependency", "requirement") + assert_equal RUBY_VERSION, document.fetch("ruby_runtime_version") + assert_nil document.dig("workflow", "run_id") + assert_equal expected_json, JSON.generate(JSON.parse(expected_json).sort.to_h) + end + + def test_rejects_checkout_commit_mismatch_for_either_client + { + @ruby_path => "opencode-ruby checkout commit", + @rails_path => "opencode-rails checkout commit" + }.each do |path, message| + original = @heads.fetch(path) + @heads[path] = "f" * 40 + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match message, error.message + ensure + @heads[path] = original + end + end + + def test_rejects_non_annotated_tag_object_for_either_client + { + @ruby_path => RUBY_TAG_OBJECT, + @rails_path => RAILS_TAG_OBJECT + }.each do |path, object| + set_git_result(path, ["cat-file", "-t", object], "commit") + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/annotated tag object type must equal "tag"/, error.message) + ensure + set_git_result(path, ["cat-file", "-t", object], "tag") + end + end + + def test_rejects_tampered_annotated_tag_object_coordinate + tampered_object = "f" * 40 + @env["OPENCODE_RUBY_TAG_OBJECT"] = tampered_object + set_git_result(@ruby_path, ["cat-file", "-t", tampered_object], "tag") + set_git_result(@ruby_path, ["rev-parse", "--verify", "#{tampered_object}^{commit}"], RUBY_COMMIT) + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + + assert_match(/opencode-ruby local tag ref object/, error.message) + assert_match(/#{RUBY_TAG_OBJECT}/, error.message) + end + + def test_rejects_local_tag_ref_resolving_to_another_object + set_git_result( + @rails_path, + ["rev-parse", "--verify", "refs/tags/#{RAILS_TAG}"], + "e" * 40 + ) + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + + assert_match(/opencode-rails local tag ref object/, error.message) + end + + def test_rejects_wrong_object_or_ref_peel + object_peel_key = ["rev-parse", "--verify", "#{RUBY_TAG_OBJECT}^{commit}"] + set_git_result(@ruby_path, object_peel_key, "e" * 40) + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/opencode-ruby annotated tag peeled commit/, error.message) + + set_git_result(@ruby_path, object_peel_key, RUBY_COMMIT) + set_git_result( + @ruby_path, + ["rev-parse", "--verify", "refs/tags/#{RUBY_TAG}^{commit}"], + "e" * 40 + ) + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/opencode-ruby local tag ref peeled commit/, error.message) + end + + def test_rejects_missing_local_tag + @git_results.delete([@ruby_path, ["rev-parse", "--verify", "refs/tags/#{RUBY_TAG}"]]) + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + + assert_match(/could not resolve opencode-ruby local tag ref/, error.message) + end + + def test_rejects_loaded_or_gem_version_drift + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) do + contract(ruby_version: "0.0.1.alpha8").verify + end + assert_match(/loaded Opencode::VERSION/, error.message) + + @rails_spec.version = Gem::Version.new("0.0.1.alpha8") + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/loaded opencode-rails gem version/, error.message) + end + + def test_rejects_non_lockstep_expected_versions + @env["OPENCODE_RAILS_VERSION"] = "0.0.1.alpha8" + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) do + contract(rails_version: "0.0.1.alpha8").verify + end + + assert_match(/lockstep client version/, error.message) + end + + def test_rejects_non_exact_or_wrong_rails_runtime_dependency + ["~> #{VERSION}", "= 0.0.1.alpha6"].each do |requirement| + @rails_spec.runtime_dependencies = [Gem::Dependency.new("opencode-ruby", requirement)] + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/must require opencode-ruby = #{Regexp.escape(VERSION)}/, error.message) + end + end + + def test_rejects_missing_or_duplicate_rails_runtime_dependency + @rails_spec.runtime_dependencies = [] + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/exactly one runtime dependency/, error.message) + + @rails_spec.runtime_dependencies = [ + Gem::Dependency.new("opencode-ruby", "= #{VERSION}"), + Gem::Dependency.new("opencode-ruby", "= #{VERSION}") + ] + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/exactly one runtime dependency/, error.message) + end + + def test_rejects_non_git_or_mismatched_bundler_source + @bundle_ruby_spec.source = LockstepClientContractTestSources::Path.new + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/must be a Git source/, error.message) + + @bundle_ruby_spec.source = LockstepClientContractTestSources::Git.new("f" * 40) + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/Bundler opencode-ruby revision/, error.message) + end + + def test_rejects_loaded_rails_gem_from_another_checkout + other_path = File.join(@root, "other-opencode-rails") + Dir.mkdir(other_path) + @rails_spec.full_gem_path = other_path + + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + + assert_match(/loaded opencode-rails checkout/, error.message) + end + + def test_requires_full_exact_candidate_coordinates + @env.delete("OPENCODE_RUBY_COMMIT") + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/missing required environment: OPENCODE_RUBY_COMMIT/, error.message) + + @env["OPENCODE_RUBY_COMMIT"] = "main" + error = assert_raises(OpenCodeCompat::LockstepClientContract::ContractError) { contract.verify } + assert_match(/full lowercase 40-character Git commit/, error.message) + end + + def test_cli_failure_writes_failure_evidence + stub_path = File.join(@root, "cli-stubs") + evidence_path = File.join(@root, "cli-evidence", "lockstep.json") + FileUtils.mkdir_p(stub_path) + File.binwrite(File.join(stub_path, "bundler.rb"), <<~RUBY) + module Bundler + LoadedDefinition = Struct.new(:specs) + + def self.load + LoadedDefinition.new([]) + end + end + RUBY + File.binwrite(File.join(stub_path, "opencode-rails.rb"), <<~RUBY) + module Opencode + VERSION = #{VERSION.inspect} + RAILS_VERSION = #{VERSION.inspect} + end + RUBY + + _stdout, stderr, status = Open3.capture3( + {"OPENCODE_COMPAT_EVIDENCE_PATH" => evidence_path}, + RbConfig.ruby, + "-I", + stub_path, + File.expand_path("../ruby/lockstep_client_contract.rb", __dir__), + unsetenv_others: true + ) + + refute status.success? + assert_equal 1, status.exitstatus + failure = JSON.parse(File.binread(evidence_path)) + assert_equal "opencode-client-lockstep", failure.fetch("contract") + assert_equal "fail", failure.fetch("status") + assert_match(/missing required environment/, failure.fetch("error")) + assert_equal failure, JSON.parse(stderr) + end + + private + + def contract(ruby_version: VERSION, rails_version: VERSION) + OpenCodeCompat::LockstepClientContract.new( + env: @env, + git_resolver: method(:resolve_git), + loaded_specs: { + "opencode-ruby" => @ruby_spec, + "opencode-rails" => @rails_spec + }, + bundle_specs: [@bundle_ruby_spec], + ruby_version: ruby_version, + rails_version: rails_version + ) + end + + def resolve_git(path, *arguments) + return @heads.fetch(path) if arguments == ["rev-parse", "--verify", "HEAD^{commit}"] + + @git_results.fetch([path, arguments]) + end + + def seed_annotated_tag(path, tag, object, commit) + set_git_result(path, ["cat-file", "-t", object], "tag") + set_git_result(path, ["rev-parse", "--verify", "refs/tags/#{tag}"], object) + set_git_result(path, ["rev-parse", "--verify", "#{object}^{commit}"], commit) + set_git_result(path, ["rev-parse", "--verify", "refs/tags/#{tag}^{commit}"], commit) + end + + def set_git_result(path, arguments, result) + @git_results[[path, arguments]] = result + end +end diff --git a/test/matrix_json_test.rb b/test/matrix_json_test.rb new file mode 100644 index 0000000..5967e3c --- /dev/null +++ b/test/matrix_json_test.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "minitest/autorun" +require "open3" +require "rbconfig" +require "tmpdir" + +class MatrixJsonTest < Minitest::Test + ROOT = File.expand_path("..", __dir__) + + def setup + @tmp = Dir.mktmpdir("opencode-matrix-json") + FileUtils.mkdir_p(File.join(@tmp, "scripts")) + FileUtils.mkdir_p(File.join(@tmp, "manifests")) + FileUtils.cp(File.join(ROOT, "scripts/matrix_json.rb"), File.join(@tmp, "scripts")) + @manifest = JSON.parse(File.read(File.join(ROOT, "manifests/image-matrix.json"))) + end + + def teardown + FileUtils.remove_entry(@tmp) + end + + def test_emits_only_the_actual_shared_client_contract_profile + output, error, status = run_matrix(@manifest) + + assert status.success?, error + JSON.parse(output).fetch("include").each do |target| + assert_equal ["ruby-rest-sse"], target.fetch("profiles") + assert_equal "shared-client-contract-only", target.fetch("certification_scope") + end + end + + def test_rejects_a_manifest_that_overstates_the_executed_profile + @manifest.fetch("public_ci").first["profiles"] = ["rails-persisted-turn"] + + _output, error, status = run_matrix(@manifest) + + refute status.success? + assert_match(/executes exactly ruby-rest-sse/, error) + end + + def test_rejects_a_manifest_that_overstates_the_certification_scope + @manifest.fetch("public_ci").first["certification_scope"] = "consumer-runtime" + + _output, error, status = run_matrix(@manifest) + + refute status.success? + assert_match(/certification_scope must be shared-client-contract-only/, error) + end + + private + + def run_matrix(manifest) + File.write( + File.join(@tmp, "manifests/image-matrix.json"), + JSON.pretty_generate(manifest) + "\n" + ) + Open3.capture3(RbConfig.ruby, File.join(@tmp, "scripts/matrix_json.rb")) + end +end diff --git a/test/repository_test.rb b/test/repository_test.rb index 8611293..de374c6 100644 --- a/test/repository_test.rb +++ b/test/repository_test.rb @@ -42,7 +42,9 @@ class RepositoryTest < Minitest::Test targets.each do |target| assert_match %r{\Aghcr\.io/anomalyco/opencode@sha256:[0-9a-f]{64}\z}, target.fetch("image") refute_includes target.fetch("image"), ":latest" - refute_empty target.fetch("profiles") + assert_equal ["ruby-rest-sse"], target.fetch("profiles") + assert_equal "shared-client-contract-only", + target.fetch("certification_scope", "shared-client-contract-only") next unless target.fetch("certification_status") == "certified" assert_match(/\A[0-9a-f]{40}\z/, target.fetch("certified_client_commit")) @@ -51,14 +53,28 @@ class RepositoryTest < Minitest::Test end end - def test_candidate_is_bound_to_an_annotated_tag + def test_candidate_client_train_is_lockstep_and_bound_to_annotated_tags candidate = json("manifests/client-candidate.json") - provenance = candidate.fetch("tag_provenance") + clients = candidate.fetch("clients") + release_train = candidate.fetch("release_train") - assert_match(/\A[0-9a-f]{40}\z/, candidate.fetch("ref")) - assert_match(/\Av\d+\.\d+\.\d+\.alpha\d+\z/, provenance.fetch("tag")) - assert_match(/\A[0-9a-f]{40}\z/, provenance.fetch("annotated_tag_object")) - assert_equal candidate.fetch("ref"), provenance.fetch("peeled_commit") + assert_equal 2, candidate.fetch("schema_version") + assert_equal "candidate", candidate.fetch("status") + assert_equal %w[opencode-rails opencode-ruby], clients.keys.sort + + clients.each do |name, client| + provenance = client.fetch("tag_provenance") + assert_equal "ajaynomics/#{name}", client.fetch("repository") + assert_equal release_train, client.fetch("version") + assert_match(/\A[0-9a-f]{40}\z/, client.fetch("ref")) + assert_equal "v#{release_train}", provenance.fetch("tag") + assert_match(/\A[0-9a-f]{40}\z/, provenance.fetch("annotated_tag_object")) + assert_equal client.fetch("ref"), provenance.fetch("peeled_commit") + end + + dependency = clients.fetch("opencode-rails").fetch("runtime_dependencies").fetch("opencode-ruby") + assert_equal "= #{release_train}", dependency.fetch("requirement") + assert_equal clients.fetch("opencode-ruby").fetch("ref"), dependency.fetch("ref") end def test_certified_migration_keeps_previous_tuple @@ -147,6 +163,41 @@ class RepositoryTest < Minitest::Test refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service)\b/i, workflow) end + def test_workflow_actions_are_pinned_to_exact_commits + workflows = Dir.glob(File.join(ROOT, ".github/workflows/*.{yml,yaml}")) + refute_empty workflows + + workflows.each do |path| + File.foreach(path).with_index(1) do |line, number| + next unless (match = line.match(/\buses:\s+[^\s@]+@([^\s#]+)/)) + + assert_match(/\A[0-9a-f]{40}\z/, match[1], "#{path}:#{number} must pin an exact action commit") + end + end + end + + def test_candidate_workflow_verifies_and_preserves_the_lockstep_client_tuple + workflow = File.read(File.join(ROOT, ".github/workflows/candidate.yml")) + + refute_includes workflow, "inputs.opencode_ruby_ref" + refute_includes workflow, "inputs.opencode_rails_ref" + assert_includes workflow, "repository: ajaynomics/opencode-ruby" + assert_includes workflow, "repository: ajaynomics/opencode-rails" + assert_includes workflow, "ruby: [\"3.2\", \"3.3\", \"3.4\"]" + assert_includes workflow, "ruby/lockstep_client_contract.rb" + assert_includes workflow, "OPENCODE_RUBY_TAG_OBJECT" + assert_includes workflow, "OPENCODE_RAILS_TAG_OBJECT" + assert_operator workflow.scan("fetch-tags: true").length, :>=, 2 + assert_operator workflow.scan("actions/upload-artifact@").length, :>=, 3 + assert_includes workflow, "OPENCODE_COMPAT_EVIDENCE_PATH" + assert_includes workflow, "OPENCODE_REQUIRED_CONSUMER_PROFILES" + assert_includes workflow, 'BUNDLE_PATH: ${{ runner.temp }}/opencode-ruby-bundle' + assert_includes workflow, "Install candidate dependencies outside the checkout" + refute_includes workflow, "OPENCODE_MATRIX_PROFILES" + refute_includes workflow, "OPENCODE_CERTIFICATION_SCOPE" + refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service|gh\s+pr\s+merge)\b/i, workflow) + end + def test_tuple_promotion_has_no_command_execution_or_deployment_client paths = %w[ lib/opencode_compat/runtime_tuple_promoter.rb @@ -164,8 +215,11 @@ class RepositoryTest < Minitest::Test runner = File.read(File.join(ROOT, "scripts/run_image_contract.sh")) assert_includes probe, "ExactLiveContract.assert_final_text!" + assert_includes probe, "ExactLiveContract.assert_authoritative_assistant_count!" refute_includes probe, "full_text.include?" assert_includes runner, "exact_live_contract.rb" + assert_includes runner, "OPENCODE_COMPAT_EVIDENCE_PATH" + assert_includes runner, "OPENCODE_EXPECTED_VERSION" refute_match(/request_count.*-lt\s+1/, runner) end end diff --git a/test/watcher_test.rb b/test/watcher_test.rb new file mode 100644 index 0000000..dd3dfa4 --- /dev/null +++ b/test/watcher_test.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "minitest/autorun" +require "open3" +require "rbconfig" +require "tmpdir" + +class WatcherTest < Minitest::Test + ROOT = File.expand_path("..", __dir__) + RUBY = RbConfig.ruby + + def setup + @tmp = Dir.mktmpdir("opencode-watcher") + FileUtils.mkdir_p(File.join(@tmp, "scripts")) + FileUtils.mkdir_p(File.join(@tmp, "manifests")) + FileUtils.cp(File.join(ROOT, "scripts/record_upstream_candidate.rb"), File.join(@tmp, "scripts")) + %w[upstream.json image-matrix.json].each do |name| + FileUtils.cp(File.join(ROOT, "manifests", name), File.join(@tmp, "manifests")) + end + end + + def teardown + FileUtils.remove_entry(@tmp) + end + + def test_same_tag_and_digest_is_an_exact_noop + upstream = read_json("manifests/upstream.json") + before = watched_bytes + + run_recorder!( + upstream.fetch("release_tag"), + upstream.fetch("published_at"), + upstream.fetch("release_url"), + upstream.fetch("image").split("@").last + ) + + assert_equal before, watched_bytes + end + + def test_same_tag_and_digest_repairs_a_missing_public_target_then_noops + upstream = read_json("manifests/upstream.json") + matrix = read_json("manifests/image-matrix.json") + image = upstream.fetch("image") + matrix.fetch("public_ci").reject! { |target| target.fetch("image") == image } + write_json("manifests/image-matrix.json", matrix) + upstream_before = File.binread(File.join(@tmp, "manifests/upstream.json")) + + arguments = [ + upstream.fetch("release_tag"), + upstream.fetch("published_at"), + upstream.fetch("release_url"), + image.split("@").last + ] + run_recorder!(*arguments) + + repaired = read_json("manifests/image-matrix.json") + targets = repaired.fetch("public_ci").select { |target| target.fetch("image") == image } + assert_equal 1, targets.length + assert_equal "pending", targets.first.fetch("certification_status") + assert_equal ["ruby-rest-sse"], targets.first.fetch("profiles") + assert_equal upstream_before, File.binread(File.join(@tmp, "manifests/upstream.json")) + + after = watched_bytes + run_recorder!(*arguments) + assert_equal after, watched_bytes + end + + def test_duplicate_public_targets_are_rejected_without_writing_either_manifest + upstream = read_json("manifests/upstream.json") + matrix = read_json("manifests/image-matrix.json") + target = matrix.fetch("public_ci").find { |entry| entry.fetch("image") == upstream.fetch("image") } + duplicate = target.merge("id" => "#{target.fetch('id')}-duplicate") + matrix.fetch("public_ci") << duplicate + write_json("manifests/image-matrix.json", matrix) + before = watched_bytes + + _output, error, status = capture_recorder( + upstream.fetch("release_tag"), + upstream.fetch("published_at"), + upstream.fetch("release_url"), + upstream.fetch("image").split("@").last + ) + + refute status.success? + assert_match(/must appear exactly once in public_ci; found 2/, error) + assert_equal before, watched_bytes + end + + def test_missing_public_target_with_conflicting_generated_id_is_rejected_without_writing + upstream = read_json("manifests/upstream.json") + matrix = read_json("manifests/image-matrix.json") + image = upstream.fetch("image") + removed = matrix.fetch("public_ci").find { |target| target.fetch("image") == image } + matrix.fetch("public_ci").reject! { |target| target.fetch("image") == image } + matrix.fetch("public_ci") << removed.merge( + "id" => "upstream-#{upstream.fetch('version')}-#{image.split('sha256:').last}", + "image" => "ghcr.io/anomalyco/opencode@sha256:#{'e' * 64}" + ) + write_json("manifests/image-matrix.json", matrix) + before = watched_bytes + + _output, error, status = capture_recorder( + upstream.fetch("release_tag"), + upstream.fetch("published_at"), + upstream.fetch("release_url"), + image.split("@").last + ) + + refute status.success? + assert_match(/public_ci id .* is already used by another image/, error) + assert_equal before, watched_bytes + end + + def test_same_tag_with_new_digest_records_a_distinct_pending_target_once + upstream = read_json("manifests/upstream.json") + replacement = "sha256:#{'f' * 64}" + arguments = [ + upstream.fetch("release_tag"), + upstream.fetch("published_at"), + upstream.fetch("release_url"), + replacement + ] + + run_recorder!(*arguments) + changed_upstream = read_json("manifests/upstream.json") + matrix = read_json("manifests/image-matrix.json") + target = matrix.fetch("public_ci").last + + assert_equal "ghcr.io/anomalyco/opencode@#{replacement}", changed_upstream.fetch("image") + expected_version = upstream.fetch("release_tag").delete_prefix("v") + assert_equal "upstream-#{expected_version}-#{'f' * 64}", target.fetch("id") + assert_equal "ghcr.io/anomalyco/opencode@#{replacement}", target.fetch("image") + assert_equal "pending", target.fetch("certification_status") + assert_equal "shared-client-contract-only", target.fetch("certification_scope") + assert_empty target.fetch("consumers") + assert_empty target.fetch("required_consumer_profiles") + + after = watched_bytes + run_recorder!(*arguments) + assert_equal after, watched_bytes + end + + def test_recorder_rejects_untrusted_release_metadata + upstream = read_json("manifests/upstream.json") + digest = upstream.fetch("image").split("@").last + + refute recorder_status("branch-name", upstream.fetch("published_at"), upstream.fetch("release_url"), digest).success? + refute recorder_status(upstream.fetch("release_tag"), "yesterday", upstream.fetch("release_url"), digest).success? + refute recorder_status(upstream.fetch("release_tag"), upstream.fetch("published_at"), "https://example.test/release", digest).success? + refute recorder_status(upstream.fetch("release_tag"), upstream.fetch("published_at"), upstream.fetch("release_url"), "sha256:nope").success? + end + + def test_branch_names_bind_the_full_tag_and_digest_and_support_retries + script = File.join(ROOT, "scripts/upstream_branch_name.rb") + digest = "sha256:#{'a' * 64}" + replacement = "sha256:#{'b' * 64}" + + base = run_script!(script, "v1.18.3", digest).strip + changed = run_script!(script, "v1.18.3", replacement).strip + retry_branch = run_script!(script, "v1.18.3", digest, "123-2").strip + + assert_equal "compat/upstream-1.18.3-#{'a' * 64}", base + refute_equal base, changed + assert_equal "#{base}-r123-2", retry_branch + refute run_status(script, "v1.18.3", digest, "bad/value").success? + end + + def test_workflow_can_only_update_manifests_and_open_a_pr + workflow = File.read(File.join(ROOT, ".github/workflows/watch-upstream.yml")) + + assert_includes workflow, "contents: write" + assert_includes workflow, "pull-requests: write" + assert_includes workflow, "git add manifests/image-matrix.json manifests/upstream.json" + assert_includes workflow, "gh pr create" + assert_includes workflow, "scripts/upstream_branch_name.rb" + assert_includes workflow, "git ls-remote --exit-code --heads" + assert_includes workflow, "--base main" + assert_includes workflow, ".baseRefName == \"main\"" + assert_includes workflow, "isCrossRepository == false" + assert_includes workflow, "headRefOid" + assert_includes workflow, "application/vnd.github.raw+json" + assert_includes workflow, "current_image_count" + assert_includes workflow, '[.public_ci[]? | select(.image == $image)] | length' + assert_includes workflow, '[ "$current_image_count" -eq 1 ]' + assert_includes workflow, ".release_tag == $tag and .image == $image" + assert_includes workflow, "([.public_ci[]? | select(.image == $image)] | length) == 1" + content_check = workflow.index(".release_tag == $tag and .image == $image") + skip = workflow.index('echo "skip=true"') + assert_operator content_check, :<, skip + refute_match(/gh\s+pr\s+merge|repository_dispatch|workflow_dispatches|git\s+push\s+--force/i, workflow) + refute_match(/\b(kamal|kubectl|helm|nomad|docker\s+service)\b/i, workflow) + end + + private + + def run_recorder!(*arguments) + run_script!(File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments) + end + + def recorder_status(*arguments) + run_status(File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments) + end + + def capture_recorder(*arguments) + Open3.capture3(RUBY, File.join(@tmp, "scripts/record_upstream_candidate.rb"), *arguments) + end + + def run_script!(script, *arguments) + output, error, status = Open3.capture3(RUBY, script, *arguments) + assert status.success?, error + output + end + + def run_status(script, *arguments) + _output, _error, status = Open3.capture3(RUBY, script, *arguments) + status + end + + def read_json(relative_path) + JSON.parse(File.read(File.join(@tmp, relative_path))) + end + + def write_json(relative_path, value) + File.write(File.join(@tmp, relative_path), JSON.pretty_generate(value) + "\n") + end + + def watched_bytes + %w[manifests/upstream.json manifests/image-matrix.json].map do |relative_path| + File.binread(File.join(@tmp, relative_path)) + end + end +end