Establish OpenCode compatibility certification corpus

2026-07-18 13:35:32 -07:00
commit 51122eb000
39 changed files with 1218 additions and 0 deletions
--- a/scripts/fake_llm.py
+++ b/scripts/fake_llm.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Deterministic OpenAI-compatible streaming server for image contracts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+
+class State:
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.requests: list[dict[str, object]] = []
+
+    def record(self, path: str, body: object) -> None:
+        with self.lock:
+            self.requests.append({"path": path, "body": body, "at": time.time()})
+
+    def snapshot(self) -> dict[str, object]:
+        with self.lock:
+            return {"request_count": len(self.requests), "requests": list(self.requests)}
+
+
+STATE = State()
+
+
+def chat_stream(text: str) -> bytes:
+    chunks = [
+        {
+            "id": "chatcmpl-opencode-compat",
+            "object": "chat.completion.chunk",
+            "choices": [{"delta": {"role": "assistant"}}],
+        },
+        {
+            "id": "chatcmpl-opencode-compat",
+            "object": "chat.completion.chunk",
+            "choices": [{"delta": {"content": text}}],
+        },
+        {
+            "id": "chatcmpl-opencode-compat",
+            "object": "chat.completion.chunk",
+            "choices": [{"delta": {}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
+        },
+    ]
+    payload = "".join(f"data: {json.dumps(chunk, separators=(',', ':'))}\n\n" for chunk in chunks)
+    payload += "data: [DONE]\n\n"
+    return payload.encode()
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "OpenCodeCompatLLM/1"
+
+    def log_message(self, _format: str, *_args: object) -> None:
+        return
+
+    def send_bytes(self, status: int, body: bytes, content_type: str) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self) -> None:  # noqa: N802
+        if self.path == "/health":
+            self.send_bytes(200, b'{"ok":true}', "application/json")
+            return
+        if self.path == "/stats":
+            self.send_bytes(200, json.dumps(STATE.snapshot()).encode(), "application/json")
+            return
+        if self.path == "/v1/models":
+            body = {"object": "list", "data": [{"id": "compat-model", "object": "model"}]}
+            self.send_bytes(200, json.dumps(body).encode(), "application/json")
+            return
+        self.send_bytes(404, b'{"error":"not found"}', "application/json")
+
+    def do_POST(self) -> None:  # noqa: N802
+        length = int(self.headers.get("Content-Length", "0"))
+        raw = self.rfile.read(length) if length else b"{}"
+        try:
+            body = json.loads(raw)
+        except json.JSONDecodeError:
+            body = {"invalid_json": raw.decode(errors="replace")}
+        STATE.record(self.path, body)
+
+        if self.path == "/v1/chat/completions":
+            self.send_bytes(200, chat_stream("compat-ok"), "text/event-stream")
+            return
+
+        self.send_bytes(404, b'{"error":"unsupported endpoint"}', "application/json")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=0)
+    parser.add_argument("--port-file", required=True)
+    args = parser.parse_args()
+
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+    Path(args.port_file).write_text(str(server.server_port), encoding="utf-8")
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/matrix_json.rb
+++ b/scripts/matrix_json.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "json"
+
+root = File.expand_path("..", __dir__)
+manifest = JSON.parse(File.read(File.join(root, "manifests/image-matrix.json")))
+matrix = manifest.fetch("public_ci").map do |target|
+  {
+    "id" => target.fetch("id"),
+    "version" => target.fetch("version"),
+    "image" => target.fetch("image")
+  }
+end
+
+puts JSON.generate("include" => matrix)
--- a/scripts/record_upstream_candidate.rb
+++ b/scripts/record_upstream_candidate.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+root = File.expand_path("..", __dir__)
+tag, published_at, release_url, digest = ARGV
+abort "usage: record_upstream_candidate.rb TAG PUBLISHED_AT RELEASE_URL DIGEST" unless digest
+abort "invalid upstream release tag: #{tag.inspect}" unless tag.match?(/\Av\d+\.\d+\.\d+\z/)
+abort "invalid OCI digest: #{digest.inspect}" unless digest.match?(/\Asha256:[0-9a-f]{64}\z/)
+
+version = tag.delete_prefix("v")
+image = "ghcr.io/anomalyco/opencode@#{digest}"
+upstream_path = File.join(root, "manifests/upstream.json")
+matrix_path = File.join(root, "manifests/image-matrix.json")
+upstream = JSON.parse(File.read(upstream_path))
+matrix = JSON.parse(File.read(matrix_path))
+
+exit 0 if upstream.fetch("release_tag") == tag
+
+upstream.merge!(
+  "release_tag" => tag,
+  "version" => version,
+  "published_at" => published_at,
+  "release_url" => release_url,
+  "image" => image,
+  "observed_at" => Time.now.utc.iso8601
+)
+
+unless matrix.fetch("public_ci").any? { |target| target.fetch("image") == image }
+  matrix.fetch("public_ci") << {
+    "id" => "upstream-#{version}",
+    "version" => version,
+    "image" => image,
+    "tag_provenance" => "ghcr.io/anomalyco/opencode:#{version}",
+    "consumers" => ["upstream-candidate"],
+    "profiles" => ["ruby-rest-sse", "rails-persisted-turn", "voice-stream", "strict-v2", "plugin-ledger", "provider-hooks"],
+    "certification_status" => "pending"
+  }
+end
+
+File.write(upstream_path, JSON.pretty_generate(upstream) + "\n")
+File.write(matrix_path, JSON.pretty_generate(matrix) + "\n")
--- a/scripts/run_image_contract.sh
+++ b/scripts/run_image_contract.sh
@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+image="${OPENCODE_IMAGE:?set OPENCODE_IMAGE to an immutable image digest}"
+gem_path="${OPENCODE_RUBY_PATH:?set OPENCODE_RUBY_PATH to the candidate checkout}"
+
+if [[ "$image" != *@sha256:* ]]; then
+  if [[ "${ALLOW_EXACT_IMAGE_ID:-0}" != "1" || "$image" != sha256:* ]]; then
+    echo "OPENCODE_IMAGE must be an OCI digest (or an exact local image ID with ALLOW_EXACT_IMAGE_ID=1)" >&2
+    exit 2
+  fi
+fi
+
+container_name="opencode-compat-${RANDOM}-$$"
+llm_container_name="opencode-compat-llm-${RANDOM}-$$"
+network_name="opencode-compat-net-${RANDOM}-$$"
+python_image="python@sha256:399babc8b49529dabfd9c922f2b5eea81d611e4512e3ed250d75bd2e7683f4b0"
+
+cleanup() {
+  status=$?
+  if [[ "$status" != "0" ]]; then
+    echo "OpenCode compatibility probe failed; container log follows" >&2
+    docker logs "$container_name" >&2 2>/dev/null || true
+    echo "Deterministic model request summary follows" >&2
+    docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats >&2 2>/dev/null || true
+    echo >&2
+  fi
+  docker rm -f "$container_name" >/dev/null 2>&1 || true
+  docker rm -f "$llm_container_name" >/dev/null 2>&1 || true
+  docker network rm "$network_name" >/dev/null 2>&1 || true
+  return "$status"
+}
+trap cleanup EXIT
+
+docker network create "$network_name" >/dev/null
+docker run --detach \
+  --name "$llm_container_name" \
+  --network "$network_name" \
+  --network-alias compat-llm \
+  --volume "$repo_root/scripts:/compat:ro" \
+  "$python_image" \
+  python /compat/fake_llm.py --port 8080 --port-file /tmp/compat-port \
+  >/dev/null
+
+for _ in $(seq 1 100); do
+  if docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/health >/dev/null 2>&1; then
+    llm_ready=1
+    break
+  fi
+  sleep 0.05
+done
+[[ "${llm_ready:-0}" == "1" ]] || { echo "fake LLM did not start" >&2; exit 1; }
+
+config_json="$(jq -cn --arg url "http://compat-llm:8080/v1" '{
+  "$schema": "https://opencode.ai/config.json",
+  formatter: false,
+  lsp: false,
+  provider: {
+    compat: {
+      name: "Compatibility fixture",
+      id: "compat",
+      env: [],
+      npm: "@ai-sdk/openai-compatible",
+      models: {
+        "compat-model": {
+          id: "compat-model",
+          name: "Compatibility model",
+          attachment: false,
+          reasoning: false,
+          temperature: false,
+          tool_call: true,
+          release_date: "2026-01-01",
+          limit: {context: 100000, output: 10000},
+          cost: {input: 0, output: 0},
+          options: {}
+        }
+      },
+      options: {apiKey: "compat-key", baseURL: $url}
+    }
+  }
+}')"
+
+docker run --detach \
+  --name "$container_name" \
+  --network "$network_name" \
+  --publish 127.0.0.1::4096 \
+  --env "OPENCODE_CONFIG_CONTENT=$config_json" \
+  --env OPENCODE_DISABLE_AUTOUPDATE=1 \
+  --env OPENCODE_DISABLE_AUTOCOMPACT=1 \
+  --env OPENCODE_DISABLE_MODELS_FETCH=1 \
+  --env OPENCODE_DISABLE_PROJECT_CONFIG=1 \
+  --env OPENCODE_PURE=1 \
+  "$image" serve --hostname 0.0.0.0 --port 4096 \
+  >/dev/null
+
+host_port="$(docker port "$container_name" 4096/tcp | sed -E 's/.*:([0-9]+)$/\1/' | head -1)"
+base_url="http://127.0.0.1:${host_port}"
+
+ready=0
+for _ in $(seq 1 120); do
+  if curl --fail --silent --connect-timeout 1 --max-time 2 "$base_url/global/health" >/dev/null; then
+    ready=1
+    break
+  fi
+  if ! docker inspect "$container_name" --format '{{.State.Running}}' 2>/dev/null | grep -qx true; then
+    docker logs "$container_name" >&2 || true
+    exit 1
+  fi
+  sleep 0.25
+done
+
+if [[ "$ready" != "1" ]]; then
+  docker logs "$container_name" >&2 || true
+  echo "OpenCode did not become ready" >&2
+  exit 1
+fi
+
+server_version="$(docker exec "$container_name" opencode --version)"
+image_id="$(docker inspect "$container_name" --format '{{.Image}}')"
+gem_commit="$(git -C "$gem_path" rev-parse HEAD 2>/dev/null || true)"
+
+OPENCODE_BASE_URL="$base_url" \
+OPENCODE_RUBY_PATH="$gem_path" \
+OPENCODE_RUBY_COMMIT="$gem_commit" \
+  ruby "$repo_root/ruby/live_probe.rb"
+
+llm_stats="$(docker exec "$llm_container_name" wget -qO- http://127.0.0.1:8080/stats)"
+request_count="$(jq -r '.request_count' <<<"$llm_stats")"
+if [[ "$request_count" -lt 1 ]]; then
+  echo "OpenCode never called the deterministic model" >&2
+  exit 1
+fi
+
+jq -cn \
+  --arg status pass \
+  --arg image "$image" \
+  --arg image_id "$image_id" \
+  --arg server_version "$server_version" \
+  --arg gem_commit "$gem_commit" \
+  --argjson llm_request_count "$request_count" \
+  '{status:$status,image:$image,image_id:$image_id,server_version:$server_version,opencode_ruby_commit:$gem_commit,llm_request_count:$llm_request_count}'