Ajay Krishnan
51dceee224
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
327 lines
13 KiB
Bash
Executable File
327 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
cleanup_ephemeral_lifecycle_lock() {
|
|
local project="${1:-}" lock_dir lock_file owner mode lock_owner
|
|
[[ "${project}" =~ ^context-kit-release-[0-9]+$ ]] || {
|
|
printf 'release-check: refusing non-release lifecycle lock project: %s\n' "${project}" >&2
|
|
return 64
|
|
}
|
|
command -v flock >/dev/null 2>&1 || {
|
|
printf 'release-check: flock is required for lifecycle lock cleanup\n' >&2
|
|
return 1
|
|
}
|
|
command -v stat >/dev/null 2>&1 || {
|
|
printf 'release-check: stat is required for lifecycle lock cleanup\n' >&2
|
|
return 1
|
|
}
|
|
|
|
lock_dir="/tmp/context-kit-${project}.lock"
|
|
lock_file="${lock_dir}/lifecycle"
|
|
[[ -e "${lock_dir}" || -L "${lock_dir}" ]] || return 0
|
|
[[ -d "${lock_dir}" && ! -L "${lock_dir}" ]] || {
|
|
printf 'release-check: refusing unsafe lifecycle lock path: %s\n' "${lock_dir}" >&2
|
|
return 1
|
|
}
|
|
if [[ -e "${lock_file}" || -L "${lock_file}" ]]; then
|
|
[[ -f "${lock_file}" && ! -L "${lock_file}" ]] || {
|
|
printf 'release-check: refusing unsafe lifecycle lock file: %s\n' "${lock_file}" >&2
|
|
return 1
|
|
}
|
|
fi
|
|
|
|
(
|
|
exec 9>"${lock_file}" || return 1
|
|
if ! flock -n 9; then
|
|
printf 'release-check: lifecycle lock is still held: %s\n' "${lock_dir}" >&2
|
|
return 1
|
|
fi
|
|
|
|
[[ -d "${lock_dir}" && ! -L "${lock_dir}" ]] || {
|
|
printf 'release-check: lifecycle lock path changed while acquiring it: %s\n' "${lock_dir}" >&2
|
|
return 1
|
|
}
|
|
owner="$(stat -c %u "${lock_dir}")"
|
|
mode="$(stat -c %a "${lock_dir}")"
|
|
[[ "${owner}" == "$(id -u)" && "${mode}" == "700" ]] || {
|
|
printf 'release-check: refusing lifecycle lock with uid %s and mode %s: %s\n' "${owner}" "${mode}" "${lock_dir}" >&2
|
|
return 1
|
|
}
|
|
[[ -f "${lock_file}" && ! -L "${lock_file}" ]] || {
|
|
printf 'release-check: lifecycle lock file changed while acquiring it: %s\n' "${lock_file}" >&2
|
|
return 1
|
|
}
|
|
lock_owner="$(stat -c %u "${lock_file}")"
|
|
[[ "${lock_owner}" == "$(id -u)" ]] || {
|
|
printf 'release-check: refusing lifecycle lock file owned by uid %s: %s\n' "${lock_owner}" "${lock_file}" >&2
|
|
return 1
|
|
}
|
|
|
|
rm -f -- "${lock_file}"
|
|
if ! rmdir -- "${lock_dir}"; then
|
|
printf 'release-check: lifecycle lock directory contains unexpected entries: %s\n' "${lock_dir}" >&2
|
|
return 1
|
|
fi
|
|
)
|
|
}
|
|
|
|
if [[ "${1:-}" == "--cleanup-ephemeral-lock" ]]; then
|
|
[[ "$#" -eq 2 ]] || {
|
|
printf 'usage: scripts/release-check --cleanup-ephemeral-lock context-kit-release-PID\n' >&2
|
|
exit 64
|
|
}
|
|
cleanup_ephemeral_lifecycle_lock "$2"
|
|
exit
|
|
fi
|
|
[[ "$#" -eq 0 ]] || {
|
|
printf 'usage: scripts/release-check\n' >&2
|
|
exit 64
|
|
}
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "${ROOT}"
|
|
|
|
tmp_dir="$(mktemp -d)"
|
|
pick_port() {
|
|
python - <<'PY'
|
|
import socket
|
|
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
print(sock.getsockname()[1])
|
|
PY
|
|
}
|
|
|
|
release_id="release-$$"
|
|
export CONTEXT_KIT_COMPOSE_PROJECT="context-kit-${release_id}"
|
|
export CONTEXT_KIT_DATA_DIR="${tmp_dir}/data"
|
|
export CONTEXT_KIT_PROJECT_DIR="${ROOT}"
|
|
CONTEXT_KIT_SEARXNG_PORT="$(pick_port)"
|
|
CONTEXT_KIT_WEB_SEARCH_PORT="$(pick_port)"
|
|
CONTEXT_KIT_DOCS_PORT="$(pick_port)"
|
|
export CONTEXT_KIT_SEARXNG_PORT CONTEXT_KIT_WEB_SEARCH_PORT CONTEXT_KIT_DOCS_PORT
|
|
export CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR="${tmp_dir}/local-sources"
|
|
export CONTEXT_KIT_DOCS_LOCAL_SOURCES_PORT="8769"
|
|
local_sources_profile="${tmp_dir}/sources.local.txt"
|
|
export CONTEXT_KIT_DOCS_SOURCES="config/sources.default.txt ${local_sources_profile}"
|
|
export CONTEXT_KIT_LOCAL_SOURCE_SMOKE_URL="http://127.0.0.1:${CONTEXT_KIT_DOCS_LOCAL_SOURCES_PORT}/release-fixture/llms.txt"
|
|
export CONTEXT_KIT_WEB_SEARCH_IMAGE="context-kit/web-search-mcp:${release_id}"
|
|
export CONTEXT_KIT_DOCS_IMAGE="context-kit/docs-mcp:${release_id}"
|
|
|
|
mkdir -p "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/release-fixture"
|
|
printf '%s\n' \
|
|
'# Release Check Local Source' \
|
|
'' \
|
|
'> Exercises machine-local llms.txt serving and refresh behavior.' \
|
|
'' \
|
|
'## MCP' \
|
|
'' \
|
|
'- [Model Context Protocol](https://modelcontextprotocol.io/)' \
|
|
> "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/release-fixture/llms.txt"
|
|
printf '%s\n' "${CONTEXT_KIT_LOCAL_SOURCE_SMOKE_URL}" > "${local_sources_profile}"
|
|
|
|
cleanup() {
|
|
local status="$?" lock_status=0
|
|
trap - EXIT
|
|
docker compose -p "${CONTEXT_KIT_COMPOSE_PROJECT}" -f compose.yml down -v --remove-orphans >/dev/null 2>&1 || true
|
|
docker image rm "${CONTEXT_KIT_WEB_SEARCH_IMAGE}" "${CONTEXT_KIT_DOCS_IMAGE}" >/dev/null 2>&1 || true
|
|
cleanup_ephemeral_lifecycle_lock "${CONTEXT_KIT_COMPOSE_PROJECT}" || lock_status=$?
|
|
rm -rf "${tmp_dir}"
|
|
if [[ "${status}" -ne 0 ]]; then
|
|
exit "${status}"
|
|
fi
|
|
exit "${lock_status}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
check_node() {
|
|
local file
|
|
for file in "$@"; do
|
|
node --check "${file}"
|
|
done
|
|
}
|
|
|
|
assert_redaction_check_does_not_disclose_matches() {
|
|
local fixture="${tmp_dir}/redaction-fixture.txt"
|
|
local output="${tmp_dir}/redaction-output.txt"
|
|
local blocked_path="/data/proj""ects/context-kit-private-fixture"
|
|
printf 'blocked=%s\n' "${blocked_path}" > "${fixture}"
|
|
if bin/context-kit redaction-check "${fixture}" >"${output}" 2>&1; then
|
|
printf 'redaction-check test unexpectedly passed\n' >&2
|
|
return 1
|
|
fi
|
|
if grep -F "${blocked_path}" "${output}" >/dev/null; then
|
|
printf 'redaction-check disclosed matched content\n' >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
assert_web_search_image() {
|
|
docker run --rm --entrypoint node \
|
|
-e EXPECTED_MAX_BYTES="${CONTEXT_KIT_WEB_SEARCH_MAX_BYTES:-52428800}" \
|
|
"${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
-e '
|
|
const fs = require("node:fs");
|
|
const expected = Number(process.env.EXPECTED_MAX_BYTES) || 0;
|
|
const actual = Number(process.env.MAX_BYTES) || 0;
|
|
if (process.getuid && process.getuid() === 0) process.exit(1);
|
|
if (actual !== expected) process.exit(1);
|
|
|
|
const serverPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/server.js";
|
|
const server = fs.readFileSync(serverPath, "utf8");
|
|
if (!server.includes("max_download_bytes: z.number().int().min(1).max(MAX_BYTES).optional()")) process.exit(1);
|
|
|
|
const bingPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/bing.js";
|
|
const bing = fs.readFileSync(bingPath, "utf8");
|
|
if (!bing.includes("Context Kit override for @zhafron/mcp-web-search 1.3.0")) process.exit(1);
|
|
if (!bing.includes("waitForSelector")) process.exit(1);
|
|
if (!bing.includes("decodeBingRedirect")) process.exit(1);
|
|
' >/dev/null
|
|
|
|
docker run --rm --entrypoint /usr/bin/test \
|
|
"${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
-x "${CONTEXT_KIT_WEB_SEARCH_CHROME_PATH:-/usr/bin/chromium}"
|
|
|
|
docker run --rm --entrypoint /usr/bin/test \
|
|
"${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
-x /opt/mcp-proxy/bin/mcp-proxy
|
|
|
|
docker run --rm --entrypoint /usr/bin/test \
|
|
"${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
-r /usr/local/lib/context-kit/http-entrypoint.mjs
|
|
|
|
docker run --rm --entrypoint /usr/bin/test \
|
|
"${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
-r /usr/local/lib/context-kit/mcp-probe.mjs
|
|
}
|
|
|
|
assert_hostile_requests_rejected() {
|
|
local status
|
|
status="$(curl -sS -o /dev/null -w '%{http_code}' \
|
|
-H 'Host: attacker.example' \
|
|
-H 'Content-Type: application/json' \
|
|
--data '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \
|
|
"http://127.0.0.1:${CONTEXT_KIT_WEB_SEARCH_PORT}/mcp")"
|
|
[[ "${status}" == 421 ]] || {
|
|
printf 'hostile Host returned HTTP %s instead of 421\n' "${status}" >&2
|
|
return 1
|
|
}
|
|
status="$(curl -sS -o /dev/null -w '%{http_code}' \
|
|
-H 'Origin: https://attacker.example' \
|
|
-H 'Content-Type: application/json' \
|
|
--data '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \
|
|
"http://127.0.0.1:${CONTEXT_KIT_WEB_SEARCH_PORT}/mcp")"
|
|
[[ "${status}" == 403 ]] || {
|
|
printf 'hostile Origin returned HTTP %s instead of 403\n' "${status}" >&2
|
|
return 1
|
|
}
|
|
}
|
|
|
|
assert_web_search_backend_supervision() {
|
|
local container_id before after attempt
|
|
container_id="$(docker compose -p "${CONTEXT_KIT_COMPOSE_PROJECT}" -f compose.yml ps -q web-search-mcp)"
|
|
before="$(docker inspect -f '{{.RestartCount}}' "${container_id}")"
|
|
docker exec "${container_id}" sh -eu -c '
|
|
for command_path in /proc/[0-9]*/cmdline; do
|
|
command="$(tr "\000" " " < "${command_path}")"
|
|
case "${command}" in
|
|
*node*mcp-web-search*)
|
|
pid="${command_path#/proc/}"
|
|
pid="${pid%/cmdline}"
|
|
kill -KILL "${pid}"
|
|
exit 0
|
|
;;
|
|
esac
|
|
done
|
|
exit 1
|
|
'
|
|
for ((attempt=1; attempt <= 60; attempt++)); do
|
|
after="$(docker inspect -f '{{.RestartCount}}' "${container_id}")"
|
|
if [[ "${after}" -gt "${before}" ]] && node docker/web-search/mcp-probe.mjs "http://127.0.0.1:${CONTEXT_KIT_WEB_SEARCH_PORT}/mcp" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
printf 'web-search container did not restart after backend death\n' >&2
|
|
return 1
|
|
}
|
|
|
|
git diff --check HEAD
|
|
git show --check --format= HEAD >/dev/null
|
|
git ls-files --cached --error-unmatch \
|
|
docker/web-search/patch-mcp-web-search.mjs \
|
|
docker/web-search/overrides/bing.js \
|
|
docker/docs/constraints.txt \
|
|
scripts/mcp-smoke-client.mjs \
|
|
scripts/smoke-web-search.mjs \
|
|
scripts/smoke-docs.mjs \
|
|
scripts/smoke-repomix.mjs \
|
|
scripts/test-compose-upgrade.sh \
|
|
scripts/test-lifecycle.sh \
|
|
scripts/test-web-search-http.mjs \
|
|
docker/web-search/mcp-probe.mjs \
|
|
docker/web-search/http-entrypoint.mjs \
|
|
scripts/release-check >/dev/null
|
|
bash -n bin/context-kit
|
|
bash -n scripts/release-check
|
|
bash -n scripts/test-compose-upgrade.sh
|
|
bash -n scripts/test-lifecycle.sh
|
|
sh -n docker/docs/entrypoint.sh
|
|
check_node docker/web-search/patch-mcp-web-search.mjs docker/web-search/overrides/bing.js docker/web-search/overrides/brave.js docker/web-search/overrides/browser-fetch.js docker/web-search/overrides/registry.js docker/web-search/mcp-probe.mjs docker/web-search/http-entrypoint.mjs scripts/docs-rebuild.mjs scripts/mcp-smoke-client.mjs scripts/smoke-web-search.mjs scripts/smoke-docs.mjs scripts/smoke-repomix.mjs scripts/test-docs-candidate.mjs scripts/test-web-search-candidate.mjs scripts/test-web-search-http.mjs scripts/test-web-search-quality.mjs
|
|
|
|
node -e 'const fs=require("node:fs"); JSON.parse(fs.readFileSync("snippets/opencode.json", "utf8")); JSON.parse(fs.readFileSync("snippets/claude.mcp.json", "utf8"));'
|
|
CONTEXT_KIT_WEB_SEARCH_HTTP_URL="http://127.0.0.1:8777/mcp" CONTEXT_KIT_DOCS_HTTP_URL="http://127.0.0.1:8776/mcp" bin/context-kit install opencode > "${tmp_dir}/opencode-default.json"
|
|
cmp -s snippets/opencode.json "${tmp_dir}/opencode-default.json" || {
|
|
printf 'snippets/opencode.json differs from bin/context-kit install opencode output\n' >&2
|
|
diff -u snippets/opencode.json "${tmp_dir}/opencode-default.json" >&2 || true
|
|
exit 1
|
|
}
|
|
CONTEXT_KIT_WEB_SEARCH_HTTP_URL="http://127.0.0.1:8777/mcp" CONTEXT_KIT_DOCS_HTTP_URL="http://127.0.0.1:8776/mcp" bin/context-kit install claude > "${tmp_dir}/claude-default.json"
|
|
cmp -s snippets/claude.mcp.json "${tmp_dir}/claude-default.json" || {
|
|
printf 'snippets/claude.mcp.json differs from bin/context-kit install claude output\n' >&2
|
|
diff -u snippets/claude.mcp.json "${tmp_dir}/claude-default.json" >&2 || true
|
|
exit 1
|
|
}
|
|
bin/context-kit install opencode > "${tmp_dir}/opencode.json"
|
|
bin/context-kit install opencode --absolute > "${tmp_dir}/opencode-absolute.json"
|
|
bin/context-kit install claude > "${tmp_dir}/claude.json"
|
|
bin/context-kit install claude --absolute > "${tmp_dir}/claude-absolute.json"
|
|
node -e 'const fs=require("node:fs"); for (const file of process.argv.slice(1)) JSON.parse(fs.readFileSync(file, "utf8"));' \
|
|
"${tmp_dir}/opencode.json" \
|
|
"${tmp_dir}/opencode-absolute.json" \
|
|
"${tmp_dir}/claude.json" \
|
|
"${tmp_dir}/claude-absolute.json"
|
|
bin/context-kit redaction-check "${tmp_dir}/opencode.json" "${tmp_dir}/claude.json"
|
|
assert_redaction_check_does_not_disclose_matches
|
|
bash scripts/test-compose-upgrade.sh
|
|
bash scripts/test-lifecycle.sh
|
|
node scripts/test-web-search-http.mjs
|
|
node scripts/test-web-search-quality.mjs
|
|
python3 scripts/test-doc-snapshots.py
|
|
|
|
bin/context-kit redaction-check
|
|
docker compose -p "${CONTEXT_KIT_COMPOSE_PROJECT}" -f compose.yml config >/dev/null
|
|
if env -u HOME -u CONTEXT_KIT_DATA_DIR -u CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR docker compose --env-file /dev/null -p context-kit-release-home-check -f compose.yml config >"${tmp_dir}/compose-no-home.out" 2>"${tmp_dir}/compose-no-home.err"; then
|
|
printf 'compose config unexpectedly succeeded without HOME or CONTEXT_KIT_DATA_DIR\n' >&2
|
|
exit 1
|
|
fi
|
|
CONTEXT_KIT_DATA_DIR="${tmp_dir}/compose-data" env -u HOME docker compose --env-file /dev/null -p context-kit-release-home-check -f compose.yml config >/dev/null
|
|
bin/context-kit build
|
|
assert_web_search_image
|
|
bin/context-kit start
|
|
bin/context-kit doctor
|
|
node docker/web-search/mcp-probe.mjs "http://127.0.0.1:${CONTEXT_KIT_WEB_SEARCH_PORT}/mcp"
|
|
assert_hostile_requests_rejected
|
|
node scripts/smoke-web-search.mjs bin/context-kit web-search
|
|
node scripts/smoke-docs.mjs bin/context-kit docs
|
|
node scripts/smoke-repomix.mjs bin/context-kit repomix
|
|
docker run --rm --entrypoint python "${CONTEXT_KIT_DOCS_IMAGE}" -m unittest discover -s /opt/context-kit/tests -t /opt/context-kit
|
|
CONTEXT_KIT_DOCS_CANDIDATE_IMAGE="${CONTEXT_KIT_DOCS_IMAGE}" \
|
|
CONTEXT_KIT_DOCS_TEST_MODELS="${CONTEXT_KIT_DATA_DIR}/models" \
|
|
bash scripts/test-docs-candidate.sh
|
|
CONTEXT_KIT_WEB_SEARCH_CANDIDATE_IMAGE="${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \
|
|
bash scripts/test-web-search-candidate.sh
|
|
assert_web_search_backend_supervision
|
|
|
|
printf 'pass release-check\n'
|