Files
context-kit/bin/context-kit
Ajay Krishnan 51dceee224 Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.
2026-07-25 08:49:26 -07:00

1049 lines
34 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_PATH="${BASH_SOURCE[0]}"
while [[ -L "${SCRIPT_PATH}" ]]; do
SCRIPT_DIR="$(cd -P "$(dirname "${SCRIPT_PATH}")" && pwd)"
SCRIPT_TARGET="$(readlink "${SCRIPT_PATH}")"
if [[ "${SCRIPT_TARGET}" = /* ]]; then
SCRIPT_PATH="${SCRIPT_TARGET}"
else
SCRIPT_PATH="${SCRIPT_DIR}/${SCRIPT_TARGET}"
fi
done
ROOT="$(cd -P "$(dirname "${SCRIPT_PATH}")/.." && pwd)"
ENV_FILE="${ROOT}/.env"
load_env_file() {
[[ -f "${ENV_FILE}" ]] || return 0
local line key value
while IFS= read -r line || [[ -n "${line}" ]]; do
line="${line%$'\r'}"
[[ -z "${line}" || "${line}" =~ ^[[:space:]]*# ]] && continue
[[ "${line}" =~ ^([A-Za-z_][A-Za-z0-9_]*)=(.*)$ ]] || fail "unsupported .env line: ${line}"
key="${BASH_REMATCH[1]}"
value="${BASH_REMATCH[2]}"
[[ "${key}" == CONTEXT_KIT_* ]] || fail ".env may only set CONTEXT_KIT_* variables: ${key}"
[[ "${!key+x}" == "x" ]] && continue
if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then
value="${value:1:${#value}-2}"
elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then
value="${value:1:${#value}-2}"
fi
export "${key}=${value}"
done < "${ENV_FILE}"
}
fail() {
printf 'context-kit: %s\n' "$*" >&2
exit 1
}
load_env_file
if [[ -z "${CONTEXT_KIT_DATA_DIR:-}" && -z "${HOME:-}" ]]; then
fail "HOME or CONTEXT_KIT_DATA_DIR must be set"
fi
DEFAULT_DATA_DIR="${HOME:-}/.local/share/context-kit"
PROJECT="${CONTEXT_KIT_COMPOSE_PROJECT:-context-kit}"
[[ "${PROJECT}" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "invalid CONTEXT_KIT_COMPOSE_PROJECT: ${PROJECT}"
HOST_UID="$(id -u)"
COMPOSE_FILE="${ROOT}/compose.yml"
DATA_DIR="${CONTEXT_KIT_DATA_DIR:-${DEFAULT_DATA_DIR}}"
NETWORK="${PROJECT}_default"
SEARXNG_PORT="${CONTEXT_KIT_SEARXNG_PORT:-8099}"
WEB_SEARCH_PORT="${CONTEXT_KIT_WEB_SEARCH_PORT:-8777}"
WEB_SEARCH_HTTP_URL="${CONTEXT_KIT_WEB_SEARCH_HTTP_URL:-http://127.0.0.1:${WEB_SEARCH_PORT}/mcp}"
DOCS_PORT="${CONTEXT_KIT_DOCS_PORT:-8776}"
DOCS_HTTP_URL="${CONTEXT_KIT_DOCS_HTTP_URL:-http://127.0.0.1:${DOCS_PORT}/mcp}"
WEB_SEARCH_SERVICE_NAME="web-search-mcp"
DOCS_SERVICE_NAME="docs-mcp"
SHARED_SERVICES=(searxng "${WEB_SEARCH_SERVICE_NAME}" "${DOCS_SERVICE_NAME}")
SNAPSHOT_IDS=()
SNAPSHOT_RUNNING=()
DOCS_SOURCES_FILE="${DATA_DIR}/docs-sources.txt"
DOCS_SOURCES_TRANSACTION_ACTIVE=0
DOCS_SOURCES_PRIOR_PRESENT=0
DOCS_SOURCES_BACKUP_DIR=''
DOCS_SOURCES_BACKUP_FILE=''
DOCS_SOURCES_RENDER_TMP=''
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=false
DOCS_DATA_DIR="${DATA_DIR}/docs"
MODELS_DATA_DIR="${DATA_DIR}/models"
DOCS_LOCAL_SOURCES_DIR="${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR:-${DATA_DIR}/local-sources}"
DOCS_LOCAL_SOURCES_PORT="${CONTEXT_KIT_DOCS_LOCAL_SOURCES_PORT:-8769}"
WEB_SEARCH_IMAGE="${CONTEXT_KIT_WEB_SEARCH_IMAGE:-context-kit/web-search-mcp:latest}"
DOCS_IMAGE="${CONTEXT_KIT_DOCS_IMAGE:-context-kit/docs-mcp:latest}"
REPOMIX_IMAGE="${CONTEXT_KIT_REPOMIX_IMAGE:-ghcr.io/yamadashy/repomix@sha256:62fb288a3f031f99bc332b73c22acb9ff1cf2a5d8ef2f0196185d5926d9edb2a}"
usage() {
cat <<'USAGE'
context-kit: local context tools for coding agents
Usage:
context-kit start Start the shared SearXNG and HTTP MCP services
context-kit stop Stop the shared services without removing them
context-kit restart Restart the shared services
context-kit build Build MCP images
context-kit status Show services, images, sources, and shared HTTP endpoints
context-kit doctor Check Docker, services, images, sources, and HTTP endpoints
context-kit redaction-check Scan this repo for local paths and secret patterns
context-kit docs-snapshot Build deterministic llms-full.txt local snapshots
context-kit docs-rebuild [URL...] Rebuild all or selected configured docs sources
MCP server commands:
context-kit web-search Stdio bridge to the shared web-search service
context-kit docs Stdio bridge to the long-lived docs-mcp service
(clients that speak HTTP MCP should connect
directly to the URL printed by `status`)
context-kit repomix Per-call Repomix MCP for the current project (stdio)
Assistant snippets:
context-kit install claude Print a project .mcp.json snippet using context-kit on PATH
context-kit install opencode Print an opencode.json MCP snippet using context-kit on PATH
Configuration is via .env or environment variables. See .env.example.
USAGE
}
compose() {
CONTEXT_KIT_DATA_DIR="${DATA_DIR}" \
CONTEXT_KIT_COMPOSE_PROJECT="${PROJECT}" \
CONTEXT_KIT_HOST_UID="${HOST_UID}" \
CONTEXT_KIT_SEARXNG_PORT="${SEARXNG_PORT}" \
CONTEXT_KIT_WEB_SEARCH_PORT="${WEB_SEARCH_PORT}" \
CONTEXT_KIT_DOCS_PORT="${DOCS_PORT}" \
CONTEXT_KIT_DOCS_UID="$(id -u)" \
CONTEXT_KIT_DOCS_GID="$(id -g)" \
CONTEXT_KIT_DOCS_TTL="${CONTEXT_KIT_DOCS_TTL:-24h}" \
CONTEXT_KIT_DOCS_MAX_GET_BYTES="${CONTEXT_KIT_DOCS_MAX_GET_BYTES:-75000}" \
CONTEXT_KIT_DOCS_EMBED_MODEL="${CONTEXT_KIT_DOCS_EMBED_MODEL:-BAAI/bge-small-en-v1.5}" \
CONTEXT_KIT_DOCS_PREINDEX="${CONTEXT_KIT_DOCS_PREINDEX:-0}" \
CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR="${DOCS_LOCAL_SOURCES_DIR}" \
CONTEXT_KIT_DOCS_LOCAL_SOURCES_PORT="${DOCS_LOCAL_SOURCES_PORT}" \
CONTEXT_KIT_WEB_SEARCH_IMAGE="${WEB_SEARCH_IMAGE}" \
CONTEXT_KIT_DOCS_IMAGE="${DOCS_IMAGE}" \
BUILDX_BUILDER="${CONTEXT_KIT_BUILDX_BUILDER:-default}" \
docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"
}
require_no_args() {
local usage_text="$1"
shift
[[ "$#" -eq 0 ]] || fail "${usage_text}"
}
write_docs_sources_file() {
mkdir -p "$(dirname "${DOCS_SOURCES_FILE}")"
local tmp="${DOCS_SOURCES_FILE}.tmp.${BASHPID}"
DOCS_SOURCES_RENDER_TMP="${tmp}"
if ! {
printf '# generated by context-kit lifecycle commands; edit your CONTEXT_KIT_DOCS_SOURCES file(s) instead\n'
resolved_sources
} > "${tmp}"; then
rm -f -- "${tmp}"
DOCS_SOURCES_RENDER_TMP=''
return 1
fi
if ! mv -fT -- "${tmp}" "${DOCS_SOURCES_FILE}"; then
rm -f -- "${tmp}"
DOCS_SOURCES_RENDER_TMP=''
return 1
fi
DOCS_SOURCES_RENDER_TMP=''
}
reset_docs_sources_transaction() {
DOCS_SOURCES_TRANSACTION_ACTIVE=0
DOCS_SOURCES_PRIOR_PRESENT=0
DOCS_SOURCES_BACKUP_DIR=''
DOCS_SOURCES_BACKUP_FILE=''
DOCS_SOURCES_RENDER_TMP=''
}
begin_docs_sources_transaction() {
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 0 ]] || return 1
mkdir -p "$(dirname "${DOCS_SOURCES_FILE}")"
local backup_dir
backup_dir="$(mktemp -d "${DOCS_SOURCES_FILE}.lifecycle-backup.XXXXXX")" || return 1
DOCS_SOURCES_BACKUP_DIR="${backup_dir}"
DOCS_SOURCES_BACKUP_FILE="${backup_dir}/docs-sources.txt"
DOCS_SOURCES_PRIOR_PRESENT=0
DOCS_SOURCES_TRANSACTION_ACTIVE=1
if [[ -e "${DOCS_SOURCES_FILE}" || -L "${DOCS_SOURCES_FILE}" ]]; then
DOCS_SOURCES_PRIOR_PRESENT=1
if ! mv -T -- "${DOCS_SOURCES_FILE}" "${DOCS_SOURCES_BACKUP_FILE}"; then
rmdir -- "${DOCS_SOURCES_BACKUP_DIR}" 2>/dev/null || true
reset_docs_sources_transaction
return 1
fi
fi
}
restore_docs_sources_transaction() {
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 1 ]] || return 0
local failed=0
if [[ -n "${DOCS_SOURCES_RENDER_TMP}" ]]; then
rm -f -- "${DOCS_SOURCES_RENDER_TMP}" || failed=1
DOCS_SOURCES_RENDER_TMP=''
fi
if [[ "${DOCS_SOURCES_PRIOR_PRESENT}" -eq 1 ]]; then
if [[ -e "${DOCS_SOURCES_BACKUP_FILE}" || -L "${DOCS_SOURCES_BACKUP_FILE}" ]]; then
if ! mv -fT -- "${DOCS_SOURCES_BACKUP_FILE}" "${DOCS_SOURCES_FILE}"; then
warn "failed to restore prior docs sources file: ${DOCS_SOURCES_FILE}"
failed=1
fi
else
warn "prior docs sources backup is missing: ${DOCS_SOURCES_BACKUP_FILE}"
failed=1
fi
elif ! rm -f -- "${DOCS_SOURCES_FILE}"; then
warn "failed to restore prior absence of docs sources file: ${DOCS_SOURCES_FILE}"
failed=1
fi
if [[ "${failed}" -eq 0 ]] && ! rmdir -- "${DOCS_SOURCES_BACKUP_DIR}"; then
warn "failed to remove docs sources backup directory: ${DOCS_SOURCES_BACKUP_DIR}"
failed=1
fi
if [[ "${failed}" -eq 0 ]]; then
reset_docs_sources_transaction
fi
return "${failed}"
}
discard_docs_sources_transaction() {
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 1 ]] || return 0
local failed=0
if [[ -n "${DOCS_SOURCES_RENDER_TMP}" ]]; then
rm -f -- "${DOCS_SOURCES_RENDER_TMP}" || failed=1
DOCS_SOURCES_RENDER_TMP=''
fi
if [[ "${DOCS_SOURCES_PRIOR_PRESENT}" -eq 1 ]]; then
rm -f -- "${DOCS_SOURCES_BACKUP_FILE}" || failed=1
fi
if [[ "${failed}" -eq 0 ]] && ! rmdir -- "${DOCS_SOURCES_BACKUP_DIR}"; then
failed=1
fi
if [[ "${failed}" -eq 0 ]]; then
reset_docs_sources_transaction
fi
return "${failed}"
}
ensure_writable_dir() {
local dir="$1"
mkdir -p "${dir}"
if [[ ! -w "${dir}" || ! -x "${dir}" ]]; then
fail "data directory is not writable by uid $(id -u): ${dir}; fix ownership or set CONTEXT_KIT_DATA_DIR"
fi
}
prepare_data_dirs() {
ensure_writable_dir "${DATA_DIR}"
ensure_writable_dir "${DOCS_DATA_DIR}"
ensure_writable_dir "${MODELS_DATA_DIR}"
ensure_writable_dir "${DOCS_LOCAL_SOURCES_DIR}"
}
check_data_dirs() {
local ok=0 dir
for dir in "${DATA_DIR}" "${DOCS_DATA_DIR}" "${MODELS_DATA_DIR}" "${DOCS_LOCAL_SOURCES_DIR}"; do
if [[ ! -d "${dir}" ]]; then
printf 'warn data directory missing: %s (run context-kit start)\n' "${dir}"
elif [[ -w "${dir}" && -x "${dir}" ]]; then
printf 'pass data directory writable: %s\n' "${dir}"
else
printf 'fail data directory not writable by uid %s: %s\n' "$(id -u)" "${dir}"
ok=1
fi
done
return "${ok}"
}
warn() {
printf 'warn: %s\n' "$*" >&2
}
print_relative_paths() {
local path
while IFS= read -r path; do
[[ -n "${path}" ]] || continue
if [[ "${path}" == "${ROOT}/"* ]]; then
path="${path#"${ROOT}/"}"
fi
printf '%s\n' "${path}"
done
}
json_escape() {
local s="$1"
s="${s//\\/\\\\}"
s="${s//\"/\\\"}"
s="${s//$'\n'/\\n}"
s="${s//$'\r'/\\r}"
s="${s//$'\t'/\\t}"
printf '%s' "${s}"
}
require_docker() {
command -v docker >/dev/null 2>&1 || fail "Docker is required"
docker info >/dev/null 2>&1 || fail "Docker is not running or not reachable"
}
require_image() {
local image="$1"
local hint="$2"
docker image inspect "${image}" >/dev/null 2>&1 || fail "missing image ${image}; run: ${hint}"
}
require_network() {
docker network inspect "${NETWORK}" >/dev/null 2>&1 || fail "missing Docker network ${NETWORK}; run: context-kit start"
}
with_lifecycle_lock() {
command -v flock >/dev/null 2>&1 || fail "flock is required for shared service lifecycle operations"
command -v stat >/dev/null 2>&1 || fail "stat is required for shared service lifecycle operations"
local lock_dir="/tmp/context-kit-${PROJECT}.lock" owner mode
[[ -d /tmp && ! -L /tmp ]] || fail "canonical lifecycle lock parent /tmp is unsafe"
if mkdir -m 700 "${lock_dir}" 2>/dev/null; then
chmod 700 "${lock_dir}"
fi
[[ -d "${lock_dir}" && ! -L "${lock_dir}" ]] || fail "unsafe lifecycle lock path: ${lock_dir}"
owner="$(stat -c %u "${lock_dir}")"
mode="$(stat -c %a "${lock_dir}")"
[[ "${owner}" == "${HOST_UID}" && "${mode}" == "700" ]] \
|| fail "shared Compose project ${PROJECT} lock has uid ${owner} and mode ${mode}; expected uid ${HOST_UID} and mode 700"
(
flock -x 9
assert_project_owner
"$@"
) 9>"${lock_dir}/lifecycle"
}
service_container_id() {
local service="$1" ids
ids="$(compose ps --all --quiet "${service}")"
[[ "${ids}" != *$'\n'* ]] || fail "multiple containers found for shared service ${service}"
printf '%s' "${ids}"
}
container_running() {
docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -qx true
}
container_exists() {
docker inspect "$1" >/dev/null 2>&1
}
shared_service_running() {
local container_id
container_id="$(service_container_id "$1" 2>/dev/null || true)"
[[ -n "${container_id}" ]] && container_running "${container_id}"
}
container_matches_shared_service() {
local container_id="$1" service="$2" labels
labels="$(docker inspect -f '{{ index .Config.Labels "com.docker.compose.project" }}:{{ index .Config.Labels "com.docker.compose.service" }}' "${container_id}" 2>/dev/null || true)"
[[ "${labels}" == "${PROJECT}:${service}" ]]
}
assert_project_owner() {
local container_id configured_uid
container_id="$(service_container_id "${DOCS_SERVICE_NAME}" 2>/dev/null || true)"
if [[ -n "${container_id}" ]]; then
configured_uid="$(docker inspect -f '{{.Config.User}}' "${container_id}" 2>/dev/null || true)"
configured_uid="${configured_uid%%:*}"
[[ "${configured_uid}" == "${HOST_UID}" ]] \
|| fail "shared Compose project ${PROJECT} docs service belongs to uid ${configured_uid:-unknown}; cross-user ownership is unsupported"
fi
container_id="$(service_container_id "${WEB_SEARCH_SERVICE_NAME}" 2>/dev/null || true)"
if [[ -n "${container_id}" ]]; then
configured_uid="$(docker inspect -f '{{ index .Config.Labels "dev.context-kit.uid" }}' "${container_id}" 2>/dev/null || true)"
[[ "${configured_uid}" == "${HOST_UID}" ]] \
|| fail "shared Compose project ${PROJECT} web-search service belongs to uid ${configured_uid:-unknown}; cross-user ownership is unsupported"
fi
}
cleanup_owned_container() {
local container_id="$1" owner="$2" actual_owner
[[ -n "${container_id}" ]] || return 0
actual_owner="$(docker inspect -f '{{ index .Config.Labels "dev.context-kit.owner" }}' "${container_id}" 2>/dev/null || true)"
[[ "${actual_owner}" == "${owner}" ]] || return 0
docker rm -f "${container_id}" >/dev/null 2>&1 || true
}
run_owned_stdio_container() {
local role="$1"
shift
local uid owner name container_id='' status=0
uid="$(id -u)"
owner="${PROJECT}:${role}:${uid}:$$"
name="${PROJECT}-${role}-${uid}-$$"
trap 'cleanup_owned_container "${container_id}" "${owner}"' EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
container_id="$(docker create -i --rm \
--name "${name}" \
--label dev.context-kit=true \
--label dev.context-kit.lifecycle=client \
--label "dev.context-kit.owner=${owner}" \
--label "dev.context-kit.role=${role}" \
"$@")" || status=$?
if [[ "${status}" -ne 0 ]]; then
cleanup_owned_container "${container_id}" "${owner}"
trap - EXIT HUP INT TERM
return "${status}"
fi
if [[ -n "${CONTEXT_KIT_DOCKER_CIDFILE:-}" ]]; then
printf '%s\n' "${container_id}" > "${CONTEXT_KIT_DOCKER_CIDFILE}"
fi
docker start -ai "${container_id}" <&0 || status=$?
cleanup_owned_container "${container_id}" "${owner}"
trap - EXIT HUP INT TERM
return "${status}"
}
wait_for_searxng() {
command -v curl >/dev/null 2>&1 || return 0
local attempt
for attempt in {1..30}; do
if curl -fsS "http://127.0.0.1:${SEARXNG_PORT}/healthz" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
warn "SearXNG did not become ready on 127.0.0.1:${SEARXNG_PORT} after 30s"
return 1
}
wait_for_web_search_mcp() {
command -v curl >/dev/null 2>&1 || return 1
local attempt
for attempt in {1..60}; do
if probe_web_search_mcp; then
return 0
fi
sleep 1
done
warn "web-search-mcp did not become ready on 127.0.0.1:${WEB_SEARCH_PORT} after 60s (check: docker compose logs ${WEB_SEARCH_SERVICE_NAME})"
return 1
}
probe_web_search_mcp() {
local url="http://127.0.0.1:${WEB_SEARCH_PORT}/mcp" initialized tools
initialized="$(curl -fsS --max-time 10 \
-H 'Accept: application/json, text/event-stream' \
-H 'Content-Type: application/json' \
-H 'MCP-Protocol-Version: 2024-11-05' \
--data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"context-kit-doctor","version":"1"}}}' \
"${url}" 2>/dev/null)" || return 1
printf '%s' "${initialized}" | grep -Eq '"serverInfo"[[:space:]]*:' || return 1
tools="$(curl -fsS --max-time 10 \
-H 'Accept: application/json, text/event-stream' \
-H 'Content-Type: application/json' \
-H 'MCP-Protocol-Version: 2024-11-05' \
--data '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' \
"${url}" 2>/dev/null)" || return 1
printf '%s' "${tools}" | grep -Eq '"name"[[:space:]]*:[[:space:]]*"search_web"' || return 1
printf '%s' "${tools}" | grep -Eq '"name"[[:space:]]*:[[:space:]]*"fetch_url"'
}
wait_for_docs_mcp() {
command -v curl >/dev/null 2>&1 || return 0
# First run can take a while: model download plus optional eager preindexing.
local attempt http_ready=0
for ((attempt=1; attempt <= 180; attempt++)); do
if curl -fsS -o /dev/null "http://127.0.0.1:${DOCS_PORT}/status" 2>/dev/null; then
http_ready=1
break
fi
sleep 1
done
if [[ "${http_ready}" -ne 1 ]]; then
warn "docs-mcp did not become ready on 127.0.0.1:${DOCS_PORT} after 180s (check: docker compose logs ${DOCS_SERVICE_NAME})"
return 1
fi
return 0
}
abs_dir() {
local path="$1"
mkdir -p "${path}"
(cd "${path}" && pwd -P)
}
project_dir() {
local dir="${CONTEXT_KIT_PROJECT_DIR:-${CLAUDE_PROJECT_DIR:-${PWD}}}"
(cd "${dir}" && pwd -P)
}
source_files() {
local configured="${CONTEXT_KIT_DOCS_SOURCES:-config/sources.default.txt}"
local file
for file in ${configured}; do
if [[ "${file}" = /* ]]; then
printf '%s\n' "${file}"
else
printf '%s\n' "${ROOT}/${file}"
fi
done
}
resolved_sources() {
local file line local_prefix relative full_path
local_prefix="http://127.0.0.1:${DOCS_LOCAL_SOURCES_PORT}/"
while IFS= read -r file; do
[[ -f "${file}" ]] || fail "docs source file not found: ${file}"
while IFS= read -r line; do
line="${line%%#*}"
line="${line//[$'\t\r\n ']/}"
[[ -z "${line}" ]] && continue
if [[ "${line}" == "${local_prefix}"*'/llms.txt' ]]; then
relative="${line#"${local_prefix}"}"
full_path="${DOCS_LOCAL_SOURCES_DIR}/${relative%llms.txt}llms-full.txt"
if [[ -f "${full_path}" ]] \
&& python3 "${ROOT}/scripts/docs_snapshot.py" --validate-output "${full_path}" >/dev/null 2>&1; then
line="${line%llms.txt}llms-full.txt"
fi
fi
printf '%s\n' "${line}"
done < "${file}"
done < <(source_files)
}
cmd_docs_snapshot() {
python3 "${ROOT}/scripts/docs_snapshot.py" \
--source-root "${DOCS_LOCAL_SOURCES_DIR}" \
--cache-dir "${DATA_DIR}/snapshot-cache" \
"$@"
}
cmd_docs_rebuild() {
require_docker
require_network
if ! shared_service_running "${DOCS_SERVICE_NAME}"; then
fail "long-lived docs-mcp not running; start it with: context-kit start"
fi
node "${ROOT}/scripts/docs-rebuild.mjs" "${DOCS_HTTP_URL}" "$@"
}
cmd_build() {
require_no_args "usage: context-kit build" "$@"
require_docker
compose build web-search-mcp docs-mcp
docker pull "${REPOMIX_IMAGE}"
}
snapshot_shared_services() {
local service container_id
SNAPSHOT_IDS=()
SNAPSHOT_RUNNING=()
for service in "${SHARED_SERVICES[@]}"; do
container_id="$(service_container_id "${service}")"
SNAPSHOT_IDS+=("${container_id}")
if [[ -n "${container_id}" ]] && container_running "${container_id}"; then
SNAPSHOT_RUNNING+=(1)
else
SNAPSHOT_RUNNING+=(0)
fi
done
}
restore_shared_service_states() {
local remove_new="$1" index service current_id prior_id failed=0
for ((index=0; index < ${#SHARED_SERVICES[@]}; index++)); do
service="${SHARED_SERVICES[index]}"
prior_id="${SNAPSHOT_IDS[index]}"
current_id="$(service_container_id "${service}" 2>/dev/null || true)"
if [[ -z "${prior_id}" && "${remove_new}" == true && -n "${current_id}" ]]; then
if container_matches_shared_service "${current_id}" "${service}"; then
if ! docker rm -f "${current_id}" >/dev/null 2>&1; then
warn "failed to remove newly-created ${service} container ${current_id} during rollback"
failed=1
fi
else
warn "new ${service} container ${current_id} lacks the expected Compose ownership; leaving it untouched"
failed=1
fi
continue
fi
[[ -n "${prior_id}" ]] || continue
if [[ -n "${current_id}" && "${current_id}" != "${prior_id}" ]]; then
warn "${service} unexpectedly changed from ${prior_id} to ${current_id}; leaving the replacement untouched"
failed=1
continue
fi
if ! container_exists "${prior_id}"; then
warn "cannot restore missing pre-start ${service} container ${prior_id}"
failed=1
continue
fi
if [[ "${SNAPSHOT_RUNNING[index]}" -eq 1 ]] && ! container_running "${prior_id}"; then
if ! docker start "${prior_id}" >/dev/null 2>&1; then
warn "failed to restart prior ${service} container ${prior_id} during rollback"
failed=1
fi
elif [[ "${SNAPSHOT_RUNNING[index]}" -eq 0 ]] && container_running "${prior_id}"; then
if ! docker stop "${prior_id}" >/dev/null 2>&1; then
warn "failed to stop prior ${service} container ${prior_id} during rollback"
failed=1
fi
fi
done
return "${failed}"
}
rollback_lifecycle_transaction() {
local failed=0
restore_docs_sources_transaction || failed=1
if [[ "${LIFECYCLE_STATE_ROLLBACK_ACTIVE}" -eq 1 ]]; then
restore_shared_service_states "${LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW}" || failed=1
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
fi
return "${failed}"
}
lifecycle_transaction_exit() {
local status="$1"
trap - EXIT HUP INT TERM
rollback_lifecycle_transaction || true
exit "${status}"
}
arm_lifecycle_transaction_traps() {
trap 'lifecycle_transaction_exit "$?"' EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
}
disarm_lifecycle_transaction_traps() {
trap - EXIT HUP INT TERM
}
shared_services_ready() {
wait_for_searxng && wait_for_web_search_mcp && wait_for_docs_mcp
}
start_locked() {
prepare_data_dirs
if ! docker image inspect "${WEB_SEARCH_IMAGE}" >/dev/null 2>&1 || ! docker image inspect "${DOCS_IMAGE}" >/dev/null 2>&1; then
cmd_build
fi
local result=0 rollback_failed=0
arm_lifecycle_transaction_traps
if begin_docs_sources_transaction; then
:
else
result=$?
disarm_lifecycle_transaction_traps
return "${result}"
fi
if write_docs_sources_file; then
:
else
result=$?
rollback_lifecycle_transaction || rollback_failed=1
disarm_lifecycle_transaction_traps
[[ "${rollback_failed}" -eq 0 ]] || return 1
return "${result}"
fi
snapshot_shared_services
LIFECYCLE_STATE_ROLLBACK_ACTIVE=1
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=true
compose up -d --no-recreate "${SHARED_SERVICES[@]}" || result=$?
if [[ "${result}" -eq 0 ]] && ! shared_services_ready; then
result=1
fi
if [[ "${result}" -eq 0 ]] && discard_docs_sources_transaction; then
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
disarm_lifecycle_transaction_traps
return 0
fi
[[ "${result}" -ne 0 ]] || result=1
warn "shared service startup failed; restoring prior docs sources and container states"
rollback_lifecycle_transaction || rollback_failed=1
disarm_lifecycle_transaction_traps
[[ "${rollback_failed}" -eq 0 ]] || return 1
return "${result}"
}
cmd_start() {
require_no_args "usage: context-kit start" "$@"
require_docker
with_lifecycle_lock start_locked
}
stop_locked() {
compose stop "${SHARED_SERVICES[@]}"
}
cmd_stop() {
require_no_args "usage: context-kit stop" "$@"
require_docker
with_lifecycle_lock stop_locked
}
cmd_restart() {
require_no_args "usage: context-kit restart" "$@"
require_docker
with_lifecycle_lock restart_locked
}
restart_locked() {
local result=0 rollback_failed=0
arm_lifecycle_transaction_traps
if begin_docs_sources_transaction; then
:
else
result=$?
disarm_lifecycle_transaction_traps
return "${result}"
fi
if write_docs_sources_file; then
:
else
result=$?
rollback_lifecycle_transaction || rollback_failed=1
disarm_lifecycle_transaction_traps
[[ "${rollback_failed}" -eq 0 ]] || return 1
return "${result}"
fi
snapshot_shared_services
LIFECYCLE_STATE_ROLLBACK_ACTIVE=1
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=false
compose restart "${SHARED_SERVICES[@]}" || result=$?
if [[ "${result}" -eq 0 ]] && ! shared_services_ready; then
result=1
fi
if [[ "${result}" -eq 0 ]] && discard_docs_sources_transaction; then
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
disarm_lifecycle_transaction_traps
return 0
fi
[[ "${result}" -ne 0 ]] || result=1
warn "shared service restart failed; restoring prior docs sources and container states"
rollback_lifecycle_transaction || rollback_failed=1
disarm_lifecycle_transaction_traps
[[ "${rollback_failed}" -eq 0 ]] || return 1
return "${result}"
}
cmd_status() {
require_no_args "usage: context-kit status" "$@"
require_docker
printf 'Services\n'
compose ps
printf '\nImages\n'
docker image ls --format '{{.Repository}}:{{.Tag}}\t{{.Size}}' \
| grep -E '^(context-kit/|ghcr.io/yamadashy/repomix:)' || true
printf '\nClient-owned stdio MCP containers\n'
docker ps -a --filter label=dev.context-kit.lifecycle=client --format 'table {{.Names}}\t{{.Status}}\t{{.Label "dev.context-kit.role"}}\t{{.Label "dev.context-kit.owner"}}'
printf '\nLegacy unlabeled Context Kit containers (diagnostic only; never auto-removed)\n'
docker ps -a --filter label=dev.context-kit=true \
--format '{{.Names}}\t{{.Status}}\t{{.Label "dev.context-kit.lifecycle"}}\t{{.Label "com.docker.compose.service"}}' \
| awk -F '\t' 'BEGIN { print "NAMES\tSTATUS" } $3 == "" && $4 == "" { print $1 "\t" $2 }'
printf '\nShared MCP endpoints\n- %s (service: %s)\n- %s (service: %s)\n' \
"${WEB_SEARCH_HTTP_URL}" "${WEB_SEARCH_SERVICE_NAME}" \
"${DOCS_HTTP_URL}" "${DOCS_SERVICE_NAME}"
printf '\nShared Docker ownership\n- project: %s\n- uid: %s\n- network: %s\n- cache volume: %s_searxng-cache\n' \
"${PROJECT}" "${HOST_UID}" "${NETWORK}" "${PROJECT}"
printf '\nDocs sources\n'
resolved_sources | sed 's/^/- /'
printf '\nLocal docs source directory\n- %s (served inside docs-mcp at http://127.0.0.1:%s/)\n' "${DOCS_LOCAL_SOURCES_DIR}" "${DOCS_LOCAL_SOURCES_PORT}"
printf '\nData directory\n- %s\n' "${DATA_DIR}"
}
cmd_doctor() {
require_no_args "usage: context-kit doctor" "$@"
local ok=0
printf 'Context Kit doctor\n'
if command -v docker >/dev/null 2>&1; then
printf 'pass docker command found\n'
else
printf 'fail docker command not found\n'; ok=1
fi
if docker info >/dev/null 2>&1; then
printf 'pass docker daemon reachable\n'
else
printf 'fail docker daemon not reachable\n'; ok=1
fi
if docker compose version >/dev/null 2>&1; then
printf 'pass docker compose available\n'
else
printf 'fail docker compose unavailable\n'; ok=1
fi
if ! check_data_dirs; then
ok=1
fi
if docker network inspect "${NETWORK}" >/dev/null 2>&1; then
printf 'pass docker network exists: %s\n' "${NETWORK}"
else
printf 'warn docker network missing: %s (run context-kit start)\n' "${NETWORK}"
fi
for image in "${WEB_SEARCH_IMAGE}" "${DOCS_IMAGE}" "${REPOMIX_IMAGE}"; do
if docker image inspect "${image}" >/dev/null 2>&1; then
printf 'pass image exists: %s\n' "${image}"
else
printf 'warn image missing: %s\n' "${image}"
fi
done
if command -v curl >/dev/null 2>&1 && curl -fsS "http://127.0.0.1:${SEARXNG_PORT}/healthz" >/dev/null 2>&1; then
printf 'pass SearXNG responds on 127.0.0.1:%s\n' "${SEARXNG_PORT}"
else
printf 'fail SearXNG not responding on 127.0.0.1:%s\n' "${SEARXNG_PORT}"
ok=1
fi
if command -v curl >/dev/null 2>&1 && probe_web_search_mcp; then
printf 'pass web-search-mcp initialize and tools/list on 127.0.0.1:%s\n' "${WEB_SEARCH_PORT}"
else
printf 'fail web-search-mcp MCP protocol probe failed on 127.0.0.1:%s (run context-kit start)\n' "${WEB_SEARCH_PORT}"
ok=1
fi
if command -v curl >/dev/null 2>&1 && curl -fsS -o /dev/null "http://127.0.0.1:${DOCS_PORT}/status" 2>/dev/null; then
printf 'pass docs-mcp HTTP responds on 127.0.0.1:%s\n' "${DOCS_PORT}"
else
printf 'fail docs-mcp HTTP not responding on 127.0.0.1:%s (run context-kit start)\n' "${DOCS_PORT}"
ok=1
fi
if [[ "$(resolved_sources | wc -l | tr -d ' ')" -gt 0 ]]; then
printf 'pass docs sources resolve\n'
else
printf 'fail no docs sources configured\n'; ok=1
fi
return "${ok}"
}
cmd_web_search() {
require_no_args "usage: context-kit web-search" "$@"
require_docker
require_network
require_image "${WEB_SEARCH_IMAGE}" "context-kit build"
if ! shared_service_running "${WEB_SEARCH_SERVICE_NAME}"; then
fail "long-lived web-search-mcp not running; start it with: context-kit start"
fi
run_owned_stdio_container web-search-bridge \
--network "${NETWORK}" \
--entrypoint mcp-proxy \
"${WEB_SEARCH_IMAGE}" \
--transport streamablehttp \
"http://${WEB_SEARCH_SERVICE_NAME}:8000/mcp"
}
cmd_docs() {
require_no_args "usage: context-kit docs" "$@"
# Prefer the `type: remote` MCP config pointing at ${DOCS_HTTP_URL}.
# This stdio entrypoint is kept for clients that cannot speak HTTP MCP:
# it spawns a thin mcp-proxy bridge per call but all calls multiplex onto
# the single long-lived docs-mcp container over the Context Kit Docker
# network (no concurrent index writers, no host networking).
require_docker
require_network
require_image "${DOCS_IMAGE}" "context-kit build"
if ! shared_service_running "${DOCS_SERVICE_NAME}"; then
fail "long-lived docs-mcp not running; start it with: context-kit start"
fi
local bridge_url="http://${DOCS_SERVICE_NAME}:8000/mcp"
run_owned_stdio_container docs-bridge \
--network "${NETWORK}" \
--entrypoint mcp-proxy \
"${DOCS_IMAGE}" \
--transport streamablehttp \
"${bridge_url}"
}
cmd_repomix() {
require_no_args "usage: context-kit repomix" "$@"
require_docker
require_image "${REPOMIX_IMAGE}" "docker pull ${REPOMIX_IMAGE}"
local dir mount_dir
dir="$(project_dir)"
mount_dir="${CONTEXT_KIT_REPOMIX_MOUNT_DIR:-${dir}}"
mount_dir="$(cd "${mount_dir}" && pwd -P)"
run_owned_stdio_container repomix \
-v "${mount_dir}:${mount_dir}:ro" \
--workdir "${dir}" \
"${REPOMIX_IMAGE}" --mcp
}
snippet_command() {
case "${1:-}" in
--absolute) printf '%s' "${ROOT}/bin/context-kit" ;;
"") printf '%s' "context-kit" ;;
*) fail "unknown install option: ${1}" ;;
esac
}
print_opencode() {
local bin docs_url web_search_url
bin="$(json_escape "$(snippet_command "${1:-}")")"
web_search_url="$(json_escape "${WEB_SEARCH_HTTP_URL}")"
docs_url="$(json_escape "${DOCS_HTTP_URL}")"
cat <<JSON
{
"\$schema": "https://opencode.ai/config.json",
"mcp": {
"context-web-search": {
"type": "remote",
"url": "${web_search_url}",
"enabled": true,
"timeout": 150000
},
"context-docs": {
"type": "remote",
"url": "${docs_url}",
"enabled": true,
"timeout": 150000
},
"context-repomix": {
"type": "local",
"command": ["${bin}", "repomix"],
"enabled": true,
"timeout": 120000
}
}
}
JSON
}
print_claude() {
local bin docs_url web_search_url
bin="$(json_escape "$(snippet_command "${1:-}")")"
web_search_url="$(json_escape "${WEB_SEARCH_HTTP_URL}")"
docs_url="$(json_escape "${DOCS_HTTP_URL}")"
cat <<JSON
{
"mcpServers": {
"context-web-search": {
"type": "http",
"url": "${web_search_url}"
},
"context-docs": {
"type": "http",
"url": "${docs_url}"
},
"context-repomix": {
"command": "${bin}",
"args": ["repomix"]
}
}
}
JSON
}
cmd_install() {
local target="${1:-}"
shift || true
local option="${1:-}"
shift || true
[[ "$#" -eq 0 ]] || fail "usage: context-kit install claude|opencode [--absolute]"
case "${target}" in
opencode) print_opencode "${option}" ;;
claude) print_claude "${option}" ;;
*) fail "usage: context-kit install claude|opencode [--absolute]" ;;
esac
}
cmd_redaction_check() {
local bad=0
local scan_paths=("${ROOT}")
if [[ "$#" -gt 0 ]]; then
scan_paths=("$@")
fi
local local_path_terms='/(home|Users)/[^/[:space:]]+|/data/(projects|opencode-mcp)[^[:space:]]*|[A-Za-z]:\\Users\\[^\\[:space:]]+'
local secret_terms='AKIA[0-9A-Z]{16}|BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY|xox[baprs]-|sk-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|gitea_[A-Za-z0-9_-]{20,}'
# Scan only what would be published: skip .git plus everything .gitignore
# excludes by convention (local .env files, caches, logs).
local grep_opts=(
-RInE
--exclude-dir=.git
--exclude=.git
--exclude-dir=.cache
--exclude-dir=tmp
--exclude=.env
--exclude=.env.local
--exclude=*.log
)
local matches
matches="$(grep "${grep_opts[@]}" --files-with-matches "${local_path_terms}" "${scan_paths[@]}" 2>/dev/null || true)"
if [[ -n "${matches}" ]]; then
printf 'fail redaction-check found local path patterns in:\n' >&2
printf '%s\n' "${matches}" | print_relative_paths | sed 's/^/- /' >&2
bad=1
fi
matches="$(grep "${grep_opts[@]}" --files-with-matches "${secret_terms}" "${scan_paths[@]}" 2>/dev/null || true)"
if [[ -n "${matches}" ]]; then
printf 'fail redaction-check found secret-like patterns in:\n' >&2
printf '%s\n' "${matches}" | print_relative_paths | sed 's/^/- /' >&2
bad=1
fi
if [[ "${bad}" -eq 0 ]]; then
printf 'pass redaction-check found no local absolute paths or common secret patterns\n'
else
printf 'fail redaction-check found blocked content\n' >&2
fi
return "${bad}"
}
case "${1:-}" in
start) shift; cmd_start "$@" ;;
stop) shift; cmd_stop "$@" ;;
restart) shift; cmd_restart "$@" ;;
build) shift; cmd_build "$@" ;;
status) shift; cmd_status "$@" ;;
doctor) shift; cmd_doctor "$@" ;;
web-search) shift; cmd_web_search "$@" ;;
docs) shift; cmd_docs "$@" ;;
docs-snapshot) shift; cmd_docs_snapshot "$@" ;;
docs-rebuild) shift; cmd_docs_rebuild "$@" ;;
repomix) shift; cmd_repomix "$@" ;;
install) shift; cmd_install "$@" ;;
redaction-check) shift; cmd_redaction_check "$@" ;;
-h|--help|help|"") usage ;;
*) usage >&2; exit 64 ;;
esac