ajaynomics/context-kit
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Overhaul docs retrieval and web search quality

Browse Source 
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.
This commit is contained in:
Ajay Krishnan
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
scripts/test-lifecycle.sh
View File

@@ -259,7 +259,7 @@ new_case() {
unset CONTEXT_KIT_DOCKER_CIDFILE CONTEXT_KIT_RUNTIME_DIR FAKE_DOCS_UID FAKE_WEB_UID \
FAKE_REPLACEMENT_REQUIRED FAKE_RESTART_FAIL FAKE_DROP_RUNNING FAKE_SEARXNG_FAIL \
FAKE_WEB_SEARCH_FAIL FAKE_DOCS_FAIL FAKE_LEGACY_CONTAINER FAKE_CLIENT_OWNER_MISMATCH \
FAKE_EXPECT_DOCS_SOURCES FAKE_EXPECT_DOCS_SOURCES_ABSENT
FAKE_EXPECT_DOCS_SOURCES FAKE_EXPECT_DOCS_SOURCES_ABSENT CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR
mkdir -p "${FAKE_DOCKER_STATE}" "${HOME}"
: > "${FAKE_DOCKER_LOG}"
}
@@ -432,6 +432,34 @@ grep -F 'https://example.test/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.tx
|| fail_test "restart did not regenerate the bind-mounted docs source list"
assert_no_docs_sources_artifacts
new_case snapshot-promotion
seed_service searxng
seed_service web-search-mcp
seed_service docs-mcp
export CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR="${CASE_ROOT}/local-sources"
mkdir -p "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich"
printf '# source menu\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms.txt"
printf '# generated snapshot\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt"
menu_hash="$(sha256sum "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms.txt")"
menu_hash="${menu_hash%% *}"
output_hash="$(sha256sum "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt")"
output_hash="${output_hash%% *}"
printf '{"menu":"llms.txt","menu_sha256":"%s","output_sha256":"%s"}\n' \
"${menu_hash}" "${output_hash}" \
> "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.provenance.json"
printf 'http://127.0.0.1:8769/immich/llms.txt\n' > "${CASE_ROOT}/sources.txt"
export CONTEXT_KIT_DOCS_SOURCES="${CASE_ROOT}/sources.txt"
"${CONTEXT_KIT}" restart
grep -F 'http://127.0.0.1:8769/immich/llms-full.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null \
|| fail_test "restart did not promote a local menu to its generated full snapshot"
if grep -Fx 'http://127.0.0.1:8769/immich/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null; then
fail_test "restart retained the menu URL despite an available full snapshot"
fi
printf '# inconsistent snapshot\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt"
"${CONTEXT_KIT}" restart
grep -Fx 'http://127.0.0.1:8769/immich/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null \
|| fail_test "restart promoted a snapshot whose provenance hash did not match"
new_case restart-failure
seed_service searxng stopped
seed_service web-search-mcp