ajaynomics/context-kit
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Overhaul docs retrieval and web search quality

Browse Source 
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.
This commit is contained in:
Ajay Krishnan
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
scripts/fixtures/docs/local-sources/fixture/llms-full.txt Normal file
View File

@@ -0,0 +1,11 @@
# Environment Variables
Source: https://docs.example.test/environment
`CONTEXT_KIT_EXACT_IDENTIFIER_20260724` enables the deterministic candidate fixture.
# Durable Persistence
Source: https://docs.example.test/persistence
Checkpoints preserve graph state across process restarts.

1
scripts/fixtures/docs/sources.txt Normal file
View File

@@ -0,0 +1 @@
http://127.0.0.1:8769/fixture/llms-full.txt

52
scripts/fixtures/web/mock-server.mjs Normal file
View File

@@ -0,0 +1,52 @@
import http from "node:http";
let websocketUpgrades = 0;
const server = http.createServer((request, response) => {
const url = new URL(request.url, "http://mock-search.test");
if (url.pathname === "/search") {
response.writeHead(200, { "Content-Type": "application/json" });
response.end(JSON.stringify({
results: [{
title: "Deterministic Search Result",
url: "https://example.test/result",
content: `fixture result for ${url.searchParams.get("q")}`
}]
}));
return;
}
if (url.pathname === "/dynamic") {
response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
response.end(`<!doctype html><html><head><title>Dynamic Fixture</title></head>
<body><main id="content">initial content</main>
<script>document.getElementById("content").textContent = "BROWSER_RENDERED_MARKER";</script>
</body></html>`);
return;
}
if (url.pathname === "/redirect-private") {
response.writeHead(302, { Location: "http://127.0.0.1:8765/private" });
response.end();
return;
}
if (url.pathname === "/websocket-attempt") {
response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
response.end(`<!doctype html><html><body><main id="content">starting</main>
<script>
const socket = new WebSocket("ws://mock-search.test:8080/socket");
socket.onerror = () => { document.getElementById("content").textContent = "WEBSOCKET_BLOCKED"; };
</script></body></html>`);
return;
}
if (url.pathname === "/ws-count") {
response.writeHead(200, { "Content-Type": "text/plain" });
response.end(String(websocketUpgrades));
return;
}
response.writeHead(404).end();
});
server.listen(8080, "0.0.0.0");
server.on("upgrade", (_request, socket) => {
websocketUpgrades += 1;
socket.destroy();
});