ajaynomics/context-kit
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Overhaul docs retrieval and web search quality

Browse Source 
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.
This commit is contained in:
Ajay Krishnan
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
docker/web-search/patch-mcp-web-search.mjs
View File

@@ -15,6 +15,14 @@ const replacements = [
[
"max_download_bytes: z.number().int().min(1).max(26214400).optional()",
"max_download_bytes: z.number().int().min(1).max(MAX_BYTES).optional()"
],
[
'provider: z.enum(["duckduckgo", "bing", "searxng"]).optional()',
'provider: z.enum(["duckduckgo", "bing", "searxng", "brave"]).optional()'
],
[
"Search the web using multiple providers (DuckDuckGo, Bing, SearXNG). Automatically falls back to other providers if the default fails. No API keys required for DuckDuckGo and SearXNG.",
"Search the web with bounded provider fallback and per-attempt diagnostics. SearXNG is local; Brave is available when BRAVE_SEARCH_API_KEY is configured."
]
];
@@ -26,3 +34,44 @@ for (const [before, after] of replacements) {
}
fs.writeFileSync(serverPath, source);
const httpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/http.js";
let httpSource = fs.readFileSync(httpPath, "utf8");
const privateTransport = "async function fetchViaVettedAddress(url, timeoutMs)";
if (!httpSource.includes(privateTransport)) throw new Error(`mcp-web-search patch target not found: ${privateTransport}`);
httpSource = httpSource.replace(privateTransport, "export async function fetchViaVettedAddress(url, timeoutMs)");
fs.writeFileSync(httpPath, httpSource);
const utilityHttpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/utils/http.js";
let utilityHttpSource = fs.readFileSync(utilityHttpPath, "utf8");
const uncombinedSignal = 'return await fetch(input, { ...init, signal: controller.signal });';
const combinedSignal = 'const signal = init.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;\n return await fetch(input, { ...init, signal });';
if (!utilityHttpSource.includes(uncombinedSignal)) throw new Error(`mcp-web-search patch target not found: ${uncombinedSignal}`);
utilityHttpSource = utilityHttpSource.replace(uncombinedSignal, combinedSignal);
fs.writeFileSync(utilityHttpPath, utilityHttpSource);
const extractPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/extract.js";
let extractSource = fs.readFileSync(extractPath, "utf8");
const extractReplacements = [
[
'import { assertSafeUrl } from "./fetch/security.js";',
'import { assertSafeUrl } from "./fetch/security.js";\nimport { fetchBrowserResource } from "./fetch/browser.js";\nimport { boundFetchCollections } from "./fetch/bounds.js";'
],
[
"fetchCache.set(cacheKey, siteResult);\n return siteResult;",
"const boundedSiteResult = boundFetchCollections(siteResult);\n fetchCache.set(cacheKey, boundedSiteResult);\n return boundedSiteResult;"
],
[
"const resource = await fetchResource(parsedUrl, options?.timeout_ms, transport, options);",
'const resource = options?.engine === "browser"\n ? await fetchBrowserResource(parsedUrl, options?.timeout_ms)\n : await fetchResource(parsedUrl, options?.timeout_ms, transport, options);'
],
[
"fetchCache.set(cacheKey, result);\n return result;",
"result = boundFetchCollections(result);\n fetchCache.set(cacheKey, result);\n return result;"
]
];
for (const [before, after] of extractReplacements) {
if (!extractSource.includes(before)) throw new Error(`mcp-web-search extract patch target not found: ${before}`);
extractSource = extractSource.replace(before, after);
}
fs.writeFileSync(extractPath, extractSource);