Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
43
scripts/test-docs-candidate.mjs
Normal file
43
scripts/test-docs-candidate.mjs
Normal file
@@ -0,0 +1,43 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { probeMcp, rpc } from "../docker/web-search/mcp-probe.mjs";
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) throw new Error("usage: node scripts/test-docs-candidate.mjs <mcp-url>");
|
||||
|
||||
const required = ["docs_query", "docs_rebuild", "docs_refresh", "docs_sources"];
|
||||
const tools = await probeMcp(url, { timeoutMs: 10_000, expectedTools: required });
|
||||
assert.deepEqual(tools, required);
|
||||
|
||||
function structured(result) {
|
||||
if (result.structuredContent) return result.structuredContent;
|
||||
const text = (result.content || []).find(part => part.type === "text")?.text;
|
||||
return text ? JSON.parse(text) : null;
|
||||
}
|
||||
|
||||
const refreshed = structured(await rpc(url, 3, "tools/call", {
|
||||
name: "docs_refresh",
|
||||
arguments: { force: true }
|
||||
}, 120_000));
|
||||
assert.equal(refreshed.sources[0].status, "updated", JSON.stringify(refreshed.sources[0]));
|
||||
assert(refreshed.sources[0].document_count >= 2);
|
||||
|
||||
const search = structured(await rpc(url, 4, "tools/call", {
|
||||
name: "docs_query",
|
||||
arguments: { query: "CONTEXT_KIT_EXACT_IDENTIFIER_20260724", limit: 3 }
|
||||
}, 30_000));
|
||||
assert.equal(search.search_results[0].title, "Environment Variables");
|
||||
assert.deepEqual(search.retrieved_content, {});
|
||||
|
||||
const identifier = search.search_results[0].id;
|
||||
const retrieved = structured(await rpc(url, 5, "tools/call", {
|
||||
name: "docs_query",
|
||||
arguments: {
|
||||
query: "CONTEXT_KIT_EXACT_IDENTIFIER_20260724",
|
||||
retrieve_ids: [identifier],
|
||||
max_bytes: 12_000
|
||||
}
|
||||
}, 30_000));
|
||||
assert(retrieved.retrieved_content[identifier].content.includes("CONTEXT_KIT_EXACT_IDENTIFIER_20260724"));
|
||||
|
||||
console.log("pass docs candidate transport, refresh, hybrid search, and explicit retrieval");
|
||||
Reference in New Issue
Block a user