Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions
--- a/scripts/fixtures/web/mock-server.mjs
+++ b/scripts/fixtures/web/mock-server.mjs
@@ -0,0 +1,52 @@
+import http from "node:http";
+
+let websocketUpgrades = 0;
+
+const server = http.createServer((request, response) => {
+  const url = new URL(request.url, "http://mock-search.test");
+  if (url.pathname === "/search") {
+    response.writeHead(200, { "Content-Type": "application/json" });
+    response.end(JSON.stringify({
+      results: [{
+        title: "Deterministic Search Result",
+        url: "https://example.test/result",
+        content: `fixture result for ${url.searchParams.get("q")}`
+      }]
+    }));
+    return;
+  }
+  if (url.pathname === "/dynamic") {
+    response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
+    response.end(`<!doctype html><html><head><title>Dynamic Fixture</title></head>
+      <body><main id="content">initial content</main>
+      <script>document.getElementById("content").textContent = "BROWSER_RENDERED_MARKER";</script>
+      </body></html>`);
+    return;
+  }
+  if (url.pathname === "/redirect-private") {
+    response.writeHead(302, { Location: "http://127.0.0.1:8765/private" });
+    response.end();
+    return;
+  }
+  if (url.pathname === "/websocket-attempt") {
+    response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
+    response.end(`<!doctype html><html><body><main id="content">starting</main>
+      <script>
+        const socket = new WebSocket("ws://mock-search.test:8080/socket");
+        socket.onerror = () => { document.getElementById("content").textContent = "WEBSOCKET_BLOCKED"; };
+      </script></body></html>`);
+    return;
+  }
+  if (url.pathname === "/ws-count") {
+    response.writeHead(200, { "Content-Type": "text/plain" });
+    response.end(String(websocketUpgrades));
+    return;
+  }
+  response.writeHead(404).end();
+});
+
+server.listen(8080, "0.0.0.0");
+server.on("upgrade", (_request, socket) => {
+  websocketUpgrades += 1;
+  socket.destroy();
+});