Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.

docker/web-search/overrides/bing.js

@@ -43,7 +43,7 @@ export class BingProvider {
     }
   }
 
-  async search(q, limit, lang) {
+  async search(q, limit, lang, signal) {
     const cacheKey = createCacheKey("bing", q, limit, lang);
     const cached = searchCache.get(cacheKey);
     if (cached)
@@ -51,7 +51,10 @@ export class BingProvider {
     const market = getMarketFromLang(lang);
     const results = await browserPool.withBrowser(async (browser) => {
       const page = await browser.newPage();
+      const abort = () => void page.close().catch(() => undefined);
+      signal?.addEventListener("abort", abort, { once: true });
       try {
+        signal?.throwIfAborted();
         await page.setViewport({ width: 1365, height: 768 });
         await page.setUserAgent(DEFAULT_BROWSER_SEARCH_USER_AGENT);
         await page.setExtraHTTPHeaders(getAcceptLanguageHeader(lang));
@@ -95,7 +98,8 @@ export class BingProvider {
         });
       }
       finally {
-        await page.close();
+        signal?.removeEventListener("abort", abort);
+        if (!page.isClosed()) await page.close();
       }
     });
     searchCache.set(cacheKey, results);

docker/web-search/overrides/bounds.mjs

@@ -0,0 +1,23 @@
+const MAX_LINKS = 500;
+const MAX_IMAGES = 200;
+const MAX_VIDEO = 50;
+const MAX_AUDIO = 50;
+const MAX_ATTACHMENTS = 10;
+
+export function boundFetchCollections(result) {
+  const warnings = [...(result.warnings || [])];
+  const trim = (value, maximum, label) => {
+    if (!Array.isArray(value)) return value;
+    if (value.length > maximum) warnings.push(`${label} truncated from ${value.length} to ${maximum}`);
+    return value.slice(0, maximum);
+  };
+  if (result.links) result.links = trim(result.links, MAX_LINKS, "links");
+  if (result.media) {
+    result.media.images = trim(result.media.images, MAX_IMAGES, "images");
+    result.media.videos = trim(result.media.videos, MAX_VIDEO, "videos");
+    result.media.audio = trim(result.media.audio, MAX_AUDIO, "audio");
+  }
+  if (result.attachments) result.attachments = trim(result.attachments, MAX_ATTACHMENTS, "attachments");
+  result.warnings = warnings;
+  return result;
+}

docker/web-search/overrides/brave.js

@@ -0,0 +1,42 @@
+import { HTTP_TIMEOUT } from "../constants.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class BraveProvider {
+  name = "brave";
+  configured = Boolean(process.env.BRAVE_SEARCH_API_KEY);
+
+  async search(q, limit, lang, signal) {
+    if (!this.configured) return [];
+    const cacheKey = createCacheKey("brave", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const url = new URL("https://api.search.brave.com/res/v1/web/search");
+    url.searchParams.set("q", q);
+    url.searchParams.set("count", String(Math.min(limit, 20)));
+    url.searchParams.set("search_lang", lang.split(/[-_]/)[0] || "en");
+    const response = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        "X-Subscription-Token": process.env.BRAVE_SEARCH_API_KEY
+      },
+      signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(HTTP_TIMEOUT)]) : AbortSignal.timeout(HTTP_TIMEOUT)
+    });
+    if (!response.ok) throw new Error(`Brave HTTP ${response.status}`);
+    const data = await response.json();
+    const items = (data.web?.results || []).slice(0, limit).flatMap(result => {
+      if (!result.title || !result.url) return [];
+      return [{
+        title: result.title,
+        url: result.url,
+        snippet: result.description || undefined,
+        source: "brave"
+      }];
+    });
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    return this.configured;
+  }
+}

docker/web-search/overrides/browser-fetch.js

@@ -0,0 +1,95 @@
+import { HTTP_TIMEOUT, MAX_BYTES } from "../constants.js";
+import { browserPool } from "../utils/browser-pool.js";
+import { assertSafeUrl } from "./security.js";
+import { fetchViaVettedAddress } from "./http.js";
+
+const MAX_BROWSER_REQUESTS = 100;
+const MAX_BROWSER_TOTAL_BYTES = Math.min(MAX_BYTES, 20 * 1024 * 1024);
+
+function responseHeaders(headers) {
+  const record = {};
+  headers.forEach((value, key) => { record[key] = value; });
+  return record;
+}
+
+export async function fetchBrowserResource(url, timeoutMs = HTTP_TIMEOUT) {
+  await assertSafeUrl(url);
+  return browserPool.withBrowser(async browser => {
+    const page = await browser.newPage();
+    const devtools = await page.target().createCDPSession();
+    await devtools.send("Network.enable");
+    await devtools.send("Network.setBlockedURLs", {
+      urls: ["ws://*", "wss://*", "file://*", "ftp://*"]
+    });
+    await page.evaluateOnNewDocument(() => {
+      const blockedTransport = name => class {
+        constructor() {
+          throw new DOMException(`${name} is disabled by the safe browser fetcher`, "SecurityError");
+        }
+      };
+      for (const name of ["WebSocket", "WebTransport", "RTCPeerConnection", "webkitRTCPeerConnection"]) {
+        if (name in globalThis) {
+          Object.defineProperty(globalThis, name, {
+            configurable: false,
+            writable: false,
+            value: blockedTransport(name)
+          });
+        }
+      }
+    });
+    await page.setBypassServiceWorker(true);
+    let requests = 0;
+    let totalBytes = 0;
+    let blockedError;
+    await page.setRequestInterception(true);
+    page.on("request", request => {
+      void (async () => {
+        try {
+          const requestUrl = new URL(request.url());
+          if (!["http:", "https:"].includes(requestUrl.protocol)) throw new Error("unsupported browser request scheme");
+          if (request.method() !== "GET") throw new Error("browser fetch blocks non-GET requests");
+          requests += 1;
+          if (requests > MAX_BROWSER_REQUESTS) throw new Error("browser request limit exceeded");
+          await assertSafeUrl(requestUrl);
+          const upstream = await fetchViaVettedAddress(requestUrl, timeoutMs);
+          const body = Buffer.from(await upstream.arrayBuffer());
+          totalBytes += body.byteLength;
+          if (totalBytes > MAX_BROWSER_TOTAL_BYTES) throw new Error("browser byte limit exceeded");
+          await request.respond({
+            status: upstream.status,
+            headers: responseHeaders(upstream.headers),
+            body
+          });
+        } catch (error) {
+          blockedError ||= error;
+          await request.abort("blockedbyclient").catch(() => undefined);
+        }
+      })();
+    });
+    try {
+      const navigation = await page.goto(url.toString(), {
+        waitUntil: "networkidle2",
+        timeout: timeoutMs
+      });
+      if (blockedError && !navigation) throw blockedError;
+      const finalUrl = new URL(page.url());
+      await assertSafeUrl(finalUrl);
+      const html = await page.content();
+      const buffer = Buffer.from(html);
+      if (buffer.byteLength > MAX_BYTES) throw new Error("rendered content too large");
+      const headers = new Headers({ "content-type": "text/html; charset=utf-8" });
+      const status = navigation?.status() || 200;
+      const response = new Response(new Uint8Array(buffer), { status, headers });
+      Object.defineProperty(response, "url", { value: finalUrl.toString() });
+      return {
+        response,
+        finalUrl: finalUrl.toString(),
+        contentType: headers.get("content-type"),
+        buffer,
+        byteLength: buffer.byteLength
+      };
+    } finally {
+      await page.close();
+    }
+  });
+}

docker/web-search/overrides/diagnostics.mjs

@@ -0,0 +1,63 @@
+const DEFAULT_TIMEOUT_MS = 15_000;
+const MAX_ERROR_LENGTH = 240;
+
+export function classifyProviderError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const lower = message.toLowerCase();
+  let category = "provider_error";
+  if (lower.includes("timed out") || lower.includes("timeout")) category = "timeout";
+  else if (lower.includes("429") || lower.includes("rate limit")) category = "rate_limited";
+  else if (lower.includes("captcha") || lower.includes("challenge")) category = "blocked";
+  else if (lower.includes("403") || lower.includes("401") || lower.includes("denied")) category = "forbidden";
+  else if (lower.includes("network") || lower.includes("fetch") || lower.includes("socket")) category = "network";
+  return { category, message: message.replace(/\s+/g, " ").slice(0, MAX_ERROR_LENGTH) };
+}
+
+export async function attemptProvider(provider, query, limit, lang, options = {}) {
+  const timeoutMs = Math.max(10, Math.min(options.timeoutMs || DEFAULT_TIMEOUT_MS, 60_000));
+  const now = options.now || (() => performance.now());
+  const started = now();
+  if (provider.configured === false) {
+    return {
+      items: [],
+      diagnostic: { provider: provider.name, status: "unavailable", duration_ms: 0, result_count: 0 }
+    };
+  }
+  let timer;
+  const controller = new AbortController();
+  try {
+    const items = await Promise.race([
+      provider.search(query, limit, lang, controller.signal),
+      new Promise((_, reject) => {
+        timer = setTimeout(() => {
+          controller.abort(new Error(`provider timed out after ${timeoutMs}ms`));
+          reject(new Error(`provider timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+      })
+    ]);
+    const bounded = Array.isArray(items) ? items.slice(0, limit) : [];
+    return {
+      items: bounded,
+      diagnostic: {
+        provider: provider.name,
+        status: bounded.length ? "success" : "empty",
+        duration_ms: Math.max(0, Math.round(now() - started)),
+        result_count: bounded.length
+      }
+    };
+  } catch (error) {
+    return {
+      items: [],
+      diagnostic: {
+        provider: provider.name,
+        status: "error",
+        duration_ms: Math.max(0, Math.round(now() - started)),
+        result_count: 0,
+        error: classifyProviderError(error)
+      }
+    };
+  } finally {
+    controller.abort();
+    clearTimeout(timer);
+  }
+}

docker/web-search/overrides/duckduckgo.js

@@ -0,0 +1,56 @@
+import { JSDOM } from "jsdom";
+import { HTTP_TIMEOUT } from "../constants.js";
+import { fetchWithTimeout } from "../utils/http.js";
+import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class DuckDuckGoProvider {
+  name = "duckduckgo";
+
+  decodeDuckDuckGoRedirect(href) {
+    try {
+      const url = new URL(href, "https://duckduckgo.com/");
+      if (url.hostname === "duckduckgo.com" && url.pathname.startsWith("/l/")) {
+        const target = url.searchParams.get("uddg");
+        if (target) return decodeURIComponent(target);
+      }
+      return url.toString();
+    } catch {
+      return href;
+    }
+  }
+
+  async search(q, limit, lang, signal) {
+    const cacheKey = createCacheKey("ddg", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const url = new URL("https://html.duckduckgo.com/html/");
+    url.searchParams.set("q", q);
+    const headers = { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) };
+    const response = await fetchWithTimeout(url, { headers, signal }, HTTP_TIMEOUT);
+    if (!response.ok) throw new Error(`DuckDuckGo HTML ${response.status}`);
+    const dom = new JSDOM(await response.text(), { url: `https://duckduckgo.com/?q=${encodeURIComponent(q)}` });
+    const anchors = Array.from(dom.window.document.querySelectorAll("a.result__a"));
+    const snippets = Array.from(dom.window.document.querySelectorAll(".result__snippet"));
+    const items = [];
+    for (let index = 0; index < anchors.length && items.length < limit; index += 1) {
+      const title = (anchors[index].textContent || "").trim();
+      const href = this.decodeDuckDuckGoRedirect(anchors[index].getAttribute("href") || "");
+      if (!title || !href) continue;
+      try {
+        items.push({
+          title,
+          url: new URL(href).toString(),
+          snippet: (snippets[index]?.textContent || "").trim() || undefined,
+          source: "duckduckgo"
+        });
+      } catch {}
+    }
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    return true;
+  }
+}

docker/web-search/overrides/registry.js

@@ -0,0 +1,65 @@
+import { DuckDuckGoProvider } from "./duckduckgo.js";
+import { BingProvider } from "./bing.js";
+import { SearXNGProvider } from "./searxng.js";
+import { BraveProvider } from "./brave.js";
+import { DEFAULT_SEARCH_PROVIDER } from "../constants.js";
+import { attemptProvider } from "./diagnostics.js";
+
+const PROVIDERS = ["searxng", "brave", "duckduckgo", "bing"];
+const PROVIDER_TIMEOUT_MS = Number(process.env.SEARCH_PROVIDER_TIMEOUT_MS || "15000");
+const MAX_PROVIDER_ATTEMPTS = Math.max(1, Math.min(Number(process.env.MAX_PROVIDER_ATTEMPTS || "4"), 4));
+
+export class ProviderRegistry {
+  constructor(providers) {
+    this.providers = providers || new Map([
+      ["duckduckgo", new DuckDuckGoProvider()],
+      ["bing", new BingProvider()],
+      ["searxng", new SearXNGProvider()],
+      ["brave", new BraveProvider()]
+    ]);
+  }
+
+  get(name) {
+    return this.providers.get(name);
+  }
+
+  async searchWithFallback(q, limit, lang, preferredProvider) {
+    const defaultProvider = preferredProvider || DEFAULT_SEARCH_PROVIDER;
+    const order = [defaultProvider, ...PROVIDERS.filter(name => name !== defaultProvider)].slice(0, MAX_PROVIDER_ATTEMPTS);
+    const attempts = [];
+    const started = performance.now();
+    for (const providerName of order) {
+      const provider = this.providers.get(providerName);
+      if (!provider) continue;
+      const attempt = await attemptProvider(provider, q, limit, lang, { timeoutMs: PROVIDER_TIMEOUT_MS });
+      attempts.push(attempt.diagnostic);
+      if (attempt.items.length) {
+        return {
+          items: attempt.items,
+          providerUsed: providerName,
+          fallbackUsed: providerName !== defaultProvider,
+          triedProviders: attempts.map(item => item.provider),
+          diagnostics: {
+            attempts,
+            elapsed_ms: Math.round(performance.now() - started),
+            exhausted: false
+          }
+        };
+      }
+    }
+    return {
+      items: [],
+      providerUsed: defaultProvider,
+      fallbackUsed: attempts.length > 1,
+      triedProviders: attempts.map(item => item.provider),
+      diagnostics: {
+        attempts,
+        elapsed_ms: Math.round(performance.now() - started),
+        exhausted: true
+      }
+    };
+  }
+}
+
+export const providerRegistry = new ProviderRegistry();
+export { DuckDuckGoProvider, BingProvider, SearXNGProvider, BraveProvider };

docker/web-search/overrides/searxng.js

@@ -0,0 +1,47 @@
+import { HTTP_TIMEOUT, SEARXNG_URL } from "../constants.js";
+import { fetchWithTimeout } from "../utils/http.js";
+import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class SearXNGProvider {
+  name = "searxng";
+
+  constructor(instanceUrl) {
+    this.instanceUrl = instanceUrl || SEARXNG_URL;
+  }
+
+  async search(q, limit, lang, signal) {
+    const cacheKey = createCacheKey("searxng", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const params = new URLSearchParams({ q, format: "json", language: lang, safesearch: "0" });
+    const response = await fetchWithTimeout(`${this.instanceUrl}/search?${params}`, {
+      headers: { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) },
+      signal
+    }, HTTP_TIMEOUT);
+    if (!response.ok) {
+      if (response.status === 403) throw new Error("SearXNG JSON API disabled");
+      throw new Error(`SearXNG error: ${response.status}`);
+    }
+    const data = await response.json();
+    const items = (data.results || []).slice(0, limit).map(result => ({
+      title: result.title || "",
+      url: result.url || "",
+      snippet: result.content || undefined,
+      source: "searxng"
+    }));
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    try {
+      const response = await fetchWithTimeout(`${this.instanceUrl}/search?q=test&format=json`, {
+        headers: { Accept: "application/json", "User-Agent": getRandomUserAgent() }
+      }, 5000);
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+}