Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
42
docker/web-search/overrides/brave.js
Normal file
42
docker/web-search/overrides/brave.js
Normal file
@@ -0,0 +1,42 @@
|
||||
import { HTTP_TIMEOUT } from "../constants.js";
|
||||
import { searchCache, createCacheKey } from "../utils/cache.js";
|
||||
|
||||
export class BraveProvider {
|
||||
name = "brave";
|
||||
configured = Boolean(process.env.BRAVE_SEARCH_API_KEY);
|
||||
|
||||
async search(q, limit, lang, signal) {
|
||||
if (!this.configured) return [];
|
||||
const cacheKey = createCacheKey("brave", q, limit, lang);
|
||||
const cached = searchCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
const url = new URL("https://api.search.brave.com/res/v1/web/search");
|
||||
url.searchParams.set("q", q);
|
||||
url.searchParams.set("count", String(Math.min(limit, 20)));
|
||||
url.searchParams.set("search_lang", lang.split(/[-_]/)[0] || "en");
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"X-Subscription-Token": process.env.BRAVE_SEARCH_API_KEY
|
||||
},
|
||||
signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(HTTP_TIMEOUT)]) : AbortSignal.timeout(HTTP_TIMEOUT)
|
||||
});
|
||||
if (!response.ok) throw new Error(`Brave HTTP ${response.status}`);
|
||||
const data = await response.json();
|
||||
const items = (data.web?.results || []).slice(0, limit).flatMap(result => {
|
||||
if (!result.title || !result.url) return [];
|
||||
return [{
|
||||
title: result.title,
|
||||
url: result.url,
|
||||
snippet: result.description || undefined,
|
||||
source: "brave"
|
||||
}];
|
||||
});
|
||||
searchCache.set(cacheKey, items);
|
||||
return items;
|
||||
}
|
||||
|
||||
async isAvailable() {
|
||||
return this.configured;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user