Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
@@ -43,7 +43,7 @@ export class BingProvider {
|
||||
}
|
||||
}
|
||||
|
||||
async search(q, limit, lang) {
|
||||
async search(q, limit, lang, signal) {
|
||||
const cacheKey = createCacheKey("bing", q, limit, lang);
|
||||
const cached = searchCache.get(cacheKey);
|
||||
if (cached)
|
||||
@@ -51,7 +51,10 @@ export class BingProvider {
|
||||
const market = getMarketFromLang(lang);
|
||||
const results = await browserPool.withBrowser(async (browser) => {
|
||||
const page = await browser.newPage();
|
||||
const abort = () => void page.close().catch(() => undefined);
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
await page.setViewport({ width: 1365, height: 768 });
|
||||
await page.setUserAgent(DEFAULT_BROWSER_SEARCH_USER_AGENT);
|
||||
await page.setExtraHTTPHeaders(getAcceptLanguageHeader(lang));
|
||||
@@ -95,7 +98,8 @@ export class BingProvider {
|
||||
});
|
||||
}
|
||||
finally {
|
||||
await page.close();
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (!page.isClosed()) await page.close();
|
||||
}
|
||||
});
|
||||
searchCache.set(cacheKey, results);
|
||||
|
||||
Reference in New Issue
Block a user