Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
23
docker/web-search/overrides/bounds.mjs
Normal file
23
docker/web-search/overrides/bounds.mjs
Normal file
@@ -0,0 +1,23 @@
|
||||
const MAX_LINKS = 500;
|
||||
const MAX_IMAGES = 200;
|
||||
const MAX_VIDEO = 50;
|
||||
const MAX_AUDIO = 50;
|
||||
const MAX_ATTACHMENTS = 10;
|
||||
|
||||
export function boundFetchCollections(result) {
|
||||
const warnings = [...(result.warnings || [])];
|
||||
const trim = (value, maximum, label) => {
|
||||
if (!Array.isArray(value)) return value;
|
||||
if (value.length > maximum) warnings.push(`${label} truncated from ${value.length} to ${maximum}`);
|
||||
return value.slice(0, maximum);
|
||||
};
|
||||
if (result.links) result.links = trim(result.links, MAX_LINKS, "links");
|
||||
if (result.media) {
|
||||
result.media.images = trim(result.media.images, MAX_IMAGES, "images");
|
||||
result.media.videos = trim(result.media.videos, MAX_VIDEO, "videos");
|
||||
result.media.audio = trim(result.media.audio, MAX_AUDIO, "audio");
|
||||
}
|
||||
if (result.attachments) result.attachments = trim(result.attachments, MAX_ATTACHMENTS, "attachments");
|
||||
result.warnings = warnings;
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user