Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
47
docker/web-search/overrides/searxng.js
Normal file
47
docker/web-search/overrides/searxng.js
Normal file
@@ -0,0 +1,47 @@
|
||||
import { HTTP_TIMEOUT, SEARXNG_URL } from "../constants.js";
|
||||
import { fetchWithTimeout } from "../utils/http.js";
|
||||
import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js";
|
||||
import { searchCache, createCacheKey } from "../utils/cache.js";
|
||||
|
||||
export class SearXNGProvider {
|
||||
name = "searxng";
|
||||
|
||||
constructor(instanceUrl) {
|
||||
this.instanceUrl = instanceUrl || SEARXNG_URL;
|
||||
}
|
||||
|
||||
async search(q, limit, lang, signal) {
|
||||
const cacheKey = createCacheKey("searxng", q, limit, lang);
|
||||
const cached = searchCache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
const params = new URLSearchParams({ q, format: "json", language: lang, safesearch: "0" });
|
||||
const response = await fetchWithTimeout(`${this.instanceUrl}/search?${params}`, {
|
||||
headers: { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) },
|
||||
signal
|
||||
}, HTTP_TIMEOUT);
|
||||
if (!response.ok) {
|
||||
if (response.status === 403) throw new Error("SearXNG JSON API disabled");
|
||||
throw new Error(`SearXNG error: ${response.status}`);
|
||||
}
|
||||
const data = await response.json();
|
||||
const items = (data.results || []).slice(0, limit).map(result => ({
|
||||
title: result.title || "",
|
||||
url: result.url || "",
|
||||
snippet: result.content || undefined,
|
||||
source: "searxng"
|
||||
}));
|
||||
searchCache.set(cacheKey, items);
|
||||
return items;
|
||||
}
|
||||
|
||||
async isAvailable() {
|
||||
try {
|
||||
const response = await fetchWithTimeout(`${this.instanceUrl}/search?q=test&format=json`, {
|
||||
headers: { Accept: "application/json", "User-Agent": getRandomUserAgent() }
|
||||
}, 5000);
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user