Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.

docker/docs/context_docs/service.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+
+
+class DocsService:
+    def __init__(self, store, search, refresh, max_get_bytes: int = 75_000):
+        self.store = store
+        self.search_engine = search
+        self.refresh_coordinator = refresh
+        self.max_get_bytes = max_get_bytes
+
+    async def refresh(self, sources: list[str] | None = None, force: bool = False) -> dict:
+        configured = [state.configured_source for state in self.store.list_sources()]
+        selected = configured if sources is None else sources
+        unknown = sorted(set(selected) - set(configured))
+        if unknown:
+            raise ValueError(f"unconfigured sources: {', '.join(unknown)}")
+        outcomes = await asyncio.gather(
+            *(self.refresh_coordinator.refresh(source, force=force) for source in selected)
+        )
+        return {"sources": [asdict(outcome) for outcome in outcomes]}
+
+    async def query(
+        self,
+        query: str,
+        limit: int = 10,
+        auto_retrieve: bool = False,
+        auto_retrieve_threshold: float = 0.55,
+        auto_retrieve_limit: int = 5,
+        retrieve_ids: list[str] | None = None,
+        max_bytes: int | None = None,
+        merge: bool = False,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> dict:
+        if not query.strip():
+            raise ValueError("query must not be empty")
+        if not 1 <= limit <= 100:
+            raise ValueError("limit must be between 1 and 100")
+        if not 0 <= auto_retrieve_threshold <= 1:
+            raise ValueError("auto_retrieve_threshold must be between 0 and 1")
+        if not 0 <= auto_retrieve_limit <= 25:
+            raise ValueError("auto_retrieve_limit must be between 0 and 25")
+
+        await self._refresh_missing_or_stale(sources)
+        results = await self.search_engine.search(query, limit, sources, hosts)
+        search_results = [
+            {
+                "id": item.id,
+                "source": item.configured_source,
+                "url": item.canonical_url,
+                "host": item.canonical_url.split("/", 3)[2] if "://" in item.canonical_url else "",
+                "title": item.title,
+                "description": item.description,
+                "heading_path": item.heading_path,
+                "score": round(item.score, 6),
+                "snippet": item.content[:500],
+                "duplicate_count": item.duplicate_count,
+                "alternate_sources": item.alternate_sources,
+            }
+            for item in results
+        ]
+
+        selected_ids = list(dict.fromkeys(retrieve_ids or []))
+        if auto_retrieve:
+            selected_ids.extend(
+                item.id
+                for item in results[:auto_retrieve_limit]
+                if item.score >= auto_retrieve_threshold and item.id not in selected_ids
+            )
+        byte_budget = min(max_bytes or self.max_get_bytes, self.max_get_bytes)
+        retrieved: dict[str, dict] = {}
+        used = 0
+        for identifier in selected_ids[:25]:
+            document = self.store.get_document(identifier, sources, hosts)
+            if not document:
+                continue
+            encoded = document.content.encode()
+            remaining = max(0, byte_budget - used)
+            if remaining == 0:
+                break
+            content = encoded[:remaining].decode(errors="ignore")
+            used += len(content.encode())
+            retrieved[identifier] = {
+                "id": identifier,
+                "source": document.configured_source,
+                "url": document.canonical_url,
+                "title": document.title,
+                "content": content,
+                "truncated": len(content.encode()) < len(encoded),
+            }
+
+        merged = ""
+        if merge:
+            merged = "\n\n".join(
+                f"# {item['title']}\n\nSource: {item['url']}\n\n{item['content']}"
+                for item in retrieved.values()
+            )
+        return {
+            "search_results": search_results,
+            "retrieved_content": retrieved,
+            "merged_content": merged,
+            "auto_retrieved_count": len(retrieved) - len([item for item in retrieve_ids or [] if item in retrieved]),
+            "total_results": len(search_results),
+        }
+
+    async def _refresh_missing_or_stale(self, sources: list[str] | None) -> None:
+        states = self.store.list_sources()
+        selected = [state for state in states if sources is None or state.configured_source in sources]
+        await asyncio.gather(
+            *(self.refresh_coordinator.refresh(state.configured_source) for state in selected)
+        )
+
+    def source_status(self) -> dict:
+        states = [asdict(state) for state in self.store.list_sources()]
+        for state in states:
+            state.pop("raw_body", None)
+        return {
+            "sources": states,
+            "source_count": len(states),
+            "document_count": sum(state["doc_count"] for state in states),
+        }