Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions
--- a/docker/docs/tests/test_refresh.py
+++ b/docker/docs/tests/test_refresh.py
@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import asyncio
+import tempfile
+import unittest
+from pathlib import Path
+
+from context_docs.parser import parse_llms_text
+from context_docs.refresh import RefreshCoordinator
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder, FakeFetch, FakeFetcher
+
+
+class RefreshTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.source = "https://example.test/llms.txt"
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.store.configure_sources([self.source])
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_concurrent_refresh_uses_one_fetch_and_one_publication(self) -> None:
+        fetcher = FakeFetcher([FakeFetch(200, "# API Identifier\n\nExact identifier content.")])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+
+        first, second = await asyncio.gather(
+            coordinator.refresh(self.source, force=True),
+            coordinator.refresh(self.source, force=True),
+        )
+
+        self.assertEqual(1, fetcher.calls)
+        self.assertEqual("updated", first.status)
+        self.assertEqual("updated", second.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+
+    async def test_304_updates_check_time_without_replacing_documents(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content.", etag='"v1"'),
+                FakeFetch(304, etag='"v1"'),
+            ]
+        )
+        clock = iter([100.0, 200.0])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: next(clock),
+        )
+        await coordinator.refresh(self.source, force=True)
+        original = self.store.list_sources()[0]
+        await coordinator.refresh(self.source, force=True)
+        checked = self.store.list_sources()[0]
+
+        self.assertEqual(original.indexed_at, checked.indexed_at)
+        self.assertEqual(200.0, checked.checked_at)
+        self.assertEqual(1, checked.doc_count)
+
+    async def test_refresh_error_preserves_searchable_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(500),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+    async def test_empty_success_response_preserves_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(200, ""),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertIn("zero documents", failed.detail)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+
+if __name__ == "__main__":
+    unittest.main()