Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.

docker/docs/tests/fakes.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+import numpy as np
+
+from context_docs.models import FetchResponse
+
+
+class FakeEmbedder:
+    fingerprint = "fake-embedder-v1"
+    ready = True
+
+    async def ensure_ready(self) -> None:
+        return None
+
+    async def encode_documents(self, texts: list[str]) -> np.ndarray:
+        return np.asarray([self._vector(text) for text in texts], dtype=np.float32)
+
+    async def encode_query(self, text: str) -> np.ndarray:
+        return np.asarray(self._vector(text), dtype=np.float32)
+
+    @staticmethod
+    def _vector(text: str) -> list[float]:
+        lower = text.lower()
+        return [
+            float("api" in lower or "identifier" in lower),
+            float("persistence" in lower or "checkpoint" in lower),
+            float("background" in lower or "asynchronous" in lower),
+            0.25 + (int(hashlib.sha256(text.encode()).hexdigest()[:2], 16) / 1024),
+        ]
+
+
+@dataclass
+class FakeFetch:
+    status: int
+    body: str = ""
+    final_url: str | None = None
+    etag: str | None = None
+    last_modified: str | None = None
+
+
+class FakeFetcher:
+    def __init__(self, responses: list[FakeFetch]):
+        self.responses = list(responses)
+        self.calls = 0
+
+    async def fetch(self, source_url: str, state=None) -> FetchResponse:
+        self.calls += 1
+        response = self.responses.pop(0)
+        return FetchResponse(
+            status=response.status,
+            requested_url=source_url,
+            resolved_url=response.final_url or source_url,
+            body=response.body,
+            etag=response.etag,
+            last_modified=response.last_modified,
+        )

docker/docs/tests/test_parser.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import unittest
+
+from context_docs.parser import parse_llms_text
+
+
+class ParserTest(unittest.TestCase):
+    def test_standard_menu_preserves_target_url_and_retrievable_content(self) -> None:
+        parsed = parse_llms_text(
+            """# Rails Docs
+
+> Curated official documentation.
+
+## Active Record
+
+- [Associations](https://guides.rubyonrails.org/association_basics.html): Model relationships
+""",
+            "http://127.0.0.1:8769/rails/llms.txt",
+        )
+
+        self.assertEqual("standard-menu", parsed.format)
+        self.assertEqual(1, len(parsed.documents))
+        document = parsed.documents[0]
+        self.assertEqual("https://guides.rubyonrails.org/association_basics.html", document.canonical_url)
+        self.assertIn("Model relationships", document.content)
+        self.assertIn("https://guides.rubyonrails.org/association_basics.html", document.content)
+
+    def test_full_bundle_is_not_misclassified_by_interior_yaml_or_rule(self) -> None:
+        parsed = parse_llms_text(
+            """# Build a client
+
+Some content.
+
+---
+title: This is an embedded example
+description: It is not file frontmatter
+---
+
+# Elicitation
+
+URL mode details.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("markdown-full", parsed.format)
+        self.assertEqual(["Build a client", "Elicitation"], [doc.title for doc in parsed.documents])
+
+    def test_full_bundle_with_bullet_links_keeps_prose_sections(self) -> None:
+        parsed = parse_llms_text(
+            """# Persistence
+
+This substantial section explains durable checkpoint behavior.
+
+- [Related guide](https://example.test/guide): Read more
+
+# Streaming
+
+Streaming emits incremental updates.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("markdown-full", parsed.format)
+        self.assertEqual(["Persistence", "Streaming"], [doc.title for doc in parsed.documents])
+        self.assertIn("durable checkpoint", parsed.documents[0].content)
+
+    def test_repeated_frontmatter_accepts_optional_description(self) -> None:
+        parsed = parse_llms_text(
+            """---
+title: First
+---
+First body.
+---
+title: Second
+description: Second description
+---
+Second body.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("yaml-full", parsed.format)
+        self.assertEqual(["First", "Second"], [doc.title for doc in parsed.documents])
+        self.assertEqual("Second description", parsed.documents[1].description)
+
+    def test_long_sections_are_split_without_losing_tail_identifiers(self) -> None:
+        body = "Paragraph.\n\n" * 100 + "IMMICH_IGNORE_MOUNT_CHECK_ERRORS disables mount checks."
+        parsed = parse_llms_text(
+            f"# Environment Variables\n\n{body}",
+            "https://example.test/llms-full.txt",
+            max_chunk_chars=500,
+        )
+
+        self.assertGreater(len(parsed.documents), 1)
+        self.assertTrue(any("IMMICH_IGNORE_MOUNT_CHECK_ERRORS" in doc.content for doc in parsed.documents))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_refresh.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import asyncio
+import tempfile
+import unittest
+from pathlib import Path
+
+from context_docs.parser import parse_llms_text
+from context_docs.refresh import RefreshCoordinator
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder, FakeFetch, FakeFetcher
+
+
+class RefreshTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.source = "https://example.test/llms.txt"
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.store.configure_sources([self.source])
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_concurrent_refresh_uses_one_fetch_and_one_publication(self) -> None:
+        fetcher = FakeFetcher([FakeFetch(200, "# API Identifier\n\nExact identifier content.")])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+
+        first, second = await asyncio.gather(
+            coordinator.refresh(self.source, force=True),
+            coordinator.refresh(self.source, force=True),
+        )
+
+        self.assertEqual(1, fetcher.calls)
+        self.assertEqual("updated", first.status)
+        self.assertEqual("updated", second.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+
+    async def test_304_updates_check_time_without_replacing_documents(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content.", etag='"v1"'),
+                FakeFetch(304, etag='"v1"'),
+            ]
+        )
+        clock = iter([100.0, 200.0])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: next(clock),
+        )
+        await coordinator.refresh(self.source, force=True)
+        original = self.store.list_sources()[0]
+        await coordinator.refresh(self.source, force=True)
+        checked = self.store.list_sources()[0]
+
+        self.assertEqual(original.indexed_at, checked.indexed_at)
+        self.assertEqual(200.0, checked.checked_at)
+        self.assertEqual(1, checked.doc_count)
+
+    async def test_refresh_error_preserves_searchable_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(500),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+    async def test_empty_success_response_preserves_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(200, ""),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertIn("zero documents", failed.detail)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_search.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, SourceUpdate
+from context_docs.search import HybridSearch
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder
+
+
+def prepared(identifier: str, source: str, canonical: str, title: str, content: str, vector) -> PreparedDocument:
+    return PreparedDocument(
+        id=identifier,
+        configured_source=source,
+        resolved_source=source,
+        source_host="source.test",
+        canonical_url=canonical,
+        canonical_host=canonical.split("/")[2],
+        title=title,
+        description="",
+        heading_path=title,
+        content=content,
+        content_hash=__import__("hashlib").sha256(content.encode()).hexdigest(),
+        embedding=np.asarray(vector, dtype=np.float32),
+    )
+
+
+def source_update(source: str, documents: list[PreparedDocument]) -> SourceUpdate:
+    return SourceUpdate(
+        configured_source=source,
+        resolved_source=source,
+        etag=None,
+        last_modified=None,
+        body_hash="body",
+        raw_body="# source",
+        parser_fingerprint="parser-v1",
+        embedding_fingerprint="fake-embedder-v1",
+        checked_at=1.0,
+        indexed_at=1.0,
+        stale_at=9999.0,
+        documents=documents,
+    )
+
+
+class HybridSearchTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.a = "https://a.test/llms.txt"
+        self.b = "https://b.test/llms.txt"
+        self.store.configure_sources([self.a, self.b])
+        duplicate = "Shared exact content."
+        self.store.replace_source(
+            source_update(
+                self.a,
+                [
+                    prepared("exact", self.a, "https://rails.test/exact", "Environment", "IMMICH_IGNORE_MOUNT_CHECK_ERRORS identifier", [1, 0, 0, 0]),
+                    prepared("duplicate-a", self.a, "https://docs.test/shared", "Shared", duplicate, [0, 1, 0, 0]),
+                ],
+            )
+        )
+        self.store.replace_source(
+            source_update(
+                self.b,
+                [
+                    prepared("persistence", self.b, "https://langgraph.test/persistence", "Persistence", "Durable checkpoint state", [0, 1, 0, 0]),
+                    prepared("duplicate-b", self.b, "https://docs.test/shared-copy", "Shared copy", duplicate, [0, 1, 0, 0]),
+                ],
+            )
+        )
+        self.search = HybridSearch(self.store, FakeEmbedder())
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_exact_identifier_is_ranked_first(self) -> None:
+        result = await self.search.search("IMMICH_IGNORE_MOUNT_CHECK_ERRORS", limit=5)
+        self.assertEqual("exact", result[0].id)
+        self.assertEqual(1, result[0].lexical_rank)
+
+    async def test_source_and_host_filters_apply_before_ranking(self) -> None:
+        by_source = await self.search.search("persistence", limit=5, sources=[self.b])
+        by_host = await self.search.search("identifier", limit=5, hosts=["rails.test"])
+
+        self.assertTrue(by_source)
+        self.assertTrue(all(item.configured_source == self.b for item in by_source))
+        self.assertEqual(["exact"], [item.id for item in by_host])
+
+    async def test_exact_duplicate_content_is_collapsed_with_alternates(self) -> None:
+        result = await self.search.search("Shared exact content", limit=10)
+        shared = [item for item in result if item.content_hash == __import__("hashlib").sha256("Shared exact content.".encode()).hexdigest()]
+
+        self.assertEqual(1, len(shared))
+        self.assertEqual(2, shared[0].duplicate_count)
+        self.assertEqual(1, len(shared[0].alternate_sources))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_server.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from starlette.testclient import TestClient
+
+from context_docs.server import build_server, parse_duration, read_sources
+
+
+class ServerTest(unittest.TestCase):
+    def test_duration_parser_rejects_ambiguous_values(self) -> None:
+        self.assertEqual(86_400, parse_duration("24h"))
+        with self.assertRaises(ValueError):
+            parse_duration("tomorrow")
+
+    def test_source_file_requires_supported_urls(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "sources.txt"
+            path.write_text("https://example.test/index.html\n")
+            with self.assertRaisesRegex(ValueError, "must end"):
+                read_sources(path)
+
+    def test_status_is_available_without_loading_embedding_model(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            source_file = Path(directory) / "sources.txt"
+            source_file.write_text("https://example.test/llms.txt\n")
+            environment = {
+                "DOCS_MCP_SOURCES_FILE": str(source_file),
+                "DOCS_MCP_STORE_PATH": str(Path(directory) / "docs.sqlite3"),
+                "DOCS_MCP_PREINDEX": "0",
+            }
+            with patch.dict(os.environ, environment, clear=False):
+                with TestClient(build_server()) as client:
+                    response = client.get("/status")
+
+            self.assertEqual(200, response.status_code)
+            self.assertTrue(response.json()["ready"])
+            self.assertFalse(response.json()["model_ready"])
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_service.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, RefreshOutcome, SourceUpdate
+from context_docs.search import HybridSearch
+from context_docs.service import DocsService
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder
+
+
+class NoopRefresh:
+    async def refresh(self, source: str, force: bool = False) -> RefreshOutcome:
+        return RefreshOutcome(source, "fresh", 1)
+
+
+class ServiceTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.source = "https://example.test/llms.txt"
+        self.store.configure_sources([self.source])
+        content = "IMMICH_IGNORE_MOUNT_CHECK_ERRORS " + "x" * 200
+        item = PreparedDocument(
+            id="exact",
+            configured_source=self.source,
+            resolved_source=self.source,
+            source_host="example.test",
+            canonical_url="https://docs.example.test/environment",
+            canonical_host="docs.example.test",
+            title="Environment",
+            description="",
+            heading_path="Environment",
+            content=content,
+            content_hash="content-hash",
+            embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
+        )
+        self.store.replace_source(
+            SourceUpdate(
+                configured_source=self.source,
+                resolved_source=self.source,
+                etag=None,
+                last_modified=None,
+                body_hash="body",
+                raw_body="# body",
+                parser_fingerprint="context-docs-parser-v1",
+                embedding_fingerprint="fake-embedder-v1",
+                checked_at=1.0,
+                indexed_at=1.0,
+                stale_at=9_999_999_999.0,
+                documents=[item],
+            )
+        )
+        embedder = FakeEmbedder()
+        self.service = DocsService(
+            self.store,
+            HybridSearch(self.store, embedder),
+            NoopRefresh(),
+            max_get_bytes=100,
+        )
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_query_does_not_retrieve_content_by_default(self) -> None:
+        response = await self.service.query("IMMICH_IGNORE_MOUNT_CHECK_ERRORS")
+
+        self.assertEqual({}, response["retrieved_content"])
+        self.assertEqual("exact", response["search_results"][0]["id"])
+
+    async def test_explicit_retrieval_respects_global_byte_cap(self) -> None:
+        response = await self.service.query(
+            "IMMICH_IGNORE_MOUNT_CHECK_ERRORS",
+            retrieve_ids=["exact"],
+            max_bytes=10_000,
+        )
+        retrieved = response["retrieved_content"]["exact"]
+
+        self.assertLessEqual(len(retrieved["content"].encode()), 100)
+        self.assertTrue(retrieved["truncated"])
+
+    async def test_high_default_threshold_only_retrieves_strong_hybrid_match(self) -> None:
+        response = await self.service.query(
+            "IMMICH_IGNORE_MOUNT_CHECK_ERRORS",
+            auto_retrieve=True,
+        )
+
+        self.assertIn("exact", response["retrieved_content"])
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_store.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, SourceUpdate
+from context_docs.store import IndexStore
+
+
+def document(identifier: str, source: str, title: str, content: str) -> PreparedDocument:
+    return PreparedDocument(
+        id=identifier,
+        configured_source=source,
+        resolved_source=source,
+        source_host="example.test",
+        canonical_url=f"https://docs.example.test/{identifier}",
+        canonical_host="docs.example.test",
+        title=title,
+        description="",
+        heading_path=title,
+        content=content,
+        content_hash=identifier,
+        embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
+    )
+
+
+def update(source: str, documents: list[PreparedDocument], checked_at: float = 100.0) -> SourceUpdate:
+    return SourceUpdate(
+        configured_source=source,
+        resolved_source=source,
+        etag=None,
+        last_modified=None,
+        body_hash="body-hash",
+        raw_body="# Fixture",
+        parser_fingerprint="parser-v1",
+        embedding_fingerprint="fake-embedder-v1",
+        checked_at=checked_at,
+        indexed_at=checked_at,
+        stale_at=checked_at + 3600,
+        documents=documents,
+    )
+
+
+class StoreTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.path = Path(self.tmp.name) / "docs.sqlite3"
+        self.source = "https://example.test/llms.txt"
+        self.store = IndexStore(self.path)
+        self.store.configure_sources([self.source])
+
+    def tearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    def test_replacement_removes_old_only_documents(self) -> None:
+        self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")]))
+        self.store.replace_source(update(self.source, [document("new", self.source, "New", "new content")]))
+
+        self.assertIsNone(self.store.get_document("old"))
+        self.assertEqual("new content", self.store.get_document("new").content)
+
+    def test_failed_replacement_rolls_back_to_previous_generation(self) -> None:
+        self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")]))
+        self.store.connection.execute(
+            "CREATE TRIGGER reject_failure BEFORE INSERT ON documents "
+            "WHEN NEW.title = 'FAIL' BEGIN SELECT RAISE(ABORT, 'injected failure'); END"
+        )
+
+        with self.assertRaisesRegex(Exception, "injected failure"):
+            self.store.replace_source(update(self.source, [document("bad", self.source, "FAIL", "bad")]))
+
+        self.assertEqual("old content", self.store.get_document("old").content)
+        self.assertIsNone(self.store.get_document("bad"))
+
+    def test_restart_loads_persisted_state_without_network(self) -> None:
+        self.store.replace_source(update(self.source, [document("persisted", self.source, "Persisted", "saved")]))
+        self.store.close()
+
+        self.store = IndexStore(self.path)
+        self.store.configure_sources([self.source])
+
+        states = self.store.list_sources()
+        self.assertEqual(1, states[0].doc_count)
+        self.assertEqual("saved", self.store.get_document("persisted").content)
+
+    def test_removed_source_is_not_searchable_or_retrievable(self) -> None:
+        self.store.replace_source(update(self.source, [document("retired", self.source, "Retired", "identifier")]))
+        self.store.configure_sources([])
+
+        self.assertIsNone(self.store.get_document("retired"))
+        self.assertEqual([], self.store.lexical_search("identifier", limit=5))
+
+
+if __name__ == "__main__":
+    unittest.main()