Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.
This commit is contained in:
2026-07-25 08:49:26 -07:00
parent 29bcb123fa
commit 51dceee224
60 changed files with 3207 additions and 107 deletions

View File

@@ -0,0 +1,46 @@
from __future__ import annotations
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from starlette.testclient import TestClient
from context_docs.server import build_server, parse_duration, read_sources
class ServerTest(unittest.TestCase):
def test_duration_parser_rejects_ambiguous_values(self) -> None:
self.assertEqual(86_400, parse_duration("24h"))
with self.assertRaises(ValueError):
parse_duration("tomorrow")
def test_source_file_requires_supported_urls(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "sources.txt"
path.write_text("https://example.test/index.html\n")
with self.assertRaisesRegex(ValueError, "must end"):
read_sources(path)
def test_status_is_available_without_loading_embedding_model(self) -> None:
with tempfile.TemporaryDirectory() as directory:
source_file = Path(directory) / "sources.txt"
source_file.write_text("https://example.test/llms.txt\n")
environment = {
"DOCS_MCP_SOURCES_FILE": str(source_file),
"DOCS_MCP_STORE_PATH": str(Path(directory) / "docs.sqlite3"),
"DOCS_MCP_PREINDEX": "0",
}
with patch.dict(os.environ, environment, clear=False):
with TestClient(build_server()) as client:
response = client.get("/status")
self.assertEqual(200, response.status_code)
self.assertTrue(response.json()["ready"])
self.assertFalse(response.json()["model_ready"])
if __name__ == "__main__":
unittest.main()