Overhaul docs retrieval and web search quality

Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo
MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings,
transactional source replacement, persisted state across restarts,
singleflight refresh with conditional requests, hybrid lexical/semantic
ranking with exact-duplicate collapse, source/host filters, and
explicit-by-default content retrieval. Add docs_rebuild and a
docs-rebuild CLI command.

Add deterministic llms-full.txt snapshot generation for machine-local
menus with hash-validated provenance manifests; lifecycle commands
promote a local menu to its snapshot only when the manifest validates.
Switch public source profiles to content-bearing llms-full.txt feeds.

Improve web search: bounded provider fallback with per-attempt
diagnostics and cancellation, an optional Brave Search API provider,
strict SearXNG engine selection, capped link/media extraction, and a
real engine=browser renderer that routes every request through the
existing SSRF vetting while blocking WebSockets, non-GET traffic, and
private destinations.

Extend release checks with offline unit suites and isolated candidate
container tests for both images.

docker/docs/.dockerignore

@@ -2,3 +2,7 @@
 !Dockerfile
 !entrypoint.sh
 !constraints.txt
+!context_docs/
+!context_docs/**
+!tests/
+!tests/**

docker/docs/Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.12-slim@sha256:6c4dd321d176d61ea848dc8c73a4f7dbae8f70e0ee48bb411ea2f045b599fa8e
 
-ARG LLMS_TXT_MCP_VERSION=0.2.0
+ARG MCP_VERSION=1.28.0
 ARG MCP_PROXY_VERSION=0.12.0
+ARG SENTENCE_TRANSFORMERS_VERSION=5.6.0
 ARG TORCH_VERSION=2.12.1+cpu
 
 COPY constraints.txt /tmp/context-kit-docs-constraints.txt
@@ -11,31 +12,35 @@ RUN apt-get update \
     ca-certificates \
   && rm -rf /var/lib/apt/lists/*
 
-# Install CPU-only torch first so llms-txt-mcp does not pull large CUDA wheels.
+# Install CPU-only torch first so sentence-transformers does not pull CUDA wheels.
 RUN pip install --no-cache-dir \
       --index-url https://download.pytorch.org/whl/cpu \
       -c /tmp/context-kit-docs-constraints.txt \
       "torch==${TORCH_VERSION}"
 
-# llms-txt-mcp does the indexing/search; mcp-proxy fronts its stdio transport
-# as Streamable HTTP so multiple MCP clients can share one long-lived process
-# (and therefore one Chroma DB writer).
-RUN if [ -n "${LLMS_TXT_MCP_VERSION}" ]; then \
-      pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt "llms-txt-mcp==${LLMS_TXT_MCP_VERSION}"; \
-    else \
-      pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt llms-txt-mcp; \
-    fi \
- && pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt "mcp-proxy==${MCP_PROXY_VERSION}" \
+RUN pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt \
+      "mcp==${MCP_VERSION}" \
+      "mcp-proxy==${MCP_PROXY_VERSION}" \
+      "sentence-transformers==${SENTENCE_TRANSFORMERS_VERSION}" \
+      httpx numpy PyYAML \
  && rm /tmp/context-kit-docs-constraints.txt
 
+COPY context_docs /opt/context-kit/context_docs
+COPY tests /opt/context-kit/tests
 COPY entrypoint.sh /usr/local/bin/docs-mcp-entrypoint
-RUN chmod 0555 /usr/local/bin/docs-mcp-entrypoint
+RUN chmod -R a+rX /opt/context-kit \
+  && chmod 0555 /usr/local/bin/docs-mcp-entrypoint
 
 RUN mkdir -p /data /models /etc/context-kit
 ENV HF_HOME=/models \
+    HOME=/tmp \
+    USER=context-kit \
+    LOGNAME=context-kit \
     SENTENCE_TRANSFORMERS_HOME=/models \
+    PYTHONPATH=/opt/context-kit \
     DOCS_MCP_HTTP_HOST=0.0.0.0 \
     DOCS_MCP_HTTP_PORT=8000 \
+    DOCS_MCP_STORE_PATH=/data/docs.sqlite3 \
     DOCS_MCP_SOURCES_FILE=/etc/context-kit/docs-sources.txt
 
 VOLUME ["/data", "/models"]

docker/docs/constraints.txt

@@ -10,7 +10,6 @@ build==1.5.0
 certifi==2026.6.17
 cffi==2.0.0
 charset-normalizer==3.4.7
-chromadb==1.5.9
 click==8.4.2
 cryptography==49.0.0
 durationpy==0.10
@@ -35,7 +34,6 @@ joblib==1.5.3
 jsonschema==4.26.0
 jsonschema-specifications==2025.9.1
 kubernetes==36.0.2
-llms-txt-mcp==0.2.0
 markdown-it-py==4.2.0
 MarkupSafe==3.0.3
 mcp==1.28.0

docker/docs/context_docs/__init__.py

@@ -0,0 +1,3 @@
+"""Maintained Context Kit documentation retrieval service."""
+
+__version__ = "1.0.0"

docker/docs/context_docs/__main__.py

@@ -0,0 +1,4 @@
+from .server import main
+
+
+main()

docker/docs/context_docs/embedder.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+
+import numpy as np
+
+
+class SentenceTransformerEmbedder:
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        self.fingerprint = f"sentence-transformers:{model_name}"
+        self._model = None
+        self._lock = asyncio.Lock()
+
+    @property
+    def ready(self) -> bool:
+        return self._model is not None
+
+    async def ensure_ready(self) -> None:
+        if self._model is not None:
+            return
+        async with self._lock:
+            if self._model is None:
+                self._model = await asyncio.to_thread(self._load)
+
+    def _load(self):
+        from sentence_transformers import SentenceTransformer
+
+        return SentenceTransformer(self.model_name, device="cpu")
+
+    async def encode_documents(self, texts: list[str]) -> np.ndarray:
+        await self.ensure_ready()
+        return await asyncio.to_thread(self._encode, texts)
+
+    async def encode_query(self, text: str) -> np.ndarray:
+        vectors = await self.encode_documents([text])
+        return vectors[0]
+
+    def _encode(self, texts: list[str]) -> np.ndarray:
+        return np.asarray(
+            self._model.encode(
+                texts,
+                batch_size=32,
+                show_progress_bar=False,
+                normalize_embeddings=True,
+                convert_to_numpy=True,
+            ),
+            dtype=np.float32,
+        )
+
+
+class LexicalFallbackEmbedder:
+    """Deterministic fallback used only when a model cannot be loaded."""
+
+    fingerprint = "lexical-fallback-v1"
+    ready = True
+
+    async def ensure_ready(self) -> None:
+        return None
+
+    async def encode_documents(self, texts: list[str]) -> np.ndarray:
+        return np.asarray([self._encode(text) for text in texts], dtype=np.float32)
+
+    async def encode_query(self, text: str) -> np.ndarray:
+        return np.asarray(self._encode(text), dtype=np.float32)
+
+    @staticmethod
+    def _encode(text: str, dimensions: int = 384) -> np.ndarray:
+        vector = np.zeros(dimensions, dtype=np.float32)
+        for token in text.lower().split():
+            digest = hashlib.sha256(token.encode()).digest()
+            vector[int.from_bytes(digest[:4], "big") % dimensions] += 1.0
+        norm = np.linalg.norm(vector)
+        return vector / norm if norm else vector

docker/docs/context_docs/fetcher.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import httpx
+
+from .models import FetchResponse, SourceState
+
+
+class SourceFetcher:
+    def __init__(self, timeout_seconds: float = 30, max_bytes: int = 20_000_000):
+        self.timeout_seconds = timeout_seconds
+        self.max_bytes = max_bytes
+        self._client = httpx.AsyncClient(
+            follow_redirects=True,
+            timeout=httpx.Timeout(timeout_seconds),
+            headers={"User-Agent": "context-kit-docs/1.0"},
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def fetch(self, source_url: str, state: SourceState | None = None) -> FetchResponse:
+        headers: dict[str, str] = {}
+        if state and state.etag:
+            headers["If-None-Match"] = state.etag
+        if state and state.last_modified:
+            headers["If-Modified-Since"] = state.last_modified
+        async with self._client.stream("GET", source_url, headers=headers) as response:
+            if response.status_code == 304:
+                return FetchResponse(
+                    status=304,
+                    requested_url=source_url,
+                    resolved_url=str(response.url),
+                    etag=response.headers.get("etag"),
+                    last_modified=response.headers.get("last-modified"),
+                )
+            chunks: list[bytes] = []
+            size = 0
+            async for chunk in response.aiter_bytes():
+                size += len(chunk)
+                if size > self.max_bytes:
+                    raise RuntimeError(f"source exceeds {self.max_bytes} byte limit")
+                chunks.append(chunk)
+            body = b"".join(chunks).decode(response.encoding or "utf-8", errors="replace")
+            return FetchResponse(
+                status=response.status_code,
+                requested_url=source_url,
+                resolved_url=str(response.url),
+                body=body,
+                etag=response.headers.get("etag"),
+                last_modified=response.headers.get("last-modified"),
+            )

docker/docs/context_docs/models.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class ParsedDocument:
+    title: str
+    description: str
+    content: str
+    canonical_url: str
+    heading_path: str
+    chunk_index: int = 0
+
+
+@dataclass(frozen=True)
+class ParsedSource:
+    format: str
+    documents: list[ParsedDocument]
+
+
+@dataclass(frozen=True)
+class FetchResponse:
+    status: int
+    requested_url: str
+    resolved_url: str
+    body: str = ""
+    etag: str | None = None
+    last_modified: str | None = None
+
+
+@dataclass(frozen=True)
+class PreparedDocument:
+    id: str
+    configured_source: str
+    resolved_source: str
+    source_host: str
+    canonical_url: str
+    canonical_host: str
+    title: str
+    description: str
+    heading_path: str
+    content: str
+    content_hash: str
+    embedding: np.ndarray
+
+
+@dataclass(frozen=True)
+class SourceUpdate:
+    configured_source: str
+    resolved_source: str
+    etag: str | None
+    last_modified: str | None
+    body_hash: str
+    raw_body: str
+    parser_fingerprint: str
+    embedding_fingerprint: str
+    checked_at: float
+    indexed_at: float
+    stale_at: float
+    documents: list[PreparedDocument]
+
+
+@dataclass(frozen=True)
+class SourceState:
+    configured_source: str
+    resolved_source: str | None
+    active: bool
+    etag: str | None
+    last_modified: str | None
+    body_hash: str | None
+    raw_body: str | None
+    parser_fingerprint: str | None
+    embedding_fingerprint: str | None
+    checked_at: float | None
+    indexed_at: float | None
+    stale_at: float | None
+    last_error: str | None
+    doc_count: int
+
+
+@dataclass(frozen=True)
+class StoredDocument:
+    id: str
+    configured_source: str
+    resolved_source: str
+    source_host: str
+    canonical_url: str
+    canonical_host: str
+    title: str
+    description: str
+    heading_path: str
+    content: str
+    content_hash: str
+    embedding: np.ndarray
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    id: str
+    configured_source: str
+    canonical_url: str
+    title: str
+    description: str
+    heading_path: str
+    content: str
+    content_hash: str
+    score: float
+    lexical_rank: int | None
+    semantic_rank: int | None
+    duplicate_count: int = 1
+    alternate_sources: list[dict[str, str]] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class RefreshOutcome:
+    source: str
+    status: str
+    document_count: int
+    detail: str | None = None

docker/docs/context_docs/parser.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin
+
+import yaml
+
+from .models import ParsedDocument, ParsedSource
+
+
+PARSER_FINGERPRINT = "context-docs-parser-v1"
+_MENU_LINK = re.compile(r"^\s*[-*]\s+\[([^]]+)]\(([^)]+)\)(?::\s*(.*))?\s*$")
+_HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+_FRONTMATTER = re.compile(r"(?m)^---\s*$")
+
+
+def parse_llms_text(content: str, source_url: str, max_chunk_chars: int = 6_000) -> ParsedSource:
+    normalized = content.replace("\r\n", "\n").replace("\r", "\n").strip()
+    if not normalized:
+        return ParsedSource("empty", [])
+
+    yaml_documents = _parse_repeated_frontmatter(normalized, source_url, max_chunk_chars)
+    if yaml_documents is not None:
+        return ParsedSource("yaml-full", yaml_documents)
+
+    if source_url.split("?", 1)[0].endswith("/llms.txt"):
+        menu_documents = _parse_menu(normalized, source_url)
+        if menu_documents:
+            return ParsedSource("standard-menu", menu_documents)
+
+    return ParsedSource("markdown-full", _parse_markdown_bundle(normalized, source_url, max_chunk_chars))
+
+
+def _parse_repeated_frontmatter(content: str, source_url: str, max_chunk_chars: int) -> list[ParsedDocument] | None:
+    if not content.startswith("---\n"):
+        return None
+    separators = [match.start() for match in _FRONTMATTER.finditer(content)]
+    if len(separators) < 2:
+        return None
+
+    documents: list[ParsedDocument] = []
+    cursor = 0
+    while cursor < len(content):
+        if not content.startswith("---", cursor):
+            return None
+        header_end = content.find("\n---", cursor + 3)
+        if header_end < 0:
+            return None
+        try:
+            metadata = yaml.safe_load(content[cursor + 3 : header_end]) or {}
+        except yaml.YAMLError:
+            return None
+        if not isinstance(metadata, dict) or not isinstance(metadata.get("title"), str):
+            return None
+        body_start = header_end + 4
+        if body_start < len(content) and content[body_start] == "\n":
+            body_start += 1
+        next_header = content.find("\n---\n", body_start)
+        body_end = len(content) if next_header < 0 else next_header
+        body = content[body_start:body_end].strip()
+        title = metadata["title"].strip()
+        description = str(metadata.get("description") or "").strip()
+        canonical = str(metadata.get("url") or metadata.get("canonical_url") or source_url)
+        documents.extend(_chunk_document(title, description, body, urljoin(source_url, canonical), title, max_chunk_chars))
+        if next_header < 0:
+            break
+        cursor = next_header + 1
+    return documents or None
+
+
+def _parse_menu(content: str, source_url: str) -> list[ParsedDocument]:
+    documents: list[ParsedDocument] = []
+    headings: list[tuple[int, str]] = []
+    in_fence = False
+    for line in content.splitlines():
+        if line.lstrip().startswith(("```", "~~~")):
+            in_fence = not in_fence
+            continue
+        if in_fence:
+            continue
+        heading = _HEADING.match(line)
+        if heading:
+            level = len(heading.group(1))
+            headings = [entry for entry in headings if entry[0] < level]
+            headings.append((level, heading.group(2).strip()))
+            continue
+        link = _MENU_LINK.match(line)
+        if not link:
+            continue
+        title, target, description = link.group(1).strip(), link.group(2).strip(), (link.group(3) or "").strip()
+        canonical = urljoin(source_url, target)
+        path = " > ".join([name for _, name in headings] + [title])
+        rendered = f"{title}\n\n{description}\n\nSource: {canonical}".strip()
+        documents.append(ParsedDocument(title, description, rendered, canonical, path))
+    return documents
+
+
+def _parse_markdown_bundle(content: str, source_url: str, max_chunk_chars: int) -> list[ParsedDocument]:
+    sections: list[tuple[str, str]] = []
+    current_title = "Documentation"
+    current_lines: list[str] = []
+    in_fence = False
+    for line in content.splitlines():
+        if line.lstrip().startswith(("```", "~~~")):
+            in_fence = not in_fence
+        heading = None if in_fence else _HEADING.match(line)
+        if heading and len(heading.group(1)) == 1:
+            if current_lines or sections:
+                sections.append((current_title, "\n".join(current_lines).strip()))
+            current_title = heading.group(2).strip()
+            current_lines = []
+        else:
+            current_lines.append(line)
+    if current_lines or not sections:
+        sections.append((current_title, "\n".join(current_lines).strip()))
+
+    documents: list[ParsedDocument] = []
+    for title, body in sections:
+        if not body and title == "Documentation":
+            continue
+        documents.extend(_chunk_document(title, "", body, source_url, title, max_chunk_chars))
+    return documents
+
+
+def _chunk_document(
+    title: str,
+    description: str,
+    content: str,
+    canonical_url: str,
+    heading_path: str,
+    max_chunk_chars: int,
+) -> list[ParsedDocument]:
+    if len(content) <= max_chunk_chars:
+        return [ParsedDocument(title, description, content, canonical_url, heading_path, 0)]
+
+    paragraphs = re.split(r"\n{2,}", content)
+    chunks: list[str] = []
+    current: list[str] = []
+    size = 0
+    for paragraph in paragraphs:
+        pieces = [paragraph[index : index + max_chunk_chars] for index in range(0, len(paragraph), max_chunk_chars)] or [""]
+        for piece in pieces:
+            added = len(piece) + (2 if current else 0)
+            if current and size + added > max_chunk_chars:
+                chunks.append("\n\n".join(current))
+                current, size = [], 0
+            current.append(piece)
+            size += len(piece) + (2 if len(current) > 1 else 0)
+    if current:
+        chunks.append("\n\n".join(current))
+    return [
+        ParsedDocument(title, description, chunk, canonical_url, heading_path, index)
+        for index, chunk in enumerate(chunks)
+    ]

docker/docs/context_docs/refresh.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from urllib.parse import urlparse
+
+from .models import PreparedDocument, RefreshOutcome, SourceUpdate
+from .parser import PARSER_FINGERPRINT
+
+
+class RefreshCoordinator:
+    def __init__(self, store, fetcher, embedder, parser, ttl_seconds: float, now):
+        self.store = store
+        self.fetcher = fetcher
+        self.embedder = embedder
+        self.parser = parser
+        self.ttl_seconds = ttl_seconds
+        self.now = now
+        self._inflight: dict[str, asyncio.Task] = {}
+        self._lock = asyncio.Lock()
+
+    async def refresh(self, source: str, force: bool = False) -> RefreshOutcome:
+        state = self.store.get_source(source)
+        timestamp = self.now()
+        compatible = bool(
+            state
+            and state.parser_fingerprint == PARSER_FINGERPRINT
+            and state.embedding_fingerprint == self.embedder.fingerprint
+        )
+        if not force and compatible and state.doc_count and state.stale_at and state.stale_at > timestamp:
+            return RefreshOutcome(source, "fresh", state.doc_count)
+
+        async with self._lock:
+            task = self._inflight.get(source)
+            if task is None:
+                task = asyncio.create_task(self._refresh_once(source, timestamp))
+                self._inflight[source] = task
+        try:
+            return await task
+        finally:
+            async with self._lock:
+                if self._inflight.get(source) is task and task.done():
+                    self._inflight.pop(source, None)
+
+    async def _refresh_once(self, source: str, timestamp: float) -> RefreshOutcome:
+        state = self.store.get_source(source)
+        try:
+            compatible = bool(
+                state
+                and state.parser_fingerprint == PARSER_FINGERPRINT
+                and state.embedding_fingerprint == self.embedder.fingerprint
+            )
+            response = await self.fetcher.fetch(source, state if compatible else None)
+            if response.status == 304:
+                count = state.doc_count if state else 0
+                self.store.mark_checked(source, timestamp, timestamp + self.ttl_seconds)
+                return RefreshOutcome(source, "not_modified", count)
+            if response.status != 200:
+                raise RuntimeError(f"source returned HTTP {response.status}")
+
+            parsed = self.parser(response.body, response.resolved_url)
+            if not parsed.documents:
+                raise RuntimeError("source parsed to zero documents; previous generation preserved")
+            texts = ["\n\n".join(filter(None, [doc.title, doc.description, doc.heading_path, doc.content])) for doc in parsed.documents]
+            vectors = await self.embedder.encode_documents(texts) if texts else []
+            documents: list[PreparedDocument] = []
+            source_host = (urlparse(response.resolved_url).hostname or "").lower()
+            for parsed_document, vector in zip(parsed.documents, vectors, strict=True):
+                content_hash = hashlib.sha256(parsed_document.content.encode()).hexdigest()
+                identity = "\0".join(
+                    [source, parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)]
+                )
+                documents.append(
+                    PreparedDocument(
+                        id=hashlib.sha256(identity.encode()).hexdigest()[:24],
+                        configured_source=source,
+                        resolved_source=response.resolved_url,
+                        source_host=source_host,
+                        canonical_url=parsed_document.canonical_url,
+                        canonical_host=(urlparse(parsed_document.canonical_url).hostname or source_host).lower(),
+                        title=parsed_document.title,
+                        description=parsed_document.description,
+                        heading_path=parsed_document.heading_path,
+                        content=parsed_document.content,
+                        content_hash=content_hash,
+                        embedding=vector,
+                    )
+                )
+            body_hash = hashlib.sha256(response.body.encode()).hexdigest()
+            self.store.replace_source(
+                SourceUpdate(
+                    configured_source=source,
+                    resolved_source=response.resolved_url,
+                    etag=response.etag,
+                    last_modified=response.last_modified,
+                    body_hash=body_hash,
+                    raw_body=response.body,
+                    parser_fingerprint=PARSER_FINGERPRINT,
+                    embedding_fingerprint=self.embedder.fingerprint,
+                    checked_at=timestamp,
+                    indexed_at=timestamp,
+                    stale_at=timestamp + self.ttl_seconds,
+                    documents=documents,
+                )
+            )
+            return RefreshOutcome(source, "updated", len(documents), parsed.format)
+        except Exception as error:
+            self.store.mark_checked(source, timestamp, timestamp, str(error))
+            return RefreshOutcome(source, "error", state.doc_count if state else 0, str(error))

docker/docs/context_docs/search.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from collections import defaultdict
+
+import numpy as np
+
+from .models import SearchResult, StoredDocument
+from .store import IndexStore
+
+
+class HybridSearch:
+    def __init__(self, store: IndexStore, embedder, rrf_k: int = 60):
+        self.store = store
+        self.embedder = embedder
+        self.rrf_k = rrf_k
+
+    async def search(
+        self,
+        query: str,
+        limit: int = 10,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> list[SearchResult]:
+        pool_size = max(limit * 8, 40)
+        lexical = self.store.lexical_search(query, pool_size, sources, hosts)
+        candidates = self.store.semantic_candidates(sources, hosts)
+        semantic: list[StoredDocument] = []
+        if candidates:
+            query_vector = np.asarray(await self.embedder.encode_query(query), dtype=np.float32)
+            query_norm = np.linalg.norm(query_vector)
+            scored: list[tuple[float, StoredDocument]] = []
+            for document in candidates:
+                norm = np.linalg.norm(document.embedding) * query_norm
+                score = float(np.dot(document.embedding, query_vector) / norm) if norm else 0.0
+                scored.append((score, document))
+            semantic = [document for _, document in sorted(scored, key=lambda item: (-item[0], item[1].id))[:pool_size]]
+
+        lexical_ranks = {document.id: rank for rank, document in enumerate(lexical, 1)}
+        semantic_ranks = {document.id: rank for rank, document in enumerate(semantic, 1)}
+        documents = {document.id: document for document in [*lexical, *semantic]}
+        scores = defaultdict(float)
+        for identifier, rank in lexical_ranks.items():
+            scores[identifier] += 1.0 / (self.rrf_k + rank)
+        for identifier, rank in semantic_ranks.items():
+            scores[identifier] += 1.0 / (self.rrf_k + rank)
+
+        ordered = sorted(documents.values(), key=lambda item: (-scores[item.id], item.id))
+        groups: dict[str, list[StoredDocument]] = {}
+        group_order: list[str] = []
+        for document in ordered:
+            key = document.content_hash
+            if key not in groups:
+                groups[key] = []
+                group_order.append(key)
+            groups[key].append(document)
+
+        results: list[SearchResult] = []
+        for key in group_order[:limit]:
+            group = groups[key]
+            primary = group[0]
+            alternates = [
+                {"source": document.configured_source, "url": document.canonical_url}
+                for document in group[1:]
+            ]
+            results.append(
+                SearchResult(
+                    id=primary.id,
+                    configured_source=primary.configured_source,
+                    canonical_url=primary.canonical_url,
+                    title=primary.title,
+                    description=primary.description,
+                    heading_path=primary.heading_path,
+                    content=primary.content,
+                    content_hash=primary.content_hash,
+                    score=min(1.0, scores[primary.id] / (2.0 / (self.rrf_k + 1))),
+                    lexical_rank=lexical_ranks.get(primary.id),
+                    semantic_rank=semantic_ranks.get(primary.id),
+                    duplicate_count=len(group),
+                    alternate_sources=alternates,
+                )
+            )
+        return results

docker/docs/context_docs/server.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import time
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+import uvicorn
+from mcp.server.fastmcp import FastMCP
+from starlette.middleware.cors import CORSMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.routing import Route
+
+from .embedder import SentenceTransformerEmbedder
+from .fetcher import SourceFetcher
+from .parser import parse_llms_text
+from .refresh import RefreshCoordinator
+from .search import HybridSearch
+from .service import DocsService
+from .store import IndexStore
+
+
+def parse_duration(value: str) -> float:
+    match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([smhd]?)\s*", value)
+    if not match:
+        raise ValueError(f"invalid duration: {value}")
+    multiplier = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)]
+    return float(match.group(1)) * multiplier
+
+
+def read_sources(path: str | Path) -> list[str]:
+    sources: list[str] = []
+    for raw_line in Path(path).read_text().splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if not line.endswith(("/llms.txt", "/llms-full.txt")):
+            raise ValueError(f"source URL must end with /llms.txt or /llms-full.txt: {line}")
+        sources.append(line)
+    if not sources:
+        raise ValueError(f"no sources configured in {path}")
+    return list(dict.fromkeys(sources))
+
+
+def build_server():
+    source_file = os.environ.get("DOCS_MCP_SOURCES_FILE", "/etc/context-kit/docs-sources.txt")
+    sources = read_sources(source_file)
+    store = IndexStore(os.environ.get("DOCS_MCP_STORE_PATH", "/data/docs.sqlite3"))
+    store.configure_sources(sources)
+    embedder = SentenceTransformerEmbedder(
+        os.environ.get("DOCS_MCP_EMBED_MODEL", "BAAI/bge-small-en-v1.5")
+    )
+    fetcher = SourceFetcher(
+        timeout_seconds=float(os.environ.get("DOCS_MCP_FETCH_TIMEOUT", "30")),
+        max_bytes=int(os.environ.get("DOCS_MCP_MAX_SOURCE_BYTES", "20000000")),
+    )
+    coordinator = RefreshCoordinator(
+        store=store,
+        fetcher=fetcher,
+        embedder=embedder,
+        parser=parse_llms_text,
+        ttl_seconds=parse_duration(os.environ.get("DOCS_MCP_TTL", "24h")),
+        now=time.time,
+    )
+    service = DocsService(
+        store,
+        HybridSearch(store, embedder),
+        coordinator,
+        max_get_bytes=int(os.environ.get("DOCS_MCP_MAX_GET_BYTES", "75000")),
+    )
+
+    mcp = FastMCP(
+        "Context Kit Docs",
+        instructions="Search and retrieve configured documentation using persisted hybrid retrieval.",
+        host=os.environ.get("DOCS_MCP_HTTP_HOST", "0.0.0.0"),
+        port=int(os.environ.get("DOCS_MCP_HTTP_PORT", "8000")),
+        streamable_http_path="/mcp",
+        stateless_http=True,
+    )
+
+    @mcp.tool()
+    async def docs_query(
+        query: str,
+        limit: int = 10,
+        auto_retrieve: bool = False,
+        auto_retrieve_threshold: float = 0.55,
+        auto_retrieve_limit: int = 5,
+        retrieve_ids: list[str] | None = None,
+        max_bytes: int | None = None,
+        merge: bool = False,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> dict:
+        """Search docs. Content retrieval is explicit by default; optionally filter source URLs or hosts."""
+        return await service.query(
+            query, limit, auto_retrieve, auto_retrieve_threshold, auto_retrieve_limit,
+            retrieve_ids, max_bytes, merge, sources, hosts,
+        )
+
+    @mcp.tool()
+    async def docs_refresh(
+        source: str | None = None,
+        sources: list[str] | None = None,
+        force: bool = False,
+    ) -> dict:
+        """Refresh configured sources transactionally; concurrent requests are coalesced."""
+        if source and sources:
+            raise ValueError("pass source or sources, not both")
+        if source:
+            sources = [source]
+        return await service.refresh(sources, force)
+
+    @mcp.tool()
+    async def docs_sources() -> dict:
+        """Report configured-source freshness, errors, and document counts."""
+        return service.source_status()
+
+    @mcp.tool()
+    async def docs_rebuild(
+        source: str | None = None,
+        sources: list[str] | None = None,
+    ) -> dict:
+        """Force a safe source rebuild without deleting the last good generation first."""
+        if source and sources:
+            raise ValueError("pass source or sources, not both")
+        if source:
+            sources = [source]
+        return await service.refresh(sources, force=True)
+
+    app = mcp.streamable_http_app()
+    mcp_lifespan = app.router.lifespan_context
+
+    @asynccontextmanager
+    async def application_lifespan(application):
+        preindex_task = None
+        async with mcp_lifespan(application):
+            if os.environ.get("DOCS_MCP_PREINDEX", "0") == "1":
+                preindex_task = asyncio.create_task(service.refresh())
+            try:
+                yield
+            finally:
+                if preindex_task:
+                    await preindex_task
+                await fetcher.close()
+                store.close()
+
+    app.router.lifespan_context = application_lifespan
+
+    async def status(_request: Request) -> JSONResponse:
+        state = service.source_status()
+        errors = sum(1 for source in state["sources"] if source["last_error"])
+        return JSONResponse(
+            {
+                "status": "ok" if state["document_count"] or not errors else "degraded",
+                "ready": True,
+                "model_ready": embedder.ready,
+                "source_count": state["source_count"],
+                "document_count": state["document_count"],
+                "source_errors": errors,
+            }
+        )
+
+    app.routes.insert(0, Route("/status", status, methods=["GET"]))
+    origins = os.environ.get("DOCS_MCP_ALLOW_ORIGIN", "").split()
+    if origins:
+        app = CORSMiddleware(app, allow_origins=origins, allow_methods=["POST", "GET", "DELETE"], allow_headers=["*"])
+    return app
+
+
+def main() -> None:
+    uvicorn.run(
+        build_server(),
+        host=os.environ.get("DOCS_MCP_HTTP_HOST", "0.0.0.0"),
+        port=int(os.environ.get("DOCS_MCP_HTTP_PORT", "8000")),
+        log_level=os.environ.get("DOCS_MCP_LOG_LEVEL", "info").lower(),
+    )
+
+
+if __name__ == "__main__":
+    main()

docker/docs/context_docs/service.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+
+
+class DocsService:
+    def __init__(self, store, search, refresh, max_get_bytes: int = 75_000):
+        self.store = store
+        self.search_engine = search
+        self.refresh_coordinator = refresh
+        self.max_get_bytes = max_get_bytes
+
+    async def refresh(self, sources: list[str] | None = None, force: bool = False) -> dict:
+        configured = [state.configured_source for state in self.store.list_sources()]
+        selected = configured if sources is None else sources
+        unknown = sorted(set(selected) - set(configured))
+        if unknown:
+            raise ValueError(f"unconfigured sources: {', '.join(unknown)}")
+        outcomes = await asyncio.gather(
+            *(self.refresh_coordinator.refresh(source, force=force) for source in selected)
+        )
+        return {"sources": [asdict(outcome) for outcome in outcomes]}
+
+    async def query(
+        self,
+        query: str,
+        limit: int = 10,
+        auto_retrieve: bool = False,
+        auto_retrieve_threshold: float = 0.55,
+        auto_retrieve_limit: int = 5,
+        retrieve_ids: list[str] | None = None,
+        max_bytes: int | None = None,
+        merge: bool = False,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> dict:
+        if not query.strip():
+            raise ValueError("query must not be empty")
+        if not 1 <= limit <= 100:
+            raise ValueError("limit must be between 1 and 100")
+        if not 0 <= auto_retrieve_threshold <= 1:
+            raise ValueError("auto_retrieve_threshold must be between 0 and 1")
+        if not 0 <= auto_retrieve_limit <= 25:
+            raise ValueError("auto_retrieve_limit must be between 0 and 25")
+
+        await self._refresh_missing_or_stale(sources)
+        results = await self.search_engine.search(query, limit, sources, hosts)
+        search_results = [
+            {
+                "id": item.id,
+                "source": item.configured_source,
+                "url": item.canonical_url,
+                "host": item.canonical_url.split("/", 3)[2] if "://" in item.canonical_url else "",
+                "title": item.title,
+                "description": item.description,
+                "heading_path": item.heading_path,
+                "score": round(item.score, 6),
+                "snippet": item.content[:500],
+                "duplicate_count": item.duplicate_count,
+                "alternate_sources": item.alternate_sources,
+            }
+            for item in results
+        ]
+
+        selected_ids = list(dict.fromkeys(retrieve_ids or []))
+        if auto_retrieve:
+            selected_ids.extend(
+                item.id
+                for item in results[:auto_retrieve_limit]
+                if item.score >= auto_retrieve_threshold and item.id not in selected_ids
+            )
+        byte_budget = min(max_bytes or self.max_get_bytes, self.max_get_bytes)
+        retrieved: dict[str, dict] = {}
+        used = 0
+        for identifier in selected_ids[:25]:
+            document = self.store.get_document(identifier, sources, hosts)
+            if not document:
+                continue
+            encoded = document.content.encode()
+            remaining = max(0, byte_budget - used)
+            if remaining == 0:
+                break
+            content = encoded[:remaining].decode(errors="ignore")
+            used += len(content.encode())
+            retrieved[identifier] = {
+                "id": identifier,
+                "source": document.configured_source,
+                "url": document.canonical_url,
+                "title": document.title,
+                "content": content,
+                "truncated": len(content.encode()) < len(encoded),
+            }
+
+        merged = ""
+        if merge:
+            merged = "\n\n".join(
+                f"# {item['title']}\n\nSource: {item['url']}\n\n{item['content']}"
+                for item in retrieved.values()
+            )
+        return {
+            "search_results": search_results,
+            "retrieved_content": retrieved,
+            "merged_content": merged,
+            "auto_retrieved_count": len(retrieved) - len([item for item in retrieve_ids or [] if item in retrieved]),
+            "total_results": len(search_results),
+        }
+
+    async def _refresh_missing_or_stale(self, sources: list[str] | None) -> None:
+        states = self.store.list_sources()
+        selected = [state for state in states if sources is None or state.configured_source in sources]
+        await asyncio.gather(
+            *(self.refresh_coordinator.refresh(state.configured_source) for state in selected)
+        )
+
+    def source_status(self) -> dict:
+        states = [asdict(state) for state in self.store.list_sources()]
+        for state in states:
+            state.pop("raw_body", None)
+        return {
+            "sources": states,
+            "source_count": len(states),
+            "document_count": sum(state["doc_count"] for state in states),
+        }

docker/docs/context_docs/store.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import re
+import sqlite3
+import threading
+from pathlib import Path
+
+import numpy as np
+
+from .models import PreparedDocument, SourceState, SourceUpdate, StoredDocument
+
+
+class IndexStore:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.connection = sqlite3.connect(self.path, check_same_thread=False, isolation_level=None)
+        self.connection.row_factory = sqlite3.Row
+        self._lock = threading.RLock()
+        self._initialize()
+
+    def _initialize(self) -> None:
+        with self.connection:
+            self.connection.execute("PRAGMA journal_mode=WAL")
+            self.connection.execute("PRAGMA foreign_keys=ON")
+            self.connection.execute("PRAGMA busy_timeout=5000")
+            self.connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS sources (
+                  configured_source TEXT PRIMARY KEY,
+                  resolved_source TEXT,
+                  active INTEGER NOT NULL DEFAULT 1,
+                  etag TEXT,
+                  last_modified TEXT,
+                  body_hash TEXT,
+                  raw_body TEXT,
+                  parser_fingerprint TEXT,
+                  embedding_fingerprint TEXT,
+                  checked_at REAL,
+                  indexed_at REAL,
+                  stale_at REAL,
+                  last_error TEXT
+                );
+                CREATE TABLE IF NOT EXISTS documents (
+                  id TEXT PRIMARY KEY,
+                  configured_source TEXT NOT NULL REFERENCES sources(configured_source) ON DELETE CASCADE,
+                  resolved_source TEXT NOT NULL,
+                  source_host TEXT NOT NULL,
+                  canonical_url TEXT NOT NULL,
+                  canonical_host TEXT NOT NULL,
+                  title TEXT NOT NULL,
+                  description TEXT NOT NULL,
+                  heading_path TEXT NOT NULL,
+                  content TEXT NOT NULL,
+                  content_hash TEXT NOT NULL,
+                  embedding BLOB NOT NULL,
+                  embedding_dim INTEGER NOT NULL
+                );
+                CREATE INDEX IF NOT EXISTS documents_source ON documents(configured_source);
+                CREATE INDEX IF NOT EXISTS documents_hash ON documents(content_hash);
+                CREATE INDEX IF NOT EXISTS documents_hosts ON documents(source_host, canonical_host);
+                CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+                  doc_id UNINDEXED, title, description, heading_path, content, canonical_url,
+                  tokenize='unicode61 remove_diacritics 2 tokenchars ''_-'''
+                );
+                """
+            )
+
+    def close(self) -> None:
+        self.connection.close()
+
+    def configure_sources(self, sources: list[str]) -> None:
+        with self._lock, self.connection:
+            self.connection.execute("UPDATE sources SET active = 0")
+            self.connection.executemany(
+                "INSERT INTO sources(configured_source, active) VALUES(?, 1) "
+                "ON CONFLICT(configured_source) DO UPDATE SET active = 1",
+                [(source,) for source in dict.fromkeys(sources)],
+            )
+
+    def replace_source(self, update: SourceUpdate) -> None:
+        with self._lock:
+            self.connection.execute("BEGIN IMMEDIATE")
+            try:
+                self._replace_source(update)
+            except Exception:
+                self.connection.rollback()
+                raise
+            else:
+                self.connection.commit()
+
+    def _replace_source(self, update: SourceUpdate) -> None:
+        self.connection.execute(
+                """INSERT INTO sources(
+                     configured_source, resolved_source, active, etag, last_modified,
+                     body_hash, raw_body, parser_fingerprint, embedding_fingerprint,
+                     checked_at, indexed_at, stale_at, last_error
+                   ) VALUES(?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)
+                   ON CONFLICT(configured_source) DO UPDATE SET
+                     resolved_source=excluded.resolved_source, etag=excluded.etag,
+                     last_modified=excluded.last_modified, body_hash=excluded.body_hash,
+                     raw_body=excluded.raw_body, parser_fingerprint=excluded.parser_fingerprint,
+                     embedding_fingerprint=excluded.embedding_fingerprint,
+                     checked_at=excluded.checked_at, indexed_at=excluded.indexed_at,
+                     stale_at=excluded.stale_at, last_error=NULL""",
+                (
+                    update.configured_source,
+                    update.resolved_source,
+                    update.etag,
+                    update.last_modified,
+                    update.body_hash,
+                    update.raw_body,
+                    update.parser_fingerprint,
+                    update.embedding_fingerprint,
+                    update.checked_at,
+                    update.indexed_at,
+                    update.stale_at,
+                ),
+        )
+        old_ids = [row[0] for row in self.connection.execute("SELECT id FROM documents WHERE configured_source=?", (update.configured_source,))]
+        if old_ids:
+            self.connection.executemany("DELETE FROM documents_fts WHERE doc_id=?", [(identifier,) for identifier in old_ids])
+        self.connection.execute("DELETE FROM documents WHERE configured_source=?", (update.configured_source,))
+        for document in update.documents:
+            vector = np.asarray(document.embedding, dtype=np.float32)
+            self.connection.execute(
+                    """INSERT INTO documents VALUES(
+                         ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+                       )""",
+                    (
+                        document.id,
+                        document.configured_source,
+                        document.resolved_source,
+                        document.source_host,
+                        document.canonical_url,
+                        document.canonical_host,
+                        document.title,
+                        document.description,
+                        document.heading_path,
+                        document.content,
+                        document.content_hash,
+                        vector.tobytes(),
+                        vector.size,
+                    ),
+            )
+            self.connection.execute(
+                "INSERT INTO documents_fts VALUES(?, ?, ?, ?, ?, ?)",
+                (
+                    document.id,
+                    document.title,
+                    document.description,
+                    document.heading_path,
+                    document.content,
+                    document.canonical_url,
+                ),
+            )
+
+    def mark_checked(self, source: str, checked_at: float, stale_at: float, error: str | None = None) -> None:
+        with self._lock, self.connection:
+            self.connection.execute(
+                "UPDATE sources SET checked_at=?, stale_at=?, last_error=? WHERE configured_source=?",
+                (checked_at, stale_at, error, source),
+            )
+
+    def list_sources(self, include_inactive: bool = False) -> list[SourceState]:
+        condition = "" if include_inactive else "WHERE s.active=1"
+        rows = self.connection.execute(
+            f"""SELECT s.*, COUNT(d.id) AS doc_count FROM sources s
+                LEFT JOIN documents d ON d.configured_source=s.configured_source
+                {condition} GROUP BY s.configured_source ORDER BY s.configured_source"""
+        ).fetchall()
+        return [self._source(row) for row in rows]
+
+    def get_source(self, source: str) -> SourceState | None:
+        row = self.connection.execute(
+            """SELECT s.*, COUNT(d.id) AS doc_count FROM sources s
+               LEFT JOIN documents d ON d.configured_source=s.configured_source
+               WHERE s.configured_source=? GROUP BY s.configured_source""",
+            (source,),
+        ).fetchone()
+        return self._source(row) if row else None
+
+    def get_document(
+        self,
+        identifier: str,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> StoredDocument | None:
+        where, parameters = self._filters(sources, hosts, alias="d")
+        row = self.connection.execute(
+            f"SELECT d.* FROM documents d JOIN sources s ON s.configured_source=d.configured_source "
+            f"WHERE s.active=1 AND d.id=? {where}",
+            [identifier, *parameters],
+        ).fetchone()
+        return self._document(row) if row else None
+
+    def lexical_search(
+        self,
+        query: str,
+        limit: int,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> list[StoredDocument]:
+        terms = re.findall(r"[\w.-]+", query, flags=re.UNICODE)
+        if not terms:
+            return []
+        expression = " AND ".join(f'"{term.replace(chr(34), chr(34) * 2)}"' for term in terms)
+        where, parameters = self._filters(sources, hosts, alias="d")
+        rows = self.connection.execute(
+            f"""SELECT d.* FROM documents_fts f
+                JOIN documents d ON d.id=f.doc_id
+                JOIN sources s ON s.configured_source=d.configured_source
+                WHERE documents_fts MATCH ? AND s.active=1 {where}
+                ORDER BY bm25(documents_fts, 0, 8, 3, 5, 1, 2) LIMIT ?""",
+            [expression, *parameters, limit],
+        ).fetchall()
+        return [self._document(row) for row in rows]
+
+    def semantic_candidates(
+        self,
+        sources: list[str] | None = None,
+        hosts: list[str] | None = None,
+    ) -> list[StoredDocument]:
+        where, parameters = self._filters(sources, hosts, alias="d")
+        rows = self.connection.execute(
+            f"SELECT d.* FROM documents d JOIN sources s ON s.configured_source=d.configured_source "
+            f"WHERE s.active=1 {where}",
+            parameters,
+        ).fetchall()
+        return [self._document(row) for row in rows]
+
+    @staticmethod
+    def _filters(sources: list[str] | None, hosts: list[str] | None, alias: str) -> tuple[str, list[str]]:
+        clauses: list[str] = []
+        parameters: list[str] = []
+        if sources:
+            clauses.append(f"{alias}.configured_source IN ({','.join('?' for _ in sources)})")
+            parameters.extend(sources)
+        if hosts:
+            clauses.append(
+                f"({alias}.source_host IN ({','.join('?' for _ in hosts)}) OR "
+                f"{alias}.canonical_host IN ({','.join('?' for _ in hosts)}))"
+            )
+            parameters.extend(hosts)
+            parameters.extend(hosts)
+        return (" AND " + " AND ".join(clauses) if clauses else "", parameters)
+
+    @staticmethod
+    def _source(row: sqlite3.Row) -> SourceState:
+        return SourceState(
+            configured_source=row["configured_source"],
+            resolved_source=row["resolved_source"],
+            active=bool(row["active"]),
+            etag=row["etag"],
+            last_modified=row["last_modified"],
+            body_hash=row["body_hash"],
+            raw_body=row["raw_body"],
+            parser_fingerprint=row["parser_fingerprint"],
+            embedding_fingerprint=row["embedding_fingerprint"],
+            checked_at=row["checked_at"],
+            indexed_at=row["indexed_at"],
+            stale_at=row["stale_at"],
+            last_error=row["last_error"],
+            doc_count=row["doc_count"],
+        )
+
+    @staticmethod
+    def _document(row: sqlite3.Row) -> StoredDocument:
+        return StoredDocument(
+            id=row["id"],
+            configured_source=row["configured_source"],
+            resolved_source=row["resolved_source"],
+            source_host=row["source_host"],
+            canonical_url=row["canonical_url"],
+            canonical_host=row["canonical_host"],
+            title=row["title"],
+            description=row["description"],
+            heading_path=row["heading_path"],
+            content=row["content"],
+            content_hash=row["content_hash"],
+            embedding=np.frombuffer(row["embedding"], dtype=np.float32, count=row["embedding_dim"]).copy(),
+        )

docker/docs/entrypoint.sh

@@ -1,9 +1,8 @@
 #!/bin/sh
 # context-kit docs-mcp entrypoint.
 #
-# Bridges llms-txt-mcp (stdio-only) to Streamable HTTP via mcp-proxy so that
-# multiple clients share a single long-lived indexer instead of each spawning
-# their own container (and racing on the same Chroma store).
+# Starts the in-repo Streamable HTTP server. Multiple clients share one
+# transactional SQLite/FTS index and one lazily loaded embedding model.
 #
 # Sources are read from $DOCS_MCP_SOURCES_FILE (one URL per line; `#` comments
 # and blank lines are allowed). Everything else is configured via env vars
@@ -52,19 +51,9 @@ import http.server
 import sys
 
 
-class LocalSourceHandler(http.server.SimpleHTTPRequestHandler):
-    def send_head(self):
-        # llms-txt-mcp 0.2.0 treats 304 responses from local sources as fetch
-        # failures, so serve machine-local docs as plain 200 responses.
-        for header in ("If-Modified-Since", "If-None-Match"):
-            if header in self.headers:
-                del self.headers[header]
-        return super().send_head()
-
-
 port = int(sys.argv[1])
 directory = sys.argv[2]
-handler = functools.partial(LocalSourceHandler, directory=directory)
+handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=directory)
 with http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) as server:
     server.serve_forever()
 PY
@@ -93,32 +82,4 @@ PY
   fi
 fi
 
-# By default llms-txt-mcp 0.2.0 re-embeds every source on launch (the actual
-# default is a background preindex, --no-preindex only disables the foreground
-# variant). On a long-lived container that wastes CPU per restart, so we disable
-# BOTH. Missing/stale sources still refresh on first docs_query/docs_refresh.
-# Set DOCS_MCP_PREINDEX=1 to restore eager startup indexing.
-preindex_flag="--no-preindex --no-background-preindex"
-if [ "${DOCS_MCP_PREINDEX:-0}" = "1" ]; then
-  preindex_flag=""
-fi
-
-allow_origin_args=""
-if [ -n "${DOCS_MCP_ALLOW_ORIGIN:-}" ]; then
-  allow_origin_args="--allow-origin ${DOCS_MCP_ALLOW_ORIGIN}"
-fi
-
-# shellcheck disable=SC2086  # intentional word-splitting on $sources / $preindex_flag / $allow_origin_args
-exec mcp-proxy \
-  --host "${DOCS_MCP_HTTP_HOST:-0.0.0.0}" \
-  --port "${DOCS_MCP_HTTP_PORT:-8000}" \
-  --pass-environment \
-  $allow_origin_args \
-  -- \
-  llms-txt-mcp \
-    --store-path /data \
-    --ttl "${DOCS_MCP_TTL:-24h}" \
-    --max-get-bytes "${DOCS_MCP_MAX_GET_BYTES:-75000}" \
-    --embed-model "${DOCS_MCP_EMBED_MODEL:-BAAI/bge-small-en-v1.5}" \
-    $preindex_flag \
-    $sources
+exec python -m context_docs

docker/docs/tests/fakes.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+import numpy as np
+
+from context_docs.models import FetchResponse
+
+
+class FakeEmbedder:
+    fingerprint = "fake-embedder-v1"
+    ready = True
+
+    async def ensure_ready(self) -> None:
+        return None
+
+    async def encode_documents(self, texts: list[str]) -> np.ndarray:
+        return np.asarray([self._vector(text) for text in texts], dtype=np.float32)
+
+    async def encode_query(self, text: str) -> np.ndarray:
+        return np.asarray(self._vector(text), dtype=np.float32)
+
+    @staticmethod
+    def _vector(text: str) -> list[float]:
+        lower = text.lower()
+        return [
+            float("api" in lower or "identifier" in lower),
+            float("persistence" in lower or "checkpoint" in lower),
+            float("background" in lower or "asynchronous" in lower),
+            0.25 + (int(hashlib.sha256(text.encode()).hexdigest()[:2], 16) / 1024),
+        ]
+
+
+@dataclass
+class FakeFetch:
+    status: int
+    body: str = ""
+    final_url: str | None = None
+    etag: str | None = None
+    last_modified: str | None = None
+
+
+class FakeFetcher:
+    def __init__(self, responses: list[FakeFetch]):
+        self.responses = list(responses)
+        self.calls = 0
+
+    async def fetch(self, source_url: str, state=None) -> FetchResponse:
+        self.calls += 1
+        response = self.responses.pop(0)
+        return FetchResponse(
+            status=response.status,
+            requested_url=source_url,
+            resolved_url=response.final_url or source_url,
+            body=response.body,
+            etag=response.etag,
+            last_modified=response.last_modified,
+        )

docker/docs/tests/test_parser.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import unittest
+
+from context_docs.parser import parse_llms_text
+
+
+class ParserTest(unittest.TestCase):
+    def test_standard_menu_preserves_target_url_and_retrievable_content(self) -> None:
+        parsed = parse_llms_text(
+            """# Rails Docs
+
+> Curated official documentation.
+
+## Active Record
+
+- [Associations](https://guides.rubyonrails.org/association_basics.html): Model relationships
+""",
+            "http://127.0.0.1:8769/rails/llms.txt",
+        )
+
+        self.assertEqual("standard-menu", parsed.format)
+        self.assertEqual(1, len(parsed.documents))
+        document = parsed.documents[0]
+        self.assertEqual("https://guides.rubyonrails.org/association_basics.html", document.canonical_url)
+        self.assertIn("Model relationships", document.content)
+        self.assertIn("https://guides.rubyonrails.org/association_basics.html", document.content)
+
+    def test_full_bundle_is_not_misclassified_by_interior_yaml_or_rule(self) -> None:
+        parsed = parse_llms_text(
+            """# Build a client
+
+Some content.
+
+---
+title: This is an embedded example
+description: It is not file frontmatter
+---
+
+# Elicitation
+
+URL mode details.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("markdown-full", parsed.format)
+        self.assertEqual(["Build a client", "Elicitation"], [doc.title for doc in parsed.documents])
+
+    def test_full_bundle_with_bullet_links_keeps_prose_sections(self) -> None:
+        parsed = parse_llms_text(
+            """# Persistence
+
+This substantial section explains durable checkpoint behavior.
+
+- [Related guide](https://example.test/guide): Read more
+
+# Streaming
+
+Streaming emits incremental updates.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("markdown-full", parsed.format)
+        self.assertEqual(["Persistence", "Streaming"], [doc.title for doc in parsed.documents])
+        self.assertIn("durable checkpoint", parsed.documents[0].content)
+
+    def test_repeated_frontmatter_accepts_optional_description(self) -> None:
+        parsed = parse_llms_text(
+            """---
+title: First
+---
+First body.
+---
+title: Second
+description: Second description
+---
+Second body.
+""",
+            "https://example.test/llms-full.txt",
+        )
+
+        self.assertEqual("yaml-full", parsed.format)
+        self.assertEqual(["First", "Second"], [doc.title for doc in parsed.documents])
+        self.assertEqual("Second description", parsed.documents[1].description)
+
+    def test_long_sections_are_split_without_losing_tail_identifiers(self) -> None:
+        body = "Paragraph.\n\n" * 100 + "IMMICH_IGNORE_MOUNT_CHECK_ERRORS disables mount checks."
+        parsed = parse_llms_text(
+            f"# Environment Variables\n\n{body}",
+            "https://example.test/llms-full.txt",
+            max_chunk_chars=500,
+        )
+
+        self.assertGreater(len(parsed.documents), 1)
+        self.assertTrue(any("IMMICH_IGNORE_MOUNT_CHECK_ERRORS" in doc.content for doc in parsed.documents))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_refresh.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import asyncio
+import tempfile
+import unittest
+from pathlib import Path
+
+from context_docs.parser import parse_llms_text
+from context_docs.refresh import RefreshCoordinator
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder, FakeFetch, FakeFetcher
+
+
+class RefreshTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.source = "https://example.test/llms.txt"
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.store.configure_sources([self.source])
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_concurrent_refresh_uses_one_fetch_and_one_publication(self) -> None:
+        fetcher = FakeFetcher([FakeFetch(200, "# API Identifier\n\nExact identifier content.")])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+
+        first, second = await asyncio.gather(
+            coordinator.refresh(self.source, force=True),
+            coordinator.refresh(self.source, force=True),
+        )
+
+        self.assertEqual(1, fetcher.calls)
+        self.assertEqual("updated", first.status)
+        self.assertEqual("updated", second.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+
+    async def test_304_updates_check_time_without_replacing_documents(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content.", etag='"v1"'),
+                FakeFetch(304, etag='"v1"'),
+            ]
+        )
+        clock = iter([100.0, 200.0])
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: next(clock),
+        )
+        await coordinator.refresh(self.source, force=True)
+        original = self.store.list_sources()[0]
+        await coordinator.refresh(self.source, force=True)
+        checked = self.store.list_sources()[0]
+
+        self.assertEqual(original.indexed_at, checked.indexed_at)
+        self.assertEqual(200.0, checked.checked_at)
+        self.assertEqual(1, checked.doc_count)
+
+    async def test_refresh_error_preserves_searchable_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(500),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertEqual(1, self.store.list_sources()[0].doc_count)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+    async def test_empty_success_response_preserves_previous_content(self) -> None:
+        fetcher = FakeFetcher(
+            [
+                FakeFetch(200, "# API Identifier\n\nOriginal content."),
+                FakeFetch(200, ""),
+            ]
+        )
+        coordinator = RefreshCoordinator(
+            store=self.store,
+            fetcher=fetcher,
+            embedder=FakeEmbedder(),
+            parser=parse_llms_text,
+            ttl_seconds=3600,
+            now=lambda: 100.0,
+        )
+        await coordinator.refresh(self.source, force=True)
+        failed = await coordinator.refresh(self.source, force=True)
+
+        self.assertEqual("error", failed.status)
+        self.assertIn("zero documents", failed.detail)
+        self.assertTrue(self.store.lexical_search("Original", limit=5))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_search.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, SourceUpdate
+from context_docs.search import HybridSearch
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder
+
+
+def prepared(identifier: str, source: str, canonical: str, title: str, content: str, vector) -> PreparedDocument:
+    return PreparedDocument(
+        id=identifier,
+        configured_source=source,
+        resolved_source=source,
+        source_host="source.test",
+        canonical_url=canonical,
+        canonical_host=canonical.split("/")[2],
+        title=title,
+        description="",
+        heading_path=title,
+        content=content,
+        content_hash=__import__("hashlib").sha256(content.encode()).hexdigest(),
+        embedding=np.asarray(vector, dtype=np.float32),
+    )
+
+
+def source_update(source: str, documents: list[PreparedDocument]) -> SourceUpdate:
+    return SourceUpdate(
+        configured_source=source,
+        resolved_source=source,
+        etag=None,
+        last_modified=None,
+        body_hash="body",
+        raw_body="# source",
+        parser_fingerprint="parser-v1",
+        embedding_fingerprint="fake-embedder-v1",
+        checked_at=1.0,
+        indexed_at=1.0,
+        stale_at=9999.0,
+        documents=documents,
+    )
+
+
+class HybridSearchTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.a = "https://a.test/llms.txt"
+        self.b = "https://b.test/llms.txt"
+        self.store.configure_sources([self.a, self.b])
+        duplicate = "Shared exact content."
+        self.store.replace_source(
+            source_update(
+                self.a,
+                [
+                    prepared("exact", self.a, "https://rails.test/exact", "Environment", "IMMICH_IGNORE_MOUNT_CHECK_ERRORS identifier", [1, 0, 0, 0]),
+                    prepared("duplicate-a", self.a, "https://docs.test/shared", "Shared", duplicate, [0, 1, 0, 0]),
+                ],
+            )
+        )
+        self.store.replace_source(
+            source_update(
+                self.b,
+                [
+                    prepared("persistence", self.b, "https://langgraph.test/persistence", "Persistence", "Durable checkpoint state", [0, 1, 0, 0]),
+                    prepared("duplicate-b", self.b, "https://docs.test/shared-copy", "Shared copy", duplicate, [0, 1, 0, 0]),
+                ],
+            )
+        )
+        self.search = HybridSearch(self.store, FakeEmbedder())
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_exact_identifier_is_ranked_first(self) -> None:
+        result = await self.search.search("IMMICH_IGNORE_MOUNT_CHECK_ERRORS", limit=5)
+        self.assertEqual("exact", result[0].id)
+        self.assertEqual(1, result[0].lexical_rank)
+
+    async def test_source_and_host_filters_apply_before_ranking(self) -> None:
+        by_source = await self.search.search("persistence", limit=5, sources=[self.b])
+        by_host = await self.search.search("identifier", limit=5, hosts=["rails.test"])
+
+        self.assertTrue(by_source)
+        self.assertTrue(all(item.configured_source == self.b for item in by_source))
+        self.assertEqual(["exact"], [item.id for item in by_host])
+
+    async def test_exact_duplicate_content_is_collapsed_with_alternates(self) -> None:
+        result = await self.search.search("Shared exact content", limit=10)
+        shared = [item for item in result if item.content_hash == __import__("hashlib").sha256("Shared exact content.".encode()).hexdigest()]
+
+        self.assertEqual(1, len(shared))
+        self.assertEqual(2, shared[0].duplicate_count)
+        self.assertEqual(1, len(shared[0].alternate_sources))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_server.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from starlette.testclient import TestClient
+
+from context_docs.server import build_server, parse_duration, read_sources
+
+
+class ServerTest(unittest.TestCase):
+    def test_duration_parser_rejects_ambiguous_values(self) -> None:
+        self.assertEqual(86_400, parse_duration("24h"))
+        with self.assertRaises(ValueError):
+            parse_duration("tomorrow")
+
+    def test_source_file_requires_supported_urls(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "sources.txt"
+            path.write_text("https://example.test/index.html\n")
+            with self.assertRaisesRegex(ValueError, "must end"):
+                read_sources(path)
+
+    def test_status_is_available_without_loading_embedding_model(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            source_file = Path(directory) / "sources.txt"
+            source_file.write_text("https://example.test/llms.txt\n")
+            environment = {
+                "DOCS_MCP_SOURCES_FILE": str(source_file),
+                "DOCS_MCP_STORE_PATH": str(Path(directory) / "docs.sqlite3"),
+                "DOCS_MCP_PREINDEX": "0",
+            }
+            with patch.dict(os.environ, environment, clear=False):
+                with TestClient(build_server()) as client:
+                    response = client.get("/status")
+
+            self.assertEqual(200, response.status_code)
+            self.assertTrue(response.json()["ready"])
+            self.assertFalse(response.json()["model_ready"])
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_service.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, RefreshOutcome, SourceUpdate
+from context_docs.search import HybridSearch
+from context_docs.service import DocsService
+from context_docs.store import IndexStore
+
+from .fakes import FakeEmbedder
+
+
+class NoopRefresh:
+    async def refresh(self, source: str, force: bool = False) -> RefreshOutcome:
+        return RefreshOutcome(source, "fresh", 1)
+
+
+class ServiceTest(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3")
+        self.source = "https://example.test/llms.txt"
+        self.store.configure_sources([self.source])
+        content = "IMMICH_IGNORE_MOUNT_CHECK_ERRORS " + "x" * 200
+        item = PreparedDocument(
+            id="exact",
+            configured_source=self.source,
+            resolved_source=self.source,
+            source_host="example.test",
+            canonical_url="https://docs.example.test/environment",
+            canonical_host="docs.example.test",
+            title="Environment",
+            description="",
+            heading_path="Environment",
+            content=content,
+            content_hash="content-hash",
+            embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
+        )
+        self.store.replace_source(
+            SourceUpdate(
+                configured_source=self.source,
+                resolved_source=self.source,
+                etag=None,
+                last_modified=None,
+                body_hash="body",
+                raw_body="# body",
+                parser_fingerprint="context-docs-parser-v1",
+                embedding_fingerprint="fake-embedder-v1",
+                checked_at=1.0,
+                indexed_at=1.0,
+                stale_at=9_999_999_999.0,
+                documents=[item],
+            )
+        )
+        embedder = FakeEmbedder()
+        self.service = DocsService(
+            self.store,
+            HybridSearch(self.store, embedder),
+            NoopRefresh(),
+            max_get_bytes=100,
+        )
+
+    async def asyncTearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    async def test_query_does_not_retrieve_content_by_default(self) -> None:
+        response = await self.service.query("IMMICH_IGNORE_MOUNT_CHECK_ERRORS")
+
+        self.assertEqual({}, response["retrieved_content"])
+        self.assertEqual("exact", response["search_results"][0]["id"])
+
+    async def test_explicit_retrieval_respects_global_byte_cap(self) -> None:
+        response = await self.service.query(
+            "IMMICH_IGNORE_MOUNT_CHECK_ERRORS",
+            retrieve_ids=["exact"],
+            max_bytes=10_000,
+        )
+        retrieved = response["retrieved_content"]["exact"]
+
+        self.assertLessEqual(len(retrieved["content"].encode()), 100)
+        self.assertTrue(retrieved["truncated"])
+
+    async def test_high_default_threshold_only_retrieves_strong_hybrid_match(self) -> None:
+        response = await self.service.query(
+            "IMMICH_IGNORE_MOUNT_CHECK_ERRORS",
+            auto_retrieve=True,
+        )
+
+        self.assertIn("exact", response["retrieved_content"])
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/docs/tests/test_store.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from context_docs.models import PreparedDocument, SourceUpdate
+from context_docs.store import IndexStore
+
+
+def document(identifier: str, source: str, title: str, content: str) -> PreparedDocument:
+    return PreparedDocument(
+        id=identifier,
+        configured_source=source,
+        resolved_source=source,
+        source_host="example.test",
+        canonical_url=f"https://docs.example.test/{identifier}",
+        canonical_host="docs.example.test",
+        title=title,
+        description="",
+        heading_path=title,
+        content=content,
+        content_hash=identifier,
+        embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
+    )
+
+
+def update(source: str, documents: list[PreparedDocument], checked_at: float = 100.0) -> SourceUpdate:
+    return SourceUpdate(
+        configured_source=source,
+        resolved_source=source,
+        etag=None,
+        last_modified=None,
+        body_hash="body-hash",
+        raw_body="# Fixture",
+        parser_fingerprint="parser-v1",
+        embedding_fingerprint="fake-embedder-v1",
+        checked_at=checked_at,
+        indexed_at=checked_at,
+        stale_at=checked_at + 3600,
+        documents=documents,
+    )
+
+
+class StoreTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.path = Path(self.tmp.name) / "docs.sqlite3"
+        self.source = "https://example.test/llms.txt"
+        self.store = IndexStore(self.path)
+        self.store.configure_sources([self.source])
+
+    def tearDown(self) -> None:
+        self.store.close()
+        self.tmp.cleanup()
+
+    def test_replacement_removes_old_only_documents(self) -> None:
+        self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")]))
+        self.store.replace_source(update(self.source, [document("new", self.source, "New", "new content")]))
+
+        self.assertIsNone(self.store.get_document("old"))
+        self.assertEqual("new content", self.store.get_document("new").content)
+
+    def test_failed_replacement_rolls_back_to_previous_generation(self) -> None:
+        self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")]))
+        self.store.connection.execute(
+            "CREATE TRIGGER reject_failure BEFORE INSERT ON documents "
+            "WHEN NEW.title = 'FAIL' BEGIN SELECT RAISE(ABORT, 'injected failure'); END"
+        )
+
+        with self.assertRaisesRegex(Exception, "injected failure"):
+            self.store.replace_source(update(self.source, [document("bad", self.source, "FAIL", "bad")]))
+
+        self.assertEqual("old content", self.store.get_document("old").content)
+        self.assertIsNone(self.store.get_document("bad"))
+
+    def test_restart_loads_persisted_state_without_network(self) -> None:
+        self.store.replace_source(update(self.source, [document("persisted", self.source, "Persisted", "saved")]))
+        self.store.close()
+
+        self.store = IndexStore(self.path)
+        self.store.configure_sources([self.source])
+
+        states = self.store.list_sources()
+        self.assertEqual(1, states[0].doc_count)
+        self.assertEqual("saved", self.store.get_document("persisted").content)
+
+    def test_removed_source_is_not_searchable_or_retrievable(self) -> None:
+        self.store.replace_source(update(self.source, [document("retired", self.source, "Retired", "identifier")]))
+        self.store.configure_sources([])
+
+        self.assertIsNone(self.store.get_document("retired"))
+        self.assertEqual([], self.store.lexical_search("identifier", limit=5))
+
+
+if __name__ == "__main__":
+    unittest.main()

docker/web-search/Dockerfile

@@ -4,10 +4,6 @@ ARG MCP_WEB_SEARCH_VERSION=1.3.0
 ARG MCP_WEB_SEARCH_MAX_BYTES=52428800
 ARG MCP_PROXY_VERSION=0.12.0
 
-COPY patch-mcp-web-search.mjs /tmp/patch-mcp-web-search.mjs
-COPY overrides/bing.js /tmp/context-kit-bing-provider.js
-COPY --chmod=0444 mcp-probe.mjs http-entrypoint.mjs /usr/local/lib/context-kit/
-
 # Chromium intentionally tracks Debian security updates inside the pinned base
 # image family; Bing's browser path is more likely to break with stale Chromium
 # than with patched OS packages.
@@ -23,13 +19,27 @@ RUN python3 -m venv /opt/mcp-proxy \
   && /opt/mcp-proxy/bin/pip install --no-cache-dir "mcp-proxy==${MCP_PROXY_VERSION}" \
   && /opt/mcp-proxy/bin/mcp-proxy --version
 
+COPY patch-mcp-web-search.mjs /tmp/patch-mcp-web-search.mjs
+COPY overrides/bing.js /tmp/context-kit-bing-provider.js
+COPY overrides/brave.js overrides/duckduckgo.js overrides/searxng.js overrides/registry.js overrides/diagnostics.mjs /tmp/context-kit-providers/
+COPY overrides/browser-fetch.js overrides/bounds.mjs /tmp/context-kit-fetch/
+COPY --chmod=0444 mcp-probe.mjs http-entrypoint.mjs /usr/local/lib/context-kit/
+
 RUN npm install -g "@zhafron/mcp-web-search@${MCP_WEB_SEARCH_VERSION}" \
   && cp /tmp/context-kit-bing-provider.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/bing.js \
+  && cp /tmp/context-kit-providers/brave.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/brave.js \
+  && cp /tmp/context-kit-providers/duckduckgo.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/duckduckgo.js \
+  && cp /tmp/context-kit-providers/searxng.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/searxng.js \
+  && cp /tmp/context-kit-providers/registry.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/registry.js \
+  && cp /tmp/context-kit-providers/diagnostics.mjs /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/diagnostics.js \
+  && cp /tmp/context-kit-fetch/browser-fetch.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/browser.js \
+  && cp /tmp/context-kit-fetch/bounds.mjs /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/bounds.js \
   && node /tmp/patch-mcp-web-search.mjs \
-  && rm /tmp/patch-mcp-web-search.mjs /tmp/context-kit-bing-provider.js \
+  && rm -rf /tmp/patch-mcp-web-search.mjs /tmp/context-kit-bing-provider.js /tmp/context-kit-providers /tmp/context-kit-fetch \
   && npm cache clean --force
 
-RUN chmod 0555 /usr/local/lib/context-kit
+RUN chmod -R a+rX /usr/local/lib/context-kit \
+      /usr/local/lib/node_modules/@zhafron/mcp-web-search
 
 ENV CHROME_PATH=/usr/bin/chromium \
   DEFAULT_SEARCH_PROVIDER=searxng \
@@ -37,6 +47,8 @@ ENV CHROME_PATH=/usr/bin/chromium \
   HTTP_TIMEOUT=15000 \
   MAX_BYTES=${MCP_WEB_SEARCH_MAX_BYTES} \
   MAX_RESULTS=10 \
+  MAX_PROVIDER_ATTEMPTS=4 \
+  SEARCH_PROVIDER_TIMEOUT_MS=15000 \
   PATH=/opt/mcp-proxy/bin:$PATH \
   SEARXNG_URL=http://searxng:8080 \
   XDG_CACHE_HOME=/tmp/.cache

docker/web-search/mcp-probe.mjs

@@ -3,7 +3,7 @@ import { fileURLToPath } from "node:url";
 const protocolVersion = "2024-11-05";
 const expectedTools = ["fetch_url", "search_web"];
 
-async function rpc(url, id, method, params = {}, timeoutMs = 5000) {
+export async function rpc(url, id, method, params = {}, timeoutMs = 5000) {
   const response = await fetch(url, {
     method: "POST",
     headers: {
@@ -29,7 +29,7 @@ async function rpc(url, id, method, params = {}, timeoutMs = 5000) {
   return payload.result;
 }
 
-export async function probeMcp(url, { timeoutMs = 5000 } = {}) {
+export async function probeMcp(url, { timeoutMs = 5000, expectedTools: requiredTools = expectedTools } = {}) {
   const initialized = await rpc(url, 1, "initialize", {
     protocolVersion,
     capabilities: {},
@@ -39,7 +39,7 @@ export async function probeMcp(url, { timeoutMs = 5000 } = {}) {
 
   const listed = await rpc(url, 2, "tools/list", {}, timeoutMs);
   const names = new Set((listed?.tools || []).map(tool => tool.name));
-  for (const name of expectedTools) {
+  for (const name of requiredTools) {
     if (!names.has(name)) throw new Error(`tools/list omitted ${name}`);
   }
   return Array.from(names).sort();

docker/web-search/overrides/bing.js

@@ -43,7 +43,7 @@ export class BingProvider {
     }
   }
 
-  async search(q, limit, lang) {
+  async search(q, limit, lang, signal) {
     const cacheKey = createCacheKey("bing", q, limit, lang);
     const cached = searchCache.get(cacheKey);
     if (cached)
@@ -51,7 +51,10 @@ export class BingProvider {
     const market = getMarketFromLang(lang);
     const results = await browserPool.withBrowser(async (browser) => {
       const page = await browser.newPage();
+      const abort = () => void page.close().catch(() => undefined);
+      signal?.addEventListener("abort", abort, { once: true });
       try {
+        signal?.throwIfAborted();
         await page.setViewport({ width: 1365, height: 768 });
         await page.setUserAgent(DEFAULT_BROWSER_SEARCH_USER_AGENT);
         await page.setExtraHTTPHeaders(getAcceptLanguageHeader(lang));
@@ -95,7 +98,8 @@ export class BingProvider {
         });
       }
       finally {
-        await page.close();
+        signal?.removeEventListener("abort", abort);
+        if (!page.isClosed()) await page.close();
       }
     });
     searchCache.set(cacheKey, results);

docker/web-search/overrides/bounds.mjs

@@ -0,0 +1,23 @@
+const MAX_LINKS = 500;
+const MAX_IMAGES = 200;
+const MAX_VIDEO = 50;
+const MAX_AUDIO = 50;
+const MAX_ATTACHMENTS = 10;
+
+export function boundFetchCollections(result) {
+  const warnings = [...(result.warnings || [])];
+  const trim = (value, maximum, label) => {
+    if (!Array.isArray(value)) return value;
+    if (value.length > maximum) warnings.push(`${label} truncated from ${value.length} to ${maximum}`);
+    return value.slice(0, maximum);
+  };
+  if (result.links) result.links = trim(result.links, MAX_LINKS, "links");
+  if (result.media) {
+    result.media.images = trim(result.media.images, MAX_IMAGES, "images");
+    result.media.videos = trim(result.media.videos, MAX_VIDEO, "videos");
+    result.media.audio = trim(result.media.audio, MAX_AUDIO, "audio");
+  }
+  if (result.attachments) result.attachments = trim(result.attachments, MAX_ATTACHMENTS, "attachments");
+  result.warnings = warnings;
+  return result;
+}

docker/web-search/overrides/brave.js

@@ -0,0 +1,42 @@
+import { HTTP_TIMEOUT } from "../constants.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class BraveProvider {
+  name = "brave";
+  configured = Boolean(process.env.BRAVE_SEARCH_API_KEY);
+
+  async search(q, limit, lang, signal) {
+    if (!this.configured) return [];
+    const cacheKey = createCacheKey("brave", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const url = new URL("https://api.search.brave.com/res/v1/web/search");
+    url.searchParams.set("q", q);
+    url.searchParams.set("count", String(Math.min(limit, 20)));
+    url.searchParams.set("search_lang", lang.split(/[-_]/)[0] || "en");
+    const response = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        "X-Subscription-Token": process.env.BRAVE_SEARCH_API_KEY
+      },
+      signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(HTTP_TIMEOUT)]) : AbortSignal.timeout(HTTP_TIMEOUT)
+    });
+    if (!response.ok) throw new Error(`Brave HTTP ${response.status}`);
+    const data = await response.json();
+    const items = (data.web?.results || []).slice(0, limit).flatMap(result => {
+      if (!result.title || !result.url) return [];
+      return [{
+        title: result.title,
+        url: result.url,
+        snippet: result.description || undefined,
+        source: "brave"
+      }];
+    });
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    return this.configured;
+  }
+}

docker/web-search/overrides/browser-fetch.js

@@ -0,0 +1,95 @@
+import { HTTP_TIMEOUT, MAX_BYTES } from "../constants.js";
+import { browserPool } from "../utils/browser-pool.js";
+import { assertSafeUrl } from "./security.js";
+import { fetchViaVettedAddress } from "./http.js";
+
+const MAX_BROWSER_REQUESTS = 100;
+const MAX_BROWSER_TOTAL_BYTES = Math.min(MAX_BYTES, 20 * 1024 * 1024);
+
+function responseHeaders(headers) {
+  const record = {};
+  headers.forEach((value, key) => { record[key] = value; });
+  return record;
+}
+
+export async function fetchBrowserResource(url, timeoutMs = HTTP_TIMEOUT) {
+  await assertSafeUrl(url);
+  return browserPool.withBrowser(async browser => {
+    const page = await browser.newPage();
+    const devtools = await page.target().createCDPSession();
+    await devtools.send("Network.enable");
+    await devtools.send("Network.setBlockedURLs", {
+      urls: ["ws://*", "wss://*", "file://*", "ftp://*"]
+    });
+    await page.evaluateOnNewDocument(() => {
+      const blockedTransport = name => class {
+        constructor() {
+          throw new DOMException(`${name} is disabled by the safe browser fetcher`, "SecurityError");
+        }
+      };
+      for (const name of ["WebSocket", "WebTransport", "RTCPeerConnection", "webkitRTCPeerConnection"]) {
+        if (name in globalThis) {
+          Object.defineProperty(globalThis, name, {
+            configurable: false,
+            writable: false,
+            value: blockedTransport(name)
+          });
+        }
+      }
+    });
+    await page.setBypassServiceWorker(true);
+    let requests = 0;
+    let totalBytes = 0;
+    let blockedError;
+    await page.setRequestInterception(true);
+    page.on("request", request => {
+      void (async () => {
+        try {
+          const requestUrl = new URL(request.url());
+          if (!["http:", "https:"].includes(requestUrl.protocol)) throw new Error("unsupported browser request scheme");
+          if (request.method() !== "GET") throw new Error("browser fetch blocks non-GET requests");
+          requests += 1;
+          if (requests > MAX_BROWSER_REQUESTS) throw new Error("browser request limit exceeded");
+          await assertSafeUrl(requestUrl);
+          const upstream = await fetchViaVettedAddress(requestUrl, timeoutMs);
+          const body = Buffer.from(await upstream.arrayBuffer());
+          totalBytes += body.byteLength;
+          if (totalBytes > MAX_BROWSER_TOTAL_BYTES) throw new Error("browser byte limit exceeded");
+          await request.respond({
+            status: upstream.status,
+            headers: responseHeaders(upstream.headers),
+            body
+          });
+        } catch (error) {
+          blockedError ||= error;
+          await request.abort("blockedbyclient").catch(() => undefined);
+        }
+      })();
+    });
+    try {
+      const navigation = await page.goto(url.toString(), {
+        waitUntil: "networkidle2",
+        timeout: timeoutMs
+      });
+      if (blockedError && !navigation) throw blockedError;
+      const finalUrl = new URL(page.url());
+      await assertSafeUrl(finalUrl);
+      const html = await page.content();
+      const buffer = Buffer.from(html);
+      if (buffer.byteLength > MAX_BYTES) throw new Error("rendered content too large");
+      const headers = new Headers({ "content-type": "text/html; charset=utf-8" });
+      const status = navigation?.status() || 200;
+      const response = new Response(new Uint8Array(buffer), { status, headers });
+      Object.defineProperty(response, "url", { value: finalUrl.toString() });
+      return {
+        response,
+        finalUrl: finalUrl.toString(),
+        contentType: headers.get("content-type"),
+        buffer,
+        byteLength: buffer.byteLength
+      };
+    } finally {
+      await page.close();
+    }
+  });
+}

docker/web-search/overrides/diagnostics.mjs

@@ -0,0 +1,63 @@
+const DEFAULT_TIMEOUT_MS = 15_000;
+const MAX_ERROR_LENGTH = 240;
+
+export function classifyProviderError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const lower = message.toLowerCase();
+  let category = "provider_error";
+  if (lower.includes("timed out") || lower.includes("timeout")) category = "timeout";
+  else if (lower.includes("429") || lower.includes("rate limit")) category = "rate_limited";
+  else if (lower.includes("captcha") || lower.includes("challenge")) category = "blocked";
+  else if (lower.includes("403") || lower.includes("401") || lower.includes("denied")) category = "forbidden";
+  else if (lower.includes("network") || lower.includes("fetch") || lower.includes("socket")) category = "network";
+  return { category, message: message.replace(/\s+/g, " ").slice(0, MAX_ERROR_LENGTH) };
+}
+
+export async function attemptProvider(provider, query, limit, lang, options = {}) {
+  const timeoutMs = Math.max(10, Math.min(options.timeoutMs || DEFAULT_TIMEOUT_MS, 60_000));
+  const now = options.now || (() => performance.now());
+  const started = now();
+  if (provider.configured === false) {
+    return {
+      items: [],
+      diagnostic: { provider: provider.name, status: "unavailable", duration_ms: 0, result_count: 0 }
+    };
+  }
+  let timer;
+  const controller = new AbortController();
+  try {
+    const items = await Promise.race([
+      provider.search(query, limit, lang, controller.signal),
+      new Promise((_, reject) => {
+        timer = setTimeout(() => {
+          controller.abort(new Error(`provider timed out after ${timeoutMs}ms`));
+          reject(new Error(`provider timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+      })
+    ]);
+    const bounded = Array.isArray(items) ? items.slice(0, limit) : [];
+    return {
+      items: bounded,
+      diagnostic: {
+        provider: provider.name,
+        status: bounded.length ? "success" : "empty",
+        duration_ms: Math.max(0, Math.round(now() - started)),
+        result_count: bounded.length
+      }
+    };
+  } catch (error) {
+    return {
+      items: [],
+      diagnostic: {
+        provider: provider.name,
+        status: "error",
+        duration_ms: Math.max(0, Math.round(now() - started)),
+        result_count: 0,
+        error: classifyProviderError(error)
+      }
+    };
+  } finally {
+    controller.abort();
+    clearTimeout(timer);
+  }
+}

docker/web-search/overrides/duckduckgo.js

@@ -0,0 +1,56 @@
+import { JSDOM } from "jsdom";
+import { HTTP_TIMEOUT } from "../constants.js";
+import { fetchWithTimeout } from "../utils/http.js";
+import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class DuckDuckGoProvider {
+  name = "duckduckgo";
+
+  decodeDuckDuckGoRedirect(href) {
+    try {
+      const url = new URL(href, "https://duckduckgo.com/");
+      if (url.hostname === "duckduckgo.com" && url.pathname.startsWith("/l/")) {
+        const target = url.searchParams.get("uddg");
+        if (target) return decodeURIComponent(target);
+      }
+      return url.toString();
+    } catch {
+      return href;
+    }
+  }
+
+  async search(q, limit, lang, signal) {
+    const cacheKey = createCacheKey("ddg", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const url = new URL("https://html.duckduckgo.com/html/");
+    url.searchParams.set("q", q);
+    const headers = { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) };
+    const response = await fetchWithTimeout(url, { headers, signal }, HTTP_TIMEOUT);
+    if (!response.ok) throw new Error(`DuckDuckGo HTML ${response.status}`);
+    const dom = new JSDOM(await response.text(), { url: `https://duckduckgo.com/?q=${encodeURIComponent(q)}` });
+    const anchors = Array.from(dom.window.document.querySelectorAll("a.result__a"));
+    const snippets = Array.from(dom.window.document.querySelectorAll(".result__snippet"));
+    const items = [];
+    for (let index = 0; index < anchors.length && items.length < limit; index += 1) {
+      const title = (anchors[index].textContent || "").trim();
+      const href = this.decodeDuckDuckGoRedirect(anchors[index].getAttribute("href") || "");
+      if (!title || !href) continue;
+      try {
+        items.push({
+          title,
+          url: new URL(href).toString(),
+          snippet: (snippets[index]?.textContent || "").trim() || undefined,
+          source: "duckduckgo"
+        });
+      } catch {}
+    }
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    return true;
+  }
+}

docker/web-search/overrides/registry.js

@@ -0,0 +1,65 @@
+import { DuckDuckGoProvider } from "./duckduckgo.js";
+import { BingProvider } from "./bing.js";
+import { SearXNGProvider } from "./searxng.js";
+import { BraveProvider } from "./brave.js";
+import { DEFAULT_SEARCH_PROVIDER } from "../constants.js";
+import { attemptProvider } from "./diagnostics.js";
+
+const PROVIDERS = ["searxng", "brave", "duckduckgo", "bing"];
+const PROVIDER_TIMEOUT_MS = Number(process.env.SEARCH_PROVIDER_TIMEOUT_MS || "15000");
+const MAX_PROVIDER_ATTEMPTS = Math.max(1, Math.min(Number(process.env.MAX_PROVIDER_ATTEMPTS || "4"), 4));
+
+export class ProviderRegistry {
+  constructor(providers) {
+    this.providers = providers || new Map([
+      ["duckduckgo", new DuckDuckGoProvider()],
+      ["bing", new BingProvider()],
+      ["searxng", new SearXNGProvider()],
+      ["brave", new BraveProvider()]
+    ]);
+  }
+
+  get(name) {
+    return this.providers.get(name);
+  }
+
+  async searchWithFallback(q, limit, lang, preferredProvider) {
+    const defaultProvider = preferredProvider || DEFAULT_SEARCH_PROVIDER;
+    const order = [defaultProvider, ...PROVIDERS.filter(name => name !== defaultProvider)].slice(0, MAX_PROVIDER_ATTEMPTS);
+    const attempts = [];
+    const started = performance.now();
+    for (const providerName of order) {
+      const provider = this.providers.get(providerName);
+      if (!provider) continue;
+      const attempt = await attemptProvider(provider, q, limit, lang, { timeoutMs: PROVIDER_TIMEOUT_MS });
+      attempts.push(attempt.diagnostic);
+      if (attempt.items.length) {
+        return {
+          items: attempt.items,
+          providerUsed: providerName,
+          fallbackUsed: providerName !== defaultProvider,
+          triedProviders: attempts.map(item => item.provider),
+          diagnostics: {
+            attempts,
+            elapsed_ms: Math.round(performance.now() - started),
+            exhausted: false
+          }
+        };
+      }
+    }
+    return {
+      items: [],
+      providerUsed: defaultProvider,
+      fallbackUsed: attempts.length > 1,
+      triedProviders: attempts.map(item => item.provider),
+      diagnostics: {
+        attempts,
+        elapsed_ms: Math.round(performance.now() - started),
+        exhausted: true
+      }
+    };
+  }
+}
+
+export const providerRegistry = new ProviderRegistry();
+export { DuckDuckGoProvider, BingProvider, SearXNGProvider, BraveProvider };

docker/web-search/overrides/searxng.js

@@ -0,0 +1,47 @@
+import { HTTP_TIMEOUT, SEARXNG_URL } from "../constants.js";
+import { fetchWithTimeout } from "../utils/http.js";
+import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js";
+import { searchCache, createCacheKey } from "../utils/cache.js";
+
+export class SearXNGProvider {
+  name = "searxng";
+
+  constructor(instanceUrl) {
+    this.instanceUrl = instanceUrl || SEARXNG_URL;
+  }
+
+  async search(q, limit, lang, signal) {
+    const cacheKey = createCacheKey("searxng", q, limit, lang);
+    const cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    const params = new URLSearchParams({ q, format: "json", language: lang, safesearch: "0" });
+    const response = await fetchWithTimeout(`${this.instanceUrl}/search?${params}`, {
+      headers: { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) },
+      signal
+    }, HTTP_TIMEOUT);
+    if (!response.ok) {
+      if (response.status === 403) throw new Error("SearXNG JSON API disabled");
+      throw new Error(`SearXNG error: ${response.status}`);
+    }
+    const data = await response.json();
+    const items = (data.results || []).slice(0, limit).map(result => ({
+      title: result.title || "",
+      url: result.url || "",
+      snippet: result.content || undefined,
+      source: "searxng"
+    }));
+    searchCache.set(cacheKey, items);
+    return items;
+  }
+
+  async isAvailable() {
+    try {
+      const response = await fetchWithTimeout(`${this.instanceUrl}/search?q=test&format=json`, {
+        headers: { Accept: "application/json", "User-Agent": getRandomUserAgent() }
+      }, 5000);
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+}

docker/web-search/patch-mcp-web-search.mjs

@@ -15,6 +15,14 @@ const replacements = [
   [
     "max_download_bytes: z.number().int().min(1).max(26214400).optional()",
     "max_download_bytes: z.number().int().min(1).max(MAX_BYTES).optional()"
+  ],
+  [
+    'provider: z.enum(["duckduckgo", "bing", "searxng"]).optional()',
+    'provider: z.enum(["duckduckgo", "bing", "searxng", "brave"]).optional()'
+  ],
+  [
+    "Search the web using multiple providers (DuckDuckGo, Bing, SearXNG). Automatically falls back to other providers if the default fails. No API keys required for DuckDuckGo and SearXNG.",
+    "Search the web with bounded provider fallback and per-attempt diagnostics. SearXNG is local; Brave is available when BRAVE_SEARCH_API_KEY is configured."
   ]
 ];
 
@@ -26,3 +34,44 @@ for (const [before, after] of replacements) {
 }
 
 fs.writeFileSync(serverPath, source);
+
+const httpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/http.js";
+let httpSource = fs.readFileSync(httpPath, "utf8");
+const privateTransport = "async function fetchViaVettedAddress(url, timeoutMs)";
+if (!httpSource.includes(privateTransport)) throw new Error(`mcp-web-search patch target not found: ${privateTransport}`);
+httpSource = httpSource.replace(privateTransport, "export async function fetchViaVettedAddress(url, timeoutMs)");
+fs.writeFileSync(httpPath, httpSource);
+
+const utilityHttpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/utils/http.js";
+let utilityHttpSource = fs.readFileSync(utilityHttpPath, "utf8");
+const uncombinedSignal = 'return await fetch(input, { ...init, signal: controller.signal });';
+const combinedSignal = 'const signal = init.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;\n        return await fetch(input, { ...init, signal });';
+if (!utilityHttpSource.includes(uncombinedSignal)) throw new Error(`mcp-web-search patch target not found: ${uncombinedSignal}`);
+utilityHttpSource = utilityHttpSource.replace(uncombinedSignal, combinedSignal);
+fs.writeFileSync(utilityHttpPath, utilityHttpSource);
+
+const extractPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/extract.js";
+let extractSource = fs.readFileSync(extractPath, "utf8");
+const extractReplacements = [
+  [
+    'import { assertSafeUrl } from "./fetch/security.js";',
+    'import { assertSafeUrl } from "./fetch/security.js";\nimport { fetchBrowserResource } from "./fetch/browser.js";\nimport { boundFetchCollections } from "./fetch/bounds.js";'
+  ],
+  [
+    "fetchCache.set(cacheKey, siteResult);\n        return siteResult;",
+    "const boundedSiteResult = boundFetchCollections(siteResult);\n        fetchCache.set(cacheKey, boundedSiteResult);\n        return boundedSiteResult;"
+  ],
+  [
+    "const resource = await fetchResource(parsedUrl, options?.timeout_ms, transport, options);",
+    'const resource = options?.engine === "browser"\n        ? await fetchBrowserResource(parsedUrl, options?.timeout_ms)\n        : await fetchResource(parsedUrl, options?.timeout_ms, transport, options);'
+  ],
+  [
+    "fetchCache.set(cacheKey, result);\n    return result;",
+    "result = boundFetchCollections(result);\n    fetchCache.set(cacheKey, result);\n    return result;"
+  ]
+];
+for (const [before, after] of extractReplacements) {
+  if (!extractSource.includes(before)) throw new Error(`mcp-web-search extract patch target not found: ${before}`);
+  extractSource = extractSource.replace(before, after);
+}
+fs.writeFileSync(extractPath, extractSource);

docker/web-search/searxng/settings.yml

@@ -1,4 +1,9 @@
-use_default_settings: true
+use_default_settings:
+  engines:
+    keep_only:
+      - duckduckgo
+      - bing
+      - google
 
 general:
   debug: false