diff --git a/.env.example b/.env.example index 3612281..06f015a 100644 --- a/.env.example +++ b/.env.example @@ -14,14 +14,18 @@ CONTEXT_KIT_SEARXNG_PORT=8099 # Keep this aligned with agent tool-call defaults to avoid schema rejections. CONTEXT_KIT_WEB_SEARCH_MAX_BYTES=52428800 -# Web-search defaults. Search uses SearXNG first, then falls back to -# DuckDuckGo and Bing. Bing requires Chromium inside the web-search image. +# Web-search defaults. Search uses SearXNG first, then bounded fallbacks with +# per-attempt diagnostics. Bing and engine=browser use Chromium. CONTEXT_KIT_WEB_SEARCH_PORT=8777 # Override only for a loopback proxy that preserves the Host/Origin policy. # CONTEXT_KIT_WEB_SEARCH_HTTP_URL=http://127.0.0.1:8777/mcp CONTEXT_KIT_WEB_SEARCH_PROVIDER=searxng CONTEXT_KIT_WEB_SEARCH_HTTP_TIMEOUT=15000 CONTEXT_KIT_WEB_SEARCH_MAX_RESULTS=10 +CONTEXT_KIT_WEB_SEARCH_MAX_PROVIDER_ATTEMPTS=4 +CONTEXT_KIT_WEB_SEARCH_PROVIDER_TIMEOUT=15000 +# Optional hosted fallback. Context Kit remains fully usable without it. +# CONTEXT_KIT_BRAVE_SEARCH_API_KEY= CONTEXT_KIT_WEB_SEARCH_CHROME_PATH=/usr/bin/chromium # User agent used by the Chromium-backed Bing search fallback. # CONTEXT_KIT_WEB_SEARCH_BROWSER_USER_AGENT="Mozilla/5.0 ..." @@ -41,7 +45,7 @@ CONTEXT_KIT_DOCS_MAX_GET_BYTES=75000 CONTEXT_KIT_DOCS_EMBED_MODEL=BAAI/bge-small-en-v1.5 # Eagerly index every source on container start. Off by default so startup is -# fast; call the docs_refresh MCP tool when you want to populate the index. +# fast; call docs_refresh or `bin/context-kit docs-rebuild` to populate it. # CONTEXT_KIT_DOCS_PREINDEX=1 # One or more source files, separated by spaces. Keep committed profiles generic. diff --git a/.gitignore b/.gitignore index 0b6183f..7f07872 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .cache/ tmp/ *.log +__pycache__/ +*.py[cod] diff --git a/README.md b/README.md index 9088e49..a852dfd 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Context Kit gives coding agents three local tools: | Tool | Purpose | |---|---| -| `context-web-search` | Current web search through local SearXNG plus URL fetch/extract | -| `context-docs` | Semantic search over curated `llms.txt` documentation | +| `context-web-search` | Current web search with fallback diagnostics plus safe HTTP/browser extraction | +| `context-docs` | Persisted hybrid lexical/semantic search over curated documentation | | `context-repomix` | Pack repositories into AI-friendly context | The first public release deliberately keeps the surface area small: web search, @@ -67,9 +67,12 @@ config that will not be committed. labeled per launcher process and remove only themselves. - Web search uses stateless MCP HTTP sessions, validates Host, rejects every supplied Origin, and exits for Docker restart if its stdio backend dies. +- Explicit `fetch_url engine=browser` renders JavaScript while routing every + network GET through the same DNS/private-address checks as HTTP fetching. - `context-docs` browser CORS is disabled by default; set exact local origins only when a browser-based client needs direct access. -- Docs and model caches live in `$HOME/.local/share/context-kit`. +- Docs use a transactional SQLite WAL/FTS5 index; docs and model caches live in + `$HOME/.local/share/context-kit` and survive container replacement. - Docs refresh TTL defaults to `24h`. - Repomix mounts only the current project read-only. - No code-editing MCP server is enabled by default. @@ -98,7 +101,7 @@ machine adds extra local menus, they affect only that machine's running ## Docs Sources -The default docs index is intentionally small: +The default docs index uses the vendors' content-bearing `llms-full.txt` feeds: - Claude Code docs - OpenAI API docs and reference @@ -121,6 +124,17 @@ CONTEXT_KIT_DOCS_SOURCES="config/sources.default.txt config/sources.js.txt" \ Source changes are loaded by `start`/`restart`; `bin/context-kit docs` is only a stdio bridge to the already-running docs service. +`docs_query` searches with FTS5 plus embeddings, deduplicates exact content, +and supports source/host filters. It returns snippets but does not retrieve full +content unless IDs are requested or `auto_retrieve` is explicitly enabled. +`bin/context-kit docs-rebuild` safely replaces selected source generations only +after fetch, parse, and embedding succeed. + +For machine-local menu files, `bin/context-kit docs-snapshot` fetches their +linked pages into deterministic sibling `llms-full.txt` files with a provenance +manifest and conditional-request cache. Lifecycle commands automatically prefer +that full snapshot while preserving a prior snapshot if regeneration fails. + Large vendor feeds are opt-in because they can expand to thousands of sections and take a while to embed. @@ -135,6 +149,8 @@ bin/context-kit doctor bin/context-kit install claude bin/context-kit install opencode bin/context-kit redaction-check +bin/context-kit docs-snapshot +bin/context-kit docs-rebuild ``` MCP entrypoints: diff --git a/bin/context-kit b/bin/context-kit index 34568f3..22bd41b 100755 --- a/bin/context-kit +++ b/bin/context-kit @@ -92,6 +92,8 @@ Usage: context-kit status Show services, images, sources, and shared HTTP endpoints context-kit doctor Check Docker, services, images, sources, and HTTP endpoints context-kit redaction-check Scan this repo for local paths and secret patterns + context-kit docs-snapshot Build deterministic llms-full.txt local snapshots + context-kit docs-rebuild [URL...] Rebuild all or selected configured docs sources MCP server commands: context-kit web-search Stdio bridge to the shared web-search service @@ -510,18 +512,43 @@ source_files() { } resolved_sources() { - local file line + local file line local_prefix relative full_path + local_prefix="http://127.0.0.1:${DOCS_LOCAL_SOURCES_PORT}/" while IFS= read -r file; do [[ -f "${file}" ]] || fail "docs source file not found: ${file}" while IFS= read -r line; do line="${line%%#*}" line="${line//[$'\t\r\n ']/}" [[ -z "${line}" ]] && continue + if [[ "${line}" == "${local_prefix}"*'/llms.txt' ]]; then + relative="${line#"${local_prefix}"}" + full_path="${DOCS_LOCAL_SOURCES_DIR}/${relative%llms.txt}llms-full.txt" + if [[ -f "${full_path}" ]] \ + && python3 "${ROOT}/scripts/docs_snapshot.py" --validate-output "${full_path}" >/dev/null 2>&1; then + line="${line%llms.txt}llms-full.txt" + fi + fi printf '%s\n' "${line}" done < "${file}" done < <(source_files) } +cmd_docs_snapshot() { + python3 "${ROOT}/scripts/docs_snapshot.py" \ + --source-root "${DOCS_LOCAL_SOURCES_DIR}" \ + --cache-dir "${DATA_DIR}/snapshot-cache" \ + "$@" +} + +cmd_docs_rebuild() { + require_docker + require_network + if ! shared_service_running "${DOCS_SERVICE_NAME}"; then + fail "long-lived docs-mcp not running; start it with: context-kit start" + fi + node "${ROOT}/scripts/docs-rebuild.mjs" "${DOCS_HTTP_URL}" "$@" +} + cmd_build() { require_no_args "usage: context-kit build" "$@" require_docker @@ -847,7 +874,7 @@ cmd_docs() { # This stdio entrypoint is kept for clients that cannot speak HTTP MCP: # it spawns a thin mcp-proxy bridge per call but all calls multiplex onto # the single long-lived docs-mcp container over the Context Kit Docker - # network (no Chroma write contention, no host networking). + # network (no concurrent index writers, no host networking). require_docker require_network require_image "${DOCS_IMAGE}" "context-kit build" @@ -1011,6 +1038,8 @@ case "${1:-}" in doctor) shift; cmd_doctor "$@" ;; web-search) shift; cmd_web_search "$@" ;; docs) shift; cmd_docs "$@" ;; + docs-snapshot) shift; cmd_docs_snapshot "$@" ;; + docs-rebuild) shift; cmd_docs_rebuild "$@" ;; repomix) shift; cmd_repomix "$@" ;; install) shift; cmd_install "$@" ;; redaction-check) shift; cmd_redaction_check "$@" ;; diff --git a/compose.yml b/compose.yml index d91c9fe..00fe47d 100644 --- a/compose.yml +++ b/compose.yml @@ -36,6 +36,9 @@ services: HTTP_TIMEOUT: "${CONTEXT_KIT_WEB_SEARCH_HTTP_TIMEOUT:-15000}" MAX_BYTES: "${CONTEXT_KIT_WEB_SEARCH_MAX_BYTES:-52428800}" MAX_RESULTS: "${CONTEXT_KIT_WEB_SEARCH_MAX_RESULTS:-10}" + MAX_PROVIDER_ATTEMPTS: "${CONTEXT_KIT_WEB_SEARCH_MAX_PROVIDER_ATTEMPTS:-4}" + SEARCH_PROVIDER_TIMEOUT_MS: "${CONTEXT_KIT_WEB_SEARCH_PROVIDER_TIMEOUT:-15000}" + BRAVE_SEARCH_API_KEY: "${CONTEXT_KIT_BRAVE_SEARCH_API_KEY:-}" BROWSER_SEARCH_USER_AGENT: "${CONTEXT_KIT_WEB_SEARCH_BROWSER_USER_AGENT:-}" MCP_COMPAT_MODE: "${CONTEXT_KIT_WEB_SEARCH_MCP_COMPAT_MODE:-}" healthcheck: @@ -54,8 +57,8 @@ services: build: context: ./docker/docs image: ${CONTEXT_KIT_DOCS_IMAGE:-context-kit/docs-mcp:latest} - # Long-lived shared docs MCP. One container = one Chroma writer; clients - # connect over Streamable HTTP (mcp-proxy bridges llms-txt-mcp's stdio). + # Long-lived shared docs MCP. One container owns the transactional SQLite + # index and embedding model; clients connect over Streamable HTTP. restart: unless-stopped ports: - "127.0.0.1:${CONTEXT_KIT_DOCS_PORT:-8776}:8000" diff --git a/config/sources.default.txt b/config/sources.default.txt index 8926aae..90b5b7b 100644 --- a/config/sources.default.txt +++ b/config/sources.default.txt @@ -1,8 +1,8 @@ # Default Context Kit docs sources. # Keep this set small, useful, and quick to index. Add profiles when needed. -https://code.claude.com/docs/llms.txt -https://developers.openai.com/api/docs/llms.txt -https://developers.openai.com/api/reference/llms.txt -https://openrouter.ai/docs/llms.txt +https://code.claude.com/docs/llms-full.txt +https://developers.openai.com/api/docs/llms-full.txt +https://developers.openai.com/api/reference/llms-full.txt +https://openrouter.ai/docs/llms-full.txt https://modelcontextprotocol.io/llms-full.txt diff --git a/config/sources.js.txt b/config/sources.js.txt index bd0f809..c097249 100644 --- a/config/sources.js.txt +++ b/config/sources.js.txt @@ -1,7 +1,7 @@ # Optional JavaScript / frontend docs. -https://ai-sdk.dev/llms.txt -https://nextjs.org/docs/llms.txt -https://orm.drizzle.team/llms.txt -https://svelte.dev/llms.txt -https://hono.dev/llms.txt +https://ai-sdk.dev/llms-full.txt +https://nextjs.org/docs/llms-full.txt +https://orm.drizzle.team/llms-full.txt +https://svelte.dev/llms-full.txt +https://hono.dev/llms-full.txt diff --git a/config/sources.ruby-ai.txt b/config/sources.ruby-ai.txt index 184b82d..f9cb1d9 100644 --- a/config/sources.ruby-ai.txt +++ b/config/sources.ruby-ai.txt @@ -1,4 +1,4 @@ # Optional Ruby / AI application docs. -https://rubyllm.com/llms.txt -https://docs.langchain.com/llms.txt +https://rubyllm.com/llms-full.txt +https://docs.langchain.com/llms-full.txt diff --git a/docker/docs/.dockerignore b/docker/docs/.dockerignore index 5142853..6988673 100644 --- a/docker/docs/.dockerignore +++ b/docker/docs/.dockerignore @@ -2,3 +2,7 @@ !Dockerfile !entrypoint.sh !constraints.txt +!context_docs/ +!context_docs/** +!tests/ +!tests/** diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile index e624990..191e18b 100644 --- a/docker/docs/Dockerfile +++ b/docker/docs/Dockerfile @@ -1,7 +1,8 @@ FROM python:3.12-slim@sha256:6c4dd321d176d61ea848dc8c73a4f7dbae8f70e0ee48bb411ea2f045b599fa8e -ARG LLMS_TXT_MCP_VERSION=0.2.0 +ARG MCP_VERSION=1.28.0 ARG MCP_PROXY_VERSION=0.12.0 +ARG SENTENCE_TRANSFORMERS_VERSION=5.6.0 ARG TORCH_VERSION=2.12.1+cpu COPY constraints.txt /tmp/context-kit-docs-constraints.txt @@ -11,31 +12,35 @@ RUN apt-get update \ ca-certificates \ && rm -rf /var/lib/apt/lists/* -# Install CPU-only torch first so llms-txt-mcp does not pull large CUDA wheels. +# Install CPU-only torch first so sentence-transformers does not pull CUDA wheels. RUN pip install --no-cache-dir \ --index-url https://download.pytorch.org/whl/cpu \ -c /tmp/context-kit-docs-constraints.txt \ "torch==${TORCH_VERSION}" -# llms-txt-mcp does the indexing/search; mcp-proxy fronts its stdio transport -# as Streamable HTTP so multiple MCP clients can share one long-lived process -# (and therefore one Chroma DB writer). -RUN if [ -n "${LLMS_TXT_MCP_VERSION}" ]; then \ - pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt "llms-txt-mcp==${LLMS_TXT_MCP_VERSION}"; \ - else \ - pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt llms-txt-mcp; \ - fi \ - && pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt "mcp-proxy==${MCP_PROXY_VERSION}" \ +RUN pip install --no-cache-dir -c /tmp/context-kit-docs-constraints.txt \ + "mcp==${MCP_VERSION}" \ + "mcp-proxy==${MCP_PROXY_VERSION}" \ + "sentence-transformers==${SENTENCE_TRANSFORMERS_VERSION}" \ + httpx numpy PyYAML \ && rm /tmp/context-kit-docs-constraints.txt +COPY context_docs /opt/context-kit/context_docs +COPY tests /opt/context-kit/tests COPY entrypoint.sh /usr/local/bin/docs-mcp-entrypoint -RUN chmod 0555 /usr/local/bin/docs-mcp-entrypoint +RUN chmod -R a+rX /opt/context-kit \ + && chmod 0555 /usr/local/bin/docs-mcp-entrypoint RUN mkdir -p /data /models /etc/context-kit ENV HF_HOME=/models \ + HOME=/tmp \ + USER=context-kit \ + LOGNAME=context-kit \ SENTENCE_TRANSFORMERS_HOME=/models \ + PYTHONPATH=/opt/context-kit \ DOCS_MCP_HTTP_HOST=0.0.0.0 \ DOCS_MCP_HTTP_PORT=8000 \ + DOCS_MCP_STORE_PATH=/data/docs.sqlite3 \ DOCS_MCP_SOURCES_FILE=/etc/context-kit/docs-sources.txt VOLUME ["/data", "/models"] diff --git a/docker/docs/constraints.txt b/docker/docs/constraints.txt index 9ac9bec..c289c81 100644 --- a/docker/docs/constraints.txt +++ b/docker/docs/constraints.txt @@ -10,7 +10,6 @@ build==1.5.0 certifi==2026.6.17 cffi==2.0.0 charset-normalizer==3.4.7 -chromadb==1.5.9 click==8.4.2 cryptography==49.0.0 durationpy==0.10 @@ -35,7 +34,6 @@ joblib==1.5.3 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 kubernetes==36.0.2 -llms-txt-mcp==0.2.0 markdown-it-py==4.2.0 MarkupSafe==3.0.3 mcp==1.28.0 diff --git a/docker/docs/context_docs/__init__.py b/docker/docs/context_docs/__init__.py new file mode 100644 index 0000000..4715643 --- /dev/null +++ b/docker/docs/context_docs/__init__.py @@ -0,0 +1,3 @@ +"""Maintained Context Kit documentation retrieval service.""" + +__version__ = "1.0.0" diff --git a/docker/docs/context_docs/__main__.py b/docker/docs/context_docs/__main__.py new file mode 100644 index 0000000..1f8d26f --- /dev/null +++ b/docker/docs/context_docs/__main__.py @@ -0,0 +1,4 @@ +from .server import main + + +main() diff --git a/docker/docs/context_docs/embedder.py b/docker/docs/context_docs/embedder.py new file mode 100644 index 0000000..f88ee96 --- /dev/null +++ b/docker/docs/context_docs/embedder.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import asyncio +import hashlib + +import numpy as np + + +class SentenceTransformerEmbedder: + def __init__(self, model_name: str): + self.model_name = model_name + self.fingerprint = f"sentence-transformers:{model_name}" + self._model = None + self._lock = asyncio.Lock() + + @property + def ready(self) -> bool: + return self._model is not None + + async def ensure_ready(self) -> None: + if self._model is not None: + return + async with self._lock: + if self._model is None: + self._model = await asyncio.to_thread(self._load) + + def _load(self): + from sentence_transformers import SentenceTransformer + + return SentenceTransformer(self.model_name, device="cpu") + + async def encode_documents(self, texts: list[str]) -> np.ndarray: + await self.ensure_ready() + return await asyncio.to_thread(self._encode, texts) + + async def encode_query(self, text: str) -> np.ndarray: + vectors = await self.encode_documents([text]) + return vectors[0] + + def _encode(self, texts: list[str]) -> np.ndarray: + return np.asarray( + self._model.encode( + texts, + batch_size=32, + show_progress_bar=False, + normalize_embeddings=True, + convert_to_numpy=True, + ), + dtype=np.float32, + ) + + +class LexicalFallbackEmbedder: + """Deterministic fallback used only when a model cannot be loaded.""" + + fingerprint = "lexical-fallback-v1" + ready = True + + async def ensure_ready(self) -> None: + return None + + async def encode_documents(self, texts: list[str]) -> np.ndarray: + return np.asarray([self._encode(text) for text in texts], dtype=np.float32) + + async def encode_query(self, text: str) -> np.ndarray: + return np.asarray(self._encode(text), dtype=np.float32) + + @staticmethod + def _encode(text: str, dimensions: int = 384) -> np.ndarray: + vector = np.zeros(dimensions, dtype=np.float32) + for token in text.lower().split(): + digest = hashlib.sha256(token.encode()).digest() + vector[int.from_bytes(digest[:4], "big") % dimensions] += 1.0 + norm = np.linalg.norm(vector) + return vector / norm if norm else vector diff --git a/docker/docs/context_docs/fetcher.py b/docker/docs/context_docs/fetcher.py new file mode 100644 index 0000000..6274232 --- /dev/null +++ b/docker/docs/context_docs/fetcher.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import httpx + +from .models import FetchResponse, SourceState + + +class SourceFetcher: + def __init__(self, timeout_seconds: float = 30, max_bytes: int = 20_000_000): + self.timeout_seconds = timeout_seconds + self.max_bytes = max_bytes + self._client = httpx.AsyncClient( + follow_redirects=True, + timeout=httpx.Timeout(timeout_seconds), + headers={"User-Agent": "context-kit-docs/1.0"}, + ) + + async def close(self) -> None: + await self._client.aclose() + + async def fetch(self, source_url: str, state: SourceState | None = None) -> FetchResponse: + headers: dict[str, str] = {} + if state and state.etag: + headers["If-None-Match"] = state.etag + if state and state.last_modified: + headers["If-Modified-Since"] = state.last_modified + async with self._client.stream("GET", source_url, headers=headers) as response: + if response.status_code == 304: + return FetchResponse( + status=304, + requested_url=source_url, + resolved_url=str(response.url), + etag=response.headers.get("etag"), + last_modified=response.headers.get("last-modified"), + ) + chunks: list[bytes] = [] + size = 0 + async for chunk in response.aiter_bytes(): + size += len(chunk) + if size > self.max_bytes: + raise RuntimeError(f"source exceeds {self.max_bytes} byte limit") + chunks.append(chunk) + body = b"".join(chunks).decode(response.encoding or "utf-8", errors="replace") + return FetchResponse( + status=response.status_code, + requested_url=source_url, + resolved_url=str(response.url), + body=body, + etag=response.headers.get("etag"), + last_modified=response.headers.get("last-modified"), + ) diff --git a/docker/docs/context_docs/models.py b/docker/docs/context_docs/models.py new file mode 100644 index 0000000..0c52e96 --- /dev/null +++ b/docker/docs/context_docs/models.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np + + +@dataclass(frozen=True) +class ParsedDocument: + title: str + description: str + content: str + canonical_url: str + heading_path: str + chunk_index: int = 0 + + +@dataclass(frozen=True) +class ParsedSource: + format: str + documents: list[ParsedDocument] + + +@dataclass(frozen=True) +class FetchResponse: + status: int + requested_url: str + resolved_url: str + body: str = "" + etag: str | None = None + last_modified: str | None = None + + +@dataclass(frozen=True) +class PreparedDocument: + id: str + configured_source: str + resolved_source: str + source_host: str + canonical_url: str + canonical_host: str + title: str + description: str + heading_path: str + content: str + content_hash: str + embedding: np.ndarray + + +@dataclass(frozen=True) +class SourceUpdate: + configured_source: str + resolved_source: str + etag: str | None + last_modified: str | None + body_hash: str + raw_body: str + parser_fingerprint: str + embedding_fingerprint: str + checked_at: float + indexed_at: float + stale_at: float + documents: list[PreparedDocument] + + +@dataclass(frozen=True) +class SourceState: + configured_source: str + resolved_source: str | None + active: bool + etag: str | None + last_modified: str | None + body_hash: str | None + raw_body: str | None + parser_fingerprint: str | None + embedding_fingerprint: str | None + checked_at: float | None + indexed_at: float | None + stale_at: float | None + last_error: str | None + doc_count: int + + +@dataclass(frozen=True) +class StoredDocument: + id: str + configured_source: str + resolved_source: str + source_host: str + canonical_url: str + canonical_host: str + title: str + description: str + heading_path: str + content: str + content_hash: str + embedding: np.ndarray + + +@dataclass(frozen=True) +class SearchResult: + id: str + configured_source: str + canonical_url: str + title: str + description: str + heading_path: str + content: str + content_hash: str + score: float + lexical_rank: int | None + semantic_rank: int | None + duplicate_count: int = 1 + alternate_sources: list[dict[str, str]] = field(default_factory=list) + + +@dataclass(frozen=True) +class RefreshOutcome: + source: str + status: str + document_count: int + detail: str | None = None diff --git a/docker/docs/context_docs/parser.py b/docker/docs/context_docs/parser.py new file mode 100644 index 0000000..5a6d5f1 --- /dev/null +++ b/docker/docs/context_docs/parser.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import re +from urllib.parse import urljoin + +import yaml + +from .models import ParsedDocument, ParsedSource + + +PARSER_FINGERPRINT = "context-docs-parser-v1" +_MENU_LINK = re.compile(r"^\s*[-*]\s+\[([^]]+)]\(([^)]+)\)(?::\s*(.*))?\s*$") +_HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +_FRONTMATTER = re.compile(r"(?m)^---\s*$") + + +def parse_llms_text(content: str, source_url: str, max_chunk_chars: int = 6_000) -> ParsedSource: + normalized = content.replace("\r\n", "\n").replace("\r", "\n").strip() + if not normalized: + return ParsedSource("empty", []) + + yaml_documents = _parse_repeated_frontmatter(normalized, source_url, max_chunk_chars) + if yaml_documents is not None: + return ParsedSource("yaml-full", yaml_documents) + + if source_url.split("?", 1)[0].endswith("/llms.txt"): + menu_documents = _parse_menu(normalized, source_url) + if menu_documents: + return ParsedSource("standard-menu", menu_documents) + + return ParsedSource("markdown-full", _parse_markdown_bundle(normalized, source_url, max_chunk_chars)) + + +def _parse_repeated_frontmatter(content: str, source_url: str, max_chunk_chars: int) -> list[ParsedDocument] | None: + if not content.startswith("---\n"): + return None + separators = [match.start() for match in _FRONTMATTER.finditer(content)] + if len(separators) < 2: + return None + + documents: list[ParsedDocument] = [] + cursor = 0 + while cursor < len(content): + if not content.startswith("---", cursor): + return None + header_end = content.find("\n---", cursor + 3) + if header_end < 0: + return None + try: + metadata = yaml.safe_load(content[cursor + 3 : header_end]) or {} + except yaml.YAMLError: + return None + if not isinstance(metadata, dict) or not isinstance(metadata.get("title"), str): + return None + body_start = header_end + 4 + if body_start < len(content) and content[body_start] == "\n": + body_start += 1 + next_header = content.find("\n---\n", body_start) + body_end = len(content) if next_header < 0 else next_header + body = content[body_start:body_end].strip() + title = metadata["title"].strip() + description = str(metadata.get("description") or "").strip() + canonical = str(metadata.get("url") or metadata.get("canonical_url") or source_url) + documents.extend(_chunk_document(title, description, body, urljoin(source_url, canonical), title, max_chunk_chars)) + if next_header < 0: + break + cursor = next_header + 1 + return documents or None + + +def _parse_menu(content: str, source_url: str) -> list[ParsedDocument]: + documents: list[ParsedDocument] = [] + headings: list[tuple[int, str]] = [] + in_fence = False + for line in content.splitlines(): + if line.lstrip().startswith(("```", "~~~")): + in_fence = not in_fence + continue + if in_fence: + continue + heading = _HEADING.match(line) + if heading: + level = len(heading.group(1)) + headings = [entry for entry in headings if entry[0] < level] + headings.append((level, heading.group(2).strip())) + continue + link = _MENU_LINK.match(line) + if not link: + continue + title, target, description = link.group(1).strip(), link.group(2).strip(), (link.group(3) or "").strip() + canonical = urljoin(source_url, target) + path = " > ".join([name for _, name in headings] + [title]) + rendered = f"{title}\n\n{description}\n\nSource: {canonical}".strip() + documents.append(ParsedDocument(title, description, rendered, canonical, path)) + return documents + + +def _parse_markdown_bundle(content: str, source_url: str, max_chunk_chars: int) -> list[ParsedDocument]: + sections: list[tuple[str, str]] = [] + current_title = "Documentation" + current_lines: list[str] = [] + in_fence = False + for line in content.splitlines(): + if line.lstrip().startswith(("```", "~~~")): + in_fence = not in_fence + heading = None if in_fence else _HEADING.match(line) + if heading and len(heading.group(1)) == 1: + if current_lines or sections: + sections.append((current_title, "\n".join(current_lines).strip())) + current_title = heading.group(2).strip() + current_lines = [] + else: + current_lines.append(line) + if current_lines or not sections: + sections.append((current_title, "\n".join(current_lines).strip())) + + documents: list[ParsedDocument] = [] + for title, body in sections: + if not body and title == "Documentation": + continue + documents.extend(_chunk_document(title, "", body, source_url, title, max_chunk_chars)) + return documents + + +def _chunk_document( + title: str, + description: str, + content: str, + canonical_url: str, + heading_path: str, + max_chunk_chars: int, +) -> list[ParsedDocument]: + if len(content) <= max_chunk_chars: + return [ParsedDocument(title, description, content, canonical_url, heading_path, 0)] + + paragraphs = re.split(r"\n{2,}", content) + chunks: list[str] = [] + current: list[str] = [] + size = 0 + for paragraph in paragraphs: + pieces = [paragraph[index : index + max_chunk_chars] for index in range(0, len(paragraph), max_chunk_chars)] or [""] + for piece in pieces: + added = len(piece) + (2 if current else 0) + if current and size + added > max_chunk_chars: + chunks.append("\n\n".join(current)) + current, size = [], 0 + current.append(piece) + size += len(piece) + (2 if len(current) > 1 else 0) + if current: + chunks.append("\n\n".join(current)) + return [ + ParsedDocument(title, description, chunk, canonical_url, heading_path, index) + for index, chunk in enumerate(chunks) + ] diff --git a/docker/docs/context_docs/refresh.py b/docker/docs/context_docs/refresh.py new file mode 100644 index 0000000..92259d6 --- /dev/null +++ b/docker/docs/context_docs/refresh.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import asyncio +import hashlib +from urllib.parse import urlparse + +from .models import PreparedDocument, RefreshOutcome, SourceUpdate +from .parser import PARSER_FINGERPRINT + + +class RefreshCoordinator: + def __init__(self, store, fetcher, embedder, parser, ttl_seconds: float, now): + self.store = store + self.fetcher = fetcher + self.embedder = embedder + self.parser = parser + self.ttl_seconds = ttl_seconds + self.now = now + self._inflight: dict[str, asyncio.Task] = {} + self._lock = asyncio.Lock() + + async def refresh(self, source: str, force: bool = False) -> RefreshOutcome: + state = self.store.get_source(source) + timestamp = self.now() + compatible = bool( + state + and state.parser_fingerprint == PARSER_FINGERPRINT + and state.embedding_fingerprint == self.embedder.fingerprint + ) + if not force and compatible and state.doc_count and state.stale_at and state.stale_at > timestamp: + return RefreshOutcome(source, "fresh", state.doc_count) + + async with self._lock: + task = self._inflight.get(source) + if task is None: + task = asyncio.create_task(self._refresh_once(source, timestamp)) + self._inflight[source] = task + try: + return await task + finally: + async with self._lock: + if self._inflight.get(source) is task and task.done(): + self._inflight.pop(source, None) + + async def _refresh_once(self, source: str, timestamp: float) -> RefreshOutcome: + state = self.store.get_source(source) + try: + compatible = bool( + state + and state.parser_fingerprint == PARSER_FINGERPRINT + and state.embedding_fingerprint == self.embedder.fingerprint + ) + response = await self.fetcher.fetch(source, state if compatible else None) + if response.status == 304: + count = state.doc_count if state else 0 + self.store.mark_checked(source, timestamp, timestamp + self.ttl_seconds) + return RefreshOutcome(source, "not_modified", count) + if response.status != 200: + raise RuntimeError(f"source returned HTTP {response.status}") + + parsed = self.parser(response.body, response.resolved_url) + if not parsed.documents: + raise RuntimeError("source parsed to zero documents; previous generation preserved") + texts = ["\n\n".join(filter(None, [doc.title, doc.description, doc.heading_path, doc.content])) for doc in parsed.documents] + vectors = await self.embedder.encode_documents(texts) if texts else [] + documents: list[PreparedDocument] = [] + source_host = (urlparse(response.resolved_url).hostname or "").lower() + for parsed_document, vector in zip(parsed.documents, vectors, strict=True): + content_hash = hashlib.sha256(parsed_document.content.encode()).hexdigest() + identity = "\0".join( + [source, parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)] + ) + documents.append( + PreparedDocument( + id=hashlib.sha256(identity.encode()).hexdigest()[:24], + configured_source=source, + resolved_source=response.resolved_url, + source_host=source_host, + canonical_url=parsed_document.canonical_url, + canonical_host=(urlparse(parsed_document.canonical_url).hostname or source_host).lower(), + title=parsed_document.title, + description=parsed_document.description, + heading_path=parsed_document.heading_path, + content=parsed_document.content, + content_hash=content_hash, + embedding=vector, + ) + ) + body_hash = hashlib.sha256(response.body.encode()).hexdigest() + self.store.replace_source( + SourceUpdate( + configured_source=source, + resolved_source=response.resolved_url, + etag=response.etag, + last_modified=response.last_modified, + body_hash=body_hash, + raw_body=response.body, + parser_fingerprint=PARSER_FINGERPRINT, + embedding_fingerprint=self.embedder.fingerprint, + checked_at=timestamp, + indexed_at=timestamp, + stale_at=timestamp + self.ttl_seconds, + documents=documents, + ) + ) + return RefreshOutcome(source, "updated", len(documents), parsed.format) + except Exception as error: + self.store.mark_checked(source, timestamp, timestamp, str(error)) + return RefreshOutcome(source, "error", state.doc_count if state else 0, str(error)) diff --git a/docker/docs/context_docs/search.py b/docker/docs/context_docs/search.py new file mode 100644 index 0000000..5057cef --- /dev/null +++ b/docker/docs/context_docs/search.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections import defaultdict + +import numpy as np + +from .models import SearchResult, StoredDocument +from .store import IndexStore + + +class HybridSearch: + def __init__(self, store: IndexStore, embedder, rrf_k: int = 60): + self.store = store + self.embedder = embedder + self.rrf_k = rrf_k + + async def search( + self, + query: str, + limit: int = 10, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> list[SearchResult]: + pool_size = max(limit * 8, 40) + lexical = self.store.lexical_search(query, pool_size, sources, hosts) + candidates = self.store.semantic_candidates(sources, hosts) + semantic: list[StoredDocument] = [] + if candidates: + query_vector = np.asarray(await self.embedder.encode_query(query), dtype=np.float32) + query_norm = np.linalg.norm(query_vector) + scored: list[tuple[float, StoredDocument]] = [] + for document in candidates: + norm = np.linalg.norm(document.embedding) * query_norm + score = float(np.dot(document.embedding, query_vector) / norm) if norm else 0.0 + scored.append((score, document)) + semantic = [document for _, document in sorted(scored, key=lambda item: (-item[0], item[1].id))[:pool_size]] + + lexical_ranks = {document.id: rank for rank, document in enumerate(lexical, 1)} + semantic_ranks = {document.id: rank for rank, document in enumerate(semantic, 1)} + documents = {document.id: document for document in [*lexical, *semantic]} + scores = defaultdict(float) + for identifier, rank in lexical_ranks.items(): + scores[identifier] += 1.0 / (self.rrf_k + rank) + for identifier, rank in semantic_ranks.items(): + scores[identifier] += 1.0 / (self.rrf_k + rank) + + ordered = sorted(documents.values(), key=lambda item: (-scores[item.id], item.id)) + groups: dict[str, list[StoredDocument]] = {} + group_order: list[str] = [] + for document in ordered: + key = document.content_hash + if key not in groups: + groups[key] = [] + group_order.append(key) + groups[key].append(document) + + results: list[SearchResult] = [] + for key in group_order[:limit]: + group = groups[key] + primary = group[0] + alternates = [ + {"source": document.configured_source, "url": document.canonical_url} + for document in group[1:] + ] + results.append( + SearchResult( + id=primary.id, + configured_source=primary.configured_source, + canonical_url=primary.canonical_url, + title=primary.title, + description=primary.description, + heading_path=primary.heading_path, + content=primary.content, + content_hash=primary.content_hash, + score=min(1.0, scores[primary.id] / (2.0 / (self.rrf_k + 1))), + lexical_rank=lexical_ranks.get(primary.id), + semantic_rank=semantic_ranks.get(primary.id), + duplicate_count=len(group), + alternate_sources=alternates, + ) + ) + return results diff --git a/docker/docs/context_docs/server.py b/docker/docs/context_docs/server.py new file mode 100644 index 0000000..d849eb2 --- /dev/null +++ b/docker/docs/context_docs/server.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import asyncio +import os +import re +import time +from contextlib import asynccontextmanager +from pathlib import Path + +import uvicorn +from mcp.server.fastmcp import FastMCP +from starlette.middleware.cors import CORSMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route + +from .embedder import SentenceTransformerEmbedder +from .fetcher import SourceFetcher +from .parser import parse_llms_text +from .refresh import RefreshCoordinator +from .search import HybridSearch +from .service import DocsService +from .store import IndexStore + + +def parse_duration(value: str) -> float: + match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([smhd]?)\s*", value) + if not match: + raise ValueError(f"invalid duration: {value}") + multiplier = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)] + return float(match.group(1)) * multiplier + + +def read_sources(path: str | Path) -> list[str]: + sources: list[str] = [] + for raw_line in Path(path).read_text().splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if not line.endswith(("/llms.txt", "/llms-full.txt")): + raise ValueError(f"source URL must end with /llms.txt or /llms-full.txt: {line}") + sources.append(line) + if not sources: + raise ValueError(f"no sources configured in {path}") + return list(dict.fromkeys(sources)) + + +def build_server(): + source_file = os.environ.get("DOCS_MCP_SOURCES_FILE", "/etc/context-kit/docs-sources.txt") + sources = read_sources(source_file) + store = IndexStore(os.environ.get("DOCS_MCP_STORE_PATH", "/data/docs.sqlite3")) + store.configure_sources(sources) + embedder = SentenceTransformerEmbedder( + os.environ.get("DOCS_MCP_EMBED_MODEL", "BAAI/bge-small-en-v1.5") + ) + fetcher = SourceFetcher( + timeout_seconds=float(os.environ.get("DOCS_MCP_FETCH_TIMEOUT", "30")), + max_bytes=int(os.environ.get("DOCS_MCP_MAX_SOURCE_BYTES", "20000000")), + ) + coordinator = RefreshCoordinator( + store=store, + fetcher=fetcher, + embedder=embedder, + parser=parse_llms_text, + ttl_seconds=parse_duration(os.environ.get("DOCS_MCP_TTL", "24h")), + now=time.time, + ) + service = DocsService( + store, + HybridSearch(store, embedder), + coordinator, + max_get_bytes=int(os.environ.get("DOCS_MCP_MAX_GET_BYTES", "75000")), + ) + + mcp = FastMCP( + "Context Kit Docs", + instructions="Search and retrieve configured documentation using persisted hybrid retrieval.", + host=os.environ.get("DOCS_MCP_HTTP_HOST", "0.0.0.0"), + port=int(os.environ.get("DOCS_MCP_HTTP_PORT", "8000")), + streamable_http_path="/mcp", + stateless_http=True, + ) + + @mcp.tool() + async def docs_query( + query: str, + limit: int = 10, + auto_retrieve: bool = False, + auto_retrieve_threshold: float = 0.55, + auto_retrieve_limit: int = 5, + retrieve_ids: list[str] | None = None, + max_bytes: int | None = None, + merge: bool = False, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> dict: + """Search docs. Content retrieval is explicit by default; optionally filter source URLs or hosts.""" + return await service.query( + query, limit, auto_retrieve, auto_retrieve_threshold, auto_retrieve_limit, + retrieve_ids, max_bytes, merge, sources, hosts, + ) + + @mcp.tool() + async def docs_refresh( + source: str | None = None, + sources: list[str] | None = None, + force: bool = False, + ) -> dict: + """Refresh configured sources transactionally; concurrent requests are coalesced.""" + if source and sources: + raise ValueError("pass source or sources, not both") + if source: + sources = [source] + return await service.refresh(sources, force) + + @mcp.tool() + async def docs_sources() -> dict: + """Report configured-source freshness, errors, and document counts.""" + return service.source_status() + + @mcp.tool() + async def docs_rebuild( + source: str | None = None, + sources: list[str] | None = None, + ) -> dict: + """Force a safe source rebuild without deleting the last good generation first.""" + if source and sources: + raise ValueError("pass source or sources, not both") + if source: + sources = [source] + return await service.refresh(sources, force=True) + + app = mcp.streamable_http_app() + mcp_lifespan = app.router.lifespan_context + + @asynccontextmanager + async def application_lifespan(application): + preindex_task = None + async with mcp_lifespan(application): + if os.environ.get("DOCS_MCP_PREINDEX", "0") == "1": + preindex_task = asyncio.create_task(service.refresh()) + try: + yield + finally: + if preindex_task: + await preindex_task + await fetcher.close() + store.close() + + app.router.lifespan_context = application_lifespan + + async def status(_request: Request) -> JSONResponse: + state = service.source_status() + errors = sum(1 for source in state["sources"] if source["last_error"]) + return JSONResponse( + { + "status": "ok" if state["document_count"] or not errors else "degraded", + "ready": True, + "model_ready": embedder.ready, + "source_count": state["source_count"], + "document_count": state["document_count"], + "source_errors": errors, + } + ) + + app.routes.insert(0, Route("/status", status, methods=["GET"])) + origins = os.environ.get("DOCS_MCP_ALLOW_ORIGIN", "").split() + if origins: + app = CORSMiddleware(app, allow_origins=origins, allow_methods=["POST", "GET", "DELETE"], allow_headers=["*"]) + return app + + +def main() -> None: + uvicorn.run( + build_server(), + host=os.environ.get("DOCS_MCP_HTTP_HOST", "0.0.0.0"), + port=int(os.environ.get("DOCS_MCP_HTTP_PORT", "8000")), + log_level=os.environ.get("DOCS_MCP_LOG_LEVEL", "info").lower(), + ) + + +if __name__ == "__main__": + main() diff --git a/docker/docs/context_docs/service.py b/docker/docs/context_docs/service.py new file mode 100644 index 0000000..37ca49f --- /dev/null +++ b/docker/docs/context_docs/service.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +from dataclasses import asdict + + +class DocsService: + def __init__(self, store, search, refresh, max_get_bytes: int = 75_000): + self.store = store + self.search_engine = search + self.refresh_coordinator = refresh + self.max_get_bytes = max_get_bytes + + async def refresh(self, sources: list[str] | None = None, force: bool = False) -> dict: + configured = [state.configured_source for state in self.store.list_sources()] + selected = configured if sources is None else sources + unknown = sorted(set(selected) - set(configured)) + if unknown: + raise ValueError(f"unconfigured sources: {', '.join(unknown)}") + outcomes = await asyncio.gather( + *(self.refresh_coordinator.refresh(source, force=force) for source in selected) + ) + return {"sources": [asdict(outcome) for outcome in outcomes]} + + async def query( + self, + query: str, + limit: int = 10, + auto_retrieve: bool = False, + auto_retrieve_threshold: float = 0.55, + auto_retrieve_limit: int = 5, + retrieve_ids: list[str] | None = None, + max_bytes: int | None = None, + merge: bool = False, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> dict: + if not query.strip(): + raise ValueError("query must not be empty") + if not 1 <= limit <= 100: + raise ValueError("limit must be between 1 and 100") + if not 0 <= auto_retrieve_threshold <= 1: + raise ValueError("auto_retrieve_threshold must be between 0 and 1") + if not 0 <= auto_retrieve_limit <= 25: + raise ValueError("auto_retrieve_limit must be between 0 and 25") + + await self._refresh_missing_or_stale(sources) + results = await self.search_engine.search(query, limit, sources, hosts) + search_results = [ + { + "id": item.id, + "source": item.configured_source, + "url": item.canonical_url, + "host": item.canonical_url.split("/", 3)[2] if "://" in item.canonical_url else "", + "title": item.title, + "description": item.description, + "heading_path": item.heading_path, + "score": round(item.score, 6), + "snippet": item.content[:500], + "duplicate_count": item.duplicate_count, + "alternate_sources": item.alternate_sources, + } + for item in results + ] + + selected_ids = list(dict.fromkeys(retrieve_ids or [])) + if auto_retrieve: + selected_ids.extend( + item.id + for item in results[:auto_retrieve_limit] + if item.score >= auto_retrieve_threshold and item.id not in selected_ids + ) + byte_budget = min(max_bytes or self.max_get_bytes, self.max_get_bytes) + retrieved: dict[str, dict] = {} + used = 0 + for identifier in selected_ids[:25]: + document = self.store.get_document(identifier, sources, hosts) + if not document: + continue + encoded = document.content.encode() + remaining = max(0, byte_budget - used) + if remaining == 0: + break + content = encoded[:remaining].decode(errors="ignore") + used += len(content.encode()) + retrieved[identifier] = { + "id": identifier, + "source": document.configured_source, + "url": document.canonical_url, + "title": document.title, + "content": content, + "truncated": len(content.encode()) < len(encoded), + } + + merged = "" + if merge: + merged = "\n\n".join( + f"# {item['title']}\n\nSource: {item['url']}\n\n{item['content']}" + for item in retrieved.values() + ) + return { + "search_results": search_results, + "retrieved_content": retrieved, + "merged_content": merged, + "auto_retrieved_count": len(retrieved) - len([item for item in retrieve_ids or [] if item in retrieved]), + "total_results": len(search_results), + } + + async def _refresh_missing_or_stale(self, sources: list[str] | None) -> None: + states = self.store.list_sources() + selected = [state for state in states if sources is None or state.configured_source in sources] + await asyncio.gather( + *(self.refresh_coordinator.refresh(state.configured_source) for state in selected) + ) + + def source_status(self) -> dict: + states = [asdict(state) for state in self.store.list_sources()] + for state in states: + state.pop("raw_body", None) + return { + "sources": states, + "source_count": len(states), + "document_count": sum(state["doc_count"] for state in states), + } diff --git a/docker/docs/context_docs/store.py b/docker/docs/context_docs/store.py new file mode 100644 index 0000000..4d2f689 --- /dev/null +++ b/docker/docs/context_docs/store.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import re +import sqlite3 +import threading +from pathlib import Path + +import numpy as np + +from .models import PreparedDocument, SourceState, SourceUpdate, StoredDocument + + +class IndexStore: + def __init__(self, path: str | Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.connection = sqlite3.connect(self.path, check_same_thread=False, isolation_level=None) + self.connection.row_factory = sqlite3.Row + self._lock = threading.RLock() + self._initialize() + + def _initialize(self) -> None: + with self.connection: + self.connection.execute("PRAGMA journal_mode=WAL") + self.connection.execute("PRAGMA foreign_keys=ON") + self.connection.execute("PRAGMA busy_timeout=5000") + self.connection.executescript( + """ + CREATE TABLE IF NOT EXISTS sources ( + configured_source TEXT PRIMARY KEY, + resolved_source TEXT, + active INTEGER NOT NULL DEFAULT 1, + etag TEXT, + last_modified TEXT, + body_hash TEXT, + raw_body TEXT, + parser_fingerprint TEXT, + embedding_fingerprint TEXT, + checked_at REAL, + indexed_at REAL, + stale_at REAL, + last_error TEXT + ); + CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, + configured_source TEXT NOT NULL REFERENCES sources(configured_source) ON DELETE CASCADE, + resolved_source TEXT NOT NULL, + source_host TEXT NOT NULL, + canonical_url TEXT NOT NULL, + canonical_host TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + heading_path TEXT NOT NULL, + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS documents_source ON documents(configured_source); + CREATE INDEX IF NOT EXISTS documents_hash ON documents(content_hash); + CREATE INDEX IF NOT EXISTS documents_hosts ON documents(source_host, canonical_host); + CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( + doc_id UNINDEXED, title, description, heading_path, content, canonical_url, + tokenize='unicode61 remove_diacritics 2 tokenchars ''_-''' + ); + """ + ) + + def close(self) -> None: + self.connection.close() + + def configure_sources(self, sources: list[str]) -> None: + with self._lock, self.connection: + self.connection.execute("UPDATE sources SET active = 0") + self.connection.executemany( + "INSERT INTO sources(configured_source, active) VALUES(?, 1) " + "ON CONFLICT(configured_source) DO UPDATE SET active = 1", + [(source,) for source in dict.fromkeys(sources)], + ) + + def replace_source(self, update: SourceUpdate) -> None: + with self._lock: + self.connection.execute("BEGIN IMMEDIATE") + try: + self._replace_source(update) + except Exception: + self.connection.rollback() + raise + else: + self.connection.commit() + + def _replace_source(self, update: SourceUpdate) -> None: + self.connection.execute( + """INSERT INTO sources( + configured_source, resolved_source, active, etag, last_modified, + body_hash, raw_body, parser_fingerprint, embedding_fingerprint, + checked_at, indexed_at, stale_at, last_error + ) VALUES(?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL) + ON CONFLICT(configured_source) DO UPDATE SET + resolved_source=excluded.resolved_source, etag=excluded.etag, + last_modified=excluded.last_modified, body_hash=excluded.body_hash, + raw_body=excluded.raw_body, parser_fingerprint=excluded.parser_fingerprint, + embedding_fingerprint=excluded.embedding_fingerprint, + checked_at=excluded.checked_at, indexed_at=excluded.indexed_at, + stale_at=excluded.stale_at, last_error=NULL""", + ( + update.configured_source, + update.resolved_source, + update.etag, + update.last_modified, + update.body_hash, + update.raw_body, + update.parser_fingerprint, + update.embedding_fingerprint, + update.checked_at, + update.indexed_at, + update.stale_at, + ), + ) + old_ids = [row[0] for row in self.connection.execute("SELECT id FROM documents WHERE configured_source=?", (update.configured_source,))] + if old_ids: + self.connection.executemany("DELETE FROM documents_fts WHERE doc_id=?", [(identifier,) for identifier in old_ids]) + self.connection.execute("DELETE FROM documents WHERE configured_source=?", (update.configured_source,)) + for document in update.documents: + vector = np.asarray(document.embedding, dtype=np.float32) + self.connection.execute( + """INSERT INTO documents VALUES( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + )""", + ( + document.id, + document.configured_source, + document.resolved_source, + document.source_host, + document.canonical_url, + document.canonical_host, + document.title, + document.description, + document.heading_path, + document.content, + document.content_hash, + vector.tobytes(), + vector.size, + ), + ) + self.connection.execute( + "INSERT INTO documents_fts VALUES(?, ?, ?, ?, ?, ?)", + ( + document.id, + document.title, + document.description, + document.heading_path, + document.content, + document.canonical_url, + ), + ) + + def mark_checked(self, source: str, checked_at: float, stale_at: float, error: str | None = None) -> None: + with self._lock, self.connection: + self.connection.execute( + "UPDATE sources SET checked_at=?, stale_at=?, last_error=? WHERE configured_source=?", + (checked_at, stale_at, error, source), + ) + + def list_sources(self, include_inactive: bool = False) -> list[SourceState]: + condition = "" if include_inactive else "WHERE s.active=1" + rows = self.connection.execute( + f"""SELECT s.*, COUNT(d.id) AS doc_count FROM sources s + LEFT JOIN documents d ON d.configured_source=s.configured_source + {condition} GROUP BY s.configured_source ORDER BY s.configured_source""" + ).fetchall() + return [self._source(row) for row in rows] + + def get_source(self, source: str) -> SourceState | None: + row = self.connection.execute( + """SELECT s.*, COUNT(d.id) AS doc_count FROM sources s + LEFT JOIN documents d ON d.configured_source=s.configured_source + WHERE s.configured_source=? GROUP BY s.configured_source""", + (source,), + ).fetchone() + return self._source(row) if row else None + + def get_document( + self, + identifier: str, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> StoredDocument | None: + where, parameters = self._filters(sources, hosts, alias="d") + row = self.connection.execute( + f"SELECT d.* FROM documents d JOIN sources s ON s.configured_source=d.configured_source " + f"WHERE s.active=1 AND d.id=? {where}", + [identifier, *parameters], + ).fetchone() + return self._document(row) if row else None + + def lexical_search( + self, + query: str, + limit: int, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> list[StoredDocument]: + terms = re.findall(r"[\w.-]+", query, flags=re.UNICODE) + if not terms: + return [] + expression = " AND ".join(f'"{term.replace(chr(34), chr(34) * 2)}"' for term in terms) + where, parameters = self._filters(sources, hosts, alias="d") + rows = self.connection.execute( + f"""SELECT d.* FROM documents_fts f + JOIN documents d ON d.id=f.doc_id + JOIN sources s ON s.configured_source=d.configured_source + WHERE documents_fts MATCH ? AND s.active=1 {where} + ORDER BY bm25(documents_fts, 0, 8, 3, 5, 1, 2) LIMIT ?""", + [expression, *parameters, limit], + ).fetchall() + return [self._document(row) for row in rows] + + def semantic_candidates( + self, + sources: list[str] | None = None, + hosts: list[str] | None = None, + ) -> list[StoredDocument]: + where, parameters = self._filters(sources, hosts, alias="d") + rows = self.connection.execute( + f"SELECT d.* FROM documents d JOIN sources s ON s.configured_source=d.configured_source " + f"WHERE s.active=1 {where}", + parameters, + ).fetchall() + return [self._document(row) for row in rows] + + @staticmethod + def _filters(sources: list[str] | None, hosts: list[str] | None, alias: str) -> tuple[str, list[str]]: + clauses: list[str] = [] + parameters: list[str] = [] + if sources: + clauses.append(f"{alias}.configured_source IN ({','.join('?' for _ in sources)})") + parameters.extend(sources) + if hosts: + clauses.append( + f"({alias}.source_host IN ({','.join('?' for _ in hosts)}) OR " + f"{alias}.canonical_host IN ({','.join('?' for _ in hosts)}))" + ) + parameters.extend(hosts) + parameters.extend(hosts) + return (" AND " + " AND ".join(clauses) if clauses else "", parameters) + + @staticmethod + def _source(row: sqlite3.Row) -> SourceState: + return SourceState( + configured_source=row["configured_source"], + resolved_source=row["resolved_source"], + active=bool(row["active"]), + etag=row["etag"], + last_modified=row["last_modified"], + body_hash=row["body_hash"], + raw_body=row["raw_body"], + parser_fingerprint=row["parser_fingerprint"], + embedding_fingerprint=row["embedding_fingerprint"], + checked_at=row["checked_at"], + indexed_at=row["indexed_at"], + stale_at=row["stale_at"], + last_error=row["last_error"], + doc_count=row["doc_count"], + ) + + @staticmethod + def _document(row: sqlite3.Row) -> StoredDocument: + return StoredDocument( + id=row["id"], + configured_source=row["configured_source"], + resolved_source=row["resolved_source"], + source_host=row["source_host"], + canonical_url=row["canonical_url"], + canonical_host=row["canonical_host"], + title=row["title"], + description=row["description"], + heading_path=row["heading_path"], + content=row["content"], + content_hash=row["content_hash"], + embedding=np.frombuffer(row["embedding"], dtype=np.float32, count=row["embedding_dim"]).copy(), + ) diff --git a/docker/docs/entrypoint.sh b/docker/docs/entrypoint.sh index a36526a..33f267f 100644 --- a/docker/docs/entrypoint.sh +++ b/docker/docs/entrypoint.sh @@ -1,9 +1,8 @@ #!/bin/sh # context-kit docs-mcp entrypoint. # -# Bridges llms-txt-mcp (stdio-only) to Streamable HTTP via mcp-proxy so that -# multiple clients share a single long-lived indexer instead of each spawning -# their own container (and racing on the same Chroma store). +# Starts the in-repo Streamable HTTP server. Multiple clients share one +# transactional SQLite/FTS index and one lazily loaded embedding model. # # Sources are read from $DOCS_MCP_SOURCES_FILE (one URL per line; `#` comments # and blank lines are allowed). Everything else is configured via env vars @@ -52,19 +51,9 @@ import http.server import sys -class LocalSourceHandler(http.server.SimpleHTTPRequestHandler): - def send_head(self): - # llms-txt-mcp 0.2.0 treats 304 responses from local sources as fetch - # failures, so serve machine-local docs as plain 200 responses. - for header in ("If-Modified-Since", "If-None-Match"): - if header in self.headers: - del self.headers[header] - return super().send_head() - - port = int(sys.argv[1]) directory = sys.argv[2] -handler = functools.partial(LocalSourceHandler, directory=directory) +handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=directory) with http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) as server: server.serve_forever() PY @@ -93,32 +82,4 @@ PY fi fi -# By default llms-txt-mcp 0.2.0 re-embeds every source on launch (the actual -# default is a background preindex, --no-preindex only disables the foreground -# variant). On a long-lived container that wastes CPU per restart, so we disable -# BOTH. Missing/stale sources still refresh on first docs_query/docs_refresh. -# Set DOCS_MCP_PREINDEX=1 to restore eager startup indexing. -preindex_flag="--no-preindex --no-background-preindex" -if [ "${DOCS_MCP_PREINDEX:-0}" = "1" ]; then - preindex_flag="" -fi - -allow_origin_args="" -if [ -n "${DOCS_MCP_ALLOW_ORIGIN:-}" ]; then - allow_origin_args="--allow-origin ${DOCS_MCP_ALLOW_ORIGIN}" -fi - -# shellcheck disable=SC2086 # intentional word-splitting on $sources / $preindex_flag / $allow_origin_args -exec mcp-proxy \ - --host "${DOCS_MCP_HTTP_HOST:-0.0.0.0}" \ - --port "${DOCS_MCP_HTTP_PORT:-8000}" \ - --pass-environment \ - $allow_origin_args \ - -- \ - llms-txt-mcp \ - --store-path /data \ - --ttl "${DOCS_MCP_TTL:-24h}" \ - --max-get-bytes "${DOCS_MCP_MAX_GET_BYTES:-75000}" \ - --embed-model "${DOCS_MCP_EMBED_MODEL:-BAAI/bge-small-en-v1.5}" \ - $preindex_flag \ - $sources +exec python -m context_docs diff --git a/docker/docs/tests/__init__.py b/docker/docs/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docker/docs/tests/fakes.py b/docker/docs/tests/fakes.py new file mode 100644 index 0000000..31e76f7 --- /dev/null +++ b/docker/docs/tests/fakes.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +import numpy as np + +from context_docs.models import FetchResponse + + +class FakeEmbedder: + fingerprint = "fake-embedder-v1" + ready = True + + async def ensure_ready(self) -> None: + return None + + async def encode_documents(self, texts: list[str]) -> np.ndarray: + return np.asarray([self._vector(text) for text in texts], dtype=np.float32) + + async def encode_query(self, text: str) -> np.ndarray: + return np.asarray(self._vector(text), dtype=np.float32) + + @staticmethod + def _vector(text: str) -> list[float]: + lower = text.lower() + return [ + float("api" in lower or "identifier" in lower), + float("persistence" in lower or "checkpoint" in lower), + float("background" in lower or "asynchronous" in lower), + 0.25 + (int(hashlib.sha256(text.encode()).hexdigest()[:2], 16) / 1024), + ] + + +@dataclass +class FakeFetch: + status: int + body: str = "" + final_url: str | None = None + etag: str | None = None + last_modified: str | None = None + + +class FakeFetcher: + def __init__(self, responses: list[FakeFetch]): + self.responses = list(responses) + self.calls = 0 + + async def fetch(self, source_url: str, state=None) -> FetchResponse: + self.calls += 1 + response = self.responses.pop(0) + return FetchResponse( + status=response.status, + requested_url=source_url, + resolved_url=response.final_url or source_url, + body=response.body, + etag=response.etag, + last_modified=response.last_modified, + ) diff --git a/docker/docs/tests/test_parser.py b/docker/docs/tests/test_parser.py new file mode 100644 index 0000000..6c57ea3 --- /dev/null +++ b/docker/docs/tests/test_parser.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import unittest + +from context_docs.parser import parse_llms_text + + +class ParserTest(unittest.TestCase): + def test_standard_menu_preserves_target_url_and_retrievable_content(self) -> None: + parsed = parse_llms_text( + """# Rails Docs + +> Curated official documentation. + +## Active Record + +- [Associations](https://guides.rubyonrails.org/association_basics.html): Model relationships +""", + "http://127.0.0.1:8769/rails/llms.txt", + ) + + self.assertEqual("standard-menu", parsed.format) + self.assertEqual(1, len(parsed.documents)) + document = parsed.documents[0] + self.assertEqual("https://guides.rubyonrails.org/association_basics.html", document.canonical_url) + self.assertIn("Model relationships", document.content) + self.assertIn("https://guides.rubyonrails.org/association_basics.html", document.content) + + def test_full_bundle_is_not_misclassified_by_interior_yaml_or_rule(self) -> None: + parsed = parse_llms_text( + """# Build a client + +Some content. + +--- +title: This is an embedded example +description: It is not file frontmatter +--- + +# Elicitation + +URL mode details. +""", + "https://example.test/llms-full.txt", + ) + + self.assertEqual("markdown-full", parsed.format) + self.assertEqual(["Build a client", "Elicitation"], [doc.title for doc in parsed.documents]) + + def test_full_bundle_with_bullet_links_keeps_prose_sections(self) -> None: + parsed = parse_llms_text( + """# Persistence + +This substantial section explains durable checkpoint behavior. + +- [Related guide](https://example.test/guide): Read more + +# Streaming + +Streaming emits incremental updates. +""", + "https://example.test/llms-full.txt", + ) + + self.assertEqual("markdown-full", parsed.format) + self.assertEqual(["Persistence", "Streaming"], [doc.title for doc in parsed.documents]) + self.assertIn("durable checkpoint", parsed.documents[0].content) + + def test_repeated_frontmatter_accepts_optional_description(self) -> None: + parsed = parse_llms_text( + """--- +title: First +--- +First body. +--- +title: Second +description: Second description +--- +Second body. +""", + "https://example.test/llms-full.txt", + ) + + self.assertEqual("yaml-full", parsed.format) + self.assertEqual(["First", "Second"], [doc.title for doc in parsed.documents]) + self.assertEqual("Second description", parsed.documents[1].description) + + def test_long_sections_are_split_without_losing_tail_identifiers(self) -> None: + body = "Paragraph.\n\n" * 100 + "IMMICH_IGNORE_MOUNT_CHECK_ERRORS disables mount checks." + parsed = parse_llms_text( + f"# Environment Variables\n\n{body}", + "https://example.test/llms-full.txt", + max_chunk_chars=500, + ) + + self.assertGreater(len(parsed.documents), 1) + self.assertTrue(any("IMMICH_IGNORE_MOUNT_CHECK_ERRORS" in doc.content for doc in parsed.documents)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docs/tests/test_refresh.py b/docker/docs/tests/test_refresh.py new file mode 100644 index 0000000..fbffd2c --- /dev/null +++ b/docker/docs/tests/test_refresh.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import asyncio +import tempfile +import unittest +from pathlib import Path + +from context_docs.parser import parse_llms_text +from context_docs.refresh import RefreshCoordinator +from context_docs.store import IndexStore + +from .fakes import FakeEmbedder, FakeFetch, FakeFetcher + + +class RefreshTest(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.source = "https://example.test/llms.txt" + self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3") + self.store.configure_sources([self.source]) + + async def asyncTearDown(self) -> None: + self.store.close() + self.tmp.cleanup() + + async def test_concurrent_refresh_uses_one_fetch_and_one_publication(self) -> None: + fetcher = FakeFetcher([FakeFetch(200, "# API Identifier\n\nExact identifier content.")]) + coordinator = RefreshCoordinator( + store=self.store, + fetcher=fetcher, + embedder=FakeEmbedder(), + parser=parse_llms_text, + ttl_seconds=3600, + now=lambda: 100.0, + ) + + first, second = await asyncio.gather( + coordinator.refresh(self.source, force=True), + coordinator.refresh(self.source, force=True), + ) + + self.assertEqual(1, fetcher.calls) + self.assertEqual("updated", first.status) + self.assertEqual("updated", second.status) + self.assertEqual(1, self.store.list_sources()[0].doc_count) + + async def test_304_updates_check_time_without_replacing_documents(self) -> None: + fetcher = FakeFetcher( + [ + FakeFetch(200, "# API Identifier\n\nOriginal content.", etag='"v1"'), + FakeFetch(304, etag='"v1"'), + ] + ) + clock = iter([100.0, 200.0]) + coordinator = RefreshCoordinator( + store=self.store, + fetcher=fetcher, + embedder=FakeEmbedder(), + parser=parse_llms_text, + ttl_seconds=3600, + now=lambda: next(clock), + ) + await coordinator.refresh(self.source, force=True) + original = self.store.list_sources()[0] + await coordinator.refresh(self.source, force=True) + checked = self.store.list_sources()[0] + + self.assertEqual(original.indexed_at, checked.indexed_at) + self.assertEqual(200.0, checked.checked_at) + self.assertEqual(1, checked.doc_count) + + async def test_refresh_error_preserves_searchable_previous_content(self) -> None: + fetcher = FakeFetcher( + [ + FakeFetch(200, "# API Identifier\n\nOriginal content."), + FakeFetch(500), + ] + ) + coordinator = RefreshCoordinator( + store=self.store, + fetcher=fetcher, + embedder=FakeEmbedder(), + parser=parse_llms_text, + ttl_seconds=3600, + now=lambda: 100.0, + ) + await coordinator.refresh(self.source, force=True) + failed = await coordinator.refresh(self.source, force=True) + + self.assertEqual("error", failed.status) + self.assertEqual(1, self.store.list_sources()[0].doc_count) + self.assertTrue(self.store.lexical_search("Original", limit=5)) + + async def test_empty_success_response_preserves_previous_content(self) -> None: + fetcher = FakeFetcher( + [ + FakeFetch(200, "# API Identifier\n\nOriginal content."), + FakeFetch(200, ""), + ] + ) + coordinator = RefreshCoordinator( + store=self.store, + fetcher=fetcher, + embedder=FakeEmbedder(), + parser=parse_llms_text, + ttl_seconds=3600, + now=lambda: 100.0, + ) + await coordinator.refresh(self.source, force=True) + failed = await coordinator.refresh(self.source, force=True) + + self.assertEqual("error", failed.status) + self.assertIn("zero documents", failed.detail) + self.assertTrue(self.store.lexical_search("Original", limit=5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docs/tests/test_search.py b/docker/docs/tests/test_search.py new file mode 100644 index 0000000..bf5ecfd --- /dev/null +++ b/docker/docs/tests/test_search.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +from context_docs.models import PreparedDocument, SourceUpdate +from context_docs.search import HybridSearch +from context_docs.store import IndexStore + +from .fakes import FakeEmbedder + + +def prepared(identifier: str, source: str, canonical: str, title: str, content: str, vector) -> PreparedDocument: + return PreparedDocument( + id=identifier, + configured_source=source, + resolved_source=source, + source_host="source.test", + canonical_url=canonical, + canonical_host=canonical.split("/")[2], + title=title, + description="", + heading_path=title, + content=content, + content_hash=__import__("hashlib").sha256(content.encode()).hexdigest(), + embedding=np.asarray(vector, dtype=np.float32), + ) + + +def source_update(source: str, documents: list[PreparedDocument]) -> SourceUpdate: + return SourceUpdate( + configured_source=source, + resolved_source=source, + etag=None, + last_modified=None, + body_hash="body", + raw_body="# source", + parser_fingerprint="parser-v1", + embedding_fingerprint="fake-embedder-v1", + checked_at=1.0, + indexed_at=1.0, + stale_at=9999.0, + documents=documents, + ) + + +class HybridSearchTest(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3") + self.a = "https://a.test/llms.txt" + self.b = "https://b.test/llms.txt" + self.store.configure_sources([self.a, self.b]) + duplicate = "Shared exact content." + self.store.replace_source( + source_update( + self.a, + [ + prepared("exact", self.a, "https://rails.test/exact", "Environment", "IMMICH_IGNORE_MOUNT_CHECK_ERRORS identifier", [1, 0, 0, 0]), + prepared("duplicate-a", self.a, "https://docs.test/shared", "Shared", duplicate, [0, 1, 0, 0]), + ], + ) + ) + self.store.replace_source( + source_update( + self.b, + [ + prepared("persistence", self.b, "https://langgraph.test/persistence", "Persistence", "Durable checkpoint state", [0, 1, 0, 0]), + prepared("duplicate-b", self.b, "https://docs.test/shared-copy", "Shared copy", duplicate, [0, 1, 0, 0]), + ], + ) + ) + self.search = HybridSearch(self.store, FakeEmbedder()) + + async def asyncTearDown(self) -> None: + self.store.close() + self.tmp.cleanup() + + async def test_exact_identifier_is_ranked_first(self) -> None: + result = await self.search.search("IMMICH_IGNORE_MOUNT_CHECK_ERRORS", limit=5) + self.assertEqual("exact", result[0].id) + self.assertEqual(1, result[0].lexical_rank) + + async def test_source_and_host_filters_apply_before_ranking(self) -> None: + by_source = await self.search.search("persistence", limit=5, sources=[self.b]) + by_host = await self.search.search("identifier", limit=5, hosts=["rails.test"]) + + self.assertTrue(by_source) + self.assertTrue(all(item.configured_source == self.b for item in by_source)) + self.assertEqual(["exact"], [item.id for item in by_host]) + + async def test_exact_duplicate_content_is_collapsed_with_alternates(self) -> None: + result = await self.search.search("Shared exact content", limit=10) + shared = [item for item in result if item.content_hash == __import__("hashlib").sha256("Shared exact content.".encode()).hexdigest()] + + self.assertEqual(1, len(shared)) + self.assertEqual(2, shared[0].duplicate_count) + self.assertEqual(1, len(shared[0].alternate_sources)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docs/tests/test_server.py b/docker/docs/tests/test_server.py new file mode 100644 index 0000000..6b2c45b --- /dev/null +++ b/docker/docs/tests/test_server.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from starlette.testclient import TestClient + +from context_docs.server import build_server, parse_duration, read_sources + + +class ServerTest(unittest.TestCase): + def test_duration_parser_rejects_ambiguous_values(self) -> None: + self.assertEqual(86_400, parse_duration("24h")) + with self.assertRaises(ValueError): + parse_duration("tomorrow") + + def test_source_file_requires_supported_urls(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "sources.txt" + path.write_text("https://example.test/index.html\n") + with self.assertRaisesRegex(ValueError, "must end"): + read_sources(path) + + def test_status_is_available_without_loading_embedding_model(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source_file = Path(directory) / "sources.txt" + source_file.write_text("https://example.test/llms.txt\n") + environment = { + "DOCS_MCP_SOURCES_FILE": str(source_file), + "DOCS_MCP_STORE_PATH": str(Path(directory) / "docs.sqlite3"), + "DOCS_MCP_PREINDEX": "0", + } + with patch.dict(os.environ, environment, clear=False): + with TestClient(build_server()) as client: + response = client.get("/status") + + self.assertEqual(200, response.status_code) + self.assertTrue(response.json()["ready"]) + self.assertFalse(response.json()["model_ready"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docs/tests/test_service.py b/docker/docs/tests/test_service.py new file mode 100644 index 0000000..23d951c --- /dev/null +++ b/docker/docs/tests/test_service.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +from context_docs.models import PreparedDocument, RefreshOutcome, SourceUpdate +from context_docs.search import HybridSearch +from context_docs.service import DocsService +from context_docs.store import IndexStore + +from .fakes import FakeEmbedder + + +class NoopRefresh: + async def refresh(self, source: str, force: bool = False) -> RefreshOutcome: + return RefreshOutcome(source, "fresh", 1) + + +class ServiceTest(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.store = IndexStore(Path(self.tmp.name) / "docs.sqlite3") + self.source = "https://example.test/llms.txt" + self.store.configure_sources([self.source]) + content = "IMMICH_IGNORE_MOUNT_CHECK_ERRORS " + "x" * 200 + item = PreparedDocument( + id="exact", + configured_source=self.source, + resolved_source=self.source, + source_host="example.test", + canonical_url="https://docs.example.test/environment", + canonical_host="docs.example.test", + title="Environment", + description="", + heading_path="Environment", + content=content, + content_hash="content-hash", + embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32), + ) + self.store.replace_source( + SourceUpdate( + configured_source=self.source, + resolved_source=self.source, + etag=None, + last_modified=None, + body_hash="body", + raw_body="# body", + parser_fingerprint="context-docs-parser-v1", + embedding_fingerprint="fake-embedder-v1", + checked_at=1.0, + indexed_at=1.0, + stale_at=9_999_999_999.0, + documents=[item], + ) + ) + embedder = FakeEmbedder() + self.service = DocsService( + self.store, + HybridSearch(self.store, embedder), + NoopRefresh(), + max_get_bytes=100, + ) + + async def asyncTearDown(self) -> None: + self.store.close() + self.tmp.cleanup() + + async def test_query_does_not_retrieve_content_by_default(self) -> None: + response = await self.service.query("IMMICH_IGNORE_MOUNT_CHECK_ERRORS") + + self.assertEqual({}, response["retrieved_content"]) + self.assertEqual("exact", response["search_results"][0]["id"]) + + async def test_explicit_retrieval_respects_global_byte_cap(self) -> None: + response = await self.service.query( + "IMMICH_IGNORE_MOUNT_CHECK_ERRORS", + retrieve_ids=["exact"], + max_bytes=10_000, + ) + retrieved = response["retrieved_content"]["exact"] + + self.assertLessEqual(len(retrieved["content"].encode()), 100) + self.assertTrue(retrieved["truncated"]) + + async def test_high_default_threshold_only_retrieves_strong_hybrid_match(self) -> None: + response = await self.service.query( + "IMMICH_IGNORE_MOUNT_CHECK_ERRORS", + auto_retrieve=True, + ) + + self.assertIn("exact", response["retrieved_content"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/docs/tests/test_store.py b/docker/docs/tests/test_store.py new file mode 100644 index 0000000..525e8ff --- /dev/null +++ b/docker/docs/tests/test_store.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +from context_docs.models import PreparedDocument, SourceUpdate +from context_docs.store import IndexStore + + +def document(identifier: str, source: str, title: str, content: str) -> PreparedDocument: + return PreparedDocument( + id=identifier, + configured_source=source, + resolved_source=source, + source_host="example.test", + canonical_url=f"https://docs.example.test/{identifier}", + canonical_host="docs.example.test", + title=title, + description="", + heading_path=title, + content=content, + content_hash=identifier, + embedding=np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32), + ) + + +def update(source: str, documents: list[PreparedDocument], checked_at: float = 100.0) -> SourceUpdate: + return SourceUpdate( + configured_source=source, + resolved_source=source, + etag=None, + last_modified=None, + body_hash="body-hash", + raw_body="# Fixture", + parser_fingerprint="parser-v1", + embedding_fingerprint="fake-embedder-v1", + checked_at=checked_at, + indexed_at=checked_at, + stale_at=checked_at + 3600, + documents=documents, + ) + + +class StoreTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.path = Path(self.tmp.name) / "docs.sqlite3" + self.source = "https://example.test/llms.txt" + self.store = IndexStore(self.path) + self.store.configure_sources([self.source]) + + def tearDown(self) -> None: + self.store.close() + self.tmp.cleanup() + + def test_replacement_removes_old_only_documents(self) -> None: + self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")])) + self.store.replace_source(update(self.source, [document("new", self.source, "New", "new content")])) + + self.assertIsNone(self.store.get_document("old")) + self.assertEqual("new content", self.store.get_document("new").content) + + def test_failed_replacement_rolls_back_to_previous_generation(self) -> None: + self.store.replace_source(update(self.source, [document("old", self.source, "Old", "old content")])) + self.store.connection.execute( + "CREATE TRIGGER reject_failure BEFORE INSERT ON documents " + "WHEN NEW.title = 'FAIL' BEGIN SELECT RAISE(ABORT, 'injected failure'); END" + ) + + with self.assertRaisesRegex(Exception, "injected failure"): + self.store.replace_source(update(self.source, [document("bad", self.source, "FAIL", "bad")])) + + self.assertEqual("old content", self.store.get_document("old").content) + self.assertIsNone(self.store.get_document("bad")) + + def test_restart_loads_persisted_state_without_network(self) -> None: + self.store.replace_source(update(self.source, [document("persisted", self.source, "Persisted", "saved")])) + self.store.close() + + self.store = IndexStore(self.path) + self.store.configure_sources([self.source]) + + states = self.store.list_sources() + self.assertEqual(1, states[0].doc_count) + self.assertEqual("saved", self.store.get_document("persisted").content) + + def test_removed_source_is_not_searchable_or_retrievable(self) -> None: + self.store.replace_source(update(self.source, [document("retired", self.source, "Retired", "identifier")])) + self.store.configure_sources([]) + + self.assertIsNone(self.store.get_document("retired")) + self.assertEqual([], self.store.lexical_search("identifier", limit=5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docker/web-search/Dockerfile b/docker/web-search/Dockerfile index 86b7922..5e65bc2 100644 --- a/docker/web-search/Dockerfile +++ b/docker/web-search/Dockerfile @@ -4,10 +4,6 @@ ARG MCP_WEB_SEARCH_VERSION=1.3.0 ARG MCP_WEB_SEARCH_MAX_BYTES=52428800 ARG MCP_PROXY_VERSION=0.12.0 -COPY patch-mcp-web-search.mjs /tmp/patch-mcp-web-search.mjs -COPY overrides/bing.js /tmp/context-kit-bing-provider.js -COPY --chmod=0444 mcp-probe.mjs http-entrypoint.mjs /usr/local/lib/context-kit/ - # Chromium intentionally tracks Debian security updates inside the pinned base # image family; Bing's browser path is more likely to break with stale Chromium # than with patched OS packages. @@ -23,13 +19,27 @@ RUN python3 -m venv /opt/mcp-proxy \ && /opt/mcp-proxy/bin/pip install --no-cache-dir "mcp-proxy==${MCP_PROXY_VERSION}" \ && /opt/mcp-proxy/bin/mcp-proxy --version +COPY patch-mcp-web-search.mjs /tmp/patch-mcp-web-search.mjs +COPY overrides/bing.js /tmp/context-kit-bing-provider.js +COPY overrides/brave.js overrides/duckduckgo.js overrides/searxng.js overrides/registry.js overrides/diagnostics.mjs /tmp/context-kit-providers/ +COPY overrides/browser-fetch.js overrides/bounds.mjs /tmp/context-kit-fetch/ +COPY --chmod=0444 mcp-probe.mjs http-entrypoint.mjs /usr/local/lib/context-kit/ + RUN npm install -g "@zhafron/mcp-web-search@${MCP_WEB_SEARCH_VERSION}" \ && cp /tmp/context-kit-bing-provider.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/bing.js \ + && cp /tmp/context-kit-providers/brave.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/brave.js \ + && cp /tmp/context-kit-providers/duckduckgo.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/duckduckgo.js \ + && cp /tmp/context-kit-providers/searxng.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/searxng.js \ + && cp /tmp/context-kit-providers/registry.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/registry.js \ + && cp /tmp/context-kit-providers/diagnostics.mjs /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/providers/diagnostics.js \ + && cp /tmp/context-kit-fetch/browser-fetch.js /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/browser.js \ + && cp /tmp/context-kit-fetch/bounds.mjs /usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/bounds.js \ && node /tmp/patch-mcp-web-search.mjs \ - && rm /tmp/patch-mcp-web-search.mjs /tmp/context-kit-bing-provider.js \ + && rm -rf /tmp/patch-mcp-web-search.mjs /tmp/context-kit-bing-provider.js /tmp/context-kit-providers /tmp/context-kit-fetch \ && npm cache clean --force -RUN chmod 0555 /usr/local/lib/context-kit +RUN chmod -R a+rX /usr/local/lib/context-kit \ + /usr/local/lib/node_modules/@zhafron/mcp-web-search ENV CHROME_PATH=/usr/bin/chromium \ DEFAULT_SEARCH_PROVIDER=searxng \ @@ -37,6 +47,8 @@ ENV CHROME_PATH=/usr/bin/chromium \ HTTP_TIMEOUT=15000 \ MAX_BYTES=${MCP_WEB_SEARCH_MAX_BYTES} \ MAX_RESULTS=10 \ + MAX_PROVIDER_ATTEMPTS=4 \ + SEARCH_PROVIDER_TIMEOUT_MS=15000 \ PATH=/opt/mcp-proxy/bin:$PATH \ SEARXNG_URL=http://searxng:8080 \ XDG_CACHE_HOME=/tmp/.cache diff --git a/docker/web-search/mcp-probe.mjs b/docker/web-search/mcp-probe.mjs index 86dcb15..022460e 100644 --- a/docker/web-search/mcp-probe.mjs +++ b/docker/web-search/mcp-probe.mjs @@ -3,7 +3,7 @@ import { fileURLToPath } from "node:url"; const protocolVersion = "2024-11-05"; const expectedTools = ["fetch_url", "search_web"]; -async function rpc(url, id, method, params = {}, timeoutMs = 5000) { +export async function rpc(url, id, method, params = {}, timeoutMs = 5000) { const response = await fetch(url, { method: "POST", headers: { @@ -29,7 +29,7 @@ async function rpc(url, id, method, params = {}, timeoutMs = 5000) { return payload.result; } -export async function probeMcp(url, { timeoutMs = 5000 } = {}) { +export async function probeMcp(url, { timeoutMs = 5000, expectedTools: requiredTools = expectedTools } = {}) { const initialized = await rpc(url, 1, "initialize", { protocolVersion, capabilities: {}, @@ -39,7 +39,7 @@ export async function probeMcp(url, { timeoutMs = 5000 } = {}) { const listed = await rpc(url, 2, "tools/list", {}, timeoutMs); const names = new Set((listed?.tools || []).map(tool => tool.name)); - for (const name of expectedTools) { + for (const name of requiredTools) { if (!names.has(name)) throw new Error(`tools/list omitted ${name}`); } return Array.from(names).sort(); diff --git a/docker/web-search/overrides/bing.js b/docker/web-search/overrides/bing.js index 5c66901..ad4b018 100644 --- a/docker/web-search/overrides/bing.js +++ b/docker/web-search/overrides/bing.js @@ -43,7 +43,7 @@ export class BingProvider { } } - async search(q, limit, lang) { + async search(q, limit, lang, signal) { const cacheKey = createCacheKey("bing", q, limit, lang); const cached = searchCache.get(cacheKey); if (cached) @@ -51,7 +51,10 @@ export class BingProvider { const market = getMarketFromLang(lang); const results = await browserPool.withBrowser(async (browser) => { const page = await browser.newPage(); + const abort = () => void page.close().catch(() => undefined); + signal?.addEventListener("abort", abort, { once: true }); try { + signal?.throwIfAborted(); await page.setViewport({ width: 1365, height: 768 }); await page.setUserAgent(DEFAULT_BROWSER_SEARCH_USER_AGENT); await page.setExtraHTTPHeaders(getAcceptLanguageHeader(lang)); @@ -95,7 +98,8 @@ export class BingProvider { }); } finally { - await page.close(); + signal?.removeEventListener("abort", abort); + if (!page.isClosed()) await page.close(); } }); searchCache.set(cacheKey, results); diff --git a/docker/web-search/overrides/bounds.mjs b/docker/web-search/overrides/bounds.mjs new file mode 100644 index 0000000..bffcdeb --- /dev/null +++ b/docker/web-search/overrides/bounds.mjs @@ -0,0 +1,23 @@ +const MAX_LINKS = 500; +const MAX_IMAGES = 200; +const MAX_VIDEO = 50; +const MAX_AUDIO = 50; +const MAX_ATTACHMENTS = 10; + +export function boundFetchCollections(result) { + const warnings = [...(result.warnings || [])]; + const trim = (value, maximum, label) => { + if (!Array.isArray(value)) return value; + if (value.length > maximum) warnings.push(`${label} truncated from ${value.length} to ${maximum}`); + return value.slice(0, maximum); + }; + if (result.links) result.links = trim(result.links, MAX_LINKS, "links"); + if (result.media) { + result.media.images = trim(result.media.images, MAX_IMAGES, "images"); + result.media.videos = trim(result.media.videos, MAX_VIDEO, "videos"); + result.media.audio = trim(result.media.audio, MAX_AUDIO, "audio"); + } + if (result.attachments) result.attachments = trim(result.attachments, MAX_ATTACHMENTS, "attachments"); + result.warnings = warnings; + return result; +} diff --git a/docker/web-search/overrides/brave.js b/docker/web-search/overrides/brave.js new file mode 100644 index 0000000..eed9867 --- /dev/null +++ b/docker/web-search/overrides/brave.js @@ -0,0 +1,42 @@ +import { HTTP_TIMEOUT } from "../constants.js"; +import { searchCache, createCacheKey } from "../utils/cache.js"; + +export class BraveProvider { + name = "brave"; + configured = Boolean(process.env.BRAVE_SEARCH_API_KEY); + + async search(q, limit, lang, signal) { + if (!this.configured) return []; + const cacheKey = createCacheKey("brave", q, limit, lang); + const cached = searchCache.get(cacheKey); + if (cached) return cached; + const url = new URL("https://api.search.brave.com/res/v1/web/search"); + url.searchParams.set("q", q); + url.searchParams.set("count", String(Math.min(limit, 20))); + url.searchParams.set("search_lang", lang.split(/[-_]/)[0] || "en"); + const response = await fetch(url, { + headers: { + Accept: "application/json", + "X-Subscription-Token": process.env.BRAVE_SEARCH_API_KEY + }, + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(HTTP_TIMEOUT)]) : AbortSignal.timeout(HTTP_TIMEOUT) + }); + if (!response.ok) throw new Error(`Brave HTTP ${response.status}`); + const data = await response.json(); + const items = (data.web?.results || []).slice(0, limit).flatMap(result => { + if (!result.title || !result.url) return []; + return [{ + title: result.title, + url: result.url, + snippet: result.description || undefined, + source: "brave" + }]; + }); + searchCache.set(cacheKey, items); + return items; + } + + async isAvailable() { + return this.configured; + } +} diff --git a/docker/web-search/overrides/browser-fetch.js b/docker/web-search/overrides/browser-fetch.js new file mode 100644 index 0000000..c08e329 --- /dev/null +++ b/docker/web-search/overrides/browser-fetch.js @@ -0,0 +1,95 @@ +import { HTTP_TIMEOUT, MAX_BYTES } from "../constants.js"; +import { browserPool } from "../utils/browser-pool.js"; +import { assertSafeUrl } from "./security.js"; +import { fetchViaVettedAddress } from "./http.js"; + +const MAX_BROWSER_REQUESTS = 100; +const MAX_BROWSER_TOTAL_BYTES = Math.min(MAX_BYTES, 20 * 1024 * 1024); + +function responseHeaders(headers) { + const record = {}; + headers.forEach((value, key) => { record[key] = value; }); + return record; +} + +export async function fetchBrowserResource(url, timeoutMs = HTTP_TIMEOUT) { + await assertSafeUrl(url); + return browserPool.withBrowser(async browser => { + const page = await browser.newPage(); + const devtools = await page.target().createCDPSession(); + await devtools.send("Network.enable"); + await devtools.send("Network.setBlockedURLs", { + urls: ["ws://*", "wss://*", "file://*", "ftp://*"] + }); + await page.evaluateOnNewDocument(() => { + const blockedTransport = name => class { + constructor() { + throw new DOMException(`${name} is disabled by the safe browser fetcher`, "SecurityError"); + } + }; + for (const name of ["WebSocket", "WebTransport", "RTCPeerConnection", "webkitRTCPeerConnection"]) { + if (name in globalThis) { + Object.defineProperty(globalThis, name, { + configurable: false, + writable: false, + value: blockedTransport(name) + }); + } + } + }); + await page.setBypassServiceWorker(true); + let requests = 0; + let totalBytes = 0; + let blockedError; + await page.setRequestInterception(true); + page.on("request", request => { + void (async () => { + try { + const requestUrl = new URL(request.url()); + if (!["http:", "https:"].includes(requestUrl.protocol)) throw new Error("unsupported browser request scheme"); + if (request.method() !== "GET") throw new Error("browser fetch blocks non-GET requests"); + requests += 1; + if (requests > MAX_BROWSER_REQUESTS) throw new Error("browser request limit exceeded"); + await assertSafeUrl(requestUrl); + const upstream = await fetchViaVettedAddress(requestUrl, timeoutMs); + const body = Buffer.from(await upstream.arrayBuffer()); + totalBytes += body.byteLength; + if (totalBytes > MAX_BROWSER_TOTAL_BYTES) throw new Error("browser byte limit exceeded"); + await request.respond({ + status: upstream.status, + headers: responseHeaders(upstream.headers), + body + }); + } catch (error) { + blockedError ||= error; + await request.abort("blockedbyclient").catch(() => undefined); + } + })(); + }); + try { + const navigation = await page.goto(url.toString(), { + waitUntil: "networkidle2", + timeout: timeoutMs + }); + if (blockedError && !navigation) throw blockedError; + const finalUrl = new URL(page.url()); + await assertSafeUrl(finalUrl); + const html = await page.content(); + const buffer = Buffer.from(html); + if (buffer.byteLength > MAX_BYTES) throw new Error("rendered content too large"); + const headers = new Headers({ "content-type": "text/html; charset=utf-8" }); + const status = navigation?.status() || 200; + const response = new Response(new Uint8Array(buffer), { status, headers }); + Object.defineProperty(response, "url", { value: finalUrl.toString() }); + return { + response, + finalUrl: finalUrl.toString(), + contentType: headers.get("content-type"), + buffer, + byteLength: buffer.byteLength + }; + } finally { + await page.close(); + } + }); +} diff --git a/docker/web-search/overrides/diagnostics.mjs b/docker/web-search/overrides/diagnostics.mjs new file mode 100644 index 0000000..02e3acf --- /dev/null +++ b/docker/web-search/overrides/diagnostics.mjs @@ -0,0 +1,63 @@ +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_ERROR_LENGTH = 240; + +export function classifyProviderError(error) { + const message = error instanceof Error ? error.message : String(error); + const lower = message.toLowerCase(); + let category = "provider_error"; + if (lower.includes("timed out") || lower.includes("timeout")) category = "timeout"; + else if (lower.includes("429") || lower.includes("rate limit")) category = "rate_limited"; + else if (lower.includes("captcha") || lower.includes("challenge")) category = "blocked"; + else if (lower.includes("403") || lower.includes("401") || lower.includes("denied")) category = "forbidden"; + else if (lower.includes("network") || lower.includes("fetch") || lower.includes("socket")) category = "network"; + return { category, message: message.replace(/\s+/g, " ").slice(0, MAX_ERROR_LENGTH) }; +} + +export async function attemptProvider(provider, query, limit, lang, options = {}) { + const timeoutMs = Math.max(10, Math.min(options.timeoutMs || DEFAULT_TIMEOUT_MS, 60_000)); + const now = options.now || (() => performance.now()); + const started = now(); + if (provider.configured === false) { + return { + items: [], + diagnostic: { provider: provider.name, status: "unavailable", duration_ms: 0, result_count: 0 } + }; + } + let timer; + const controller = new AbortController(); + try { + const items = await Promise.race([ + provider.search(query, limit, lang, controller.signal), + new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(new Error(`provider timed out after ${timeoutMs}ms`)); + reject(new Error(`provider timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }) + ]); + const bounded = Array.isArray(items) ? items.slice(0, limit) : []; + return { + items: bounded, + diagnostic: { + provider: provider.name, + status: bounded.length ? "success" : "empty", + duration_ms: Math.max(0, Math.round(now() - started)), + result_count: bounded.length + } + }; + } catch (error) { + return { + items: [], + diagnostic: { + provider: provider.name, + status: "error", + duration_ms: Math.max(0, Math.round(now() - started)), + result_count: 0, + error: classifyProviderError(error) + } + }; + } finally { + controller.abort(); + clearTimeout(timer); + } +} diff --git a/docker/web-search/overrides/duckduckgo.js b/docker/web-search/overrides/duckduckgo.js new file mode 100644 index 0000000..f29803b --- /dev/null +++ b/docker/web-search/overrides/duckduckgo.js @@ -0,0 +1,56 @@ +import { JSDOM } from "jsdom"; +import { HTTP_TIMEOUT } from "../constants.js"; +import { fetchWithTimeout } from "../utils/http.js"; +import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js"; +import { searchCache, createCacheKey } from "../utils/cache.js"; + +export class DuckDuckGoProvider { + name = "duckduckgo"; + + decodeDuckDuckGoRedirect(href) { + try { + const url = new URL(href, "https://duckduckgo.com/"); + if (url.hostname === "duckduckgo.com" && url.pathname.startsWith("/l/")) { + const target = url.searchParams.get("uddg"); + if (target) return decodeURIComponent(target); + } + return url.toString(); + } catch { + return href; + } + } + + async search(q, limit, lang, signal) { + const cacheKey = createCacheKey("ddg", q, limit, lang); + const cached = searchCache.get(cacheKey); + if (cached) return cached; + const url = new URL("https://html.duckduckgo.com/html/"); + url.searchParams.set("q", q); + const headers = { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) }; + const response = await fetchWithTimeout(url, { headers, signal }, HTTP_TIMEOUT); + if (!response.ok) throw new Error(`DuckDuckGo HTML ${response.status}`); + const dom = new JSDOM(await response.text(), { url: `https://duckduckgo.com/?q=${encodeURIComponent(q)}` }); + const anchors = Array.from(dom.window.document.querySelectorAll("a.result__a")); + const snippets = Array.from(dom.window.document.querySelectorAll(".result__snippet")); + const items = []; + for (let index = 0; index < anchors.length && items.length < limit; index += 1) { + const title = (anchors[index].textContent || "").trim(); + const href = this.decodeDuckDuckGoRedirect(anchors[index].getAttribute("href") || ""); + if (!title || !href) continue; + try { + items.push({ + title, + url: new URL(href).toString(), + snippet: (snippets[index]?.textContent || "").trim() || undefined, + source: "duckduckgo" + }); + } catch {} + } + searchCache.set(cacheKey, items); + return items; + } + + async isAvailable() { + return true; + } +} diff --git a/docker/web-search/overrides/registry.js b/docker/web-search/overrides/registry.js new file mode 100644 index 0000000..b3c72da --- /dev/null +++ b/docker/web-search/overrides/registry.js @@ -0,0 +1,65 @@ +import { DuckDuckGoProvider } from "./duckduckgo.js"; +import { BingProvider } from "./bing.js"; +import { SearXNGProvider } from "./searxng.js"; +import { BraveProvider } from "./brave.js"; +import { DEFAULT_SEARCH_PROVIDER } from "../constants.js"; +import { attemptProvider } from "./diagnostics.js"; + +const PROVIDERS = ["searxng", "brave", "duckduckgo", "bing"]; +const PROVIDER_TIMEOUT_MS = Number(process.env.SEARCH_PROVIDER_TIMEOUT_MS || "15000"); +const MAX_PROVIDER_ATTEMPTS = Math.max(1, Math.min(Number(process.env.MAX_PROVIDER_ATTEMPTS || "4"), 4)); + +export class ProviderRegistry { + constructor(providers) { + this.providers = providers || new Map([ + ["duckduckgo", new DuckDuckGoProvider()], + ["bing", new BingProvider()], + ["searxng", new SearXNGProvider()], + ["brave", new BraveProvider()] + ]); + } + + get(name) { + return this.providers.get(name); + } + + async searchWithFallback(q, limit, lang, preferredProvider) { + const defaultProvider = preferredProvider || DEFAULT_SEARCH_PROVIDER; + const order = [defaultProvider, ...PROVIDERS.filter(name => name !== defaultProvider)].slice(0, MAX_PROVIDER_ATTEMPTS); + const attempts = []; + const started = performance.now(); + for (const providerName of order) { + const provider = this.providers.get(providerName); + if (!provider) continue; + const attempt = await attemptProvider(provider, q, limit, lang, { timeoutMs: PROVIDER_TIMEOUT_MS }); + attempts.push(attempt.diagnostic); + if (attempt.items.length) { + return { + items: attempt.items, + providerUsed: providerName, + fallbackUsed: providerName !== defaultProvider, + triedProviders: attempts.map(item => item.provider), + diagnostics: { + attempts, + elapsed_ms: Math.round(performance.now() - started), + exhausted: false + } + }; + } + } + return { + items: [], + providerUsed: defaultProvider, + fallbackUsed: attempts.length > 1, + triedProviders: attempts.map(item => item.provider), + diagnostics: { + attempts, + elapsed_ms: Math.round(performance.now() - started), + exhausted: true + } + }; + } +} + +export const providerRegistry = new ProviderRegistry(); +export { DuckDuckGoProvider, BingProvider, SearXNGProvider, BraveProvider }; diff --git a/docker/web-search/overrides/searxng.js b/docker/web-search/overrides/searxng.js new file mode 100644 index 0000000..a1ea602 --- /dev/null +++ b/docker/web-search/overrides/searxng.js @@ -0,0 +1,47 @@ +import { HTTP_TIMEOUT, SEARXNG_URL } from "../constants.js"; +import { fetchWithTimeout } from "../utils/http.js"; +import { getRandomUserAgent, getAcceptLanguageHeader } from "../utils/user-agent.js"; +import { searchCache, createCacheKey } from "../utils/cache.js"; + +export class SearXNGProvider { + name = "searxng"; + + constructor(instanceUrl) { + this.instanceUrl = instanceUrl || SEARXNG_URL; + } + + async search(q, limit, lang, signal) { + const cacheKey = createCacheKey("searxng", q, limit, lang); + const cached = searchCache.get(cacheKey); + if (cached) return cached; + const params = new URLSearchParams({ q, format: "json", language: lang, safesearch: "0" }); + const response = await fetchWithTimeout(`${this.instanceUrl}/search?${params}`, { + headers: { "User-Agent": getRandomUserAgent(), ...getAcceptLanguageHeader(lang) }, + signal + }, HTTP_TIMEOUT); + if (!response.ok) { + if (response.status === 403) throw new Error("SearXNG JSON API disabled"); + throw new Error(`SearXNG error: ${response.status}`); + } + const data = await response.json(); + const items = (data.results || []).slice(0, limit).map(result => ({ + title: result.title || "", + url: result.url || "", + snippet: result.content || undefined, + source: "searxng" + })); + searchCache.set(cacheKey, items); + return items; + } + + async isAvailable() { + try { + const response = await fetchWithTimeout(`${this.instanceUrl}/search?q=test&format=json`, { + headers: { Accept: "application/json", "User-Agent": getRandomUserAgent() } + }, 5000); + return response.ok; + } catch { + return false; + } + } +} diff --git a/docker/web-search/patch-mcp-web-search.mjs b/docker/web-search/patch-mcp-web-search.mjs index f5b4f8d..2cdb035 100644 --- a/docker/web-search/patch-mcp-web-search.mjs +++ b/docker/web-search/patch-mcp-web-search.mjs @@ -15,6 +15,14 @@ const replacements = [ [ "max_download_bytes: z.number().int().min(1).max(26214400).optional()", "max_download_bytes: z.number().int().min(1).max(MAX_BYTES).optional()" + ], + [ + 'provider: z.enum(["duckduckgo", "bing", "searxng"]).optional()', + 'provider: z.enum(["duckduckgo", "bing", "searxng", "brave"]).optional()' + ], + [ + "Search the web using multiple providers (DuckDuckGo, Bing, SearXNG). Automatically falls back to other providers if the default fails. No API keys required for DuckDuckGo and SearXNG.", + "Search the web with bounded provider fallback and per-attempt diagnostics. SearXNG is local; Brave is available when BRAVE_SEARCH_API_KEY is configured." ] ]; @@ -26,3 +34,44 @@ for (const [before, after] of replacements) { } fs.writeFileSync(serverPath, source); + +const httpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/fetch/http.js"; +let httpSource = fs.readFileSync(httpPath, "utf8"); +const privateTransport = "async function fetchViaVettedAddress(url, timeoutMs)"; +if (!httpSource.includes(privateTransport)) throw new Error(`mcp-web-search patch target not found: ${privateTransport}`); +httpSource = httpSource.replace(privateTransport, "export async function fetchViaVettedAddress(url, timeoutMs)"); +fs.writeFileSync(httpPath, httpSource); + +const utilityHttpPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/utils/http.js"; +let utilityHttpSource = fs.readFileSync(utilityHttpPath, "utf8"); +const uncombinedSignal = 'return await fetch(input, { ...init, signal: controller.signal });'; +const combinedSignal = 'const signal = init.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;\n return await fetch(input, { ...init, signal });'; +if (!utilityHttpSource.includes(uncombinedSignal)) throw new Error(`mcp-web-search patch target not found: ${uncombinedSignal}`); +utilityHttpSource = utilityHttpSource.replace(uncombinedSignal, combinedSignal); +fs.writeFileSync(utilityHttpPath, utilityHttpSource); + +const extractPath = "/usr/local/lib/node_modules/@zhafron/mcp-web-search/dist/src/extract.js"; +let extractSource = fs.readFileSync(extractPath, "utf8"); +const extractReplacements = [ + [ + 'import { assertSafeUrl } from "./fetch/security.js";', + 'import { assertSafeUrl } from "./fetch/security.js";\nimport { fetchBrowserResource } from "./fetch/browser.js";\nimport { boundFetchCollections } from "./fetch/bounds.js";' + ], + [ + "fetchCache.set(cacheKey, siteResult);\n return siteResult;", + "const boundedSiteResult = boundFetchCollections(siteResult);\n fetchCache.set(cacheKey, boundedSiteResult);\n return boundedSiteResult;" + ], + [ + "const resource = await fetchResource(parsedUrl, options?.timeout_ms, transport, options);", + 'const resource = options?.engine === "browser"\n ? await fetchBrowserResource(parsedUrl, options?.timeout_ms)\n : await fetchResource(parsedUrl, options?.timeout_ms, transport, options);' + ], + [ + "fetchCache.set(cacheKey, result);\n return result;", + "result = boundFetchCollections(result);\n fetchCache.set(cacheKey, result);\n return result;" + ] +]; +for (const [before, after] of extractReplacements) { + if (!extractSource.includes(before)) throw new Error(`mcp-web-search extract patch target not found: ${before}`); + extractSource = extractSource.replace(before, after); +} +fs.writeFileSync(extractPath, extractSource); diff --git a/docker/web-search/searxng/settings.yml b/docker/web-search/searxng/settings.yml index de0d4fe..63cfdf1 100644 --- a/docker/web-search/searxng/settings.yml +++ b/docker/web-search/searxng/settings.yml @@ -1,4 +1,9 @@ -use_default_settings: true +use_default_settings: + engines: + keep_only: + - duckduckgo + - bing + - google general: debug: false diff --git a/docs/configuration.md b/docs/configuration.md index 8002085..a0e952b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,6 +64,9 @@ Only the variables below are part of the public configuration surface. Other | `CONTEXT_KIT_WEB_SEARCH_PROVIDER` | `searxng` | Default `search_web` provider; fallback order depends on this provider | | `CONTEXT_KIT_WEB_SEARCH_HTTP_TIMEOUT` | `15000` | HTTP timeout in milliseconds for search providers | | `CONTEXT_KIT_WEB_SEARCH_MAX_RESULTS` | `10` | Default search result count when clients omit `limit` | +| `CONTEXT_KIT_WEB_SEARCH_MAX_PROVIDER_ATTEMPTS` | `4` | Maximum providers attempted for one search | +| `CONTEXT_KIT_WEB_SEARCH_PROVIDER_TIMEOUT` | `15000` | Per-provider diagnostic timeout in milliseconds | +| `CONTEXT_KIT_BRAVE_SEARCH_API_KEY` | unset | Optional Brave Search API fallback credential | | `CONTEXT_KIT_WEB_SEARCH_CHROME_PATH` | `/usr/bin/chromium` | Chromium path inside the web-search image for Bing fallback | | `CONTEXT_KIT_WEB_SEARCH_BROWSER_USER_AGENT` | bundled Chrome/Linux UA | User agent for the Chromium-backed Bing fallback | | `CONTEXT_KIT_WEB_SEARCH_MCP_COMPAT_MODE` | unset | Set to `legacy` for MCP clients with weak tool-schema parsers | @@ -74,7 +77,7 @@ Only the variables below are part of the public configuration surface. Other | `CONTEXT_KIT_DOCS_SOURCES` | `config/sources.default.txt` | Space-separated source profile files | | `CONTEXT_KIT_DOCS_MAX_GET_BYTES` | `75000` | Max bytes returned by docs retrieval | | `CONTEXT_KIT_DOCS_EMBED_MODEL` | `BAAI/bge-small-en-v1.5` | SentenceTransformers embedding model | -| `CONTEXT_KIT_DOCS_PREINDEX` | `0` | Set to `1` to re-embed every source on container start | +| `CONTEXT_KIT_DOCS_PREINDEX` | `0` | Set to `1` to refresh stale/missing sources in the background on startup | | `CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR` | `${CONTEXT_KIT_DATA_DIR}/local-sources` | Machine-local llms.txt tree mounted read-only into docs-mcp | | `CONTEXT_KIT_DOCS_LOCAL_SOURCES_PORT` | `8769` | Loopback port inside docs-mcp for serving local source files | @@ -136,6 +139,10 @@ same-ID `restart` does not apply a changed TTL; it takes effect only when a new container is explicitly provisioned. When freshness matters for one task, prefer `docs_refresh` instead of replacing the shared container. +Use `bin/context-kit docs-rebuild [SOURCE_URL ...]` after parser/model changes or +to force an atomic rebuild. Existing searchable generations remain available if +a source fetch, parse, or embedding step fails. + ## Browser CORS `context-docs` disables browser CORS by default. CLI assistants and server-side @@ -172,3 +179,10 @@ For local llms.txt files, place content under `http://127.0.0.1:8769/path/inside/local-sources/llms.txt` or another URL that ends in `/llms.txt` or `/llms-full.txt`; that loopback URL is inside the docs-mcp container, not exposed on the host. + +Run `bin/context-kit docs-snapshot [--only DIRECTORY]` to materialize linked +local menus. Each successful directory gets `llms-full.txt` and +`llms-full.provenance.json`; cache validators live under +`${CONTEXT_KIT_DATA_DIR}/snapshot-cache`. `--offline` rebuilds only from that +cache. During `start`/`restart`, a local `/llms.txt` URL is automatically changed +to its sibling `/llms-full.txt` when that file exists. diff --git a/docs/security.md b/docs/security.md index 273a673..cb194e7 100644 --- a/docs/security.md +++ b/docs/security.md @@ -10,6 +10,11 @@ Context Kit is designed to be safe by default for local development. - The web-search MCP image runs as the non-root `node` user. - Web-search MCP sessions are stateless. Its HTTP front end permits only loopback/internal Host values and rejects every supplied Origin with 403. +- Browser fetch intercepts each network GET, resolves it outside Chromium, and + blocks private/localhost addresses, non-GET requests, request-count overflow, + and byte-budget overflow. Redirect targets are checked independently. +- Search diagnostics contain bounded categorized error messages and never emit + the optional Brave credential. - Repomix mounts only the current project read-only. - Docs indexing stores data under `$HOME/.local/share/context-kit` unless you override it. @@ -27,6 +32,11 @@ Only index sources you trust enough to retrieve into an agent conversation. More sources are not always better. Large or noisy docs can make retrieval slower and less precise. +Docs source replacement is transactional. SQLite WAL state persists on the docs +volume, removed source profiles become inactive immediately, and full content is +not returned by default. Local snapshot provenance is stored separately from the +retrieval text so metadata does not pollute ranking. + ## Code-Editing MCP Servers Context Kit's default MCP servers either read remote content or mount the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 73c520d..477194b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -133,9 +133,15 @@ race result rendering and return no items even when Chromium sees Bing result cards. The override waits for result cards and decodes current Bing redirect URLs before handing results back to the upstream fallback registry. -`fetch_url` is different: in upstream `mcp-web-search` 1.3.0, `engine=browser` is -accepted but reserved for future support. It does not currently invoke Chromium; -URL fetching uses the HTTP extractor path. +`search_web` now returns bounded `diagnostics.attempts` entries. Check each +provider's `status`, `duration_ms`, `result_count`, and categorized error before +changing provider order. An optional Brave API fallback is enabled only when +`CONTEXT_KIT_BRAVE_SEARCH_API_KEY` is set. + +`fetch_url engine=browser` invokes Chromium for JavaScript-rendered pages. Every +HTTP(S) GET is intercepted and fetched through vetted DNS addresses; non-GET +requests, private/localhost destinations, more than 100 requests, and more than +20 MiB total browser traffic are blocked. Use `engine=http` for ordinary pages. ## Docs Indexing Is Slow @@ -147,11 +153,10 @@ Cloudflare and other large docs sets can take significantly longer than the default source profile. Set `CONTEXT_KIT_DOCS_PREINDEX=1` only if you want startup to eagerly embed every configured source. -## Docs Tools Say Index Manager Not Initialized +## Docs Sources Report Refresh Errors -If `docs_query` or `docs_refresh` returns `Index manager not initialized` while -`/status` still responds, the HTTP wrapper is up but `llms-txt-mcp` failed to -initialize its embedding model or Chroma database. Check the container logs: +If `docs_sources` reports `last_error`, the service keeps the previous generation +searchable and records the failed check. Check the container logs: ```sh docker compose -p "${CONTEXT_KIT_COMPOSE_PROJECT:-context-kit}" -f compose.yml logs docs-mcp @@ -173,7 +178,7 @@ sudo chown -R "$(id -u):$(id -g)" "$DATA_DIR/docs" "$DATA_DIR/models" bin/context-kit restart ``` -`bin/context-kit start` now pre-creates these directories and `doctor` reports -existing directories that are not writable by the current user. If an assistant -client reports `Session not found` after restarting `docs-mcp`, restart the -assistant so it opens a fresh Streamable HTTP MCP session. +`bin/context-kit start` pre-creates these directories and `doctor` reports +existing directories that are not writable by the current user. The docs MCP +uses stateless HTTP sessions, so clients do not retain a session ID across calls. +Use `bin/context-kit docs-rebuild` after fixing the underlying error. diff --git a/scripts/docs-rebuild.mjs b/scripts/docs-rebuild.mjs new file mode 100644 index 0000000..98590a7 --- /dev/null +++ b/scripts/docs-rebuild.mjs @@ -0,0 +1,22 @@ +import { rpc } from "../docker/web-search/mcp-probe.mjs"; + +const [url, ...sources] = process.argv.slice(2); +if (!url) throw new Error("usage: node scripts/docs-rebuild.mjs [source ...]"); + +await rpc(url, 1, "initialize", { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { name: "context-kit-docs-rebuild", version: "1" } +}, 10_000); +const result = await rpc(url, 2, "tools/call", { + name: "docs_rebuild", + arguments: sources.length ? { sources } : {} +}, 600_000); +if (result.isError) { + const text = (result.content || []).map(part => part.text || "").join("\n"); + throw new Error(text || "docs_rebuild failed"); +} +const structured = result.structuredContent || JSON.parse( + (result.content || []).find(part => part.type === "text")?.text || "{}" +); +console.log(JSON.stringify(structured, null, 2)); diff --git a/scripts/docs_snapshot.py b/scripts/docs_snapshot.py new file mode 100644 index 0000000..750cecc --- /dev/null +++ b/scripts/docs_snapshot.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import os +import re +import tempfile +from dataclasses import dataclass +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +GENERATOR_VERSION = "1" +_LINK = re.compile(r"^\s*[-*]\s+\[([^]]+)]\(([^)]+)\)(?::\s*(.*))?\s*$") + + +@dataclass(frozen=True) +class MenuEntry: + title: str + url: str + description: str + + +@dataclass(frozen=True) +class FetchedPage: + requested_url: str + resolved_url: str + body: bytes + content_type: str + etag: str | None + last_modified: str | None + + +def parse_menu(content: str, source_url: str = "") -> list[MenuEntry]: + entries: list[MenuEntry] = [] + for line in content.splitlines(): + match = _LINK.match(line) + if match: + entries.append( + MenuEntry( + title=match.group(1).strip(), + url=urljoin(source_url, match.group(2).strip()), + description=(match.group(3) or "").strip(), + ) + ) + return entries + + +class _ReadableHTML(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.all_parts: list[str] = [] + self.main_parts: list[str] = [] + self.main_depth = 0 + self.skip_depth = 0 + self.heading_level = 0 + + def handle_starttag(self, tag: str, attrs) -> None: + tag = tag.lower() + if tag in {"script", "style", "svg", "noscript", "nav", "footer"}: + self.skip_depth += 1 + return + if tag in {"main", "article"}: + self.main_depth += 1 + if self.skip_depth: + return + if tag in {"p", "div", "section", "br", "table", "tr", "pre"}: + self._append("\n") + elif tag == "li": + self._append("\n- ") + elif tag in {"h1", "h2", "h3", "h4", "h5", "h6"}: + self.heading_level = int(tag[1]) + self._append(f"\n\n{'#' * self.heading_level} ") + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + if tag in {"script", "style", "svg", "noscript", "nav", "footer"} and self.skip_depth: + self.skip_depth -= 1 + return + if not self.skip_depth and tag in {"p", "div", "section", "li", "tr", "pre", "h1", "h2", "h3", "h4", "h5", "h6"}: + self._append("\n") + if tag in {"main", "article"} and self.main_depth: + self.main_depth -= 1 + if tag.startswith("h"): + self.heading_level = 0 + + def handle_data(self, data: str) -> None: + if not self.skip_depth: + self._append(data) + + def _append(self, text: str) -> None: + self.all_parts.append(text) + if self.main_depth: + self.main_parts.append(text) + + def rendered(self) -> str: + preferred = self.main_parts if any(part.strip() for part in self.main_parts) else self.all_parts + text = html.unescape("".join(preferred)).replace("\r", "") + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r" *\n *", "\n", text) + return re.sub(r"\n{3,}", "\n\n", text).strip() + + +def page_to_markdown(page: FetchedPage) -> str: + text = page.body.decode("utf-8", errors="replace") + content_type = page.content_type.lower() + if "html" not in content_type and not re.search(r" None: + return None + + def fetch(self, url: str) -> FetchedPage: + key = hashlib.sha256(url.encode()).hexdigest() + body_path = self.cache_dir / f"{key}.body" + metadata_path = self.cache_dir / f"{key}.json" + metadata = json.loads(metadata_path.read_text()) if metadata_path.exists() else {} + if self.offline: + if not body_path.exists(): + raise RuntimeError(f"offline cache miss: {url}") + return self._cached(url, body_path, metadata) + + headers = {} + if metadata.get("etag"): + headers["If-None-Match"] = metadata["etag"] + if metadata.get("last_modified"): + headers["If-Modified-Since"] = metadata["last_modified"] + headers["User-Agent"] = "context-kit-snapshot/1.0" + try: + response = urlopen(Request(url, headers=headers), timeout=self.timeout) + except HTTPError as error: + if error.code != 304: + raise + response = error + if response.status == 304: + if not body_path.exists(): + raise RuntimeError(f"HTTP 304 without cached body: {url}") + return self._cached(url, body_path, metadata) + body = response.read() + metadata = { + "requested_url": url, + "resolved_url": response.geturl(), + "content_type": response.headers.get("content-type", ""), + "etag": response.headers.get("etag"), + "last_modified": response.headers.get("last-modified"), + "sha256": hashlib.sha256(body).hexdigest(), + } + atomic_write(body_path, body) + atomic_write(metadata_path, (json.dumps(metadata, sort_keys=True, indent=2) + "\n").encode()) + return self._cached(url, body_path, metadata) + + @staticmethod + def _cached(url: str, body_path: Path, metadata: dict) -> FetchedPage: + return FetchedPage( + requested_url=url, + resolved_url=metadata.get("resolved_url", url), + body=body_path.read_bytes(), + content_type=metadata.get("content_type", "text/plain"), + etag=metadata.get("etag"), + last_modified=metadata.get("last_modified"), + ) + + +def build_snapshot(menu_path: Path, fetcher) -> dict: + menu = menu_path.read_text() + entries = parse_menu(menu) + if not entries: + raise RuntimeError(f"no markdown links in {menu_path}") + sections: list[str] = [] + documents: list[dict] = [] + failures: list[str] = [] + for entry in entries: + try: + page = fetcher.fetch(entry.url) + content = page_to_markdown(page) + if not content: + raise RuntimeError("extracted content is empty") + sections.append(f"# {entry.title}\n\nSource: {page.resolved_url}\n\n{content}") + documents.append( + { + "title": entry.title, + "requested_url": entry.url, + "resolved_url": page.resolved_url, + "content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "source_sha256": hashlib.sha256(page.body).hexdigest(), + "etag": page.etag, + "last_modified": page.last_modified, + } + ) + except Exception as error: + failures.append(f"{entry.url}: {error}") + if failures: + raise RuntimeError("snapshot fetch failed; previous output preserved:\n" + "\n".join(failures)) + + output = ("\n\n".join(sections).strip() + "\n").encode() + manifest = { + "schema_version": 1, + "generator_version": GENERATOR_VERSION, + "menu": menu_path.name, + "menu_sha256": hashlib.sha256(menu.encode()).hexdigest(), + "output_sha256": hashlib.sha256(output).hexdigest(), + "document_count": len(documents), + "documents": documents, + } + output_path = menu_path.with_name("llms-full.txt") + manifest_path = menu_path.with_name("llms-full.provenance.json") + atomic_write(output_path, output) + atomic_write(manifest_path, (json.dumps(manifest, sort_keys=True, indent=2) + "\n").encode()) + return {"menu": str(menu_path), "output": str(output_path), **manifest} + + +def validate_snapshot(output_path: Path) -> dict: + manifest_path = output_path.with_name("llms-full.provenance.json") + if not output_path.is_file() or not manifest_path.is_file(): + raise RuntimeError("snapshot or provenance manifest is missing") + manifest = json.loads(manifest_path.read_text()) + output_hash = hashlib.sha256(output_path.read_bytes()).hexdigest() + if manifest.get("output_sha256") != output_hash: + raise RuntimeError("snapshot hash does not match provenance manifest") + menu_path = output_path.with_name(str(manifest.get("menu") or "llms.txt")) + if not menu_path.is_file(): + raise RuntimeError("snapshot source menu is missing") + menu_hash = hashlib.sha256(menu_path.read_bytes()).hexdigest() + if manifest.get("menu_sha256") != menu_hash: + raise RuntimeError("menu hash does not match provenance manifest") + return manifest + + +def atomic_write(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def snapshot_menus(menus: list[Path], fetcher) -> dict: + """Snapshot every menu independently so one bad directory cannot block the rest.""" + report: dict = {"snapshots": [], "skipped": [], "failures": []} + for menu in menus: + if not parse_menu(menu.read_text()): + report["skipped"].append({"menu": str(menu), "reason": "no markdown links"}) + continue + try: + report["snapshots"].append(build_snapshot(menu, fetcher)) + except Exception as error: + report["failures"].append({"menu": str(menu), "error": str(error)}) + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build deterministic content snapshots from local llms.txt menus.") + parser.add_argument("--source-root", type=Path) + parser.add_argument("--cache-dir", type=Path) + parser.add_argument("--only", action="append", default=[]) + parser.add_argument("--offline", action="store_true") + parser.add_argument("--validate-output", type=Path) + args = parser.parse_args() + if args.validate_output: + print(json.dumps(validate_snapshot(args.validate_output), sort_keys=True)) + return + if not args.source_root: + parser.error("--source-root is required unless --validate-output is used") + cache_dir = args.cache_dir or args.source_root / ".snapshot-cache" + menus = sorted(args.source_root.glob("*/llms.txt")) + if args.only: + selected = set(args.only) + menus = [menu for menu in menus if menu.parent.name in selected] + if not menus: + raise SystemExit("no matching llms.txt menus") + + fetcher = CachedFetcher(cache_dir, offline=args.offline) + try: + report = snapshot_menus(menus, fetcher) + finally: + fetcher.close() + print(json.dumps(report, sort_keys=True, indent=2)) + if report["failures"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/fixtures/docs/local-sources/fixture/llms-full.txt b/scripts/fixtures/docs/local-sources/fixture/llms-full.txt new file mode 100644 index 0000000..b869f13 --- /dev/null +++ b/scripts/fixtures/docs/local-sources/fixture/llms-full.txt @@ -0,0 +1,11 @@ +# Environment Variables + +Source: https://docs.example.test/environment + +`CONTEXT_KIT_EXACT_IDENTIFIER_20260724` enables the deterministic candidate fixture. + +# Durable Persistence + +Source: https://docs.example.test/persistence + +Checkpoints preserve graph state across process restarts. diff --git a/scripts/fixtures/docs/sources.txt b/scripts/fixtures/docs/sources.txt new file mode 100644 index 0000000..646164e --- /dev/null +++ b/scripts/fixtures/docs/sources.txt @@ -0,0 +1 @@ +http://127.0.0.1:8769/fixture/llms-full.txt diff --git a/scripts/fixtures/web/mock-server.mjs b/scripts/fixtures/web/mock-server.mjs new file mode 100644 index 0000000..617d2a7 --- /dev/null +++ b/scripts/fixtures/web/mock-server.mjs @@ -0,0 +1,52 @@ +import http from "node:http"; + +let websocketUpgrades = 0; + +const server = http.createServer((request, response) => { + const url = new URL(request.url, "http://mock-search.test"); + if (url.pathname === "/search") { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ + results: [{ + title: "Deterministic Search Result", + url: "https://example.test/result", + content: `fixture result for ${url.searchParams.get("q")}` + }] + })); + return; + } + if (url.pathname === "/dynamic") { + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + response.end(`Dynamic Fixture +
initial content
+ + `); + return; + } + if (url.pathname === "/redirect-private") { + response.writeHead(302, { Location: "http://127.0.0.1:8765/private" }); + response.end(); + return; + } + if (url.pathname === "/websocket-attempt") { + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + response.end(`
starting
+ `); + return; + } + if (url.pathname === "/ws-count") { + response.writeHead(200, { "Content-Type": "text/plain" }); + response.end(String(websocketUpgrades)); + return; + } + response.writeHead(404).end(); +}); + +server.listen(8080, "0.0.0.0"); +server.on("upgrade", (_request, socket) => { + websocketUpgrades += 1; + socket.destroy(); +}); diff --git a/scripts/release-check b/scripts/release-check index d9ee9ef..99ad07e 100755 --- a/scripts/release-check +++ b/scripts/release-check @@ -267,7 +267,7 @@ bash -n scripts/release-check bash -n scripts/test-compose-upgrade.sh bash -n scripts/test-lifecycle.sh sh -n docker/docs/entrypoint.sh -check_node docker/web-search/patch-mcp-web-search.mjs docker/web-search/overrides/bing.js docker/web-search/mcp-probe.mjs docker/web-search/http-entrypoint.mjs scripts/mcp-smoke-client.mjs scripts/smoke-web-search.mjs scripts/smoke-docs.mjs scripts/smoke-repomix.mjs scripts/test-web-search-http.mjs +check_node docker/web-search/patch-mcp-web-search.mjs docker/web-search/overrides/bing.js docker/web-search/overrides/brave.js docker/web-search/overrides/browser-fetch.js docker/web-search/overrides/registry.js docker/web-search/mcp-probe.mjs docker/web-search/http-entrypoint.mjs scripts/docs-rebuild.mjs scripts/mcp-smoke-client.mjs scripts/smoke-web-search.mjs scripts/smoke-docs.mjs scripts/smoke-repomix.mjs scripts/test-docs-candidate.mjs scripts/test-web-search-candidate.mjs scripts/test-web-search-http.mjs scripts/test-web-search-quality.mjs node -e 'const fs=require("node:fs"); JSON.parse(fs.readFileSync("snippets/opencode.json", "utf8")); JSON.parse(fs.readFileSync("snippets/claude.mcp.json", "utf8"));' CONTEXT_KIT_WEB_SEARCH_HTTP_URL="http://127.0.0.1:8777/mcp" CONTEXT_KIT_DOCS_HTTP_URL="http://127.0.0.1:8776/mcp" bin/context-kit install opencode > "${tmp_dir}/opencode-default.json" @@ -296,6 +296,8 @@ assert_redaction_check_does_not_disclose_matches bash scripts/test-compose-upgrade.sh bash scripts/test-lifecycle.sh node scripts/test-web-search-http.mjs +node scripts/test-web-search-quality.mjs +python3 scripts/test-doc-snapshots.py bin/context-kit redaction-check docker compose -p "${CONTEXT_KIT_COMPOSE_PROJECT}" -f compose.yml config >/dev/null @@ -313,6 +315,12 @@ assert_hostile_requests_rejected node scripts/smoke-web-search.mjs bin/context-kit web-search node scripts/smoke-docs.mjs bin/context-kit docs node scripts/smoke-repomix.mjs bin/context-kit repomix +docker run --rm --entrypoint python "${CONTEXT_KIT_DOCS_IMAGE}" -m unittest discover -s /opt/context-kit/tests -t /opt/context-kit +CONTEXT_KIT_DOCS_CANDIDATE_IMAGE="${CONTEXT_KIT_DOCS_IMAGE}" \ + CONTEXT_KIT_DOCS_TEST_MODELS="${CONTEXT_KIT_DATA_DIR}/models" \ + bash scripts/test-docs-candidate.sh +CONTEXT_KIT_WEB_SEARCH_CANDIDATE_IMAGE="${CONTEXT_KIT_WEB_SEARCH_IMAGE}" \ + bash scripts/test-web-search-candidate.sh assert_web_search_backend_supervision printf 'pass release-check\n' diff --git a/scripts/smoke-docs.mjs b/scripts/smoke-docs.mjs index 75d59cb..1802ab0 100644 --- a/scripts/smoke-docs.mjs +++ b/scripts/smoke-docs.mjs @@ -1,4 +1,4 @@ -import { requireToolSuccess, runSmoke } from "./mcp-smoke-client.mjs"; +import { requireToolSuccess, runSmoke, textFrom } from "./mcp-smoke-client.mjs"; const live = process.env.CONTEXT_KIT_LIVE_CHECKS === "1"; const localSourceSmokeUrl = process.env.CONTEXT_KIT_LOCAL_SOURCE_SMOKE_URL; @@ -12,7 +12,8 @@ runSmoke({ const toolNames = await client.requireTools(["docs_query", "docs_sources"]); const sources = requireToolSuccess("docs_sources", await client.callTool("docs_sources")); - if (!Array.isArray(sources?.structuredContent?.result)) { + const sourcesPayload = sources?.structuredContent || JSON.parse(textFrom(sources) || "null"); + if (typeof sourcesPayload?.source_count !== "number") { const sourcesText = JSON.stringify(sources); throw new Error(`docs_sources returned unexpected payload: ${sourcesText.slice(0, 500)}`); } diff --git a/scripts/test-doc-snapshots.py b/scripts/test-doc-snapshots.py new file mode 100644 index 0000000..8e3f4c6 --- /dev/null +++ b/scripts/test-doc-snapshots.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from docs_snapshot import FetchedPage, build_snapshot, page_to_markdown, snapshot_menus, validate_snapshot + + +class FakeFetcher: + def __init__(self, pages: dict[str, FetchedPage | Exception]): + self.pages = pages + + def fetch(self, url: str) -> FetchedPage: + result = self.pages[url] + if isinstance(result, Exception): + raise result + return result + + +def page(url: str, body: str, content_type: str = "text/html") -> FetchedPage: + return FetchedPage(url, url, body.encode(), content_type, '"fixture"', "Wed, 01 Jan 2025 00:00:00 GMT") + + +class SnapshotTest(unittest.TestCase): + def test_html_extraction_prefers_main_and_discards_navigation(self) -> None: + rendered = page_to_markdown( + page( + "https://example.test/page", + "

API

Useful content.

", + ) + ) + self.assertNotIn("Noise", rendered) + self.assertIn("# API", rendered) + self.assertIn("Useful content.", rendered) + + def test_snapshot_and_manifest_are_deterministic(self) -> None: + with tempfile.TemporaryDirectory() as directory: + menu = Path(directory) / "fixture" / "llms.txt" + menu.parent.mkdir() + menu.write_text("# Menu\n\n- [API](https://example.test/api)\n") + fetcher = FakeFetcher( + {"https://example.test/api": page("https://example.test/api", "

API

Stable.

")} + ) + + first = build_snapshot(menu, fetcher) + first_output = menu.with_name("llms-full.txt").read_bytes() + first_manifest = menu.with_name("llms-full.provenance.json").read_bytes() + second = build_snapshot(menu, fetcher) + + self.assertEqual(first["output_sha256"], second["output_sha256"]) + self.assertEqual(first_output, menu.with_name("llms-full.txt").read_bytes()) + self.assertEqual(first_manifest, menu.with_name("llms-full.provenance.json").read_bytes()) + self.assertEqual(first["output_sha256"], validate_snapshot(menu.with_name("llms-full.txt"))["output_sha256"]) + + menu.with_name("llms-full.txt").write_text("tampered\n") + with self.assertRaisesRegex(RuntimeError, "does not match"): + validate_snapshot(menu.with_name("llms-full.txt")) + + def test_failed_build_preserves_last_good_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as directory: + menu = Path(directory) / "fixture" / "llms.txt" + menu.parent.mkdir() + menu.write_text("# Menu\n\n- [API](https://example.test/api)\n") + output = menu.with_name("llms-full.txt") + output.write_text("last good\n") + + with self.assertRaisesRegex(RuntimeError, "previous output preserved"): + build_snapshot(menu, FakeFetcher({"https://example.test/api": RuntimeError("offline")})) + + self.assertEqual("last good\n", output.read_text()) + + def test_one_bad_menu_does_not_block_other_directories(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, body in [ + ("good", "# Menu\n\n- [API](https://example.test/api)\n"), + ("prose-only", "# Workspace Notes\n\nNo links here, just prose.\n"), + ("broken", "# Menu\n\n- [Down](https://example.test/down)\n"), + ]: + (root / name).mkdir() + (root / name / "llms.txt").write_text(body) + fetcher = FakeFetcher({ + "https://example.test/api": page("https://example.test/api", "

API

Stable.

"), + "https://example.test/down": RuntimeError("host unreachable"), + }) + + report = snapshot_menus(sorted(root.glob("*/llms.txt")), fetcher) + + self.assertEqual(1, len(report["snapshots"])) + self.assertTrue((root / "good" / "llms-full.txt").exists()) + self.assertEqual(1, len(report["skipped"])) + self.assertIn("prose-only", report["skipped"][0]["menu"]) + self.assertEqual(1, len(report["failures"])) + self.assertIn("broken", report["failures"][0]["menu"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-docs-candidate.mjs b/scripts/test-docs-candidate.mjs new file mode 100644 index 0000000..b6207a1 --- /dev/null +++ b/scripts/test-docs-candidate.mjs @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; + +import { probeMcp, rpc } from "../docker/web-search/mcp-probe.mjs"; + +const url = process.argv[2]; +if (!url) throw new Error("usage: node scripts/test-docs-candidate.mjs "); + +const required = ["docs_query", "docs_rebuild", "docs_refresh", "docs_sources"]; +const tools = await probeMcp(url, { timeoutMs: 10_000, expectedTools: required }); +assert.deepEqual(tools, required); + +function structured(result) { + if (result.structuredContent) return result.structuredContent; + const text = (result.content || []).find(part => part.type === "text")?.text; + return text ? JSON.parse(text) : null; +} + +const refreshed = structured(await rpc(url, 3, "tools/call", { + name: "docs_refresh", + arguments: { force: true } +}, 120_000)); +assert.equal(refreshed.sources[0].status, "updated", JSON.stringify(refreshed.sources[0])); +assert(refreshed.sources[0].document_count >= 2); + +const search = structured(await rpc(url, 4, "tools/call", { + name: "docs_query", + arguments: { query: "CONTEXT_KIT_EXACT_IDENTIFIER_20260724", limit: 3 } +}, 30_000)); +assert.equal(search.search_results[0].title, "Environment Variables"); +assert.deepEqual(search.retrieved_content, {}); + +const identifier = search.search_results[0].id; +const retrieved = structured(await rpc(url, 5, "tools/call", { + name: "docs_query", + arguments: { + query: "CONTEXT_KIT_EXACT_IDENTIFIER_20260724", + retrieve_ids: [identifier], + max_bytes: 12_000 + } +}, 30_000)); +assert(retrieved.retrieved_content[identifier].content.includes("CONTEXT_KIT_EXACT_IDENTIFIER_20260724")); + +console.log("pass docs candidate transport, refresh, hybrid search, and explicit retrieval"); diff --git a/scripts/test-docs-candidate.sh b/scripts/test-docs-candidate.sh new file mode 100644 index 0000000..05fae19 --- /dev/null +++ b/scripts/test-docs-candidate.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +IMAGE="${CONTEXT_KIT_DOCS_CANDIDATE_IMAGE:-context-kit/docs-mcp:quality-20260724}" +MODELS="${CONTEXT_KIT_DOCS_TEST_MODELS:-${CONTEXT_KIT_DATA_DIR:-${HOME}/.local/share/context-kit}/models}" +TMP_DIR="$(mktemp -d)" +CONTAINER="context-kit-docs-quality-$RANDOM-$$" + +cleanup() { + docker rm -f "${CONTAINER}" >/dev/null 2>&1 || true + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +mkdir -p "${TMP_DIR}/data" +docker run -d --name "${CONTAINER}" \ + --user "$(id -u):$(id -g)" \ + -p 127.0.0.1::8000 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e DOCS_MCP_PREINDEX=0 \ + -v "${TMP_DIR}/data:/data" \ + -v "${MODELS}:/models:ro" \ + -v "${ROOT}/scripts/fixtures/docs/sources.txt:/etc/context-kit/docs-sources.txt:ro" \ + -v "${ROOT}/scripts/fixtures/docs/local-sources:/etc/context-kit/local-sources:ro" \ + "${IMAGE}" >/dev/null + +binding="$(docker port "${CONTAINER}" 8000/tcp)" +port="${binding##*:}" +for _ in {1..120}; do + if curl -fsS "http://127.0.0.1:${port}/status" >/dev/null 2>&1; then + node "${ROOT}/scripts/test-docs-candidate.mjs" "http://127.0.0.1:${port}/mcp" + exit 0 + fi + sleep 0.25 +done + +docker logs "${CONTAINER}" >&2 +exit 1 diff --git a/scripts/test-lifecycle.sh b/scripts/test-lifecycle.sh index 3f12d34..1ab43bc 100644 --- a/scripts/test-lifecycle.sh +++ b/scripts/test-lifecycle.sh @@ -259,7 +259,7 @@ new_case() { unset CONTEXT_KIT_DOCKER_CIDFILE CONTEXT_KIT_RUNTIME_DIR FAKE_DOCS_UID FAKE_WEB_UID \ FAKE_REPLACEMENT_REQUIRED FAKE_RESTART_FAIL FAKE_DROP_RUNNING FAKE_SEARXNG_FAIL \ FAKE_WEB_SEARCH_FAIL FAKE_DOCS_FAIL FAKE_LEGACY_CONTAINER FAKE_CLIENT_OWNER_MISMATCH \ - FAKE_EXPECT_DOCS_SOURCES FAKE_EXPECT_DOCS_SOURCES_ABSENT + FAKE_EXPECT_DOCS_SOURCES FAKE_EXPECT_DOCS_SOURCES_ABSENT CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR mkdir -p "${FAKE_DOCKER_STATE}" "${HOME}" : > "${FAKE_DOCKER_LOG}" } @@ -432,6 +432,34 @@ grep -F 'https://example.test/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.tx || fail_test "restart did not regenerate the bind-mounted docs source list" assert_no_docs_sources_artifacts +new_case snapshot-promotion +seed_service searxng +seed_service web-search-mcp +seed_service docs-mcp +export CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR="${CASE_ROOT}/local-sources" +mkdir -p "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich" +printf '# source menu\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms.txt" +printf '# generated snapshot\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt" +menu_hash="$(sha256sum "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms.txt")" +menu_hash="${menu_hash%% *}" +output_hash="$(sha256sum "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt")" +output_hash="${output_hash%% *}" +printf '{"menu":"llms.txt","menu_sha256":"%s","output_sha256":"%s"}\n' \ + "${menu_hash}" "${output_hash}" \ + > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.provenance.json" +printf 'http://127.0.0.1:8769/immich/llms.txt\n' > "${CASE_ROOT}/sources.txt" +export CONTEXT_KIT_DOCS_SOURCES="${CASE_ROOT}/sources.txt" +"${CONTEXT_KIT}" restart +grep -F 'http://127.0.0.1:8769/immich/llms-full.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null \ + || fail_test "restart did not promote a local menu to its generated full snapshot" +if grep -Fx 'http://127.0.0.1:8769/immich/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null; then + fail_test "restart retained the menu URL despite an available full snapshot" +fi +printf '# inconsistent snapshot\n' > "${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR}/immich/llms-full.txt" +"${CONTEXT_KIT}" restart +grep -Fx 'http://127.0.0.1:8769/immich/llms.txt' "${CONTEXT_KIT_DATA_DIR}/docs-sources.txt" >/dev/null \ + || fail_test "restart promoted a snapshot whose provenance hash did not match" + new_case restart-failure seed_service searxng stopped seed_service web-search-mcp diff --git a/scripts/test-web-search-candidate.mjs b/scripts/test-web-search-candidate.mjs new file mode 100644 index 0000000..c836a90 --- /dev/null +++ b/scripts/test-web-search-candidate.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; + +import { probeMcp, rpc } from "../docker/web-search/mcp-probe.mjs"; + +const url = process.argv[2]; +if (!url) throw new Error("usage: node scripts/test-web-search-candidate.mjs "); + +await probeMcp(url, { timeoutMs: 10_000 }); + +function payload(result) { + const text = (result.content || []).find(part => part.type === "text")?.text; + return text ? JSON.parse(text) : result.structuredContent; +} + +const search = payload(await rpc(url, 3, "tools/call", { + name: "search_web", + arguments: { q: "candidate diagnostic fixture", limit: 3, provider: "searxng" } +}, 30_000)); +assert.equal(search.items[0].title, "Deterministic Search Result"); +assert.equal(search.providerUsed, "searxng"); +assert.equal(search.diagnostics.attempts[0].status, "success"); +assert.equal(search.diagnostics.attempts[0].result_count, 1); + +const httpFetch = payload(await rpc(url, 4, "tools/call", { + name: "fetch_url", + arguments: { url: "http://mock-search.test:8080/dynamic", engine: "http", format: "text" } +}, 30_000)); +assert(!httpFetch.content.includes("BROWSER_RENDERED_MARKER")); + +const browserFetch = payload(await rpc(url, 5, "tools/call", { + name: "fetch_url", + arguments: { url: "http://mock-search.test:8080/dynamic", engine: "browser", format: "text", timeout_ms: 20_000 } +}, 60_000)); +assert(browserFetch.content.includes("BROWSER_RENDERED_MARKER")); + +const blocked = await rpc(url, 6, "tools/call", { + name: "fetch_url", + arguments: { url: "http://127.0.0.1:8765/private", engine: "browser" } +}, 30_000); +assert.equal(blocked.isError, true); +assert((blocked.content || []).some(part => part.text?.includes("Blocked localhost/private URL"))); + +const blockedRedirect = await rpc(url, 7, "tools/call", { + name: "fetch_url", + arguments: { url: "http://mock-search.test:8080/redirect-private", engine: "browser" } +}, 30_000); +assert.equal(blockedRedirect.isError, true); + +await rpc(url, 8, "tools/call", { + name: "fetch_url", + arguments: { url: "http://mock-search.test:8080/websocket-attempt", engine: "browser", fresh: true } +}, 30_000); +const websocketCount = payload(await rpc(url, 9, "tools/call", { + name: "fetch_url", + arguments: { url: "http://mock-search.test:8080/ws-count", engine: "http", format: "text", fresh: true } +}, 30_000)); +assert.equal(websocketCount.content.trim(), "0"); + +console.log("pass web-search candidate diagnostics, browser rendering, and SSRF rejection"); diff --git a/scripts/test-web-search-candidate.sh b/scripts/test-web-search-candidate.sh new file mode 100644 index 0000000..ac71e75 --- /dev/null +++ b/scripts/test-web-search-candidate.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +IMAGE="${CONTEXT_KIT_WEB_SEARCH_CANDIDATE_IMAGE:-context-kit/web-search-mcp:quality-20260724}" +NETWORK="context-kit-web-quality-$RANDOM-$$" +MOCK="${NETWORK}-mock" +SERVER="${NETWORK}-server" + +cleanup() { + docker rm -f "${SERVER}" "${MOCK}" >/dev/null 2>&1 || true + docker network rm "${NETWORK}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker network create --subnet 203.0.113.0/24 "${NETWORK}" >/dev/null +docker run -d --name "${MOCK}" --network "${NETWORK}" --ip 203.0.113.10 \ + --network-alias mock-search.test \ + -v "${ROOT}/scripts/fixtures/web/mock-server.mjs:/fixture/mock-server.mjs:ro" \ + node:22-bookworm-slim node /fixture/mock-server.mjs >/dev/null +docker run -d --name "${SERVER}" --network "${NETWORK}" --ip 203.0.113.11 \ + -p 127.0.0.1::8000 \ + -e SEARXNG_URL=http://mock-search.test:8080 \ + -e DEFAULT_SEARCH_PROVIDER=searxng \ + "${IMAGE}" >/dev/null + +binding="$(docker port "${SERVER}" 8000/tcp)" +port="${binding##*:}" +for _ in {1..120}; do + if curl -fsS "http://127.0.0.1:${port}/healthz" >/dev/null 2>&1; then + node "${ROOT}/scripts/test-web-search-candidate.mjs" "http://127.0.0.1:${port}/mcp" + exit 0 + fi + sleep 0.25 +done + +docker logs "${SERVER}" >&2 +exit 1 diff --git a/scripts/test-web-search-quality.mjs b/scripts/test-web-search-quality.mjs new file mode 100644 index 0000000..e7a737b --- /dev/null +++ b/scripts/test-web-search-quality.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; + +import { boundFetchCollections } from "../docker/web-search/overrides/bounds.mjs"; +import { + attemptProvider, + classifyProviderError +} from "../docker/web-search/overrides/diagnostics.mjs"; + +assert.deepEqual(classifyProviderError(new Error("HTTP 429 rate limit")), { + category: "rate_limited", + message: "HTTP 429 rate limit" +}); +assert.equal(classifyProviderError(new Error("captcha challenge")).category, "blocked"); + +const unavailable = await attemptProvider({ name: "brave", configured: false }, "q", 3, "en"); +assert.equal(unavailable.diagnostic.status, "unavailable"); +assert.equal(unavailable.diagnostic.result_count, 0); + +const empty = await attemptProvider({ + name: "empty", + async search() { return []; } +}, "q", 3, "en"); +assert.equal(empty.diagnostic.status, "empty"); + +const failed = await attemptProvider({ + name: "failed", + async search() { throw new Error("network socket failed"); } +}, "q", 3, "en"); +assert.equal(failed.diagnostic.status, "error"); +assert.equal(failed.diagnostic.error.category, "network"); + +let underlyingAborted = false; +const timedOut = await attemptProvider({ + name: "slow", + async search(_query, _limit, _lang, signal) { + await new Promise((resolve, reject) => { + signal.addEventListener("abort", () => { + underlyingAborted = true; + reject(signal.reason); + }, { once: true }); + }); + } +}, "q", 3, "en", { timeoutMs: 20 }); +assert.equal(timedOut.diagnostic.error.category, "timeout"); +assert.equal(underlyingAborted, true); + +const result = boundFetchCollections({ + links: Array.from({ length: 550 }, (_, index) => ({ url: `https://example.test/${index}` })), + media: { + images: Array.from({ length: 250 }, (_, index) => ({ url: `https://example.test/${index}.png` })), + videos: [], + audio: [] + }, + warnings: [] +}); +assert.equal(result.links.length, 500); +assert.equal(result.media.images.length, 200); +assert(result.warnings.some(warning => warning.includes("links truncated"))); +assert(result.warnings.some(warning => warning.includes("images truncated"))); + +console.log("pass web-search diagnostics and collection bounds tests");