Overhaul docs retrieval and web search quality
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
This commit is contained in:
122
docker/docs/context_docs/models.py
Normal file
122
docker/docs/context_docs/models.py
Normal file
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedDocument:
|
||||
title: str
|
||||
description: str
|
||||
content: str
|
||||
canonical_url: str
|
||||
heading_path: str
|
||||
chunk_index: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedSource:
|
||||
format: str
|
||||
documents: list[ParsedDocument]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResponse:
|
||||
status: int
|
||||
requested_url: str
|
||||
resolved_url: str
|
||||
body: str = ""
|
||||
etag: str | None = None
|
||||
last_modified: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedDocument:
|
||||
id: str
|
||||
configured_source: str
|
||||
resolved_source: str
|
||||
source_host: str
|
||||
canonical_url: str
|
||||
canonical_host: str
|
||||
title: str
|
||||
description: str
|
||||
heading_path: str
|
||||
content: str
|
||||
content_hash: str
|
||||
embedding: np.ndarray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceUpdate:
|
||||
configured_source: str
|
||||
resolved_source: str
|
||||
etag: str | None
|
||||
last_modified: str | None
|
||||
body_hash: str
|
||||
raw_body: str
|
||||
parser_fingerprint: str
|
||||
embedding_fingerprint: str
|
||||
checked_at: float
|
||||
indexed_at: float
|
||||
stale_at: float
|
||||
documents: list[PreparedDocument]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceState:
|
||||
configured_source: str
|
||||
resolved_source: str | None
|
||||
active: bool
|
||||
etag: str | None
|
||||
last_modified: str | None
|
||||
body_hash: str | None
|
||||
raw_body: str | None
|
||||
parser_fingerprint: str | None
|
||||
embedding_fingerprint: str | None
|
||||
checked_at: float | None
|
||||
indexed_at: float | None
|
||||
stale_at: float | None
|
||||
last_error: str | None
|
||||
doc_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredDocument:
|
||||
id: str
|
||||
configured_source: str
|
||||
resolved_source: str
|
||||
source_host: str
|
||||
canonical_url: str
|
||||
canonical_host: str
|
||||
title: str
|
||||
description: str
|
||||
heading_path: str
|
||||
content: str
|
||||
content_hash: str
|
||||
embedding: np.ndarray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchResult:
|
||||
id: str
|
||||
configured_source: str
|
||||
canonical_url: str
|
||||
title: str
|
||||
description: str
|
||||
heading_path: str
|
||||
content: str
|
||||
content_hash: str
|
||||
score: float
|
||||
lexical_rank: int | None
|
||||
semantic_rank: int | None
|
||||
duplicate_count: int = 1
|
||||
alternate_sources: list[dict[str, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefreshOutcome:
|
||||
source: str
|
||||
status: str
|
||||
document_count: int
|
||||
detail: str | None = None
|
||||
Reference in New Issue
Block a user