Ajay Krishnan
51dceee224
Replace the abandoned llms-txt-mcp/Chroma docs backend with an in-repo MCP service: SQLite WAL + FTS5 + sentence-transformer embeddings, transactional source replacement, persisted state across restarts, singleflight refresh with conditional requests, hybrid lexical/semantic ranking with exact-duplicate collapse, source/host filters, and explicit-by-default content retrieval. Add docs_rebuild and a docs-rebuild CLI command. Add deterministic llms-full.txt snapshot generation for machine-local menus with hash-validated provenance manifests; lifecycle commands promote a local menu to its snapshot only when the manifest validates. Switch public source profiles to content-bearing llms-full.txt feeds. Improve web search: bounded provider fallback with per-attempt diagnostics and cancellation, an optional Brave Search API provider, strict SearXNG engine selection, capped link/media extraction, and a real engine=browser renderer that routes every request through the existing SSRF vetting while blocking WebSockets, non-GET traffic, and private destinations. Extend release checks with offline unit suites and isolated candidate container tests for both images.
76 lines
2.3 KiB
Python
76 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
|
|
import numpy as np
|
|
|
|
|
|
class SentenceTransformerEmbedder:
|
|
def __init__(self, model_name: str):
|
|
self.model_name = model_name
|
|
self.fingerprint = f"sentence-transformers:{model_name}"
|
|
self._model = None
|
|
self._lock = asyncio.Lock()
|
|
|
|
@property
|
|
def ready(self) -> bool:
|
|
return self._model is not None
|
|
|
|
async def ensure_ready(self) -> None:
|
|
if self._model is not None:
|
|
return
|
|
async with self._lock:
|
|
if self._model is None:
|
|
self._model = await asyncio.to_thread(self._load)
|
|
|
|
def _load(self):
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
return SentenceTransformer(self.model_name, device="cpu")
|
|
|
|
async def encode_documents(self, texts: list[str]) -> np.ndarray:
|
|
await self.ensure_ready()
|
|
return await asyncio.to_thread(self._encode, texts)
|
|
|
|
async def encode_query(self, text: str) -> np.ndarray:
|
|
vectors = await self.encode_documents([text])
|
|
return vectors[0]
|
|
|
|
def _encode(self, texts: list[str]) -> np.ndarray:
|
|
return np.asarray(
|
|
self._model.encode(
|
|
texts,
|
|
batch_size=32,
|
|
show_progress_bar=False,
|
|
normalize_embeddings=True,
|
|
convert_to_numpy=True,
|
|
),
|
|
dtype=np.float32,
|
|
)
|
|
|
|
|
|
class LexicalFallbackEmbedder:
|
|
"""Deterministic fallback used only when a model cannot be loaded."""
|
|
|
|
fingerprint = "lexical-fallback-v1"
|
|
ready = True
|
|
|
|
async def ensure_ready(self) -> None:
|
|
return None
|
|
|
|
async def encode_documents(self, texts: list[str]) -> np.ndarray:
|
|
return np.asarray([self._encode(text) for text in texts], dtype=np.float32)
|
|
|
|
async def encode_query(self, text: str) -> np.ndarray:
|
|
return np.asarray(self._encode(text), dtype=np.float32)
|
|
|
|
@staticmethod
|
|
def _encode(text: str, dimensions: int = 384) -> np.ndarray:
|
|
vector = np.zeros(dimensions, dtype=np.float32)
|
|
for token in text.lower().split():
|
|
digest = hashlib.sha256(token.encode()).digest()
|
|
vector[int.from_bytes(digest[:4], "big") % dimensions] += 1.0
|
|
norm = np.linalg.norm(vector)
|
|
return vector / norm if norm else vector
|