Fix ID collisions for repeated section titles

Large llms-full.txt feeds repeat section headings, which made document
identity hashes collide within one source and abort indexing on the
documents primary key. Include each document's ordinal in the identity.
This commit is contained in:
2026-07-25 09:00:59 -07:00
parent 51dceee224
commit ac3465c656
2 changed files with 22 additions and 2 deletions

View File

@@ -65,10 +65,12 @@ class RefreshCoordinator:
vectors = await self.embedder.encode_documents(texts) if texts else [] vectors = await self.embedder.encode_documents(texts) if texts else []
documents: list[PreparedDocument] = [] documents: list[PreparedDocument] = []
source_host = (urlparse(response.resolved_url).hostname or "").lower() source_host = (urlparse(response.resolved_url).hostname or "").lower()
for parsed_document, vector in zip(parsed.documents, vectors, strict=True): for ordinal, (parsed_document, vector) in enumerate(zip(parsed.documents, vectors, strict=True)):
content_hash = hashlib.sha256(parsed_document.content.encode()).hexdigest() content_hash = hashlib.sha256(parsed_document.content.encode()).hexdigest()
# Include the ordinal so repeated section titles (common in large
# llms-full.txt feeds) cannot collide on the primary key.
identity = "\0".join( identity = "\0".join(
[source, parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)] [source, str(ordinal), parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)]
) )
documents.append( documents.append(
PreparedDocument( PreparedDocument(

View File

@@ -91,6 +91,24 @@ class RefreshTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(1, self.store.list_sources()[0].doc_count) self.assertEqual(1, self.store.list_sources()[0].doc_count)
self.assertTrue(self.store.lexical_search("Original", limit=5)) self.assertTrue(self.store.lexical_search("Original", limit=5))
async def test_repeated_section_titles_index_without_id_collisions(self) -> None:
body = "# Basic syntax\n\nFirst variant.\n\n# Basic syntax\n\nSecond variant.\n"
fetcher = FakeFetcher([FakeFetch(200, body)])
coordinator = RefreshCoordinator(
store=self.store,
fetcher=fetcher,
embedder=FakeEmbedder(),
parser=parse_llms_text,
ttl_seconds=3600,
now=lambda: 100.0,
)
outcome = await coordinator.refresh(self.source, force=True)
self.assertEqual("updated", outcome.status)
self.assertEqual(2, outcome.document_count)
self.assertEqual(2, self.store.list_sources()[0].doc_count)
async def test_empty_success_response_preserves_previous_content(self) -> None: async def test_empty_success_response_preserves_previous_content(self) -> None:
fetcher = FakeFetcher( fetcher = FakeFetcher(
[ [