Fix ID collisions for repeated section titles
Large llms-full.txt feeds repeat section headings, which made document identity hashes collide within one source and abort indexing on the documents primary key. Include each document's ordinal in the identity.
This commit is contained in:
@@ -65,10 +65,12 @@ class RefreshCoordinator:
|
||||
vectors = await self.embedder.encode_documents(texts) if texts else []
|
||||
documents: list[PreparedDocument] = []
|
||||
source_host = (urlparse(response.resolved_url).hostname or "").lower()
|
||||
for parsed_document, vector in zip(parsed.documents, vectors, strict=True):
|
||||
for ordinal, (parsed_document, vector) in enumerate(zip(parsed.documents, vectors, strict=True)):
|
||||
content_hash = hashlib.sha256(parsed_document.content.encode()).hexdigest()
|
||||
# Include the ordinal so repeated section titles (common in large
|
||||
# llms-full.txt feeds) cannot collide on the primary key.
|
||||
identity = "\0".join(
|
||||
[source, parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)]
|
||||
[source, str(ordinal), parsed_document.canonical_url, parsed_document.heading_path, str(parsed_document.chunk_index)]
|
||||
)
|
||||
documents.append(
|
||||
PreparedDocument(
|
||||
|
||||
@@ -91,6 +91,24 @@ class RefreshTest(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(1, self.store.list_sources()[0].doc_count)
|
||||
self.assertTrue(self.store.lexical_search("Original", limit=5))
|
||||
|
||||
async def test_repeated_section_titles_index_without_id_collisions(self) -> None:
|
||||
body = "# Basic syntax\n\nFirst variant.\n\n# Basic syntax\n\nSecond variant.\n"
|
||||
fetcher = FakeFetcher([FakeFetch(200, body)])
|
||||
coordinator = RefreshCoordinator(
|
||||
store=self.store,
|
||||
fetcher=fetcher,
|
||||
embedder=FakeEmbedder(),
|
||||
parser=parse_llms_text,
|
||||
ttl_seconds=3600,
|
||||
now=lambda: 100.0,
|
||||
)
|
||||
|
||||
outcome = await coordinator.refresh(self.source, force=True)
|
||||
|
||||
self.assertEqual("updated", outcome.status)
|
||||
self.assertEqual(2, outcome.document_count)
|
||||
self.assertEqual(2, self.store.list_sources()[0].doc_count)
|
||||
|
||||
async def test_empty_success_response_preserves_previous_content(self) -> None:
|
||||
fetcher = FakeFetcher(
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user