Fix ID collisions for repeated section titles

Large llms-full.txt feeds repeat section headings, which made document
identity hashes collide within one source and abort indexing on the
documents primary key. Include each document's ordinal in the identity.
This commit is contained in:
2026-07-25 09:00:59 -07:00
parent 51dceee224
commit ac3465c656
2 changed files with 22 additions and 2 deletions

View File

@@ -91,6 +91,24 @@ class RefreshTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(1, self.store.list_sources()[0].doc_count)
self.assertTrue(self.store.lexical_search("Original", limit=5))
async def test_repeated_section_titles_index_without_id_collisions(self) -> None:
body = "# Basic syntax\n\nFirst variant.\n\n# Basic syntax\n\nSecond variant.\n"
fetcher = FakeFetcher([FakeFetch(200, body)])
coordinator = RefreshCoordinator(
store=self.store,
fetcher=fetcher,
embedder=FakeEmbedder(),
parser=parse_llms_text,
ttl_seconds=3600,
now=lambda: 100.0,
)
outcome = await coordinator.refresh(self.source, force=True)
self.assertEqual("updated", outcome.status)
self.assertEqual(2, outcome.document_count)
self.assertEqual(2, self.store.list_sources()[0].doc_count)
async def test_empty_success_response_preserves_previous_content(self) -> None:
fetcher = FakeFetcher(
[