Fix ID collisions for repeated section titles
Large llms-full.txt feeds repeat section headings, which made document identity hashes collide within one source and abort indexing on the documents primary key. Include each document's ordinal in the identity.
This commit is contained in:
1 parent
51dceee224
commit
ac3465c656
2 files changed
+22
-2
No files matched your search
@@ -91,6 +91,24 @@ class RefreshTest(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(1, self.store.list_sources()[0].doc_count)
|
||||
self.assertTrue(self.store.lexical_search("Original", limit=5))
|
||||
|
||||
async def test_repeated_section_titles_index_without_id_collisions(self) -> None:
|
||||
body = "# Basic syntax\n\nFirst variant.\n\n# Basic syntax\n\nSecond variant.\n"
|
||||
fetcher = FakeFetcher([FakeFetch(200, body)])
|
||||
coordinator = RefreshCoordinator(
|
||||
store=self.store,
|
||||
fetcher=fetcher,
|
||||
embedder=FakeEmbedder(),
|
||||
parser=parse_llms_text,
|
||||
ttl_seconds=3600,
|
||||
now=lambda: 100.0,
|
||||
)
|
||||
|
||||
outcome = await coordinator.refresh(self.source, force=True)
|
||||
|
||||
self.assertEqual("updated", outcome.status)
|
||||
self.assertEqual(2, outcome.document_count)
|
||||
self.assertEqual(2, self.store.list_sources()[0].doc_count)
|
||||
|
||||
async def test_empty_success_response_preserves_previous_content(self) -> None:
|
||||
fetcher = FakeFetcher(
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user