From 8ff3ad83d155208e7dc67a5eeee5957e322ed79e Mon Sep 17 00:00:00 2001
From: Ryan Westfall <ryan@aimloperations.com>
Date: Sun, 2 Aug 2026 13:54:58 -0500
Subject: [PATCH] Fix reindex_embeddings crash: add networkx for Unstructured
 xlsx.

unstructured.partition.xlsx imports networkx; without it reindex dies
mid-run after clearing Chroma. Also skip per-document ingest failures
so one bad file cannot abort the whole rebuild.
---
 .../management/commands/reindex_embeddings.py       | 13 ++++++++++++-
 llm_be/chat_backend/services/rag_services.py        |  7 +++++++
 pyproject.toml                                      |  1 +
 uv.lock                                             | 13 +++++++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/llm_be/chat_backend/management/commands/reindex_embeddings.py b/llm_be/chat_backend/management/commands/reindex_embeddings.py
index ed247c3..06380ad 100644
--- a/llm_be/chat_backend/management/commands/reindex_embeddings.py
+++ b/llm_be/chat_backend/management/commands/reindex_embeddings.py
@@ -46,7 +46,18 @@ class Command(BaseCommand):
         self.stdout.write("Clearing Chroma collection…")
         rag.clear_vector_store()
         self.stdout.write("Re-ingesting documents…")
-        rag.ingest_documents()
+        try:
+            rag.ingest_documents()
+        except Exception as exc:
+            # Partial ingest may still have written some chunks; report and re-raise.
+            count = rag.vector_store._collection.count()
+            self.stderr.write(
+                self.style.ERROR(
+                    f"Reindex failed after clearing Chroma ({exc}). "
+                    f"Vector chunks currently: {count}. Fix the error and re-run."
+                )
+            )
+            raise
         count = rag.vector_store._collection.count()
         self.stdout.write(
             self.style.SUCCESS(f"Reindex complete. Vector chunks now: {count}")
diff --git a/llm_be/chat_backend/services/rag_services.py b/llm_be/chat_backend/services/rag_services.py
index eeccf42..ce82d1c 100644
--- a/llm_be/chat_backend/services/rag_services.py
+++ b/llm_be/chat_backend/services/rag_services.py
@@ -161,6 +161,13 @@ class RAGService(BaseService):
                 )
                 if chunks:
                     self.vector_store.add_documents(chunks)
+            except Exception:
+                # Keep reindex/ingest moving; one bad file must not wipe progress.
+                logger.exception(
+                    "Failed to ingest document_id=%s file=%s",
+                    doc.id,
+                    doc.file.name,
+                )
             finally:
                 if os.path.exists(tmp_path):
                     os.unlink(tmp_path)
diff --git a/pyproject.toml b/pyproject.toml
index 7e4a3a8..aa107dc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,6 +47,7 @@ dependencies = [
     "python-dateutil==2.9.0.post0",
     "pytz==2025.2",
     "stripe>=12.0.0,<14.0.0",
+    "networkx>=3.0,<4",
 ]
 
 [dependency-groups]
diff --git a/uv.lock b/uv.lock
index 041098a..2f16ebb 100644
--- a/uv.lock
+++ b/uv.lock
@@ -651,6 +651,7 @@ dependencies = [
     { name = "langchain-text-splitters" },
     { name = "langgraph" },
     { name = "matplotlib" },
+    { name = "networkx" },
     { name = "numpy" },
     { name = "ollama" },
     { name = "openpyxl" },
@@ -661,6 +662,7 @@ dependencies = [
     { name = "python-dateutil" },
     { name = "python-docx" },
     { name = "pytz" },
+    { name = "requests" },
     { name = "stripe" },
     { name = "unstructured" },
     { name = "uvicorn" },
@@ -698,6 +700,7 @@ requires-dist = [
     { name = "langchain-text-splitters", specifier = "==1.1.2" },
     { name = "langgraph", specifier = ">=1.2.5,<1.3.0" },
     { name = "matplotlib", specifier = "==3.10.7" },
+    { name = "networkx", specifier = ">=3.0,<4" },
     { name = "numpy", specifier = "==2.2.6" },
     { name = "ollama", specifier = "==0.6.1" },
     { name = "openpyxl", specifier = "==3.1.5" },
@@ -708,6 +711,7 @@ requires-dist = [
     { name = "python-dateutil", specifier = "==2.9.0.post0" },
     { name = "python-docx", specifier = "==1.2.0" },
     { name = "pytz", specifier = "==2025.2" },
+    { name = "requests", specifier = ">=2.32,<3" },
     { name = "stripe", specifier = ">=12.0.0,<14.0.0" },
     { name = "unstructured", specifier = "==0.18.21" },
     { name = "uvicorn", specifier = "==0.38.0" },
@@ -2394,6 +2398,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
 ]
 
+[[package]]
+name = "networkx"
+version = "3.6.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" },
+]
+
 [[package]]
 name = "nltk"
 version = "3.10.0"