Fix reindex_embeddings: install unstructured[xlsx] extras (#66)

## Summary

`reindex_embeddings` crashed with `ModuleNotFoundError: No module named 'networkx'` when Unstructured hit spreadsheet files via `partition/xlsx.py`. Reindex clears Chroma first, so a mid-run crash leaves an empty/partial vector store.

### True fix
Depend on **`unstructured[xlsx]==0.18.21`** (not bare `unstructured`). That extra pulls the required spreadsheet partition deps: `networkx`, `msoffcrypto-tool`, `xlrd` (plus openpyxl/pandas already present).

### Resilience
Also catch per-document ingest failures so one corrupt/unsupported file cannot abort a full rebuild after Chroma was cleared. Successful files still ingest; failures are logged.

## Test plan
- [x] CI / unit tests on this PR
- [ ] Merge + redeploy image to all hosts
- [ ] On each host: `docker compose -p chat_backend_prod exec web bash -lc 'cd /app/llm_be && SKIP_RAG_INIT=1 uv run python manage.py reindex_embeddings'`
- [ ] Confirm non-zero chunk count and no `networkx` / `msoffcrypto` / `xlrd` import errors

## Ops workaround (running containers only, until deploy)
```bash
uv pip install 'unstructured[xlsx]==0.18.21'
cd /app/llm_be && SKIP_RAG_INIT=1 uv run python manage.py reindex_embeddings
```Reviewed-on: #66

llm_be/chat_backend/management/commands/reindex_embeddings.py

@@ -46,7 +46,18 @@ class Command(BaseCommand):
         self.stdout.write("Clearing Chroma collection…")
         rag.clear_vector_store()
         self.stdout.write("Re-ingesting documents…")
-        rag.ingest_documents()
+        try:
+            rag.ingest_documents()
+        except Exception as exc:
+            # Partial ingest may still have written some chunks; report and re-raise.
+            count = rag.vector_store._collection.count()
+            self.stderr.write(
+                self.style.ERROR(
+                    f"Reindex failed after clearing Chroma ({exc}). "
+                    f"Vector chunks currently: {count}. Fix the error and re-run."
+                )
+            )
+            raise
         count = rag.vector_store._collection.count()
         self.stdout.write(
             self.style.SUCCESS(f"Reindex complete. Vector chunks now: {count}")

llm_be/chat_backend/services/rag_services.py

@@ -161,6 +161,13 @@ class RAGService(BaseService):
                 )
                 if chunks:
                     self.vector_store.add_documents(chunks)
+            except Exception:
+                # Keep reindex/ingest moving; one bad file must not wipe progress.
+                logger.exception(
+                    "Failed to ingest document_id=%s file=%s",
+                    doc.id,
+                    doc.file.name,
+                )
             finally:
                 if os.path.exists(tmp_path):
                     os.unlink(tmp_path)

pyproject.toml

@@ -31,7 +31,9 @@ dependencies = [
     # Supported: langgraph>=1.2.5,<1.3 (matches langchain 1.3.x).
     "langgraph>=1.2.5,<1.3.0",
     "chromadb==1.3.5",
-    "unstructured==0.18.21",
+    # Fallback loader uses UnstructuredFileLoader; xlsx partition needs the
+    # [xlsx] extra (networkx, msoffcrypto-tool, xlrd, openpyxl, pandas).
+    "unstructured[xlsx]==0.18.21",
     "pypdf==6.4.0",
     "python-docx==1.2.0",
     "docx2txt==0.8",

uv.lock

@@ -661,8 +661,9 @@ dependencies = [
     { name = "python-dateutil" },
     { name = "python-docx" },
     { name = "pytz" },
+    { name = "requests" },
     { name = "stripe" },
-    { name = "unstructured" },
+    { name = "unstructured", extra = ["xlsx"] },
     { name = "uvicorn" },
     { name = "whitenoise" },
 ]
@@ -708,8 +709,9 @@ requires-dist = [
     { name = "python-dateutil", specifier = "==2.9.0.post0" },
     { name = "python-docx", specifier = "==1.2.0" },
     { name = "pytz", specifier = "==2025.2" },
+    { name = "requests", specifier = ">=2.32,<3" },
     { name = "stripe", specifier = ">=12.0.0,<14.0.0" },
-    { name = "unstructured", specifier = "==0.18.21" },
+    { name = "unstructured", extras = ["xlsx"], specifier = "==0.18.21" },
     { name = "uvicorn", specifier = "==0.38.0" },
     { name = "whitenoise", specifier = "==6.9.0" },
 ]
@@ -2286,6 +2288,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" },
 ]
 
+[[package]]
+name = "msoffcrypto-tool"
+version = "6.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cryptography" },
+    { name = "olefile" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a6/34/6250bdddaeaae24098e45449ea362fb3555a65fba30cad0ad5630ea48d1a/msoffcrypto_tool-6.0.0.tar.gz", hash = "sha256:9a5ebc4c0096b42e5d7ebc2350afdc92dc511061e935ca188468094fdd032bbe", size = 40593, upload-time = "2026-01-12T08:59:56.73Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/85/9e359fa9279e1d6861faaf9b6f037a3226374deb20a054c3937be6992013/msoffcrypto_tool-6.0.0-py3-none-any.whl", hash = "sha256:46c394ed5d9641e802fc79bf3fb0666a53748b23fa8c4aa634ae9d30d46fe397", size = 48791, upload-time = "2026-01-12T08:59:55.394Z" },
+]
+
 [[package]]
 name = "multidict"
 version = "6.7.1"
@@ -2394,6 +2409,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
 ]
 
+[[package]]
+name = "networkx"
+version = "3.6.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" },
+]
+
 [[package]]
 name = "nltk"
 version = "3.10.0"
@@ -4238,6 +4262,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/98/e8ddcfadd762f8f69d84e14498c28adefdd8e2008f443077495984405c45/unstructured-0.18.21-py3-none-any.whl", hash = "sha256:8611c06017199fd7f131f3fbf586a458fd83074fc6aadac23640f6450a00e377", size = 1783346, upload-time = "2025-11-24T14:57:36.863Z" },
 ]
 
+[package.optional-dependencies]
+xlsx = [
+    { name = "msoffcrypto-tool" },
+    { name = "networkx" },
+    { name = "openpyxl" },
+    { name = "pandas" },
+    { name = "xlrd" },
+]
+
 [[package]]
 name = "unstructured-client"
 version = "0.42.12"
@@ -4596,6 +4629,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" },
 ]
 
+[[package]]
+name = "xlrd"
+version = "2.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" },
+]
+
 [[package]]
 name = "xxhash"
 version = "3.8.1"