diff --git a/llm_be/chat_backend/management/commands/reindex_embeddings.py b/llm_be/chat_backend/management/commands/reindex_embeddings.py index ed247c3..06380ad 100644 --- a/llm_be/chat_backend/management/commands/reindex_embeddings.py +++ b/llm_be/chat_backend/management/commands/reindex_embeddings.py @@ -46,7 +46,18 @@ class Command(BaseCommand): self.stdout.write("Clearing Chroma collection…") rag.clear_vector_store() self.stdout.write("Re-ingesting documents…") - rag.ingest_documents() + try: + rag.ingest_documents() + except Exception as exc: + # Partial ingest may still have written some chunks; report and re-raise. + count = rag.vector_store._collection.count() + self.stderr.write( + self.style.ERROR( + f"Reindex failed after clearing Chroma ({exc}). " + f"Vector chunks currently: {count}. Fix the error and re-run." + ) + ) + raise count = rag.vector_store._collection.count() self.stdout.write( self.style.SUCCESS(f"Reindex complete. Vector chunks now: {count}") diff --git a/llm_be/chat_backend/services/rag_services.py b/llm_be/chat_backend/services/rag_services.py index eeccf42..ce82d1c 100644 --- a/llm_be/chat_backend/services/rag_services.py +++ b/llm_be/chat_backend/services/rag_services.py @@ -161,6 +161,13 @@ class RAGService(BaseService): ) if chunks: self.vector_store.add_documents(chunks) + except Exception: + # Keep reindex/ingest moving; one bad file must not wipe progress. + logger.exception( + "Failed to ingest document_id=%s file=%s", + doc.id, + doc.file.name, + ) finally: if os.path.exists(tmp_path): os.unlink(tmp_path) diff --git a/pyproject.toml b/pyproject.toml index 7e4a3a8..43cdd34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,9 @@ dependencies = [ # Supported: langgraph>=1.2.5,<1.3 (matches langchain 1.3.x). "langgraph>=1.2.5,<1.3.0", "chromadb==1.3.5", - "unstructured==0.18.21", + # Fallback loader uses UnstructuredFileLoader; xlsx partition needs the + # [xlsx] extra (networkx, msoffcrypto-tool, xlrd, openpyxl, pandas). + "unstructured[xlsx]==0.18.21", "pypdf==6.4.0", "python-docx==1.2.0", "docx2txt==0.8", diff --git a/uv.lock b/uv.lock index 041098a..3871176 100644 --- a/uv.lock +++ b/uv.lock @@ -661,8 +661,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "python-docx" }, { name = "pytz" }, + { name = "requests" }, { name = "stripe" }, - { name = "unstructured" }, + { name = "unstructured", extra = ["xlsx"] }, { name = "uvicorn" }, { name = "whitenoise" }, ] @@ -708,8 +709,9 @@ requires-dist = [ { name = "python-dateutil", specifier = "==2.9.0.post0" }, { name = "python-docx", specifier = "==1.2.0" }, { name = "pytz", specifier = "==2025.2" }, + { name = "requests", specifier = ">=2.32,<3" }, { name = "stripe", specifier = ">=12.0.0,<14.0.0" }, - { name = "unstructured", specifier = "==0.18.21" }, + { name = "unstructured", extras = ["xlsx"], specifier = "==0.18.21" }, { name = "uvicorn", specifier = "==0.38.0" }, { name = "whitenoise", specifier = "==6.9.0" }, ] @@ -2286,6 +2288,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" }, ] +[[package]] +name = "msoffcrypto-tool" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "olefile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/34/6250bdddaeaae24098e45449ea362fb3555a65fba30cad0ad5630ea48d1a/msoffcrypto_tool-6.0.0.tar.gz", hash = "sha256:9a5ebc4c0096b42e5d7ebc2350afdc92dc511061e935ca188468094fdd032bbe", size = 40593, upload-time = "2026-01-12T08:59:56.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/85/9e359fa9279e1d6861faaf9b6f037a3226374deb20a054c3937be6992013/msoffcrypto_tool-6.0.0-py3-none-any.whl", hash = "sha256:46c394ed5d9641e802fc79bf3fb0666a53748b23fa8c4aa634ae9d30d46fe397", size = 48791, upload-time = "2026-01-12T08:59:55.394Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -2394,6 +2409,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + [[package]] name = "nltk" version = "3.10.0" @@ -4238,6 +4262,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/98/e8ddcfadd762f8f69d84e14498c28adefdd8e2008f443077495984405c45/unstructured-0.18.21-py3-none-any.whl", hash = "sha256:8611c06017199fd7f131f3fbf586a458fd83074fc6aadac23640f6450a00e377", size = 1783346, upload-time = "2025-11-24T14:57:36.863Z" }, ] +[package.optional-dependencies] +xlsx = [ + { name = "msoffcrypto-tool" }, + { name = "networkx" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "xlrd" }, +] + [[package]] name = "unstructured-client" version = "0.42.12" @@ -4596,6 +4629,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +] + [[package]] name = "xxhash" version = "3.8.1"