Add Drive sync progress, paginated docs list, and prompt heatmap API.

Expose sync_total/processed counters for FE progress bars (#59), support
documents pagination/search/sort/workspace scope (#60), and add
weekday×hour user prompt heatmap analytics (#94).

llm_be/chat_backend/drive_tasks.py

@@ -85,8 +85,22 @@ def enqueue_drive_sync(
 
     connection.last_sync_status = DriveConnection.SyncStatus.PENDING
     connection.last_sync_error = ""
+    connection.sync_total = 0
+    connection.sync_processed = 0
+    connection.sync_added = 0
+    connection.sync_updated = 0
+    connection.sync_failed = 0
     connection.save(
-        update_fields=["last_sync_status", "last_sync_error", "last_modified"]
+        update_fields=[
+            "last_sync_status",
+            "last_sync_error",
+            "sync_total",
+            "sync_processed",
+            "sync_added",
+            "sync_updated",
+            "sync_failed",
+            "last_modified",
+        ]
     )
 
     transaction.on_commit(

llm_be/chat_backend/migrations/0030_drive_sync_progress_counters.py

@@ -0,0 +1,44 @@
+# Generated by Django 6.0 on 2026-08-02 11:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("chat_backend", "0029_personal_drive_rag_without_company"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="driveconnection",
+            name="sync_added",
+            field=models.PositiveIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="driveconnection",
+            name="sync_failed",
+            field=models.PositiveIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="driveconnection",
+            name="sync_processed",
+            field=models.PositiveIntegerField(
+                default=0,
+                help_text="Files finished in the current/last sync (includes skips).",
+            ),
+        ),
+        migrations.AddField(
+            model_name="driveconnection",
+            name="sync_total",
+            field=models.PositiveIntegerField(
+                default=0,
+                help_text="Remote files discovered for the current/last sync run.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="driveconnection",
+            name="sync_updated",
+            field=models.PositiveIntegerField(default=0),
+        ),
+    ]

llm_be/chat_backend/models.py

@@ -480,6 +480,18 @@ class DriveConnection(TimeInfoBase):
         max_length=16, choices=SyncStatus.choices, default=SyncStatus.NEVER
     )
     last_sync_error = models.TextField(blank=True, default="")
+    # Progress for FE progress bar while last_sync_status=pending (#59).
+    sync_total = models.PositiveIntegerField(
+        default=0,
+        help_text="Remote files discovered for the current/last sync run.",
+    )
+    sync_processed = models.PositiveIntegerField(
+        default=0,
+        help_text="Files finished in the current/last sync (includes skips).",
+    )
+    sync_added = models.PositiveIntegerField(default=0)
+    sync_updated = models.PositiveIntegerField(default=0)
+    sync_failed = models.PositiveIntegerField(default=0)
     is_active = models.BooleanField(default=True)
 
     class Meta:

llm_be/chat_backend/serializers.py

@@ -298,6 +298,11 @@ class DriveConnectionSerializer(serializers.ModelSerializer):
             "last_sync_at",
             "last_sync_status",
             "last_sync_error",
+            "sync_total",
+            "sync_processed",
+            "sync_added",
+            "sync_updated",
+            "sync_failed",
             "is_active",
             "created",
         ]

llm_be/chat_backend/services/drive_sync.py

@@ -346,11 +346,43 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
     result: dict[str, Any] = {"added": 0, "updated": 0, "removed": 0, "failed": []}
 
     connection.last_sync_status = DriveConnection.SyncStatus.PENDING
-    connection.save(update_fields=["last_sync_status", "last_modified"])
+    connection.sync_total = 0
+    connection.sync_processed = 0
+    connection.sync_added = 0
+    connection.sync_updated = 0
+    connection.sync_failed = 0
+    connection.save(
+        update_fields=[
+            "last_sync_status",
+            "sync_total",
+            "sync_processed",
+            "sync_added",
+            "sync_updated",
+            "sync_failed",
+            "last_modified",
+        ]
+    )
+
+    def _persist_progress(**extra: Any) -> None:
+        fields = [
+            "sync_total",
+            "sync_processed",
+            "sync_added",
+            "sync_updated",
+            "sync_failed",
+            "last_modified",
+            *extra.keys(),
+        ]
+        for key, value in extra.items():
+            setattr(connection, key, value)
+        connection.save(update_fields=list(dict.fromkeys(fields)))
 
     try:
         access_token = ensure_fresh_token(connection)
         remote_files = _list_remote_files(connection, access_token)
+        connection.sync_total = len(remote_files)
+        _persist_progress()
+
         if connection.kind == DriveConnection.Kind.PERSONAL:
             if connection.user_id is None:
                 raise DriveSyncError(
@@ -379,6 +411,8 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
             for remote in remote_files:
                 existing = existing_docs.get(remote.id)
                 if existing is not None and existing.remote_etag == remote.etag:
+                    connection.sync_processed += 1
+                    _persist_progress()
                     continue
 
                 try:
@@ -387,6 +421,9 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
                     )
                 except DriveSyncError as exc:
                     result["failed"].append({"file": remote.name, "error": exc.message})
+                    connection.sync_failed += 1
+                    connection.sync_processed += 1
+                    _persist_progress()
                     continue
 
                 if existing is not None:
@@ -399,6 +436,7 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
                     existing.save()
                     document = existing
                     result["updated"] += 1
+                    connection.sync_updated += 1
                 else:
                     document = Document.objects.create(
                         workspace=workspace,
@@ -411,6 +449,7 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
                     )
                     document.file.save(filename, ContentFile(content), save=True)
                     result["added"] += 1
+                    connection.sync_added += 1
 
                 try:
                     ingest_result = rag_service.add_files_to_store(
@@ -442,6 +481,10 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
                     document.sync_error = str(exc)
                     document.save(update_fields=["sync_error", "last_modified"])
                     result["failed"].append({"file": remote.name, "error": str(exc)})
+                    connection.sync_failed += 1
+
+                connection.sync_processed += 1
+                _persist_progress()
 
             for remote_id, document in existing_docs.items():
                 if remote_id not in remote_by_id:
@@ -465,7 +508,17 @@ def sync_connection(connection: DriveConnection) -> dict[str, Any]:
     finally:
         connection.last_sync_at = timezone.now()
         connection.save(
-            update_fields=["last_sync_status", "last_sync_error", "last_sync_at", "last_modified"]
+            update_fields=[
+                "last_sync_status",
+                "last_sync_error",
+                "last_sync_at",
+                "sync_total",
+                "sync_processed",
+                "sync_added",
+                "sync_updated",
+                "sync_failed",
+                "last_modified",
+            ]
         )
 
     return result

llm_be/chat_backend/tests/test_drive_tasks.py

@@ -50,6 +50,11 @@ class EnqueueDriveSyncTestCase(TransactionTestCase):
         connection.refresh_from_db()
         self.assertEqual(connection.last_sync_status, DriveConnection.SyncStatus.PENDING)
         self.assertEqual(connection.last_sync_error, "")
+        self.assertEqual(connection.sync_total, 0)
+        self.assertEqual(connection.sync_processed, 0)
+        self.assertEqual(connection.sync_added, 0)
+        self.assertEqual(connection.sync_updated, 0)
+        self.assertEqual(connection.sync_failed, 0)
         mock_dispatch.assert_called_once_with(connection_id=self.connection.id)
 
     @mock.patch("chat_backend.drive_tasks._dispatch_sync")

llm_be/chat_backend/tests/test_services_drive_sync.py

@@ -397,6 +397,11 @@ class SyncConnectionTestCase(TestCase):
         self.connection.refresh_from_db()
         self.assertEqual(self.connection.last_sync_status, DriveConnection.SyncStatus.OK)
         self.assertIsNotNone(self.connection.last_sync_at)
+        self.assertEqual(self.connection.sync_total, 1)
+        self.assertEqual(self.connection.sync_processed, 1)
+        self.assertEqual(self.connection.sync_added, 1)
+        self.assertEqual(self.connection.sync_updated, 0)
+        self.assertEqual(self.connection.sync_failed, 0)
 
     def test_skips_unchanged_file(self):
         Document.objects.create(
@@ -415,6 +420,9 @@ class SyncConnectionTestCase(TestCase):
         self.assertEqual(result["added"], 0)
         self.assertEqual(result["updated"], 0)
         self.mock_download.assert_not_called()
+        self.connection.refresh_from_db()
+        self.assertEqual(self.connection.sync_total, 1)
+        self.assertEqual(self.connection.sync_processed, 1)
 
     def test_updates_file_when_etag_changes(self):
         existing = Document.objects.create(

llm_be/chat_backend/tests/test_views_analytics.py

@@ -127,3 +127,60 @@ class AdminAnalyticsTestCase(APITestCase):
         self.assertEqual(current["range"], [10, 20])
         self.assertEqual(current["avg"], 15)
         self.assertEqual(current["median"], 20)
+
+
+class UserPromptHeatmapTestCase(APITestCase):
+    def setUp(self):
+        self.user = make_user(company=make_company())
+        self.other = make_user(email="other@example.com", company=make_company("Other"))
+        self.client.force_authenticate(user=self.user)
+        self.conversation = make_conversation(user=self.user)
+        self.url = reverse("analytics_user_prompt_heatmap")
+
+    def test_empty_heatmap(self):
+        response = self.client.get(self.url, {"tz": "UTC"})
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["total"], 0)
+        self.assertEqual(response.data["max"], 0)
+        self.assertEqual(response.data["matrix"], [[0] * 24 for _ in range(7)])
+        self.assertIsNone(response.data["peak_cell"])
+
+    def test_bins_user_prompts_in_utc(self):
+        # Wednesday 2026-07-15 14:30 UTC → weekday 2, hour 14
+        created = datetime.datetime(2026, 7, 15, 14, 30, tzinfo=datetime.timezone.utc)
+        make_prompt(self.conversation, message="a", created=created)
+        make_prompt(self.conversation, message="b", created=created)
+        make_prompt(
+            self.conversation,
+            message="llm",
+            user_created=False,
+            created=created,
+        )
+        other_convo = make_conversation(user=self.other)
+        make_prompt(other_convo, message="other", created=created)
+
+        response = self.client.get(self.url, {"tz": "UTC"})
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["total"], 2)
+        self.assertEqual(response.data["max"], 2)
+        self.assertEqual(response.data["matrix"][2][14], 2)
+        self.assertEqual(response.data["most_active_day"], "Wed")
+        self.assertEqual(response.data["most_active_hour"], 14)
+        self.assertEqual(
+            response.data["peak_cell"],
+            {"day": "Wed", "hour": 14, "count": 2},
+        )
+
+    def test_applies_timezone(self):
+        # 2026-07-15 14:30 UTC → America/Chicago CDT (UTC-5) = 09:30 Wed
+        created = datetime.datetime(2026, 7, 15, 14, 30, tzinfo=datetime.timezone.utc)
+        make_prompt(self.conversation, message="a", created=created)
+
+        response = self.client.get(self.url, {"tz": "America/Chicago"})
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["tz"], "America/Chicago")
+        self.assertEqual(response.data["matrix"][2][9], 1)
+        self.assertEqual(response.data["matrix"][2][14], 0)

llm_be/chat_backend/tests/test_views_documents.py

@@ -112,9 +112,11 @@ class DocumentUploadViewTestCase(APITestCase):
         response = self.client.get(self.url)
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), 1)
-        self.assertIn("test", response.data[0]["file"])
-        self.assertIn("pdf", response.data[0]["file"])
+        self.assertEqual(response.data["count"], 1)
+        self.assertEqual(response.data["scope"], "company")
+        self.assertEqual(len(response.data["results"]), 1)
+        self.assertIn("test", response.data["results"][0]["file"])
+        self.assertIn("pdf", response.data["results"][0]["file"])
 
     def test_list_without_workspace_creates_a_default_one(self):
         """#46: missing workspace is auto-created instead of 404ing the list."""
@@ -123,13 +125,50 @@ class DocumentUploadViewTestCase(APITestCase):
         response = self.client.get(self.url)
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.data, [])
+        self.assertEqual(response.data["count"], 0)
+        self.assertEqual(response.data["results"], [])
         self.assertTrue(
             DocumentWorkspace.objects.filter(
                 company=self.company, name="Default"
             ).exists()
         )
 
+    def test_list_paginates_searches_and_sorts(self):
+        """#60: page / search / ordering query params."""
+        make_document(self.workspace, name="alpha.pdf")
+        make_document(self.workspace, name="beta.pdf")
+        make_document(self.workspace, name="gamma.pdf")
+
+        page_one = self.client.get(
+            self.url, {"page": 1, "page_size": 2, "ordering": "name"}
+        )
+        self.assertEqual(page_one.status_code, status.HTTP_200_OK)
+        self.assertEqual(page_one.data["count"], 3)
+        self.assertEqual(page_one.data["page"], 1)
+        self.assertEqual(page_one.data["page_size"], 2)
+        self.assertEqual(len(page_one.data["results"]), 2)
+        self.assertIn("alpha", page_one.data["results"][0]["file"])
+
+        search = self.client.get(self.url, {"search": "beta"})
+        self.assertEqual(search.status_code, status.HTTP_200_OK)
+        self.assertEqual(search.data["count"], 1)
+        self.assertIn("beta", search.data["results"][0]["file"])
+
+    def test_list_personal_scope_isolated_from_company(self):
+        """#60: company members can list personal workspace separately."""
+        make_document(self.workspace, name="company.pdf")
+        personal = make_workspace(user=self.user, name="Personal")
+        make_document(personal, name="mine.pdf")
+
+        company_list = self.client.get(self.url, {"workspace": "company"})
+        personal_list = self.client.get(self.url, {"workspace": "personal"})
+
+        self.assertEqual(company_list.data["count"], 1)
+        self.assertIn("company", company_list.data["results"][0]["file"])
+        self.assertEqual(personal_list.data["count"], 1)
+        self.assertIn("mine", personal_list.data["results"][0]["file"])
+        self.assertEqual(personal_list.data["scope"], "personal")
+
 
 class DocumentDetailViewTestCase(APITestCase):
     def setUp(self):

llm_be/chat_backend/urls.py

@@ -22,6 +22,7 @@ from .views import (
     UserConversationAnalytics,
     CompanyUsageAnalytics,
     AdminAnalytics,
+    UserPromptHeatmap,
     DocumentWorkspaceView,
     DocumentUploadView,
     DocumentDetailView,
@@ -103,6 +104,11 @@ urlpatterns = [
         name="analytics_company_usage",
     ),
     path("analytics/admin/", AdminAnalytics.as_view(), name="analytics_admin"),
+    path(
+        "analytics/user_prompt_heatmap/",
+        UserPromptHeatmap.as_view(),
+        name="analytics_user_prompt_heatmap",
+    ),
     # document urls
     path(
         "document_workspaces/",

llm_be/chat_backend/views.py

@@ -51,6 +51,7 @@ from chat_backend.services.assistant_identity import ASSISTANT_SYSTEM_PROMPT
 from django.utils import timezone
 from django.core.files import File
 from django.core.files.base import ContentFile
+from django.db.models import Q
 import math
 import datetime
 import pytz
@@ -68,7 +69,11 @@ from .email_tasks import (
 from finance.services.quotas import FeatureNotAllowed, assert_feature_allowed
 from .services.llm_service import AsyncLLMService
 from .services.rag_services import AsyncRAGService
-from .services.chat_tenant_scope import ensure_workspace_for_user
+from .services.chat_tenant_scope import (
+    ensure_company_workspace,
+    ensure_personal_workspace,
+    ensure_workspace_for_user,
+)
 from .services.title_generator import title_generator
 from .services.moderation_classifier import moderation_classifier, ModerationLabel
 from .services.prompt_classifier.prompt_classifier import PromptClassifier, PromptType
@@ -730,6 +735,76 @@ class CompanyUsageAnalytics(APIView):
         return Response(result[::-1], status=status.HTTP_200_OK)
 
 
+        return Response(result[::-1], status=status.HTTP_200_OK)
+
+
+class UserPromptHeatmap(APIView):
+    """Weekday × hour bins of user-entered prompts (local timezone) (#94)."""
+
+    DAY_LABELS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
+
+    def get(self, request, format="json"):
+        tz_name = request.query_params.get("tz", "UTC")
+        try:
+            user_tz = pytz.timezone(tz_name)
+        except pytz.UnknownTimeZoneError:
+            user_tz = pytz.UTC
+            tz_name = "UTC"
+
+        matrix = [[0 for _ in range(24)] for _ in range(7)]
+        total = 0
+        prompts = Prompt.objects.filter(
+            conversation__user=request.user,
+            conversation__deleted=False,
+            user_created=True,
+        ).only("created")
+
+        for prompt in prompts.iterator():
+            created = prompt.created
+            if timezone.is_naive(created):
+                created = timezone.make_aware(created, datetime.timezone.utc)
+            local = created.astimezone(user_tz)
+            matrix[local.weekday()][local.hour] += 1
+            total += 1
+
+        max_count = max((count for row in matrix for count in row), default=0)
+        most_active_day = None
+        most_active_hour = None
+        peak_cell = None
+        if max_count > 0:
+            day_totals = [sum(row) for row in matrix]
+            most_active_day = self.DAY_LABELS[day_totals.index(max(day_totals))]
+            hour_totals = [sum(matrix[day][hour] for day in range(7)) for hour in range(24)]
+            most_active_hour = hour_totals.index(max(hour_totals))
+            peak_day = 0
+            peak_hour = 0
+            for day_idx, row in enumerate(matrix):
+                for hour_idx, count in enumerate(row):
+                    if count > matrix[peak_day][peak_hour]:
+                        peak_day = day_idx
+                        peak_hour = hour_idx
+            peak_cell = {
+                "day": self.DAY_LABELS[peak_day],
+                "hour": peak_hour,
+                "count": matrix[peak_day][peak_hour],
+            }
+
+        return Response(
+            {
+                "tz": tz_name,
+                "total": total,
+                "max": max_count,
+                "days": list(self.DAY_LABELS),
+                "hours": list(range(24)),
+                "matrix": matrix,
+                "most_active_day": most_active_day,
+                "most_active_hour": most_active_hour,
+                "peak_cell": peak_cell,
+            },
+            status=status.HTTP_200_OK,
+        )
+
+
 class AdminAnalytics(APIView):
     def get(self, request, format="json"):
         number_of_months = 3
@@ -835,6 +910,33 @@ class DocumentWorkspaceView(APIView):
 class DocumentUploadView(APIView):
     # permission_classes = [permissions.IsAuthenticated]Z
 
+    DOCUMENT_SORT_FIELDS = {
+        "name": "file",
+        "-name": "-file",
+        "created": "created",
+        "-created": "-created",
+        "date_uploaded": "created",
+        "-date_uploaded": "-created",
+        "processed": "processed",
+        "-processed": "-processed",
+        "active": "active",
+        "-active": "-active",
+    }
+
+    def _resolve_list_workspace(self, request):
+        """Return (workspace, scope) for personal|company document lists (#60)."""
+        raw = (request.query_params.get("workspace") or "").strip().lower()
+        if not raw:
+            raw = "company" if request.user.company_id else "personal"
+
+        if raw == "personal":
+            return ensure_personal_workspace(request.user), "personal"
+        if raw == "company":
+            if not request.user.company_id:
+                return None, "company"
+            return ensure_company_workspace(request.user.company), "company"
+        return False, raw
+
     def get(self, request):
         logger.debug(f"request_3: {request}")
         try:
@@ -842,11 +944,54 @@ class DocumentUploadView(APIView):
         except FeatureNotAllowed as exc:
             return _feature_gate_response(exc)
 
-        workspace = ensure_workspace_for_user(request.user)
-        serializer = DocumentSerializer(
-            Document.objects.filter(workspace=workspace), many=True
+        workspace, scope = self._resolve_list_workspace(request)
+        if workspace is False:
+            return Response(
+                {"error": "workspace must be 'personal' or 'company'."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        if workspace is None:
+            return Response(
+                {"error": "Company workspace requires a company membership."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        queryset = Document.objects.filter(workspace=workspace)
+
+        search = (request.query_params.get("search") or request.query_params.get("q") or "").strip()
+        if search:
+            queryset = queryset.filter(
+                Q(file__icontains=search) | Q(remote_name__icontains=search)
+            )
+
+        ordering = (request.query_params.get("ordering") or "-created").strip()
+        order_by = self.DOCUMENT_SORT_FIELDS.get(ordering, "-created")
+        queryset = queryset.order_by(order_by, "id")
+
+        try:
+            page = max(int(request.query_params.get("page", 1)), 1)
+        except (TypeError, ValueError):
+            page = 1
+        try:
+            page_size = int(request.query_params.get("page_size", 20))
+        except (TypeError, ValueError):
+            page_size = 20
+        page_size = min(max(page_size, 1), 100)
+
+        total = queryset.count()
+        start = (page - 1) * page_size
+        end = start + page_size
+        serializer = DocumentSerializer(queryset[start:end], many=True)
+        return Response(
+            {
+                "count": total,
+                "page": page,
+                "page_size": page_size,
+                "scope": scope,
+                "results": serializer.data,
+            },
+            status=status.HTTP_200_OK,
         )
-        return Response(serializer.data, status=status.HTTP_200_OK)
 
     def post(self, request):
         logger.debug(f"request: {request}")
@@ -856,6 +1001,17 @@ class DocumentUploadView(APIView):
         except FeatureNotAllowed as exc:
             return _feature_gate_response(exc)
 
+        scope = (request.query_params.get("workspace") or request.data.get("workspace") or "").strip().lower()
+        if scope == "personal":
+            workspace = ensure_personal_workspace(request.user)
+        elif scope == "company":
+            if not request.user.company_id:
+                return Response(
+                    {"error": "Company workspace requires a company membership."},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+            workspace = ensure_company_workspace(request.user.company)
+        else:
             workspace = ensure_workspace_for_user(request.user)
 
         logger.info(request.FILES)
@@ -899,8 +1055,24 @@ class DocumentDetailView(APIView):
     # permission_classes = [permissions.IsAuthenticated]
 
     def _get_document(self, request, document_id):
-        workspace = ensure_workspace_for_user(request.user)
-        return Document.objects.filter(workspace=workspace, id=document_id).first()
+        user = request.user
+        if user.company_id:
+            return (
+                Document.objects.filter(id=document_id)
+                .filter(
+                    Q(workspace__user=user, workspace__company__isnull=True)
+                    | Q(
+                        workspace__company_id=user.company_id,
+                        workspace__user__isnull=True,
+                    )
+                )
+                .first()
+            )
+        return Document.objects.filter(
+            id=document_id,
+            workspace__user=user,
+            workspace__company__isnull=True,
+        ).first()
 
     def get(self, request, document_id):
         logger.info(f"request: {request}")