Read attached chat images with a vision model.

Image uploads were treated as unknown documents and never shown to the model. PNG, JPEG, WebP, and GIF now go to Ollama vision, and history returns the image bytes.

Co-authored-by: Cursor <cursoragent@cursor.com>

llm_be/chat_backend/consumers_graph.py

@@ -1,5 +1,4 @@
 import json
-import base64
 import logging
 import pandas as pd
 from datetime import datetime
@@ -29,6 +28,7 @@ from .services.title_generator import title_generator
 from .services.moderation_classifier import moderation_classifier, ModerationLabel
 from .services.prompt_classifier.prompt_classifier import PromptClassifier, PromptType
 from .services.data_analysis_service import AsyncDataAnalysisService
+from .services.vision import AsyncVisionService, is_image_type, prepare_attachment
 from .services.grounded_chat import prepare_grounded_chat
 from .services.status_context import (
     emit_status,
@@ -36,7 +36,11 @@ from .services.status_context import (
     set_status_emitter,
 )
 from .services.ws_frames import citations_frame, status_frame
-from chat_backend.ollama_config import ollama_model_for_role, resolve_chat_role
+from chat_backend.ollama_config import (
+    ROLE_VISION,
+    ollama_model_for_role,
+    resolve_chat_role,
+)
 from .utils import (
     TokenUsageCollector,
     aiter_text_chunks,
@@ -202,7 +206,9 @@ async def get_conversation_file_async(conversation_id):
     try:
         prompt_with_file = await Prompt.objects.filter(
             conversation_id=conversation_id
-        ).exclude(file='').order_by('created').afirst()
+        ).exclude(file='').exclude(
+            file_type__in=["png", "jpeg", "jpg", "webp", "gif"]
+        ).order_by('created').afirst()
 
         if prompt_with_file and prompt_with_file.file:
             # Opening a DatabaseStorage file hits the DB, so read inside the thread.
@@ -249,7 +255,11 @@ async def classification_node(state: ChatState) -> ChatState:
     decoded_file = state.get("decoded_file")
     
     prompt_type = await PROMPT_CLASSIFIER.classify_async(msg)
-    
+
+    # Images are read by the vision model, not the spreadsheet parser.
+    if is_image_type(state.get("file_type")):
+        return {"prompt_type": PromptType.GENERAL_CHAT}
+
     # Override logic
     if decoded_file and (prompt_type == PromptType.DATA_ANALYSIS or 'analyze' in msg.lower() or 'data' in msg.lower()):
         prompt_type = PromptType.DATA_ANALYSIS
@@ -269,7 +279,18 @@ async def generation_node(state: ChatState) -> ChatState:
     conversation_id = state["conversation_id"]
     decoded_file = state.get("decoded_file")
     file_type = state.get("file_type")
-    
+
+    if is_image_type(file_type) and decoded_file:
+        service = AsyncVisionService()
+        await emit_status("reading_image")
+        generator = service.generate_response(
+            prompt_instance.message, decoded_file, file_type
+        )
+        return {
+            "response_generator": generator,
+            "resolved_model": service.model_name,
+        }
+
     # Feature Flag + plan gate: Image Generation
     if prompt_type == PromptType.IMAGE_GENERATION:
         if not getattr(settings, "ALLOW_IMAGE_GENERATION", False):
@@ -485,13 +506,19 @@ class ChatConsumerGraph(AsyncWebsocketConsumer):
                 print("Conversation ID: ", conversation_id)
                 decoded_file = None
                 if file:
-                    decoded_file = base64.b64decode(file)
-                    if "csv" in file_type: file_type = "csv"
-                    elif "xmlformats-officedocument" in file_type: file_type = "xlsx"
-                    elif "word" in file_type: file_type = "docx"
-                    elif "pdf" in file_type: file_type = "pdf"
-                    elif "text" in file_type: file_type = "txt"
-                    else: file_type = "Not Sure"
+                    decoded_file, file_type, attachment_error = prepare_attachment(
+                        file, file_type
+                    )
+                    if attachment_error:
+                        await self.send_json_message(
+                            json.dumps(
+                                {
+                                    "type": "error",
+                                    "content": attachment_error,
+                                }
+                            )
+                        )
+                        return
 
                 # Pre-fetch messages and file
                 messages, prompt_instance = await get_messages(
@@ -501,7 +528,9 @@ class ChatConsumerGraph(AsyncWebsocketConsumer):
                 if not decoded_file:
                     decoded_file, file_type = await get_conversation_file_async(conversation_id)
 
-                resolved_model = ollama_model_for_role(resolve_chat_role(model))
+                resolved_model = ollama_model_for_role(
+                    ROLE_VISION if is_image_type(file_type) else resolve_chat_role(model)
+                )
                 prompt_metric = await create_prompt_metric(
                     prompt_instance.id,
                     prompt_instance.message,
@@ -556,7 +585,7 @@ class ChatConsumerGraph(AsyncWebsocketConsumer):
                     citations = []
                     final_model = resolved_model
 
-                    if should_use_agent(message):
+                    if should_use_agent(message) and not is_image_type(file_type):
                         try:
                             await enforce_feature_gate(chat_user, "agentic_tasks")
                         except FeatureNotAllowed as exc: