From 89228719b59d4e93e71b2a54294c5af39b9f0885 Mon Sep 17 00:00:00 2001
From: Ryan Westfall <ryan@aimloperations.com>
Date: Tue, 20 Oct 2026 06:13:42 -0700
Subject: [PATCH] Vision: attach an image with a prompt and have the model read
 it (#129)

## Summary
- Closes #128.
- Chat composer accepts PNG, JPEG, WebP, and GIF along with existing documents.
- Selected image shows a thumbnail before send. Filename stays hidden for images. Documents still show the filename.
- User turn keeps the image after reload, from the prompt `image` field the backend returns.
- Oversized or unsupported images show the websocket error on the assistant bubble.
- Landing copy and `llms.txt` no longer list vision as roadmap-only.

Needs the matching `chat_backend` branch `feat/128-vision-image-prompt` so the model actually reads the image.

## Test plan
- [ ] Attach a PNG with a prompt that asks to read text in the image. Answer uses the picture.
- [ ] Thumbnail shows before send, with no filename.
- [ ] Reload the conversation. The image is still on the user turn.
- [ ] CSV/PDF attach still works.
- [ ] Text-only prompt unchanged.
- [ ] A BMP or an image over 8 MB shows an error instead of a silent failure.

Reviewed-on: https://git.aimloperations.com/ai_ml_operations/chat_web_app/pulls/129
---
 llm-fe/public/index.html                      |  2 +-
 llm-fe/public/llms.txt                        |  2 +-
 .../ConversationDetailCard.tsx                | 15 +++++
 llm-fe/src/llm-fe/contexts/MessageContext.tsx | 19 ++++++
 llm-fe/src/llm-fe/data.ts                     |  6 ++
 .../pages/AsyncDashboard2/AsyncDashboard2.tsx | 65 +++++++++++++++----
 .../llm-fe/pages/Landing/LandingPage.test.tsx |  2 +-
 llm-fe/src/llm-fe/publicCrawlerFiles.test.ts  |  6 +-
 llm-fe/src/llm-fe/utils/pageProse.js          |  2 +-
 .../src/llm-fe/utils/visionAttachment.test.ts | 21 ++++++
 llm-fe/src/llm-fe/utils/visionAttachment.ts   | 20 ++++++
 11 files changed, 142 insertions(+), 18 deletions(-)
 create mode 100644 llm-fe/src/llm-fe/utils/visionAttachment.test.ts
 create mode 100644 llm-fe/src/llm-fe/utils/visionAttachment.ts

diff --git a/llm-fe/public/index.html b/llm-fe/public/index.html
index a7fee6f..98f530c 100644
--- a/llm-fe/public/index.html
+++ b/llm-fe/public/index.html
@@ -83,7 +83,7 @@
 </head>
 
 <body>
-  <div id="root"><h1>Hesychia is a calm AI assistant.</h1><div id="crawler-prose"><p>Quiet, focused AI chat — private by default — is how this Hesychia product is built.</p><p>It remembers what you told it and your documents, so you never start over, and conversation memory is a setting you control.</p><p>We store your account, your conversations, and the documents you upload.</p><p>We do not sell your data, and we do not use advertising cookies.</p><p>Canceling does not delete documents, and they stay until you delete them or delete the account, and account deletion is a soft-delete.</p><p>It remembers what you told it and your documents, so you never start over.</p><p>Your documents stay stored and searchable, and answers are grounded in your own files.</p><p>Business accounts are simple management for your team, and company managers can manage team accounts.</p><p>Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up.</p><p>Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.</p><p>Vision is on the roadmap, so you can ask questions about an image you upload.</p></div><nav aria-label="Related pages"><a href="/pricing/">Hesychia pricing — $10/month, no ads</a><a href="/privacy/">What Hesychia stores and does not sell</a><a href="/changelog/">Dated Hesychia product updates</a><a href="/terms_of_service/">Hesychia terms of service</a><a href="/account-deletion/">How to request Hesychia account deletion</a></nav></div>
+  <div id="root"><h1>Hesychia is a calm AI assistant.</h1><div id="crawler-prose"><p>Quiet, focused AI chat — private by default — is how this Hesychia product is built.</p><p>It remembers what you told it and your documents, so you never start over, and conversation memory is a setting you control.</p><p>We store your account, your conversations, and the documents you upload.</p><p>We do not sell your data, and we do not use advertising cookies.</p><p>Canceling does not delete documents, and they stay until you delete them or delete the account, and account deletion is a soft-delete.</p><p>It remembers what you told it and your documents, so you never start over.</p><p>Your documents stay stored and searchable, and answers are grounded in your own files.</p><p>Business accounts are simple management for your team, and company managers can manage team accounts.</p><p>Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up.</p><p>Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.</p><p>Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.</p></div><nav aria-label="Related pages"><a href="/pricing/">Hesychia pricing — $10/month, no ads</a><a href="/privacy/">What Hesychia stores and does not sell</a><a href="/changelog/">Dated Hesychia product updates</a><a href="/terms_of_service/">Hesychia terms of service</a><a href="/account-deletion/">How to request Hesychia account deletion</a></nav></div>
   <!--
       This HTML file is a template.
       If you open it directly in the browser, you will see an empty page.
diff --git a/llm-fe/public/llms.txt b/llm-fe/public/llms.txt
index 879020f..785c968 100644
--- a/llm-fe/public/llms.txt
+++ b/llm-fe/public/llms.txt
@@ -2,7 +2,7 @@
 
 > Hesychia is a calm AI assistant with quiet, focused AI chat that stays private by default.
 
-Hesychia is a $10/month AI chat product from AI ML Operations, LLC. Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up. It stores your documents so answers can use your files, and company managers can manage team accounts. Conversation memory is a setting you control, and canceling a subscription does not delete documents. Account deletion is a soft-delete, and stored account data may remain. Image generation and vision are on the roadmap.
+Hesychia is a $10/month AI chat product from AI ML Operations, LLC. Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up. It stores your documents so answers can use your files, and company managers can manage team accounts. Conversation memory is a setting you control, and canceling a subscription does not delete documents. Account deletion is a soft-delete, and stored account data may remain. Image generation is on the roadmap. Upload an image with a prompt and Hesychia reads the picture.
 
 ## Pages
 
diff --git a/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx b/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
index b9208bf..e1d234f 100644
--- a/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
+++ b/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
@@ -170,6 +170,7 @@ type ConversationDetailCardProps = {
   activityDetail?: string | null;
   activityHistory?: ActivityHistoryEntry[];
   activityInterrupted?: boolean;
+  imageSrc?: string | null;
 };
 
 const MyPlot = ({ format, image }: { format: string; image: string }) => {
@@ -207,6 +208,7 @@ const ConversationDetailCard = ({
   activityDetail = null,
   activityHistory = [],
   activityInterrupted = false,
+  imageSrc = null,
 }: ConversationDetailCardProps): JSX.Element => {
   const CitationMark = useCallback(
     ({ indices }: { indices?: string }) => {
@@ -316,6 +318,19 @@ const ConversationDetailCard = ({
   return (
     <MessageContainer $isUser={user_created}>
       <Bubble $isUser={user_created}>
+        {imageSrc && (
+          <img
+            src={imageSrc}
+            alt="Attached image"
+            style={{
+              display: 'block',
+              maxWidth: '100%',
+              maxHeight: 240,
+              borderRadius: 8,
+              marginBottom: '0.75rem',
+            }}
+          />
+        )}
         <Markdown
           className="display-linebreak"
           style={{ whiteSpace: 'pre-line' }}
diff --git a/llm-fe/src/llm-fe/contexts/MessageContext.tsx b/llm-fe/src/llm-fe/contexts/MessageContext.tsx
index bc2733b..7372f13 100644
--- a/llm-fe/src/llm-fe/contexts/MessageContext.tsx
+++ b/llm-fe/src/llm-fe/contexts/MessageContext.tsx
@@ -12,6 +12,7 @@ import {
     parseStatusData,
     parseVersionedFrame,
 } from "../utils/wsFrames";
+import { parseChatErrorPayload } from "../utils/chatErrors";
 
 type MessageProviderProps ={
     children? : ReactNode;
@@ -113,6 +114,8 @@ const MessageProvider = ( {children}: MessageProviderProps) => {
                         tokens_out: item.tokens_out ?? null,
                         citations: item.citations ?? [],
                         feedback: item.feedback ?? null,
+                        file_type: item.file_type ?? null,
+                        image: item.image ?? null,
                     }),
             )
             if (tempConversations.length === 1) {
@@ -288,6 +291,22 @@ const MessageProvider = ( {children}: MessageProviderProps) => {
                 setStreamInterrupted(false)
                 messageResponsePart.current = 1
             }else{
+                if (messageResponsePart.current === 0) {
+                    const err = parseChatErrorPayload(message)
+                    if (err?.content) {
+                        const details = [...conversationRef.current]
+                        const last = details[details.length - 1]
+                        if (last && !last.user_created) {
+                            details[details.length - 1] = new ConversationPrompt({
+                                ...last,
+                                message,
+                            })
+                            conversationRef.current = details
+                            setConversationDetails(details)
+                        }
+                    }
+                    return
+                }
                 if (messageResponsePart.current === 1){
                     isGeneratingRef.current = true
                     setIsGeneratingMessage(true)
diff --git a/llm-fe/src/llm-fe/data.ts b/llm-fe/src/llm-fe/data.ts
index 49a96fc..3da3485 100644
--- a/llm-fe/src/llm-fe/data.ts
+++ b/llm-fe/src/llm-fe/data.ts
@@ -18,6 +18,8 @@ export interface ConversationPromptType {
     tokens_out?: number | null,
     citations?: Citation[],
     feedback?: PromptFeedbackState | null,
+    file_type?: string | null,
+    image?: string | null,
 }
 
 export class ConversationPrompt{
@@ -29,6 +31,8 @@ export class ConversationPrompt{
     tokens_out: number | null = null;
     citations: Citation[] = [];
     feedback: PromptFeedbackState | null = null;
+    file_type: string | null = null;
+    image: string | null = null;
 
     constructor(initializer?: any){
         if(!initializer) return;
@@ -41,6 +45,8 @@ export class ConversationPrompt{
         if (initializer.tokens_out !== undefined) this.tokens_out = initializer.tokens_out;
         if (Array.isArray(initializer.citations)) this.citations = initializer.citations;
         if (initializer.feedback !== undefined) this.feedback = initializer.feedback;
+        if (initializer.file_type) this.file_type = initializer.file_type;
+        if (initializer.image) this.image = initializer.image;
     }
 
 }
diff --git a/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx b/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
index 5dfdbf8..7608a5b 100644
--- a/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
+++ b/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
@@ -20,6 +20,7 @@ import { AnalyticsEvents, trackEvent } from "../../utils/analytics";
 import { exportChat, type ExportFormat } from "../../utils/export/exportChat";
 import { showToast } from "../../utils/toastBus";
 import type { PromptRating } from "../../utils/promptFeedback";
+import { CHAT_FILE_ACCEPT, imageSrcForPrompt, isImageFile } from "../../utils/visionAttachment";
 
 // Styled Components
 const PageContainer = styled.div`
@@ -334,6 +335,46 @@ const AlwaysScrollToBottom = ({ trigger }: { trigger: string | number }): JSX.El
   return <div ref={elementRef} />;
 };
 
+const AttachmentPreview = ({ file }: { file: File | null }): JSX.Element | null => {
+  const theme = useContext(ThemeContext);
+  const [url, setUrl] = useState<string | null>(null);
+  useEffect(() => {
+    if (!isImageFile(file)) {
+      setUrl(null);
+      return;
+    }
+    const next = URL.createObjectURL(file as File);
+    setUrl(next);
+    return () => URL.revokeObjectURL(next);
+  }, [file]);
+  if (url) {
+    return (
+      <img
+        src={url}
+        alt=""
+        style={{
+          display: "block",
+          margin: "0.5rem auto 0",
+          maxHeight: 120,
+          maxWidth: "100%",
+          borderRadius: 8,
+        }}
+      />
+    );
+  }
+  if (!file) return null;
+  return (
+    <div style={{
+      marginTop: "0.5rem",
+      fontSize: "0.8rem",
+      color: theme?.darkMode ? "rgba(255,255,255,0.6)" : "rgba(0,0,0,0.6)",
+      textAlign: "center",
+    }}>
+      Attached: {file.name}
+    </div>
+  );
+};
+
 const AsyncDashboardInner = (): JSX.Element => {
   const [, , , sendMessage, isConnected, connectionStatus] =
     useContext(WebSocketContext);
@@ -432,9 +473,15 @@ const AsyncDashboardInner = (): JSX.Element => {
     }
     try {
       clearActivity();
+      const attachedImage = isImageFile(file) ? URL.createObjectURL(file as File) : null;
       const tempConversations: ConversationPrompt[] = [
         ...conversationDetails,
-        new ConversationPrompt({ message: trimmedPrompt, user_created: true }),
+        new ConversationPrompt({
+          message: trimmedPrompt,
+          user_created: true,
+          image: attachedImage,
+          file_type: fileType,
+        }),
         new ConversationPrompt({ message: "", user_created: false }),
       ];
 
@@ -457,6 +504,8 @@ const AsyncDashboardInner = (): JSX.Element => {
       if (textareaRef.current) {
         textareaRef.current.style.height = 'auto';
       }
+      const fileInput = document.getElementById("file-upload") as HTMLInputElement | null;
+      if (fileInput) fileInput.value = "";
     } catch (e) {
       console.log(`error ${e}`);
     }
@@ -591,6 +640,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                   activityDetail={isLiveStream ? activityDetail : null}
                   activityHistory={isLiveStream ? activityHistory : []}
                   activityInterrupted={isLiveStream && streamInterrupted}
+                  imageSrc={imageSrcForPrompt(convo_detail.image, convo_detail.file_type)}
                 />
               );
             })
@@ -652,7 +702,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                   <VisuallyHiddenInput
                     id="file-upload"
                     type="file"
-                    accept=".csv,.xlsx,.txt,.pdf,.PDF"
+                    accept={CHAT_FILE_ACCEPT}
                     onChange={(event) => {
                       const file = event.target.files?.[0];
                       if (file) {
@@ -703,16 +753,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                     </span>
                   </Tooltip>
                 </StyledInputContainer>
-                {formik.values.file && (
-                  <div style={{
-                    marginTop: '0.5rem',
-                    fontSize: '0.8rem',
-                    color: theme?.darkMode ? 'rgba(255,255,255,0.6)' : 'rgba(0,0,0,0.6)',
-                    textAlign: 'center'
-                  }}>
-                    Attached: {(formik.values.file as File).name}
-                  </div>
-                )}
+                <AttachmentPreview file={formik.values.file as File | null} />
               </Form>
             )}
           </Formik>
diff --git a/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx b/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
index 197c00b..dec2ce5 100644
--- a/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
+++ b/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
@@ -52,7 +52,7 @@ describe('LandingPage', () => {
     expect(screen.getByRole('heading', { name: 'Business accounts' })).toBeInTheDocument();
     expect(screen.getByRole('heading', { name: 'Roadmap' })).toBeInTheDocument();
     expect(screen.getByText(/Image generation is on the roadmap/i)).toBeInTheDocument();
-    expect(screen.getByText(/Vision is on the roadmap/i)).toBeInTheDocument();
+    expect(screen.getByText(/Hesychia reads the picture/i)).toBeInTheDocument();
     expect(
       screen.getByText(/Early adopters pay \$10 per month, locked for the lifetime of the account/i),
     ).toBeInTheDocument();
diff --git a/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts b/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
index 034a45f..c0d654d 100644
--- a/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
+++ b/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
@@ -27,7 +27,7 @@ describe('public crawler files', () => {
     expect(sitemap.startsWith('<?xml')).toBe(true);
     expect(llms).toContain('$10/month');
     expect(llms).toContain('locked for the lifetime of the account');
-    expect(llms).toContain('Image generation and vision are on the roadmap');
+    expect(llms).toContain('Upload an image with a prompt and Hesychia reads the picture');
     expect(llms).not.toContain('<html');
 
     expect(html).toContain('<title>Hesychia — quiet, focused AI chat</title>');
@@ -56,7 +56,9 @@ describe('public crawler files', () => {
     expect(html).toContain(
       'Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.',
     );
-    expect(html).toContain('Vision is on the roadmap, so you can ask questions about an image you upload.');
+    expect(html).toContain(
+      'Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.',
+    );
     expect(html).not.toContain('>here<');
     expect(html.match(/application\/ld\+json/g)).toHaveLength(1);
     expect(html).toContain('"@type": "WebPage"');
diff --git a/llm-fe/src/llm-fe/utils/pageProse.js b/llm-fe/src/llm-fe/utils/pageProse.js
index e6524b2..ed79b8e 100644
--- a/llm-fe/src/llm-fe/utils/pageProse.js
+++ b/llm-fe/src/llm-fe/utils/pageProse.js
@@ -32,7 +32,7 @@ const HOME = {
   roadmapImage:
     'Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.',
   roadmapVision:
-    'Vision is on the roadmap, so you can ask questions about an image you upload.',
+    'Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.',
 };
 
 const PRICING = {
diff --git a/llm-fe/src/llm-fe/utils/visionAttachment.test.ts b/llm-fe/src/llm-fe/utils/visionAttachment.test.ts
new file mode 100644
index 0000000..fb8ea27
--- /dev/null
+++ b/llm-fe/src/llm-fe/utils/visionAttachment.test.ts
@@ -0,0 +1,21 @@
+import { imageSrcForPrompt, isImageFile } from './visionAttachment';
+
+describe('visionAttachment', () => {
+  it('builds a data url from stored image bytes', () => {
+    expect(imageSrcForPrompt('abc', 'png')).toBe('data:image/png;base64,abc');
+    expect(imageSrcForPrompt('abc', 'jpeg')).toBe('data:image/jpeg;base64,abc');
+  });
+
+  it('keeps blob and data urls', () => {
+    expect(imageSrcForPrompt('blob:http://local/1', 'png')).toBe('blob:http://local/1');
+    expect(imageSrcForPrompt('data:image/png;base64,abc', 'png')).toBe(
+      'data:image/png;base64,abc',
+    );
+  });
+
+  it('detects image files', () => {
+    expect(isImageFile(new File(['x'], 'a.png', { type: 'image/png' }))).toBe(true);
+    expect(isImageFile(new File(['x'], 'a.pdf', { type: 'application/pdf' }))).toBe(false);
+    expect(isImageFile(null)).toBe(false);
+  });
+});
diff --git a/llm-fe/src/llm-fe/utils/visionAttachment.ts b/llm-fe/src/llm-fe/utils/visionAttachment.ts
new file mode 100644
index 0000000..380039c
--- /dev/null
+++ b/llm-fe/src/llm-fe/utils/visionAttachment.ts
@@ -0,0 +1,20 @@
+export const CHAT_FILE_ACCEPT =
+  '.csv,.xlsx,.txt,.pdf,.PDF,image/png,image/jpeg,image/webp,image/gif';
+
+export function isImageFile(file: File | null | undefined): boolean {
+  return Boolean(file && file.type.startsWith('image/'));
+}
+
+/** Preview URL or data URL for a stored prompt image. */
+export function imageSrcForPrompt(
+  image?: string | null,
+  fileType?: string | null,
+): string | null {
+  if (!image) return null;
+  if (image.startsWith('blob:') || image.startsWith('data:')) return image;
+  const raw = (fileType || 'png').toLowerCase();
+  const mime = raw.startsWith('image/')
+    ? raw
+    : `image/${raw === 'jpg' ? 'jpeg' : raw}`;
+  return `data:${mime};base64,${image}`;
+}