From 760b06f3f4df21d772a42c6bc8eb6b42203727e5 Mon Sep 17 00:00:00 2001
From: Ryan Westfall <ryan@aimloperations.com>
Date: Tue, 20 Oct 2026 08:11:56 -0500
Subject: [PATCH] Let a chat prompt include an image the model can read.

The composer accepts PNG, JPEG, WebP, and GIF, shows a thumbnail before send, and keeps the image on the user turn after reload. Closes #128.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 llm-fe/public/index.html                      |  2 +-
 llm-fe/public/llms.txt                        |  2 +-
 .../ConversationDetailCard.tsx                | 15 +++++
 llm-fe/src/llm-fe/contexts/MessageContext.tsx | 19 ++++++
 llm-fe/src/llm-fe/data.ts                     |  6 ++
 .../pages/AsyncDashboard2/AsyncDashboard2.tsx | 65 +++++++++++++++----
 .../llm-fe/pages/Landing/LandingPage.test.tsx |  2 +-
 llm-fe/src/llm-fe/publicCrawlerFiles.test.ts  |  6 +-
 llm-fe/src/llm-fe/utils/pageProse.js          |  2 +-
 .../src/llm-fe/utils/visionAttachment.test.ts | 21 ++++++
 llm-fe/src/llm-fe/utils/visionAttachment.ts   | 20 ++++++
 11 files changed, 142 insertions(+), 18 deletions(-)
 create mode 100644 llm-fe/src/llm-fe/utils/visionAttachment.test.ts
 create mode 100644 llm-fe/src/llm-fe/utils/visionAttachment.ts

diff --git a/llm-fe/public/index.html b/llm-fe/public/index.html
index a7fee6f..98f530c 100644
--- a/llm-fe/public/index.html
+++ b/llm-fe/public/index.html
@@ -83,7 +83,7 @@
 </head>
 
 <body>
-  <div id="root"><h1>Hesychia is a calm AI assistant.</h1><div id="crawler-prose"><p>Quiet, focused AI chat — private by default — is how this Hesychia product is built.</p><p>It remembers what you told it and your documents, so you never start over, and conversation memory is a setting you control.</p><p>We store your account, your conversations, and the documents you upload.</p><p>We do not sell your data, and we do not use advertising cookies.</p><p>Canceling does not delete documents, and they stay until you delete them or delete the account, and account deletion is a soft-delete.</p><p>It remembers what you told it and your documents, so you never start over.</p><p>Your documents stay stored and searchable, and answers are grounded in your own files.</p><p>Business accounts are simple management for your team, and company managers can manage team accounts.</p><p>Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up.</p><p>Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.</p><p>Vision is on the roadmap, so you can ask questions about an image you upload.</p></div><nav aria-label="Related pages"><a href="/pricing/">Hesychia pricing — $10/month, no ads</a><a href="/privacy/">What Hesychia stores and does not sell</a><a href="/changelog/">Dated Hesychia product updates</a><a href="/terms_of_service/">Hesychia terms of service</a><a href="/account-deletion/">How to request Hesychia account deletion</a></nav></div>
+  <div id="root"><h1>Hesychia is a calm AI assistant.</h1><div id="crawler-prose"><p>Quiet, focused AI chat — private by default — is how this Hesychia product is built.</p><p>It remembers what you told it and your documents, so you never start over, and conversation memory is a setting you control.</p><p>We store your account, your conversations, and the documents you upload.</p><p>We do not sell your data, and we do not use advertising cookies.</p><p>Canceling does not delete documents, and they stay until you delete them or delete the account, and account deletion is a soft-delete.</p><p>It remembers what you told it and your documents, so you never start over.</p><p>Your documents stay stored and searchable, and answers are grounded in your own files.</p><p>Business accounts are simple management for your team, and company managers can manage team accounts.</p><p>Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up.</p><p>Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.</p><p>Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.</p></div><nav aria-label="Related pages"><a href="/pricing/">Hesychia pricing — $10/month, no ads</a><a href="/privacy/">What Hesychia stores and does not sell</a><a href="/changelog/">Dated Hesychia product updates</a><a href="/terms_of_service/">Hesychia terms of service</a><a href="/account-deletion/">How to request Hesychia account deletion</a></nav></div>
   <!--
       This HTML file is a template.
       If you open it directly in the browser, you will see an empty page.
diff --git a/llm-fe/public/llms.txt b/llm-fe/public/llms.txt
index 879020f..785c968 100644
--- a/llm-fe/public/llms.txt
+++ b/llm-fe/public/llms.txt
@@ -2,7 +2,7 @@
 
 > Hesychia is a calm AI assistant with quiet, focused AI chat that stays private by default.
 
-Hesychia is a $10/month AI chat product from AI ML Operations, LLC. Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up. It stores your documents so answers can use your files, and company managers can manage team accounts. Conversation memory is a setting you control, and canceling a subscription does not delete documents. Account deletion is a soft-delete, and stored account data may remain. Image generation and vision are on the roadmap.
+Hesychia is a $10/month AI chat product from AI ML Operations, LLC. Early adopters pay $10 per month, locked for the lifetime of the account, and that price never goes up. It stores your documents so answers can use your files, and company managers can manage team accounts. Conversation memory is a setting you control, and canceling a subscription does not delete documents. Account deletion is a soft-delete, and stored account data may remain. Image generation is on the roadmap. Upload an image with a prompt and Hesychia reads the picture.
 
 ## Pages
 
diff --git a/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx b/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
index b9208bf..e1d234f 100644
--- a/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
+++ b/llm-fe/src/llm-fe/components/ConversationDetailCard/ConversationDetailCard.tsx
@@ -170,6 +170,7 @@ type ConversationDetailCardProps = {
   activityDetail?: string | null;
   activityHistory?: ActivityHistoryEntry[];
   activityInterrupted?: boolean;
+  imageSrc?: string | null;
 };
 
 const MyPlot = ({ format, image }: { format: string; image: string }) => {
@@ -207,6 +208,7 @@ const ConversationDetailCard = ({
   activityDetail = null,
   activityHistory = [],
   activityInterrupted = false,
+  imageSrc = null,
 }: ConversationDetailCardProps): JSX.Element => {
   const CitationMark = useCallback(
     ({ indices }: { indices?: string }) => {
@@ -316,6 +318,19 @@ const ConversationDetailCard = ({
   return (
     <MessageContainer $isUser={user_created}>
       <Bubble $isUser={user_created}>
+        {imageSrc && (
+          <img
+            src={imageSrc}
+            alt="Attached image"
+            style={{
+              display: 'block',
+              maxWidth: '100%',
+              maxHeight: 240,
+              borderRadius: 8,
+              marginBottom: '0.75rem',
+            }}
+          />
+        )}
         <Markdown
           className="display-linebreak"
           style={{ whiteSpace: 'pre-line' }}
diff --git a/llm-fe/src/llm-fe/contexts/MessageContext.tsx b/llm-fe/src/llm-fe/contexts/MessageContext.tsx
index bc2733b..7372f13 100644
--- a/llm-fe/src/llm-fe/contexts/MessageContext.tsx
+++ b/llm-fe/src/llm-fe/contexts/MessageContext.tsx
@@ -12,6 +12,7 @@ import {
     parseStatusData,
     parseVersionedFrame,
 } from "../utils/wsFrames";
+import { parseChatErrorPayload } from "../utils/chatErrors";
 
 type MessageProviderProps ={
     children? : ReactNode;
@@ -113,6 +114,8 @@ const MessageProvider = ( {children}: MessageProviderProps) => {
                         tokens_out: item.tokens_out ?? null,
                         citations: item.citations ?? [],
                         feedback: item.feedback ?? null,
+                        file_type: item.file_type ?? null,
+                        image: item.image ?? null,
                     }),
             )
             if (tempConversations.length === 1) {
@@ -288,6 +291,22 @@ const MessageProvider = ( {children}: MessageProviderProps) => {
                 setStreamInterrupted(false)
                 messageResponsePart.current = 1
             }else{
+                if (messageResponsePart.current === 0) {
+                    const err = parseChatErrorPayload(message)
+                    if (err?.content) {
+                        const details = [...conversationRef.current]
+                        const last = details[details.length - 1]
+                        if (last && !last.user_created) {
+                            details[details.length - 1] = new ConversationPrompt({
+                                ...last,
+                                message,
+                            })
+                            conversationRef.current = details
+                            setConversationDetails(details)
+                        }
+                    }
+                    return
+                }
                 if (messageResponsePart.current === 1){
                     isGeneratingRef.current = true
                     setIsGeneratingMessage(true)
diff --git a/llm-fe/src/llm-fe/data.ts b/llm-fe/src/llm-fe/data.ts
index 49a96fc..3da3485 100644
--- a/llm-fe/src/llm-fe/data.ts
+++ b/llm-fe/src/llm-fe/data.ts
@@ -18,6 +18,8 @@ export interface ConversationPromptType {
     tokens_out?: number | null,
     citations?: Citation[],
     feedback?: PromptFeedbackState | null,
+    file_type?: string | null,
+    image?: string | null,
 }
 
 export class ConversationPrompt{
@@ -29,6 +31,8 @@ export class ConversationPrompt{
     tokens_out: number | null = null;
     citations: Citation[] = [];
     feedback: PromptFeedbackState | null = null;
+    file_type: string | null = null;
+    image: string | null = null;
 
     constructor(initializer?: any){
         if(!initializer) return;
@@ -41,6 +45,8 @@ export class ConversationPrompt{
         if (initializer.tokens_out !== undefined) this.tokens_out = initializer.tokens_out;
         if (Array.isArray(initializer.citations)) this.citations = initializer.citations;
         if (initializer.feedback !== undefined) this.feedback = initializer.feedback;
+        if (initializer.file_type) this.file_type = initializer.file_type;
+        if (initializer.image) this.image = initializer.image;
     }
 
 }
diff --git a/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx b/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
index 5dfdbf8..7608a5b 100644
--- a/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
+++ b/llm-fe/src/llm-fe/pages/AsyncDashboard2/AsyncDashboard2.tsx
@@ -20,6 +20,7 @@ import { AnalyticsEvents, trackEvent } from "../../utils/analytics";
 import { exportChat, type ExportFormat } from "../../utils/export/exportChat";
 import { showToast } from "../../utils/toastBus";
 import type { PromptRating } from "../../utils/promptFeedback";
+import { CHAT_FILE_ACCEPT, imageSrcForPrompt, isImageFile } from "../../utils/visionAttachment";
 
 // Styled Components
 const PageContainer = styled.div`
@@ -334,6 +335,46 @@ const AlwaysScrollToBottom = ({ trigger }: { trigger: string | number }): JSX.El
   return <div ref={elementRef} />;
 };
 
+const AttachmentPreview = ({ file }: { file: File | null }): JSX.Element | null => {
+  const theme = useContext(ThemeContext);
+  const [url, setUrl] = useState<string | null>(null);
+  useEffect(() => {
+    if (!isImageFile(file)) {
+      setUrl(null);
+      return;
+    }
+    const next = URL.createObjectURL(file as File);
+    setUrl(next);
+    return () => URL.revokeObjectURL(next);
+  }, [file]);
+  if (url) {
+    return (
+      <img
+        src={url}
+        alt=""
+        style={{
+          display: "block",
+          margin: "0.5rem auto 0",
+          maxHeight: 120,
+          maxWidth: "100%",
+          borderRadius: 8,
+        }}
+      />
+    );
+  }
+  if (!file) return null;
+  return (
+    <div style={{
+      marginTop: "0.5rem",
+      fontSize: "0.8rem",
+      color: theme?.darkMode ? "rgba(255,255,255,0.6)" : "rgba(0,0,0,0.6)",
+      textAlign: "center",
+    }}>
+      Attached: {file.name}
+    </div>
+  );
+};
+
 const AsyncDashboardInner = (): JSX.Element => {
   const [, , , sendMessage, isConnected, connectionStatus] =
     useContext(WebSocketContext);
@@ -432,9 +473,15 @@ const AsyncDashboardInner = (): JSX.Element => {
     }
     try {
       clearActivity();
+      const attachedImage = isImageFile(file) ? URL.createObjectURL(file as File) : null;
       const tempConversations: ConversationPrompt[] = [
         ...conversationDetails,
-        new ConversationPrompt({ message: trimmedPrompt, user_created: true }),
+        new ConversationPrompt({
+          message: trimmedPrompt,
+          user_created: true,
+          image: attachedImage,
+          file_type: fileType,
+        }),
         new ConversationPrompt({ message: "", user_created: false }),
       ];
 
@@ -457,6 +504,8 @@ const AsyncDashboardInner = (): JSX.Element => {
       if (textareaRef.current) {
         textareaRef.current.style.height = 'auto';
       }
+      const fileInput = document.getElementById("file-upload") as HTMLInputElement | null;
+      if (fileInput) fileInput.value = "";
     } catch (e) {
       console.log(`error ${e}`);
     }
@@ -591,6 +640,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                   activityDetail={isLiveStream ? activityDetail : null}
                   activityHistory={isLiveStream ? activityHistory : []}
                   activityInterrupted={isLiveStream && streamInterrupted}
+                  imageSrc={imageSrcForPrompt(convo_detail.image, convo_detail.file_type)}
                 />
               );
             })
@@ -652,7 +702,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                   <VisuallyHiddenInput
                     id="file-upload"
                     type="file"
-                    accept=".csv,.xlsx,.txt,.pdf,.PDF"
+                    accept={CHAT_FILE_ACCEPT}
                     onChange={(event) => {
                       const file = event.target.files?.[0];
                       if (file) {
@@ -703,16 +753,7 @@ const AsyncDashboardInner = (): JSX.Element => {
                     </span>
                   </Tooltip>
                 </StyledInputContainer>
-                {formik.values.file && (
-                  <div style={{
-                    marginTop: '0.5rem',
-                    fontSize: '0.8rem',
-                    color: theme?.darkMode ? 'rgba(255,255,255,0.6)' : 'rgba(0,0,0,0.6)',
-                    textAlign: 'center'
-                  }}>
-                    Attached: {(formik.values.file as File).name}
-                  </div>
-                )}
+                <AttachmentPreview file={formik.values.file as File | null} />
               </Form>
             )}
           </Formik>
diff --git a/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx b/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
index 197c00b..dec2ce5 100644
--- a/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
+++ b/llm-fe/src/llm-fe/pages/Landing/LandingPage.test.tsx
@@ -52,7 +52,7 @@ describe('LandingPage', () => {
     expect(screen.getByRole('heading', { name: 'Business accounts' })).toBeInTheDocument();
     expect(screen.getByRole('heading', { name: 'Roadmap' })).toBeInTheDocument();
     expect(screen.getByText(/Image generation is on the roadmap/i)).toBeInTheDocument();
-    expect(screen.getByText(/Vision is on the roadmap/i)).toBeInTheDocument();
+    expect(screen.getByText(/Hesychia reads the picture/i)).toBeInTheDocument();
     expect(
       screen.getByText(/Early adopters pay \$10 per month, locked for the lifetime of the account/i),
     ).toBeInTheDocument();
diff --git a/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts b/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
index 034a45f..c0d654d 100644
--- a/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
+++ b/llm-fe/src/llm-fe/publicCrawlerFiles.test.ts
@@ -27,7 +27,7 @@ describe('public crawler files', () => {
     expect(sitemap.startsWith('<?xml')).toBe(true);
     expect(llms).toContain('$10/month');
     expect(llms).toContain('locked for the lifetime of the account');
-    expect(llms).toContain('Image generation and vision are on the roadmap');
+    expect(llms).toContain('Upload an image with a prompt and Hesychia reads the picture');
     expect(llms).not.toContain('<html');
 
     expect(html).toContain('<title>Hesychia — quiet, focused AI chat</title>');
@@ -56,7 +56,9 @@ describe('public crawler files', () => {
     expect(html).toContain(
       'Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.',
     );
-    expect(html).toContain('Vision is on the roadmap, so you can ask questions about an image you upload.');
+    expect(html).toContain(
+      'Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.',
+    );
     expect(html).not.toContain('>here<');
     expect(html.match(/application\/ld\+json/g)).toHaveLength(1);
     expect(html).toContain('"@type": "WebPage"');
diff --git a/llm-fe/src/llm-fe/utils/pageProse.js b/llm-fe/src/llm-fe/utils/pageProse.js
index e6524b2..ed79b8e 100644
--- a/llm-fe/src/llm-fe/utils/pageProse.js
+++ b/llm-fe/src/llm-fe/utils/pageProse.js
@@ -32,7 +32,7 @@ const HOME = {
   roadmapImage:
     'Image generation is on the roadmap, so a prompt can create a picture inside Hesychia.',
   roadmapVision:
-    'Vision is on the roadmap, so you can ask questions about an image you upload.',
+    'Upload an image with your prompt and Hesychia reads the picture, so you can ask about what is in it.',
 };
 
 const PRICING = {
diff --git a/llm-fe/src/llm-fe/utils/visionAttachment.test.ts b/llm-fe/src/llm-fe/utils/visionAttachment.test.ts
new file mode 100644
index 0000000..fb8ea27
--- /dev/null
+++ b/llm-fe/src/llm-fe/utils/visionAttachment.test.ts
@@ -0,0 +1,21 @@
+import { imageSrcForPrompt, isImageFile } from './visionAttachment';
+
+describe('visionAttachment', () => {
+  it('builds a data url from stored image bytes', () => {
+    expect(imageSrcForPrompt('abc', 'png')).toBe('data:image/png;base64,abc');
+    expect(imageSrcForPrompt('abc', 'jpeg')).toBe('data:image/jpeg;base64,abc');
+  });
+
+  it('keeps blob and data urls', () => {
+    expect(imageSrcForPrompt('blob:http://local/1', 'png')).toBe('blob:http://local/1');
+    expect(imageSrcForPrompt('data:image/png;base64,abc', 'png')).toBe(
+      'data:image/png;base64,abc',
+    );
+  });
+
+  it('detects image files', () => {
+    expect(isImageFile(new File(['x'], 'a.png', { type: 'image/png' }))).toBe(true);
+    expect(isImageFile(new File(['x'], 'a.pdf', { type: 'application/pdf' }))).toBe(false);
+    expect(isImageFile(null)).toBe(false);
+  });
+});
diff --git a/llm-fe/src/llm-fe/utils/visionAttachment.ts b/llm-fe/src/llm-fe/utils/visionAttachment.ts
new file mode 100644
index 0000000..380039c
--- /dev/null
+++ b/llm-fe/src/llm-fe/utils/visionAttachment.ts
@@ -0,0 +1,20 @@
+export const CHAT_FILE_ACCEPT =
+  '.csv,.xlsx,.txt,.pdf,.PDF,image/png,image/jpeg,image/webp,image/gif';
+
+export function isImageFile(file: File | null | undefined): boolean {
+  return Boolean(file && file.type.startsWith('image/'));
+}
+
+/** Preview URL or data URL for a stored prompt image. */
+export function imageSrcForPrompt(
+  image?: string | null,
+  fileType?: string | null,
+): string | null {
+  if (!image) return null;
+  if (image.startsWith('blob:') || image.startsWith('data:')) return image;
+  const raw = (fileType || 'png').toLowerCase();
+  const mime = raw.startsWith('image/')
+    ? raw
+    : `image/${raw === 'jpg' ? 'jpeg' : raw}`;
+  return `data:${mime};base64,${image}`;
+}