From 7853b5f5e2be6c324d9c98ff037ae65c8a86154d Mon Sep 17 00:00:00 2001 From: Matthew Wang Date: Wed, 18 Mar 2026 16:57:45 -0700 Subject: [PATCH 1/3] Working now --- sdk/ATTACHMENTS.md | 11 + sdk/README.md | 11 +- sdk/sandbox/server/agents/default.ts | 4 +- sdk/widget/src/OpenChatWidget.tsx | 57 ++- sdk/widget/src/components/Composer.tsx | 409 ++++++++++++++++++++-- sdk/widget/src/components/MessageList.tsx | 66 +++- sdk/widget/src/index.ts | 4 + sdk/widget/src/server.ts | 48 +++ sdk/widget/src/utils/chat.ts | 8 + 9 files changed, 572 insertions(+), 46 deletions(-) create mode 100644 sdk/ATTACHMENTS.md create mode 100644 sdk/widget/src/server.ts diff --git a/sdk/ATTACHMENTS.md b/sdk/ATTACHMENTS.md new file mode 100644 index 0000000..16c29c5 --- /dev/null +++ b/sdk/ATTACHMENTS.md @@ -0,0 +1,11 @@ +# Objective +I want the AI to be able to handle uploading attachments. Things such as upload images and PDFs, etc. + +The user should be able to upload the attachment by clicking on the "+" sign in sdk/widget/src/components/Composer.tsx and selectting on the file. They should also be able to drag and drop a file from their computer into the chat widget. + +Upon uploading the file, the Composer should show a very small attachment preview of what they've attached. + +Upon hitting submit with the attachment, the attachments, along with the message, should be sent to Vercel AI-SDK on the backend. + +# Consider +Also consider how we can handle images on the backend too. I want to be able to do cool workflows like ask the AI "What is in this image" and it be able to tell me what it is by analyzing the image. \ No newline at end of file diff --git a/sdk/README.md b/sdk/README.md index f38204c..7b5d28d 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -51,6 +51,13 @@ The next step is to set up your AI agent backend. Create an API endpoint with yo Here's a simple AI SDK UI stream agent: ```tsx +import { + createOpenAI, + convertWidgetMessagesToModelMessages, + streamText, + type UIMessage, +} from "@openchatwidget/sdk"; + app.use(express.json()); app.post("/api/chat", async (request, response) => { const { messages } = request.body as { messages: UIMessage[] }; @@ -62,13 +69,15 @@ app.post("/api/chat", async (request, response) => { const result = streamText({ model: openai("gpt-5-mini"), system: "You are the OpenChatWidget example assistant. Keep answers concise and useful.", - messages: await convertToModelMessages(messages), + messages: await convertWidgetMessagesToModelMessages(messages), }); result.pipeUIMessageStreamToResponse(response); }); ``` +`convertWidgetMessagesToModelMessages` normalizes uploaded file parts (including browser `data:` URLs) so server implementors do not need to manually preprocess attachments. + ### 3. Connect the widget to the agent. Grab the exact URL of your agent endpoint and paste it into ``, for example `http://localhost:8787/api/chat`. Make sure to start both your front end and Node backend. You should be able to start chatting. diff --git a/sdk/sandbox/server/agents/default.ts b/sdk/sandbox/server/agents/default.ts index f61195e..2c11ddf 100644 --- a/sdk/sandbox/server/agents/default.ts +++ b/sdk/sandbox/server/agents/default.ts @@ -1,5 +1,5 @@ import { - convertToModelMessages, + convertWidgetMessagesToModelMessages, createOpenAI, stepCountIs, streamText, @@ -27,7 +27,7 @@ export async function runDefaultAgent(messages: UIMessage[]) { `${baseSystemPrompt} ` + "Use the weather tool for any weather question. Use the run_command tool only when the user explicitly asks to run a command. " + "If a tool execution is denied, do not retry it. When you use a tool, briefly summarize the result for the user.", - messages: await convertToModelMessages(messages), + messages: await convertWidgetMessagesToModelMessages(messages), stopWhen: stepCountIs(5), tools: { fetch_weather_data: tool({ diff --git a/sdk/widget/src/OpenChatWidget.tsx b/sdk/widget/src/OpenChatWidget.tsx index 6ac803b..273e313 100644 --- a/sdk/widget/src/OpenChatWidget.tsx +++ b/sdk/widget/src/OpenChatWidget.tsx @@ -30,6 +30,7 @@ export function OpenChatWidget({ disableReasoning = false, }: OpenChatWidgetProps) { const [input, setInput] = React.useState(""); + const [attachments, setAttachments] = React.useState([]); const [isOpen, setIsOpen] = React.useState(false); const [isInputFocused, setIsInputFocused] = React.useState(false); const [isMobileViewport, setIsMobileViewport] = React.useState(() => { @@ -79,7 +80,8 @@ export function OpenChatWidget({ }); const isGenerating = status === "submitted" || status === "streaming"; - const canSend = input.trim().length > 0 && !isGenerating; + const canSend = + (input.trim().length > 0 || attachments.length > 0) && !isGenerating; React.useEffect(() => { const handleResize = () => { @@ -187,15 +189,36 @@ export function OpenChatWidget({ const submitMessage = React.useCallback(() => { const nextInput = input.trim(); - if (!nextInput || isGenerating) { + if ((nextInput.length === 0 && attachments.length === 0) || isGenerating) { return; } - void sendMessage({ - text: nextInput, - }); + + if (attachments.length > 0) { + const transfer = new DataTransfer(); + attachments.forEach((file) => { + transfer.items.add(file); + }); + + if (nextInput.length > 0) { + void sendMessage({ + text: nextInput, + files: transfer.files, + }); + } else { + void sendMessage({ + files: transfer.files, + }); + } + } else { + void sendMessage({ + text: nextInput, + }); + } + setInput(""); + setAttachments([]); textareaRef.current?.focus(); - }, [sendMessage, input, isGenerating, setInput]); + }, [attachments, sendMessage, input, isGenerating, setInput]); const handleSubmit = React.useCallback( (event: React.FormEvent) => { @@ -316,6 +339,28 @@ export function OpenChatWidget({ { + const deduped = new Map(); + for (const file of attachments) { + deduped.set( + `${file.name}:${file.size}:${file.lastModified}`, + file, + ); + } + for (const file of nextFiles) { + deduped.set( + `${file.name}:${file.size}:${file.lastModified}`, + file, + ); + } + setAttachments(Array.from(deduped.values())); + }} + onRemoveAttachment={(fileIndex) => { + setAttachments((prev) => + prev.filter((_, index) => index !== fileIndex), + ); + }} placeholder={DEFAULT_PLACEHOLDER} isGenerating={isGenerating} canSend={canSend} diff --git a/sdk/widget/src/components/Composer.tsx b/sdk/widget/src/components/Composer.tsx index bad1cc8..5e46ff9 100644 --- a/sdk/widget/src/components/Composer.tsx +++ b/sdk/widget/src/components/Composer.tsx @@ -4,6 +4,9 @@ import { HELPFUL_CHAT_LOGO_DATA_URI } from "../theme"; type WidgetComposerProps = { input: string; setInput: (nextInput: string) => void; + attachments: File[]; + onAddAttachments: (nextFiles: File[]) => void; + onRemoveAttachment: (attachmentIndex: number) => void; placeholder: string; isGenerating: boolean; canSend: boolean; @@ -20,6 +23,16 @@ type WidgetComposerProps = { const POWERED_BY_LOGO_SRC = HELPFUL_CHAT_LOGO_DATA_URI; +function formatFileSize(sizeInBytes: number) { + if (sizeInBytes < 1024) { + return `${sizeInBytes} B`; + } + if (sizeInBytes < 1024 * 1024) { + return `${(sizeInBytes / 1024).toFixed(1)} KB`; + } + return `${(sizeInBytes / (1024 * 1024)).toFixed(1)} MB`; +} + function SendIcon() { return ( + ); +} + export function Composer({ input, setInput, + attachments, + onAddAttachments, + onRemoveAttachment, placeholder, isGenerating, canSend, @@ -72,9 +108,102 @@ export function Composer({ onBlur, logoSrc = POWERED_BY_LOGO_SRC, }: WidgetComposerProps) { + const [isAttachmentMenuOpen, setIsAttachmentMenuOpen] = React.useState(false); + const [isDraggingFiles, setIsDraggingFiles] = React.useState(false); + const attachmentMenuRef = React.useRef(null); + const fileInputRef = React.useRef(null); + const dragDepthRef = React.useRef(0); + + const imagePreviewUrls = React.useMemo( + () => + attachments.map((file) => + file.type.startsWith("image/") ? URL.createObjectURL(file) : null, + ), + [attachments], + ); + + React.useEffect(() => { + return () => { + imagePreviewUrls.forEach((previewUrl) => { + if (previewUrl) { + URL.revokeObjectURL(previewUrl); + } + }); + }; + }, [imagePreviewUrls]); + + const resizeTextarea = React.useCallback( + (node: HTMLTextAreaElement | null) => { + if (!node) { + return; + } + const computedStyle = window.getComputedStyle(node); + const lineHeight = Number.parseFloat(computedStyle.lineHeight) || 22; + const paddingTop = Number.parseFloat(computedStyle.paddingTop) || 0; + const paddingBottom = Number.parseFloat(computedStyle.paddingBottom) || 0; + const maxHeight = lineHeight * 10 + paddingTop + paddingBottom; + + node.style.height = "auto"; + node.style.height = `${Math.min(node.scrollHeight, maxHeight)}px`; + node.style.overflowY = node.scrollHeight > maxHeight ? "auto" : "hidden"; + }, + [], + ); + + React.useEffect(() => { + resizeTextarea(textareaRef.current); + }, [input, textareaRef, resizeTextarea]); + + React.useEffect(() => { + if (!isAttachmentMenuOpen || typeof document === "undefined") { + return; + } + + const handlePointerDown = (event: PointerEvent) => { + const target = event.target; + if (!(target instanceof Node)) { + return; + } + if (!attachmentMenuRef.current?.contains(target)) { + setIsAttachmentMenuOpen(false); + } + }; + + document.addEventListener("pointerdown", handlePointerDown); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + }; + }, [isAttachmentMenuOpen]); + return (
{ + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + }} + onDragEnter={(event) => { + event.preventDefault(); + dragDepthRef.current += 1; + setIsDraggingFiles(true); + }} + onDragLeave={(event) => { + event.preventDefault(); + dragDepthRef.current = Math.max(0, dragDepthRef.current - 1); + if (dragDepthRef.current === 0) { + setIsDraggingFiles(false); + } + }} + onDrop={(event) => { + event.preventDefault(); + dragDepthRef.current = 0; + setIsDraggingFiles(false); + + const droppedFiles = Array.from(event.dataTransfer.files ?? []); + if (droppedFiles.length > 0) { + onAddAttachments(droppedFiles); + } + }} style={{ display: "flex", flexDirection: "column", @@ -87,69 +216,279 @@ export function Composer({ >
+ { + const selectedFiles = Array.from(event.target.files ?? []); + if (selectedFiles.length > 0) { + onAddAttachments(selectedFiles); + } + + // Allow selecting the same file again after removing it. + event.currentTarget.value = ""; + setIsAttachmentMenuOpen(false); + }} + /> + + {isDraggingFiles ? ( +
+ Drop files to attach +
+ ) : null} + + {attachments.length > 0 ? ( +
+ {attachments.map((file, index) => { + const imagePreviewUrl = imagePreviewUrls[index]; + return ( +
+ {imagePreviewUrl ? ( + {file.name} + ) : ( + + )} + + + {file.name} + + + + {formatFileSize(file.size)} + + + +
+ ); + })} +
+ ) : null} +