Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ These are captures of real Electron interactions with a temporary Git workspace
- Group existing local folders into named workspaces and discover Pi sessions by their original working directory.
- Start, stop, steer, resume, and fork sessions; preview history before launching Pi.
- Browse linked session archives and branch entries; preview and continue the selected path.
- Render streamed Markdown, thinking, and expandable tool output.
- Render streamed Markdown, thinking, and expandable tool output with on-demand image previews.
- Browse files, review diffs, stage changes, and commit.
- Discover workspace and global skills, switch models, and save model and reasoning defaults per workspace.
- Switch roots while sessions run in the background; retain per-root drafts, views and model defaults.
Expand Down
2 changes: 2 additions & 0 deletions docs/pidex-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ Per-root views, new-session/history drafts, unsaved settings drafts, open tabs,

Markdown and tool output use native browser layout. ResizeObserver supplies actual row heights, including expanded thinking and tools. Width changes and font readiness invalidate measurements. Observer work is scheduled through animation frames to avoid resize feedback loops.

Tool results retain inline PNG, JPEG, GIF and WebP images alongside text in both saved history and live events. Each image loads through a separate disclosure control and fits within the card width and 320 pixels in height. Unsupported formats and decoding failures show an explanation. Text copies, exports and handoff drafts include numbered image notices without image bytes.

Active sessions initially open at the end. Appends and row growth follow the tail when the reader is near the bottom. Upward wheel or keyboard navigation and pointer interaction cancel a pending automatic scroll. History previews start at the beginning. Selected text ranges and keyboard-focused rows remain mounted while scrolling; tool/thinking expansion survives row recycling within the view.

![Long transcript in the Electron test fixture](assets/workbench-long-session.png)
Expand Down
53 changes: 51 additions & 2 deletions src/renderer/components/WorkbenchControls.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
import { BracketButton } from './BracketButton';
import { FigureFrame } from './FigureFrame';
import { Markdown } from './Markdown';
import { toolStatus, type Entry } from '../timeline';
import { toolStatus, type Entry, type ToolImage } from '../timeline';
import type { ModelInfo, SlashCommand } from '../api';

const icons = import.meta.glob<string>('../assets/workbench/*.svg', {
Expand Down Expand Up @@ -478,11 +478,51 @@ function ToolDiffSection({
);
}

function ToolTextSection({ title, text, stateKey, expanded }: {
function ToolImagePreview({ image, stateKey, expanded }: {
image: ToolImage;
stateKey: string;
expanded: Map<string, boolean>;
}) {
const id = useId();
const [open, setOpen] = useState(() => expanded.get(stateKey) ?? false);
const [failedSource, setFailedSource] = useState<string | null>(null);
return (
<div className="wb-tool-image">
<button
type="button"
className="wb-tool-toggle"
aria-expanded={open}
aria-controls={id}
onClick={() => {
expanded.set(stateKey, !open);
setOpen(!open);
}}
>
{open ? 'Hide' : 'Show'} image {image.number}
</button>
<div id={id} hidden={!open}>
{open && (failedSource === image.source ? (
<p className="wb-tool-notice wb-muted" role="status">
Image {image.number} could not be displayed. Text output is still available.
</p>
) : (
<img
src={image.source}
alt={`Tool output image ${image.number}`}
onError={() => setFailedSource(image.source)}
/>
))}
</div>
</div>
);
}

function ToolTextSection({ title, text, stateKey, expanded, images = [] }: {
title: string;
text: string;
stateKey: string;
expanded: Map<string, boolean>;
images?: ToolImage[];
}) {
const [showAll, setShowAll] = useState(false);
const [showEnd, setShowEnd] = useState(
Expand Down Expand Up @@ -537,6 +577,14 @@ function ToolTextSection({ title, text, stateKey, expanded }: {
</button>
</div>
)}
{images.map((image) => (
<ToolImagePreview
key={image.number}
image={image}
stateKey={`${stateKey}:image:${image.number}`}
expanded={expanded}
/>
))}
</ToolSection>
);
}
Expand Down Expand Up @@ -599,6 +647,7 @@ export const TranscriptRow = memo(function TranscriptRow({
title="Output"
stateKey={`${entry.key}:output`}
expanded={expanded}
images={entry.images}
text={
entry.output ||
(entry.running ? 'Waiting for output…' : 'No output.')
Expand Down
39 changes: 29 additions & 10 deletions src/renderer/timeline.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
type ToolLine = { text: string; tone?: 'add' | 'del' | 'dim' };
type ToolImage = { number: number; source: string };

type Entry =
| { key: string; kind: 'user'; text: string }
Expand All @@ -13,6 +14,7 @@ type Entry =
oldText?: string;
newText?: string;
output: string;
images?: ToolImage[];
running: boolean;
ok: boolean;
interrupted?: boolean;
Expand Down Expand Up @@ -101,12 +103,29 @@ function toolDiff(
return undefined;
}

function resultText(content: unknown): string {
if (!Array.isArray(content)) return '';
return content
.filter(p => p && p.type === 'text' && typeof p.text === 'string')
.map(p => (p as { text: string }).text)
.join('\n');
function resultOutput(content: unknown): { output: string; images: ToolImage[] } {
const text: string[] = [];
const images: ToolImage[] = [];
let imageNumber = 0;
for (const part of Array.isArray(content) ? content : []) {
if (!part || typeof part !== 'object') continue;
const block = part as Record<string, unknown>;
if (block.type === 'text' && typeof block.text === 'string') text.push(block.text);
if (block.type !== 'image') continue;
const number = ++imageNumber;
// Only inline raster data may reach an img src. Never load a tool-supplied
// URL or active document format such as SVG.
if (
typeof block.mimeType === 'string' && /^image\/(png|jpeg|gif|webp)$/.test(block.mimeType) &&
typeof block.data === 'string' && /^[A-Za-z0-9+/]+={0,2}$/.test(block.data)
) {
images.push({ number, source: `data:${block.mimeType};base64,${block.data}` });
text.push(`Image ${number} (${block.mimeType}). Image data is omitted from text copies and exports.`);
} else {
text.push(`Image ${number} cannot be previewed (unsupported format or invalid image data).`);
}
}
return { output: text.join('\n'), images };
}

function entriesFromMessages(messages: unknown[]): Entry[] {
Expand Down Expand Up @@ -154,7 +173,7 @@ function entriesFromMessages(messages: unknown[]): Entry[] {
if (m.role === 'toolResult' && m.toolCallId) {
const tool = tools.get(m.toolCallId);
if (tool) {
tool.output = resultText(m.content);
Object.assign(tool, resultOutput(m.content));
tool.ok = m.isError !== true;
tool.interrupted = false;
}
Expand Down Expand Up @@ -244,7 +263,7 @@ function applyEvent(
const idx = next.findIndex(e => e.kind === 'tool' && e.toolId === event.toolCallId);
if (idx !== -1) {
const partial = event.partialResult as { content?: unknown } | undefined;
next[idx] = { ...next[idx], output: resultText(partial?.content) } as Entry;
next[idx] = { ...next[idx], ...resultOutput(partial?.content) } as Entry;
}
break;
}
Expand All @@ -254,7 +273,7 @@ function applyEvent(
const result = event.result as { content?: unknown } | undefined;
next[idx] = {
...next[idx],
output: resultText(result?.content),
...resultOutput(result?.content),
running: false,
ok: event.isError !== true,
interrupted: false,
Expand All @@ -273,4 +292,4 @@ function toolStatus(entry: Extract<Entry, { kind: 'tool' }>): string {
}

export { entriesFromMessages, applyEvent, toolStatus };
export type { Entry, ToolLine };
export type { Entry, ToolLine, ToolImage };
9 changes: 9 additions & 0 deletions src/renderer/workbench.css
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,15 @@ html[data-reduced-motion='true'] .wb-shell *::after {
white-space: pre-wrap;
overflow-wrap: anywhere;
}
.wb-tool-image img {
display: block;
max-width: calc(100% - 24px);
max-height: 320px;
width: auto;
height: auto;
object-fit: contain;
margin: 0 12px 12px;
}
.wb-tool-diff {
padding: 8px 0 12px;
max-height: 320px;
Expand Down
105 changes: 105 additions & 0 deletions tests/e2e/tool-preview.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,30 @@ describe('bounded tool previews in Electron', () => {
];
await fs.writeFile(path.join(archives, `${tool}.jsonl`), records.map((r) => JSON.stringify(r)).join('\n'));
}
const image = await fs.readFile(path.join(root, 'docs/assets/banner.png'), 'base64');
for (const broken of [false, true]) {
const name = broken ? 'broken-image' : 'image';
const records = [
{ type: 'session', version: 3, id: name, cwd: project, timestamp: new Date().toISOString() },
{ type: 'message', id: 'task', parentId: null, message: {
role: 'user', content: [{ type: 'text', text: broken ? 'Inspect broken image output' : 'Inspect image output' }],
} },
{ type: 'message', id: 'call', parentId: 'task', message: {
role: 'assistant', content: [{ type: 'toolCall', id: 'read', name: 'read', arguments: { path: 'banner.png' } }],
} },
{ type: 'message', id: 'result', parentId: 'call', message: {
role: 'toolResult', toolCallId: 'read', toolName: 'read',
content: broken ? [
{ type: 'image', mimeType: 'image/png', data: 'YnJva2Vu' },
{ type: 'image', mimeType: 'image/svg+xml', data: 'PHN2Zz4=' },
] : [
{ type: 'text', text: 'Pidex banner preview' },
{ type: 'image', mimeType: 'image/png', data: image },
],
} },
];
await fs.writeFile(path.join(archives, `${name}.jsonl`), records.map((r) => JSON.stringify(r)).join('\n'));
}
app = await electron.launch({
executablePath: require('electron'),
args: ['.'],
Expand Down Expand Up @@ -145,6 +169,87 @@ describe('bounded tool previews in Electron', () => {
if (fixture) await fs.rm(fixture, { recursive: true, force: true });
});

it('previews saved tool images on demand at desktop and minimum window sizes', async () => {
await page.getByRole('tree', { name: 'Session archives', exact: true })
.getByRole('treeitem', { name: /Inspect image output/ }).click();
const card = page.locator('.wb-tool');
await card.locator('summary').click();
await card.getByText('Pidex banner preview', { exact: false }).waitFor();
expect(await card.getByText('No output.', { exact: true }).count()).toBe(0);
expect(await card.getByRole('img', { name: 'Tool output image 1' }).count()).toBe(0);
await card.getByRole('button', { name: 'Show image 1' }).focus();
await page.keyboard.press('Enter');
const image = card.getByRole('img', { name: 'Tool output image 1' });
await expect.poll(() => image.evaluate((img: HTMLImageElement) => img.complete && img.naturalWidth > 0)).toBe(true);
// The image and its controls may need separate frames on a short display.
await card.locator('summary').scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(root, '.tmp/proof/tool-image-controls.png'), scale: 'css' });
await page.screenshot({ path: path.join(shots, 'tool-image-controls.png'), scale: 'css' });
const capture = async (name: string) => {
await image.scrollIntoViewIfNeeded();
const bounds = await image.boundingBox();
expect(bounds?.height).toBeLessThanOrEqual(320);
expect(await page.evaluate(() => document.body.scrollWidth <= innerWidth)).toBe(true);
const screenshot = await page.screenshot({ scale: 'css' });
await Promise.all([
fs.writeFile(path.join(root, '.tmp/proof', name), screenshot),
fs.writeFile(path.join(shots, name), screenshot),
]);
};
await capture('tool-image-desktop.png');
await resizeWindow(900, 600);
await capture('tool-image-narrow.png');
await card.getByRole('button', { name: 'Copy output' }).click();
await card.getByText('Output copied.', { exact: true }).waitFor();
expect(await app.evaluate(({ clipboard }) => clipboard.readText())).toBe(
'Pidex banner preview\nImage 1 (image/png). Image data is omitted from text copies and exports.',
);
await page.getByRole('button', { name: 'Copy context', exact: true }).click();
await page.getByRole('status').filter({ hasText: 'Context copied.' }).waitFor();
const exported = await app.evaluate(({ clipboard }) => clipboard.readText());
expect(exported).toContain('Image data is omitted from text copies and exports.');
expect(exported).not.toContain('data:image');
await card.getByRole('button', { name: 'Output details' }).click();
await card.getByRole('button', { name: 'Output details' }).click();
await card.getByRole('button', { name: 'Hide image 1' }).click();
expect(await image.count()).toBe(0);
await resizeWindow(1280, 1100);
});

it('explains broken and unsupported saved images while keeping the tool result readable', async () => {
await page.getByRole('tree', { name: 'Session archives', exact: true })
.getByRole('treeitem', { name: /Inspect broken image output/ }).click();
const card = page.locator('.wb-tool');
await card.locator('summary').click();
await card.getByText(/Image 2 cannot be previewed/).waitFor();
expect(await card.getByRole('button', { name: 'Show image 2' }).count()).toBe(0);
await card.getByRole('button', { name: 'Show image 1' }).click();
const failure = card.getByRole('status').filter({ hasText: 'Image 1 could not be displayed.' });
await failure.waitFor();
await failure.scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(root, '.tmp/proof/tool-image-unavailable.png'), scale: 'css' });
await page.screenshot({ path: path.join(shots, 'tool-image-unavailable.png'), scale: 'css' });
expect(errors).toEqual([]);
});

it('keeps an opened image visible when a live tool result replaces its partial snapshot', async () => {
await page.getByRole('button', { name: 'New session', exact: true }).click();
await page.getByLabel('Message pi').fill('Inspect tool image fixture');
await page.getByRole('button', { name: 'Send', exact: true }).click();
const card = page.locator('.wb-tool');
await card.locator('summary').click();
await card.getByRole('button', { name: 'Show image 1' }).click();
await card.getByText('Final image preview', { exact: false }).waitFor();
await card.locator('summary').getByText('Done', { exact: true }).waitFor();
const image = card.getByRole('img', { name: 'Tool output image 1' });
await expect.poll(() => image.evaluate((img: HTMLImageElement) => img.complete && img.naturalWidth > 0)).toBe(true);
await image.scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(root, '.tmp/proof/tool-image-live.png'), scale: 'css' });
await page.screenshot({ path: path.join(shots, 'tool-image-live.png'), scale: 'css' });
expect(errors).toEqual([]);
await page.getByRole('button', { name: /^Sessions/ }).click();
});

it.each(['write', 'edit'])('bounds a long %s line and keeps the footer painted through native window resizing', async (tool) => {
await page.getByRole('tree', { name: 'Session archives', exact: true })
.getByRole('treeitem', { name: new RegExp(`Inspect long ${tool} input`) }).click();
Expand Down
20 changes: 20 additions & 0 deletions tests/fixtures/pi-rpc.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,26 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => {
reply();
streaming = true;
event('agent_start');
if (command.message === 'Inspect tool image fixture') {
const image = {
type: 'image', mimeType: 'image/png',
data: fs.readFileSync(path.join(__dirname, '../../docs/assets/banner.png'), 'base64'),
};
event('tool_execution_start', {
toolCallId: 'read-image', toolName: 'read', args: { path: 'banner.png' },
});
event('tool_execution_update', {
toolCallId: 'read-image', partialResult: { content: [{ type: 'text', text: 'Streaming image preview' }, image] },
});
timers.push(setTimeout(() => {
event('tool_execution_end', {
toolCallId: 'read-image', result: { content: [{ type: 'text', text: 'Final image preview' }, image] },
});
streaming = false;
event('agent_end');
}, 1500));
break;
}
if (command.message === 'Start interrupted tool fixture') {
event('tool_execution_start', {
toolCallId: 'interrupted-build', toolName: 'bash', args: { command: 'build-project' },
Expand Down
Loading
Loading