From 7d54b5d32053fb93f842681300a9f7d1324d9ac0 Mon Sep 17 00:00:00 2001 From: Yilin Jing Date: Mon, 28 Sep 2026 00:53:24 -0400 Subject: [PATCH 1/2] Hand the agent a mockup's file, not a picture of it The + on a board shot it and sent the PNG as an image block beside its path. The file is what the agent reads and edits, so the shot now stays the tile's and the transcript's, and the agent gets only "[Image #n] ". Pictures, and the person's own shapes, still go as images. Co-Authored-By: Claude Opus 5.5 --- canvas/server/agent.ts | 23 ++++++++++++++++++++++- canvas/src/ChatPanel.tsx | 22 +++++++++++++++------- canvas/src/agents.test.ts | 22 ++++++++++++++++++++++ canvas/src/agents.ts | 23 +++++++++++++++-------- canvas/src/canvasAttach.tsx | 2 +- 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/canvas/server/agent.ts b/canvas/server/agent.ts index fb47016f..01a188a4 100644 --- a/canvas/server/agent.ts +++ b/canvas/server/agent.ts @@ -310,6 +310,8 @@ export function createAgentServer(options: { !/^[A-Za-z0-9+/]*={0,2}$/.test(i.data) ) return send(400, "bad image data"); + if (i.page !== undefined && i.page !== true) + return send(400, "bad image page"); } // The body cap above is the panel's limit in base64; a client that is not the // panel meets the limit itself here, in the bytes the files come out as. @@ -493,8 +495,26 @@ export function createAgentServer(options: { const id = randomUUID(); fs.mkdirSync(keptOf(id), { recursive: true }); const imagesDir = images.length ? keptOf(id) : ""; + // A mockup's picture is kept for the panel, and the agent is pointed at its file, + // `/.html` in whichever canvases folder holds that slug. A community + // project's is not on this machine, so it keeps the name `sp fetch` finds it by. + const pageOf = (name: string) => { + const [slug, file, ...rest] = name.split("/"); + return community === undefined && + rest.length === 0 && + SAFE_NAME.test(slug) && + SAFE_NAME.test(file ?? "") + ? path.join(folderOf(boards, examplesDir, slug), file) + : name; + }; const held: AgentImage[] = images.map( - (i: { n: number; name: string; type: string; data: string }) => ({ + (i: { + n: number; + name: string; + type: string; + data: string; + page?: true; + }) => ({ ...i, path: picture( id, @@ -502,6 +522,7 @@ export function createAgentServer(options: { i.type, Buffer.from(i.data, "base64"), ), + page: i.page && pageOf(i.name), }), ); const c = command( diff --git a/canvas/src/ChatPanel.tsx b/canvas/src/ChatPanel.tsx index de31b2a1..d6623b52 100644 --- a/canvas/src/ChatPanel.tsx +++ b/canvas/src/ChatPanel.tsx @@ -87,8 +87,9 @@ const OPEN_KEY = "sp-chat-open"; /** * What the canvas hands the chat panel when the button is pressed (canvasAttach.tsx): a picture - * to attach to the message, or the reason none was. A board comes over as a picture too. The - * file's own name says which board it is, and the panel shows it under the tile. And the start + * to attach to the message, or the reason none was. A board comes over as a picture too, for its + * tile only: the agent is handed the board's file, which its name says, and the panel shows that + * name under the tile. And the start * of a message, from the strip's "+" (CanvasStrip.tsx), because a canvas is only ever the * agent's work, and a folder with no boards in it is not one. And a whole message, sent as it is, * from the new-project dialog (AppShell.tsx), which starts the agent defining the product. @@ -104,7 +105,8 @@ const OPEN_KEY = "sp-chat-open"; export const CANVAS_ATTACH = "sp:canvas-attach"; export type CanvasAttachDetail = - | { kind: "board"; name: string; src: string } + /** `page` for a mockup, whose drawing is only the tile's: the agent is handed its file. */ + | { kind: "board"; name: string; src: string; page?: true } | { kind: "image"; file: File } | { kind: "error"; message: string } | { kind: "draft"; text: string } @@ -249,6 +251,8 @@ interface Attached { url: string; /** A board still being drawn, with no `url` yet, or one whose drawing failed. */ state?: "pending" | "failed"; + /** A mockup: the agent gets its file rather than this picture of it (agents.ts). */ + page?: true; } /** @@ -713,6 +717,7 @@ export function ChatPanel(props: { type: r.file.type, size: r.file.size, url: r.url, + page: t.page, } : t; }); @@ -758,7 +763,7 @@ export function ChatPanel(props: { * drawing asked of the server, to land in that tile. Asked for again it keeps the tile it has — * one already there or on its way is only named again, and one that failed is drawn again. */ - const addBoard = (name: string, src: string) => { + const addBoard = (name: string, src: string, page?: true) => { let tile = tray.current.find((t) => t.name === name); if (!tile && tray.current.length >= MAX_IMAGES) return setSendError( @@ -779,6 +784,7 @@ export function ChatPanel(props: { size: 0, url: "", state: "pending", + page, }; const next = tile; tray.current = [...tray.current.filter((t) => t.n !== next.n), next].sort( @@ -805,7 +811,8 @@ export function ChatPanel(props: { // What the buttons on a canvas shape hand over (canvasAttach.tsx): a picture, attached and named // in the sentence, or the reason there is none. A mockup arrives as a picture of itself, called // by its own path, so pointing at one puts the same tile and the same number in the panel that - // pointing at a picture does — and the path is what the tile is captioned with. + // pointing at a picture does — and the path is what the tile is captioned with, and all the + // agent is given of it. // No dependency list, so every render leaves a listener holding that render's `addImages` and // its numbering — a listener that stayed would be attaching to the draft the panel had at mount. useEffect(() => { @@ -819,7 +826,7 @@ export function ChatPanel(props: { // that is still being read. if (detail.kind === "board") { adds.current = adds.current.then(() => - addBoard(detail.name, detail.src), + addBoard(detail.name, detail.src, detail.page), ); return; } @@ -937,11 +944,12 @@ export function ChatPanel(props: { agent, model, effort, - images: attached.map(({ n, name, type, url }) => ({ + images: attached.map(({ n, name, type, url, page }) => ({ n, name, type, data: url.slice(url.indexOf(",") + 1), + page, })), }), }); diff --git a/canvas/src/agents.test.ts b/canvas/src/agents.test.ts index ce898f4c..f2d4ac29 100644 --- a/canvas/src/agents.test.ts +++ b/canvas/src/agents.test.ts @@ -86,6 +86,28 @@ describe("AGENTS", () => { ); }); + // A mockup's picture is only the panel's tile: the agent is pointed at its file instead. + it("hands a mockup over as its file, not its picture", () => { + const board = { + n: 2, + name: "shop/01-home.html", + type: "image/png", + data: "CCC", + path: "/tmp/sp-chat-r/2.png", + page: "/proj/canvases/shop/01-home.html", + }; + expect( + JSON.parse(def("claude").stdin("tighten #2", "P", [board])).message + .content, + ).toEqual([ + { type: "text", text: "[Image #2] /proj/canvases/shop/01-home.html" }, + { type: "text", text: "tighten #2" }, + ]); + expect(def("codex").stdin("tighten #2", "P", [board])).toBe( + "P\n\n[Image #2] /proj/canvases/shop/01-home.html\n\ntighten #2", + ); + }); + // A session is what the agent called it on its first turn, and every later turn resumes that. it("reads the session off the first turn and resumes it on the next", () => { expect( diff --git a/canvas/src/agents.ts b/canvas/src/agents.ts index 24b895a8..0ad085a2 100644 --- a/canvas/src/agents.ts +++ b/canvas/src/agents.ts @@ -80,6 +80,9 @@ export interface AgentImage { /** Where the server wrote it, for an agent that takes files rather than bytes; gone once * that agent has exited. */ path: string; + /** A mockup's HTML file, when the picture is only the panel's drawing of one: the agent is + * pointed at the file and not handed the picture, since the file is what it reads and edits. */ + page?: string; } /** What the composer chose, handed to `args`. An empty string means the CLI decides. */ @@ -199,13 +202,17 @@ export const AGENTS: AgentDef[] = [ // `[Image #2]` is the marker Claude Code writes itself when a screenshot is pasted into its // terminal, so the number arrives as something already read rather than a local convention. stdin: (message, _preamble, images) => { - const blocks = images.flatMap((i) => [ - { type: "text", text: `[Image #${i.n}] ${i.name}` }, - { - type: "image", - source: { type: "base64", media_type: i.type, data: i.data }, - }, - ]); + const blocks = images.flatMap((i) => + i.page + ? [{ type: "text", text: `[Image #${i.n}] ${i.page}` }] + : [ + { type: "text", text: `[Image #${i.n}] ${i.name}` }, + { + type: "image", + source: { type: "base64", media_type: i.type, data: i.data }, + }, + ], + ); const content = blocks.length ? [...blocks, { type: "text", text: message }] : message; @@ -308,7 +315,7 @@ export const AGENTS: AgentDef[] = [ stdin: (message, preamble, images) => [ preamble, - images.map((i) => `[Image #${i.n}] ${i.path}`).join("\n"), + images.map((i) => `[Image #${i.n}] ${i.page ?? i.path}`).join("\n"), message, ] .filter(Boolean) diff --git a/canvas/src/canvasAttach.tsx b/canvas/src/canvasAttach.tsx index 448eafab..505eea9d 100644 --- a/canvas/src/canvasAttach.tsx +++ b/canvas/src/canvasAttach.tsx @@ -54,7 +54,7 @@ async function attach(editor: Editor, target: TLShape) { `&w=${Math.max(1, Math.round(w * scale))}&h=${Math.max(1, Math.round(h * scale))}`, window.location.href, ).href; - return dispatchAttach({ kind: "board", name, src }); + return dispatchAttach({ kind: "board", name, src, page: true }); } // One of the person's own: whatever it is, the agent gets a picture of it, named by where it // reads the thing itself (canvasContent.ts). From 2ed1ab89c11ca75e25c109a1647cce2e897e72f4 Mon Sep 17 00:00:00 2001 From: Yilin Jing Date: Mon, 28 Sep 2026 00:57:30 -0400 Subject: [PATCH 2/2] Resolve an attached mockup in the project it came from The panel is one across tabs, so a tile attached in one project can be sent from another; it now carries where it was attached and the server resolves the file there. Inline the one-use helper, and add the release note. Co-Authored-By: Claude Opus 5.5 --- RELEASE-NOTES.md | 4 +++ canvas/server/agent.ts | 69 ++++++++++++++++++++++++++-------------- canvas/src/ChatPanel.tsx | 7 ++-- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 5af1f97f..699556bd 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -19,6 +19,10 @@ The app, its skills and the toolkit carry one version. Everything below is on `main` and reaches no install until a version is cut. +- **A mockup goes to the agent as its file.** Its **+** still shows a picture + of it in the chat, but the agent is handed the board's `.html` path rather + than that picture, since the file is what it reads and changes. + ## v1.7.0 2026-09-26. The community projects open in the app, and the app no longer diff --git a/canvas/server/agent.ts b/canvas/server/agent.ts index 01a188a4..f6789142 100644 --- a/canvas/server/agent.ts +++ b/canvas/server/agent.ts @@ -310,7 +310,13 @@ export function createAgentServer(options: { !/^[A-Za-z0-9+/]*={0,2}$/.test(i.data) ) return send(400, "bad image data"); - if (i.page !== undefined && i.page !== true) + if ( + i.page !== undefined && + (typeof i.page !== "object" || + !["project", "community"].every( + (k) => i.page[k] === undefined || typeof i.page[k] === "string", + )) + ) return send(400, "bad image page"); } // The body cap above is the panel's limit in base64; a client that is not the @@ -495,35 +501,50 @@ export function createAgentServer(options: { const id = randomUUID(); fs.mkdirSync(keptOf(id), { recursive: true }); const imagesDir = images.length ? keptOf(id) : ""; - // A mockup's picture is kept for the panel, and the agent is pointed at its file, - // `/.html` in whichever canvases folder holds that slug. A community - // project's is not on this machine, so it keeps the name `sp fetch` finds it by. - const pageOf = (name: string) => { - const [slug, file, ...rest] = name.split("/"); - return community === undefined && - rest.length === 0 && - SAFE_NAME.test(slug) && - SAFE_NAME.test(file ?? "") - ? path.join(folderOf(boards, examplesDir, slug), file) - : name; - }; const held: AgentImage[] = images.map( (i: { n: number; name: string; type: string; data: string; - page?: true; - }) => ({ - ...i, - path: picture( - id, - `image-${i.n}`, - i.type, - Buffer.from(i.data, "base64"), - ), - page: i.page && pageOf(i.name), - }), + page?: { project?: string; community?: string }; + }) => { + // A mockup's picture is kept for the panel, and the agent is pointed at its + // file, `/.html` in the canvases of the project it was attached + // from, which need not be the one it is sent from. A community project's is not + // on this machine, so it keeps the name `sp fetch` finds it by. + const [slug, file, ...rest] = i.name.split("/"); + const from = i.page?.project && projects().get(i.page.project); + const local = + i.page?.community === undefined && + (i.page?.project === undefined || from) && + rest.length === 0 && + SAFE_NAME.test(slug) && + SAFE_NAME.test(file ?? ""); + return { + ...i, + path: picture( + id, + `image-${i.n}`, + i.type, + Buffer.from(i.data, "base64"), + ), + page: + i.page && + (local + ? path.join( + folderOf( + from ? path.join(from, CANVASES) : examplesDir, + examplesDir, + slug, + ), + file, + ) + : i.page.community + ? `${i.name} of the community project ${i.page.community}` + : i.name), + }; + }, ); const c = command( def.bin, diff --git a/canvas/src/ChatPanel.tsx b/canvas/src/ChatPanel.tsx index d6623b52..0c6456a5 100644 --- a/canvas/src/ChatPanel.tsx +++ b/canvas/src/ChatPanel.tsx @@ -251,8 +251,9 @@ interface Attached { url: string; /** A board still being drawn, with no `url` yet, or one whose drawing failed. */ state?: "pending" | "failed"; - /** A mockup: the agent gets its file rather than this picture of it (agents.ts). */ - page?: true; + /** A mockup, and the project it was attached from, which the panel may since have left: the + * agent gets its file there rather than this picture of it (agents.ts). */ + page?: { project?: string; community?: string }; } /** @@ -784,7 +785,7 @@ export function ChatPanel(props: { size: 0, url: "", state: "pending", - page, + page: page && { project, community }, }; const next = tile; tray.current = [...tray.current.filter((t) => t.n !== next.n), next].sort(