From 8a0b4f3dbb84d28b87c51825d2752935e5df227b Mon Sep 17 00:00:00 2001 From: William Valentin Date: Mon, 16 Feb 2026 12:36:02 -0800 Subject: [PATCH] Add canvas artifact RPC foundation for A2UI --- README.md | 11 +++ docs/api/PROTOCOL.md | 38 +++++++ ...-02-16-canvas-a2ui-foundation-checklist.md | 46 +++++++++ docs/plans/state.json | 28 +++++- src/gateway/canvas-store.test.ts | 58 +++++++++++ src/gateway/canvas-store.ts | 99 +++++++++++++++++++ src/gateway/handlers/canvas.ts | 89 +++++++++++++++++ src/gateway/handlers/handlers.test.ts | 78 +++++++++++++++ src/gateway/handlers/index.ts | 2 + src/gateway/server.test.ts | 40 ++++++++ src/gateway/server.ts | 11 +++ 11 files changed, 497 insertions(+), 3 deletions(-) create mode 100644 docs/plans/2026-02-16-canvas-a2ui-foundation-checklist.md create mode 100644 src/gateway/canvas-store.test.ts create mode 100644 src/gateway/canvas-store.ts create mode 100644 src/gateway/handlers/canvas.ts diff --git a/README.md b/README.md index 3127cc5..d67c7ea 100644 --- a/README.md +++ b/README.md @@ -870,6 +870,17 @@ Methods: - `system.location` provides an operator view of registered node locations. - `system.capabilities` returns gateway protocol and node policy snapshot. +## Canvas / A2UI Foundation + +Gateway provides a session-scoped canvas artifact API for companion/UI surfaces: +- `canvas.put` upserts an artifact (`artifactId`, `type`, `content`, optional `title`/`metadata`). +- `canvas.get` retrieves a single artifact. +- `canvas.list` lists artifacts for a session (most recently updated first). +- `canvas.delete` removes one artifact. +- `canvas.clear` removes all artifacts for a session. + +This foundation is currently in-memory (runtime ephemeral) and intended as the first step for richer visual workspace flows. + ## Gateway Request Body Limit Cap inbound HTTP POST body size (webhooks and Gmail push) to reduce memory-DoS risk. diff --git a/docs/api/PROTOCOL.md b/docs/api/PROTOCOL.md index d5c93d4..9c8db12 100644 --- a/docs/api/PROTOCOL.md +++ b/docs/api/PROTOCOL.md @@ -684,6 +684,44 @@ Return gateway protocol version, node policy status, and feature-gate snapshot. Return the operator-facing snapshot of registered node locations. +### Canvas Methods + +#### `canvas.put` + +Upsert a session-scoped canvas artifact. + +**Request:** +```json +{ + "id": 12, + "method": "canvas.put", + "params": { + "sessionId": "ws:abc123", + "artifactId": "summary-card", + "type": "note", + "title": "Draft Summary", + "content": { "markdown": "## Notes" }, + "metadata": { "lane": "analysis" } + } +} +``` + +#### `canvas.get` + +Fetch a single artifact by id. + +#### `canvas.list` + +List artifacts for a session (newest first). + +#### `canvas.delete` + +Delete a single artifact. + +#### `canvas.clear` + +Delete all artifacts for a session. + #### `agent.setToolUseCallback` Set callback for tool use events (for confirmation UI). diff --git a/docs/plans/2026-02-16-canvas-a2ui-foundation-checklist.md b/docs/plans/2026-02-16-canvas-a2ui-foundation-checklist.md new file mode 100644 index 0000000..c5c19aa --- /dev/null +++ b/docs/plans/2026-02-16-canvas-a2ui-foundation-checklist.md @@ -0,0 +1,46 @@ +# Canvas / A2UI Foundation Checklist + +**Date:** 2026-02-16 +**Scope:** Close OpenClaw "Canvas / A2UI" gap with a first gateway-level artifact API. + +## Goal + +Add a minimal, testable visual-workspace foundation that companion clients/web UI can use to store and retrieve structured session artifacts. + +## Implemented + +- Added in-memory canvas artifact store: + - `src/gateway/canvas-store.ts` +- Added gateway canvas handlers: + - `canvas.put` + - `canvas.get` + - `canvas.list` + - `canvas.delete` + - `canvas.clear` +- Wired handlers into gateway routing: + - `src/gateway/server.ts` + - `src/gateway/handlers/index.ts` +- Added docs updates: + - `README.md` + - `docs/api/PROTOCOL.md` + +## Test Coverage + +- `src/gateway/canvas-store.test.ts` + - session scoping + - update semantics + - delete/clear lifecycle + - per-session cap eviction +- `src/gateway/handlers/handlers.test.ts` + - full `canvas.*` lifecycle handler flow + - request validation error paths +- `src/gateway/server.test.ts` + - end-to-end gateway RPC canvas flow + +## Validation Run + +```bash +pnpm test:run src/gateway/canvas-store.test.ts src/gateway/handlers/handlers.test.ts src/gateway/server.test.ts +pnpm typecheck +pnpm build +``` diff --git a/docs/plans/state.json b/docs/plans/state.json index 9d582fe..ab66499 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -495,6 +495,28 @@ ], "test_status": "pnpm test:run src/gateway/protocol.test.ts src/gateway/auth.test.ts src/gateway/handlers/node.test.ts src/gateway/handlers/handlers.test.ts src/gateway/server.test.ts src/config/schema.test.ts + pnpm typecheck + pnpm build passing" }, + "canvas-a2ui-foundation": { + "file": "2026-02-16-canvas-a2ui-foundation-checklist.md", + "status": "completed", + "date": "2026-02-16", + "updated": "2026-02-16", + "summary": "Implemented Canvas/A2UI gateway foundation with session-scoped artifact RPCs (`canvas.put/get/list/delete/clear`), in-memory artifact storage, handler/server wiring, and docs/tests.", + "files_created": [ + "docs/plans/2026-02-16-canvas-a2ui-foundation-checklist.md", + "src/gateway/canvas-store.ts", + "src/gateway/canvas-store.test.ts", + "src/gateway/handlers/canvas.ts" + ], + "files_modified": [ + "src/gateway/handlers/index.ts", + "src/gateway/handlers/handlers.test.ts", + "src/gateway/server.ts", + "src/gateway/server.test.ts", + "README.md", + "docs/api/PROTOCOL.md" + ], + "test_status": "pnpm test:run src/gateway/canvas-store.test.ts src/gateway/handlers/handlers.test.ts src/gateway/server.test.ts + pnpm typecheck + pnpm build passing" + }, "qmd-backend": { "file": "2026-02-16-qmd-backend-checklist.md", "status": "completed", @@ -3056,7 +3078,7 @@ } }, "overall_progress": { - "total_test_count": 1773, + "total_test_count": 1780, "all_tests_passing": true, "p0_completion": "3/3 (100%)", "p1_completion": "4/4 (100%)", @@ -3071,12 +3093,12 @@ "tier2_completion": "4/4 (100%) — inbound webhooks, vector memory search, Dockerfile, heartbeat monitor", "tier3_completion": "5/5 (100%) — lane queue, credential redaction, web UI token dashboard, xAI (Grok) provider, Voyage AI embeddings", "tier4_completion": "4/4 (100%) — gateway lock, shell completion, Tailscale Serve/Funnel, DM pairing codes", - "feature_gap_scorecard": "119/128 match (93%), 0 partial (0%), 9 missing (7%)", + "feature_gap_scorecard": "120/128 match (94%), 0 partial (0%), 8 missing (6%)", "operator_dx_milestone": "Phase 3 (Live Ops Dashboard): 2/2 plans complete — milestone done", "gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram", "native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback", "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", - "next_up": "OpenClaw gap: Canvas / A2UI (open next scoped implementation checklist)" + "next_up": "OpenClaw gap: macOS menu bar companion app (open next scoped implementation checklist)" }, "soul_md_and_cron_create": { "date": "2026-02-11", diff --git a/src/gateway/canvas-store.test.ts b/src/gateway/canvas-store.test.ts new file mode 100644 index 0000000..742b3c9 --- /dev/null +++ b/src/gateway/canvas-store.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { CanvasStore } from './canvas-store.js'; + +describe('CanvasStore', () => { + it('stores and retrieves artifacts per session', () => { + const store = new CanvasStore(); + const artifact = store.put('ws:1', { + id: 'a1', + type: 'note', + content: { text: 'hello' }, + }); + + expect(artifact.id).toBe('a1'); + expect(store.get('ws:1', 'a1')?.type).toBe('note'); + expect(store.list('ws:1')).toHaveLength(1); + }); + + it('updates existing artifacts while preserving createdAt', async () => { + const store = new CanvasStore(); + const first = store.put('ws:1', { + id: 'a1', + type: 'note', + content: { text: 'one' }, + }); + await new Promise((resolve) => setTimeout(resolve, 2)); + const second = store.put('ws:1', { + id: 'a1', + type: 'note', + content: { text: 'two' }, + }); + + expect(second.createdAt).toBe(first.createdAt); + expect(second.updatedAt).toBeGreaterThanOrEqual(first.updatedAt); + expect(store.get('ws:1', 'a1')?.content).toEqual({ text: 'two' }); + }); + + it('deletes and clears session artifacts', () => { + const store = new CanvasStore(); + store.put('ws:1', { id: 'a1', type: 'note', content: 'x' }); + store.put('ws:1', { id: 'a2', type: 'note', content: 'y' }); + + expect(store.delete('ws:1', 'a1')).toBe(true); + expect(store.delete('ws:1', 'missing')).toBe(false); + expect(store.list('ws:1')).toHaveLength(1); + expect(store.clear('ws:1')).toBe(1); + expect(store.list('ws:1')).toEqual([]); + }); + + it('evicts oldest artifact when per-session cap is exceeded', () => { + const store = new CanvasStore(2); + store.put('ws:1', { id: 'a1', type: 'note', content: 1 }); + store.put('ws:1', { id: 'a2', type: 'note', content: 2 }); + store.put('ws:1', { id: 'a3', type: 'note', content: 3 }); + + expect(store.get('ws:1', 'a1')).toBeUndefined(); + expect(store.list('ws:1').map((a) => a.id).sort()).toEqual(['a2', 'a3']); + }); +}); diff --git a/src/gateway/canvas-store.ts b/src/gateway/canvas-store.ts new file mode 100644 index 0000000..5c814c0 --- /dev/null +++ b/src/gateway/canvas-store.ts @@ -0,0 +1,99 @@ +export interface CanvasArtifact { + id: string; + type: string; + title?: string; + content: unknown; + metadata?: Record; + createdAt: number; + updatedAt: number; +} + +interface CanvasPutInput { + id?: string; + type: string; + title?: string; + content: unknown; + metadata?: Record; +} + +/** + * Ephemeral in-memory canvas store keyed by sessionId. + * Intended as a gateway runtime foundation for A2UI/canvas surfaces. + */ +export class CanvasStore { + private readonly sessions = new Map>(); + + constructor(private readonly maxArtifactsPerSession = 200) {} + + list(sessionId: string): CanvasArtifact[] { + const entries = this.sessions.get(sessionId); + if (!entries) { + return []; + } + return Array.from(entries.values()).sort((a, b) => b.updatedAt - a.updatedAt); + } + + get(sessionId: string, artifactId: string): CanvasArtifact | undefined { + return this.sessions.get(sessionId)?.get(artifactId); + } + + put(sessionId: string, input: CanvasPutInput): CanvasArtifact { + const id = sanitizeArtifactId(input.id); + const now = Date.now(); + let entries = this.sessions.get(sessionId); + if (!entries) { + entries = new Map(); + this.sessions.set(sessionId, entries); + } + + const existing = entries.get(id); + const artifact: CanvasArtifact = { + id, + type: input.type.trim(), + title: input.title?.trim() || undefined, + content: input.content, + metadata: input.metadata, + createdAt: existing?.createdAt ?? now, + updatedAt: now, + }; + entries.set(id, artifact); + + if (entries.size > this.maxArtifactsPerSession) { + const oldest = Array.from(entries.values()).sort((a, b) => a.updatedAt - b.updatedAt)[0]; + if (oldest) { + entries.delete(oldest.id); + } + } + return artifact; + } + + delete(sessionId: string, artifactId: string): boolean { + const entries = this.sessions.get(sessionId); + if (!entries) { + return false; + } + const removed = entries.delete(artifactId); + if (entries.size === 0) { + this.sessions.delete(sessionId); + } + return removed; + } + + clear(sessionId: string): number { + const entries = this.sessions.get(sessionId); + if (!entries) { + return 0; + } + const count = entries.size; + this.sessions.delete(sessionId); + return count; + } +} + +function sanitizeArtifactId(raw?: string): string { + const value = (raw ?? '').trim(); + if (value) { + return value.slice(0, 128); + } + return `art_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`; +} diff --git a/src/gateway/handlers/canvas.ts b/src/gateway/handlers/canvas.ts new file mode 100644 index 0000000..4248b5e --- /dev/null +++ b/src/gateway/handlers/canvas.ts @@ -0,0 +1,89 @@ +import type { GatewayRequest, OutboundMessage } from '../protocol.js'; +import { makeError, makeResponse, ErrorCode } from '../protocol.js'; +import type { CanvasStore } from '../canvas-store.js'; + +export interface CanvasHandlerDeps { + store: CanvasStore; +} + +export function createCanvasHandlers(deps: CanvasHandlerDeps) { + return { + 'canvas.list': async (request: GatewayRequest): Promise => { + const params = request.params as { sessionId?: string } | undefined; + if (!params?.sessionId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'sessionId is required'); + } + + const artifacts = deps.store.list(params.sessionId); + return makeResponse(request.id, { artifacts }); + }, + + 'canvas.get': async (request: GatewayRequest): Promise => { + const params = request.params as { sessionId?: string; artifactId?: string } | undefined; + if (!params?.sessionId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'sessionId is required'); + } + if (!params?.artifactId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'artifactId is required'); + } + + const artifact = deps.store.get(params.sessionId, params.artifactId); + if (!artifact) { + return makeError(request.id, ErrorCode.SessionNotFound, 'Canvas artifact not found'); + } + return makeResponse(request.id, { artifact }); + }, + + 'canvas.put': async (request: GatewayRequest): Promise => { + const params = request.params as { + sessionId?: string; + artifactId?: string; + type?: string; + title?: string; + content?: unknown; + metadata?: Record; + } | undefined; + if (!params?.sessionId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'sessionId is required'); + } + if (!params?.type || typeof params.type !== 'string' || !params.type.trim()) { + return makeError(request.id, ErrorCode.InvalidRequest, 'type is required'); + } + if (params.content === undefined) { + return makeError(request.id, ErrorCode.InvalidRequest, 'content is required'); + } + + const artifact = deps.store.put(params.sessionId, { + id: params.artifactId, + type: params.type, + title: params.title, + content: params.content, + metadata: params.metadata, + }); + return makeResponse(request.id, { artifact, upserted: true }); + }, + + 'canvas.delete': async (request: GatewayRequest): Promise => { + const params = request.params as { sessionId?: string; artifactId?: string } | undefined; + if (!params?.sessionId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'sessionId is required'); + } + if (!params?.artifactId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'artifactId is required'); + } + + const deleted = deps.store.delete(params.sessionId, params.artifactId); + return makeResponse(request.id, { deleted }); + }, + + 'canvas.clear': async (request: GatewayRequest): Promise => { + const params = request.params as { sessionId?: string } | undefined; + if (!params?.sessionId) { + return makeError(request.id, ErrorCode.InvalidRequest, 'sessionId is required'); + } + + const cleared = deps.store.clear(params.sessionId); + return makeResponse(request.id, { cleared }); + }, + }; +} diff --git a/src/gateway/handlers/handlers.test.ts b/src/gateway/handlers/handlers.test.ts index ca8d2e2..6e49f80 100644 --- a/src/gateway/handlers/handlers.test.ts +++ b/src/gateway/handlers/handlers.test.ts @@ -7,10 +7,12 @@ import { createAgentHandlers } from './agent.js'; import { createIntentHandlers } from './intents.js'; import { createRoutingHandlers } from './routing.js'; import { createHistoryHandlers } from './history.js'; +import { createCanvasHandlers } from './canvas.js'; import { createConfigHandlers, redactConfig } from './config.js'; import { createPairingHandlers } from './pairing.js'; import { PairingManager } from '../../channels/pairing.js'; import { LaneQueue } from '../lane-queue.js'; +import { CanvasStore } from '../canvas-store.js'; import { ErrorCode } from '../protocol.js'; import type { GatewayRequest, GatewayResponse, GatewayError, GatewayEvent, OutboundMessage } from '../protocol.js'; import { ComponentRegistry } from '../../intents/index.js'; @@ -350,6 +352,82 @@ describe('session handlers', () => { }); }); +describe('canvas handlers', () => { + it('supports put/get/list/delete/clear lifecycle', async () => { + const handlers = createCanvasHandlers({ store: new CanvasStore() }); + + const putReq: GatewayRequest = { + id: 1, + method: 'canvas.put', + params: { + sessionId: 'ws:abc', + artifactId: 'card-1', + type: 'note', + title: 'Draft', + content: { text: 'hello' }, + }, + }; + const putRes = await handlers['canvas.put'](putReq) as GatewayResponse; + expect(getPath(putRes.result, 'artifact', 'id')).toBe('card-1'); + + const getRes = await handlers['canvas.get']({ + id: 2, + method: 'canvas.get', + params: { sessionId: 'ws:abc', artifactId: 'card-1' }, + }) as GatewayResponse; + expect(getPath(getRes.result, 'artifact', 'title')).toBe('Draft'); + + const listRes = await handlers['canvas.list']({ + id: 3, + method: 'canvas.list', + params: { sessionId: 'ws:abc' }, + }) as GatewayResponse; + expect((getPath(listRes.result, 'artifacts') as unknown[]).length).toBe(1); + + const delRes = await handlers['canvas.delete']({ + id: 4, + method: 'canvas.delete', + params: { sessionId: 'ws:abc', artifactId: 'card-1' }, + }) as GatewayResponse; + expect(getPath(delRes.result, 'deleted')).toBe(true); + + await handlers['canvas.put']({ + id: 5, + method: 'canvas.put', + params: { sessionId: 'ws:abc', artifactId: 'card-2', type: 'note', content: 'a' }, + }); + await handlers['canvas.put']({ + id: 6, + method: 'canvas.put', + params: { sessionId: 'ws:abc', artifactId: 'card-3', type: 'note', content: 'b' }, + }); + const clearRes = await handlers['canvas.clear']({ + id: 7, + method: 'canvas.clear', + params: { sessionId: 'ws:abc' }, + }) as GatewayResponse; + expect(getPath(clearRes.result, 'cleared')).toBe(2); + }); + + it('validates required params', async () => { + const handlers = createCanvasHandlers({ store: new CanvasStore() }); + + const missingSession = await handlers['canvas.list']({ + id: 8, + method: 'canvas.list', + params: {}, + }) as GatewayError; + expect(missingSession.error.code).toBe(ErrorCode.InvalidRequest); + + const missingContent = await handlers['canvas.put']({ + id: 9, + method: 'canvas.put', + params: { sessionId: 'ws:abc', type: 'note' }, + }) as GatewayError; + expect(missingContent.error.code).toBe(ErrorCode.InvalidRequest); + }); +}); + describe('tool handlers', () => { const mockTool = { name: 'test.tool', diff --git a/src/gateway/handlers/index.ts b/src/gateway/handlers/index.ts index 9a982f7..7e5aaa1 100644 --- a/src/gateway/handlers/index.ts +++ b/src/gateway/handlers/index.ts @@ -17,5 +17,7 @@ export { createRoutingHandlers } from './routing.js'; export type { RoutingHandlerDeps } from './routing.js'; export { createHistoryHandlers } from './history.js'; export type { HistoryHandlerDeps } from './history.js'; +export { createCanvasHandlers } from './canvas.js'; +export type { CanvasHandlerDeps } from './canvas.js'; export { createNodeHandlers } from './node.js'; export type { NodeHandlerDeps, NodeRegistration, NodeConnectionState, NodeLocation } from './node.js'; diff --git a/src/gateway/server.test.ts b/src/gateway/server.test.ts index 060bc3d..8e9877c 100644 --- a/src/gateway/server.test.ts +++ b/src/gateway/server.test.ts @@ -234,6 +234,46 @@ describe('GatewayServer integration', () => { expect(methods).toContain('sessions.create'); expect(methods).toContain('tools.list'); expect(methods).toContain('tools.invoke'); + expect(methods).toContain('canvas.put'); + expect(methods).toContain('canvas.list'); + }); + + it('supports canvas artifact lifecycle via gateway RPC', async () => { + if (!LISTEN_ALLOWED) { + return; + } + const ws = await createClient(); + try { + const put = await sendAndReceive(ws, { + id: 31, + method: 'canvas.put', + params: { + sessionId: 'ws:test-canvas', + artifactId: 'a1', + type: 'note', + content: { text: 'draft' }, + }, + }); + expect((put as GatewayResponse).id).toBe(31); + + const list = await sendAndReceive(ws, { + id: 32, + method: 'canvas.list', + params: { sessionId: 'ws:test-canvas' }, + }); + const artifacts = ((list as GatewayResponse).result as { artifacts: Array<{ id: string }> }).artifacts; + expect(artifacts).toHaveLength(1); + expect(artifacts[0]?.id).toBe('a1'); + + const clear = await sendAndReceive(ws, { + id: 33, + method: 'canvas.clear', + params: { sessionId: 'ws:test-canvas' }, + }); + expect(((clear as GatewayResponse).result as { cleared: number }).cleared).toBe(1); + } finally { + ws.close(); + } }); // ── HTTP static file serving tests ──────────────────────────── diff --git a/src/gateway/server.ts b/src/gateway/server.ts index 4833735..e62a9bd 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -7,6 +7,7 @@ import { SessionBridge } from './session-bridge.js'; import type { SessionBridgeConfig } from './session-bridge.js'; import { LaneQueue } from './lane-queue.js'; import type { LaneQueueConfig } from './lane-queue.js'; +import { CanvasStore } from './canvas-store.js'; import { MetricsCollector } from './metrics.js'; import { authenticateRequest, authorizeNodeMethod } from './auth.js'; import type { AuthConfig } from './auth.js'; @@ -28,6 +29,7 @@ import { createIntentHandlers, createRoutingHandlers, createHistoryHandlers, + createCanvasHandlers, createNodeHandlers, } from './handlers/index.js'; import { discoverServices } from './handlers/services.js'; @@ -133,6 +135,7 @@ export class GatewayServer { private router: Router; private sessionBridge: SessionBridge; private laneQueue: LaneQueue; + private canvasStore: CanvasStore; private metrics: MetricsCollector; private discoveryHandle: GatewayDiscoveryHandle | null = null; private connectionMap: Map = new Map(); @@ -160,6 +163,7 @@ export class GatewayServer { }); this.laneQueue = new LaneQueue(config.queue); + this.canvasStore = new CanvasStore(); this.metrics = new MetricsCollector({ getQueueDepth: () => this.laneQueue.totalPending(), }); @@ -235,6 +239,10 @@ export class GatewayServer { sessionManager: this.config.sessionManager, }); + const canvasHandlers = createCanvasHandlers({ + store: this.canvasStore, + }); + const toolHandlers = createToolHandlers({ toolRegistry: this.config.toolRegistry, toolExecutor: this.config.toolExecutor, @@ -365,6 +373,9 @@ export class GatewayServer { for (const [method, handler] of Object.entries(historyHandlers)) { this.router.register(method, handler); } + for (const [method, handler] of Object.entries(canvasHandlers)) { + this.router.register(method, handler); + } for (const [method, handler] of Object.entries(toolHandlers)) { this.router.register(method, handler); }