Phase 1 run-control semantics and run_state events

This commit is contained in:
William Valentin
2026-02-25 10:22:44 -08:00
parent ae21681958
commit e4ee6acce8
13 changed files with 485 additions and 120 deletions
+84
View File
@@ -437,6 +437,90 @@ describe('daemon command fast-path integration', () => {
});
});
it('preempts active runs when queue mode is interrupt', async () => {
const cancelSpy = vi.spyOn(AgentOrchestrator.prototype, 'cancel');
vi.spyOn(AgentOrchestrator.prototype, 'isCancellable').mockReturnValue(true);
const processSpy = vi.spyOn(AgentOrchestrator.prototype, 'process');
let resolveFirst: ((value: string) => void) | undefined;
let markStarted: (() => void) | undefined;
const started = new Promise<void>((resolve) => { markStarted = resolve; });
processSpy
.mockImplementationOnce(() => {
markStarted?.();
return new Promise<string>((resolve) => { resolveFirst = resolve; });
})
.mockResolvedValueOnce('second');
const session = {
id: 'telegram:user-interrupt',
addMessage: vi.fn(),
getHistory: vi.fn(() => []),
clear: vi.fn(),
replaceHistory: vi.fn(),
getConfig: vi.fn((key: string) => (key === 'queue.mode' ? 'interrupt' : undefined)),
setConfig: vi.fn(),
deleteConfig: vi.fn(),
};
const router = createMessageRouter({
sessionManager: {
getSession: vi.fn(() => session),
} as unknown as MessageRouterDeps['sessionManager'],
modelRouter: {
getAvailableTiers: () => ['fast', 'default', 'complex', 'local'],
getAllLabels: () => ({ fast: 'fast', default: 'default', complex: 'complex', local: 'local' }),
getLabel: (tier: string) => tier,
} as unknown as MessageRouterDeps['modelRouter'],
systemPrompt: 'test prompt',
toolRegistry: {
clone() { return this; },
register: vi.fn(),
} as unknown as MessageRouterDeps['toolRegistry'],
toolExecutor: {} as unknown as MessageRouterDeps['toolExecutor'],
config: {
agents: {
primary_tier: 'default',
delegation: {
compaction: 'fast',
memory_extraction: 'fast',
classification: 'fast',
tool_summarisation: 'fast',
complex_reasoning: 'complex',
},
max_delegation_depth: 3,
max_iterations: 10,
},
compaction: { enabled: false },
models: { default: { provider: 'anthropic', model: 'claude' } },
server: { queue: { mode: 'collect' } },
} as unknown as MessageRouterDeps['config'],
});
const reply = vi.fn(async (_message: OutboundMessage) => {});
const firstRun = router.handler({
id: 'm-interrupt-1',
channel: 'telegram',
senderId: 'user-interrupt',
text: 'first',
timestamp: Date.now(),
} as MessageRouterInput, reply);
await started;
await router.handler({
id: 'm-interrupt-2',
channel: 'telegram',
senderId: 'user-interrupt',
text: 'second',
timestamp: Date.now(),
} as MessageRouterInput, reply);
expect(cancelSpy).toHaveBeenCalled();
resolveFirst?.('first');
await firstRun;
});
it('emits run.state start and complete for non-command channel messages', async () => {
const processSpy = vi.spyOn(AgentOrchestrator.prototype, 'process').mockResolvedValue('ok');
const mockAuditLogger = {
+72 -40
View File
@@ -443,6 +443,56 @@ export function createMessageRouter(deps: {
});
}
function requestActiveRunCancellation(input: {
sessionId: string;
channel: string;
senderId: string;
requestId: string;
}): { cancelled: boolean; latencyMs: number } {
const cancelStartedAt = Date.now();
const run = activeRuns.get(input.sessionId);
if (!run || !run.isCancellable()) {
const latencyMs = Date.now() - cancelStartedAt;
deps.metrics?.recordCancelLatency(latencyMs);
auditLogger?.runCancel?.({
session_id: input.sessionId,
channel: input.channel,
sender: input.senderId,
source: 'channel',
requested: true,
acknowledged: false,
request_id: input.requestId,
latency_ms: latencyMs,
});
return { cancelled: false, latencyMs };
}
run.cancel();
const cancelLatencyMs = Date.now() - cancelStartedAt;
deps.metrics?.recordCancelLatency(cancelLatencyMs);
auditLogger?.runCancel?.({
session_id: input.sessionId,
channel: input.channel,
sender: input.senderId,
source: 'channel',
requested: true,
acknowledged: true,
request_id: input.requestId,
latency_ms: cancelLatencyMs,
});
auditLogger?.runState?.({
session_id: input.sessionId,
channel: input.channel,
sender: input.senderId,
source: 'channel',
state: 'cancel_requested',
request_id: input.requestId,
duration_ms: cancelLatencyMs,
});
deps.metrics?.recordRunState('cancel_requested');
return { cancelled: true, latencyMs: cancelLatencyMs };
}
function executeBackendCommand(inputRaw: string, activeTier: string): string {
return executeRuntimeBackendModeCommand(inputRaw, {
getActiveTier: () => activeTier,
@@ -770,6 +820,21 @@ export function createMessageRouter(deps: {
}
}
const session = deps.sessionManager.getSession(msg.channel, msg.senderId);
const queueMode = session.getConfig('queue.mode') ?? deps.config.server?.queue?.mode ?? 'collect';
const rawCommand = msg.metadata?.isCommand
? msg.metadata.command
: incomingText.trim().startsWith('/') ? incomingText.trim().slice(1).split(/\s+/, 1)[0] : undefined;
const isCancelCommand = rawCommand === 'stop' || rawCommand === 'cancel';
if (queueMode === 'interrupt' && !isCancelCommand) {
requestActiveRunCancellation({
sessionId: sessionIdForRun,
channel: msg.channel,
senderId: msg.senderId,
requestId: msg.id,
});
}
const automationReactions = deps.config.automation?.reactions ?? [];
if (!msg.metadata?.isCommand) {
if (automationReactions.length === 0) {
@@ -896,7 +961,6 @@ export function createMessageRouter(deps: {
});
if (deps.commandRegistry && deps.commandRegistry.isCommand(commandInput)) {
const session = deps.sessionManager.getSession(msg.channel, msg.senderId);
const commandResult = await deps.commandRegistry.execute(commandInput, {
channel: msg.channel,
senderId: msg.senderId,
@@ -1066,46 +1130,15 @@ export function createMessageRouter(deps: {
return '';
},
cancelRun: () => {
const cancelStartedAt = Date.now();
const run = activeRuns.get(session.id);
if (!run || !run.isCancellable()) {
deps.metrics?.recordCancelLatency(Date.now() - cancelStartedAt);
auditLogger?.runCancel?.({
session_id: session.id,
channel: msg.channel,
sender: msg.senderId,
source: 'channel',
requested: true,
acknowledged: false,
request_id: msg.id,
latency_ms: Date.now() - cancelStartedAt,
});
return 'No active operation to cancel.';
}
run.cancel();
const cancelLatencyMs = Date.now() - cancelStartedAt;
deps.metrics?.recordCancelLatency(cancelLatencyMs);
auditLogger?.runCancel?.({
session_id: session.id,
const result = requestActiveRunCancellation({
sessionId: session.id,
channel: msg.channel,
sender: msg.senderId,
source: 'channel',
requested: true,
acknowledged: true,
request_id: msg.id,
latency_ms: cancelLatencyMs,
senderId: msg.senderId,
requestId: msg.id,
});
auditLogger?.runState?.({
session_id: session.id,
channel: msg.channel,
sender: msg.senderId,
source: 'channel',
state: 'cancel_requested',
request_id: msg.id,
duration_ms: cancelLatencyMs,
});
deps.metrics?.recordRunState('cancel_requested');
return 'Cancellation requested. The active operation will stop at the next safe point.';
return result.cancelled
? 'Cancellation requested. The active operation will stop at the next safe point.'
: 'No active operation to cancel.';
},
delegateAgent: async (agentName: string, task: string) => {
@@ -1515,7 +1548,6 @@ export function createMessageRouter(deps: {
// Determine if the active model supports native audio input
let effectiveTier: string = deps.config.agents.primary_tier ?? 'default';
const session = deps.sessionManager.getSession(msg.channel, msg.senderId);
const sessionTierOverride = session.getConfig('modelTier');
const tierFromUseCaseMetadata = tierFromUseCase(deps.config, msg.metadata?.modelFor);
+13 -10
View File
@@ -331,8 +331,9 @@ describe('createAgentHandlers command fast-path', () => {
await handlers['agent.send'](req, send);
expect(mockAgent.process).toHaveBeenCalledWith('/not-a-real-command', undefined);
expect((sent[0] as GatewayEvent).event).toBe('done');
expect(((sent[0] as GatewayEvent).data as { content: string }).content).toBe('agent response');
const doneEvent = sent.find((msg) => (msg as GatewayEvent).event === 'done') as GatewayEvent | undefined;
expect(doneEvent).toBeTruthy();
expect(((doneEvent as GatewayEvent).data as { content: string }).content).toBe('agent response');
});
it('handles /approvals command via fast-path when hook engine is available', async () => {
@@ -421,7 +422,7 @@ describe('createAgentHandlers command fast-path', () => {
state: 'cancelled',
}),
);
expect((sent[0] as GatewayEvent).event).toBe('done');
expect(sent.some((msg) => (msg as GatewayEvent).event === 'done')).toBe(true);
});
it('emits run.cancel telemetry for agent.cancel requests', async () => {
@@ -429,7 +430,7 @@ describe('createAgentHandlers command fast-path', () => {
id: 16,
method: 'agent.cancel',
params: { connectionId: 'conn-1' },
});
}, vi.fn());
expect(sessionBridge.cancel).toHaveBeenCalledWith('conn-1');
expect(mockAuditLogger.runCancel).toHaveBeenCalledWith(
@@ -492,9 +493,10 @@ describe('createAgentHandlers command fast-path', () => {
params: { message: 'hello', connectionId: 'conn-1' },
}, send);
expect(sent).toHaveLength(2);
expect((sent[0] as GatewayEvent).event).toBe('context_warning');
expect((sent[1] as GatewayEvent).event).toBe('done');
const events = sent.map((msg) => (msg as GatewayEvent).event);
expect(events).toContain('context_warning');
expect(events).toContain('done');
expect(events.indexOf('context_warning')).toBeLessThan(events.indexOf('done'));
});
});
@@ -706,8 +708,9 @@ describe('createAgentHandlers queue policy resolution', () => {
mode: 'interrupt',
cancelled_active_run: true,
}));
expect((sent[0] as GatewayEvent).event).toBe('content');
expect(((sent[0] as GatewayEvent).data as { text: string }).text).toContain('Interrupt mode');
expect((sent[1] as GatewayEvent).event).toBe('done');
const contentEvent = sent.find((msg) => (msg as GatewayEvent).event === 'content') as GatewayEvent | undefined;
expect(contentEvent).toBeTruthy();
expect(((contentEvent as GatewayEvent).data as { text: string }).text).toContain('Interrupt mode');
expect(sent.some((msg) => (msg as GatewayEvent).event === 'done')).toBe(true);
});
});
+46 -8
View File
@@ -86,6 +86,8 @@ function listEnabledExternalBackends(config?: Config): string[] {
}
export function createAgentHandlers(deps: AgentHandlerDeps) {
const activeRequestIds = new Map<string, number>();
return {
'agent.send': async (request: GatewayRequest, send: SendFn): Promise<OutboundMessage | void> => {
const params = request.params as { message?: string; connectionId?: string; attachments?: GatewayAttachment[]; metadata?: { isCommand?: boolean; command?: string; commandArgs?: string } } | undefined;
@@ -116,10 +118,13 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
const laneId = sessionId ?? connectionId;
const channel = sessionId?.split(':', 1)[0] ?? 'ws';
const resolvedPolicy = deps.resolveQueuePolicy?.({ laneId, sessionId, connectionId, channel });
const laneQueueWithProcessing = deps.laneQueue as LaneQueue & { isProcessing?: (lane: string) => boolean };
const laneIsProcessing = typeof laneQueueWithProcessing.isProcessing === 'function'
? laneQueueWithProcessing.isProcessing(laneId)
: false;
const laneQueueWithProcessing = deps.laneQueue as LaneQueue & {
isProcessing?: (lane: string) => boolean;
isActive?: (lane: string) => boolean;
};
const laneIsActive = typeof laneQueueWithProcessing.isActive === 'function'
? laneQueueWithProcessing.isActive(laneId)
: (laneQueueWithProcessing.isProcessing?.(laneId) ?? false);
const requestId = request.id.toString();
const sessionIdForAudit = sessionId ?? `ws:${connectionId}`;
const runStartedAt = Date.now();
@@ -127,7 +132,7 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
// Interrupt mode should preempt active work when a newer request arrives.
// LaneQueue itself only rejects queued entries, so we also request agent cancellation.
if (resolvedPolicy?.mode === 'interrupt' && laneIsProcessing) {
if (resolvedPolicy?.mode === 'interrupt' && laneIsActive) {
const cancelStartedAt = Date.now();
const cancelled = sessionId
? deps.sessionBridge.cancelSession(sessionId)
@@ -165,6 +170,13 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
duration_ms: cancelLatencyMs,
});
deps.metrics?.recordRunState('cancel_requested');
const activeRequestId = activeRequestIds.get(connectionId);
if (activeRequestId && activeRequestId !== request.id) {
send(makeEvent(activeRequestId, 'run_state', {
state: 'cancel_requested',
timestamp: Date.now(),
}));
}
}
}
@@ -202,6 +214,7 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
const isCommand = Boolean(commandInput && deps.commandRegistry?.isCommand(commandInput));
if (!isCommand) {
activeRequestIds.set(connectionId, request.id);
auditLogger?.runState?.({
session_id: sessionIdForAudit,
channel: 'ws',
@@ -211,9 +224,13 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
request_id: requestId,
});
deps.metrics?.recordRunState('start');
send(makeEvent(request.id, 'run_state', {
state: 'start',
timestamp: Date.now(),
}));
}
if (commandInput && deps.commandRegistry?.isCommand(commandInput)) {
if (isCommand) {
const sessionId = deps.sessionBridge.getSessionId(connectionId);
const commandResult = await deps.commandRegistry.execute(commandInput, {
channel: 'ws',
@@ -646,6 +663,10 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
if (commandResult.handled) {
send(makeEvent(request.id, 'done', { content: commandResult.text }));
deps.sessionBridge.setBusy(connectionId, false);
deps.sessionBridge.setOnToolUse(connectionId, undefined);
activeRequestIds.delete(connectionId);
deps.metrics?.endRequest(requestId);
return;
}
}
@@ -699,10 +720,14 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
context: { sessionId: laneId, level: contextAlert.level },
});
}
send(makeEvent(request.id, 'done', { content: response }));
const finalState = response.trim().toLowerCase() === 'operation cancelled by user.'
? 'cancelled'
: 'complete';
send(makeEvent(request.id, 'run_state', {
state: finalState,
timestamp: Date.now(),
}));
send(makeEvent(request.id, 'done', { content: response }));
auditLogger?.runState?.({
session_id: sessionIdForAudit,
channel: 'ws',
@@ -723,6 +748,11 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
message,
context: { sessionId: laneId },
});
send(makeEvent(request.id, 'run_state', {
state: 'error',
timestamp: Date.now(),
message,
}));
send(makeEvent(request.id, 'error', { code: ErrorCode.InternalError, message }));
auditLogger?.runState?.({
session_id: sessionIdForAudit,
@@ -738,6 +768,7 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
} finally {
deps.sessionBridge.setBusy(connectionId, false);
deps.sessionBridge.setOnToolUse(connectionId, undefined);
activeRequestIds.delete(connectionId);
deps.metrics?.endRequest(requestId);
}
}, resolvedPolicy);
@@ -754,7 +785,7 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
}
},
'agent.cancel': async (request: GatewayRequest): Promise<OutboundMessage> => {
'agent.cancel': async (request: GatewayRequest, send: SendFn): Promise<OutboundMessage> => {
const params = request.params as { connectionId?: string } | undefined;
const connectionId = params?.connectionId as string;
@@ -795,6 +826,13 @@ export function createAgentHandlers(deps: AgentHandlerDeps) {
});
deps.metrics?.recordRunState('cancel_requested');
}
const activeRequestId = activeRequestIds.get(connectionId);
if (cancelled && activeRequestId) {
send(makeEvent(activeRequestId, 'run_state', {
state: 'cancel_requested',
timestamp: Date.now(),
}));
}
return {
id: request.id,
result: {
+58 -1
View File
@@ -1,4 +1,4 @@
import { describe, it, expect } from 'vitest';
import { describe, it, expect, vi } from 'vitest';
import { LaneQueue, LaneQueueRejectedError } from './lane-queue.js';
describe('LaneQueue', () => {
@@ -230,6 +230,25 @@ describe('LaneQueue', () => {
await expect(p3).resolves.toBe('new-pending');
});
it('interrupt mode keeps only the most recent pending request', async () => {
const queue = new LaneQueue({ mode: 'interrupt' });
let resolveFirst!: () => void;
const firstBlocks = new Promise<void>((r) => { resolveFirst = r; });
const p1 = queue.enqueue('lane-a', async () => {
await firstBlocks;
return 'active';
});
const p2 = queue.enqueue('lane-a', async () => 'old-pending');
const p3 = queue.enqueue('lane-a', async () => 'latest-pending');
await expect(p2).rejects.toThrow('Superseded by newer request');
resolveFirst();
await expect(p1).resolves.toBe('active');
await expect(p3).resolves.toBe('latest-pending');
});
it('drop_new overflow rejects newest request when cap is reached', async () => {
const queue = new LaneQueue({ cap: 1, overflow: 'drop_new' });
let resolveFirst!: () => void;
@@ -249,6 +268,44 @@ describe('LaneQueue', () => {
await expect(p2).resolves.toBe('pending-1');
});
it('interrupt mode clears debounce backlog to run latest request immediately', async () => {
vi.useFakeTimers();
try {
const queue = new LaneQueue({ mode: 'interrupt', debounceMs: 50 });
const events: string[] = [];
let resolveFirst!: () => void;
const firstBlocks = new Promise<void>((r) => { resolveFirst = r; });
const p1 = queue.enqueue('lane-a', async () => {
events.push('first:start');
await firstBlocks;
events.push('first:end');
return 'first';
});
const p2 = queue.enqueue('lane-a', async () => {
events.push('second:start');
return 'second';
});
resolveFirst();
await p1;
await Promise.resolve();
const p3 = queue.enqueue('lane-a', async () => {
events.push('third:start');
return 'third';
});
await Promise.resolve();
expect(events).toContain('third:start');
await expect(p2).rejects.toThrow('Superseded by newer request');
await expect(p3).resolves.toBe('third');
} finally {
vi.useRealTimers();
}
});
it('drop_old overflow evicts oldest pending request when cap is reached', async () => {
const queue = new LaneQueue({ cap: 1, overflow: 'drop_old' });
let resolveFirst!: () => void;
+21 -10
View File
@@ -98,15 +98,9 @@ export class LaneQueue {
this.lanes.set(laneId, lane);
}
// If nothing is running on this lane, execute immediately
if (!lane.active && !lane.debounceTimer) {
lane.active = true;
try {
return await work();
} finally {
lane.active = false;
this.processNext(laneId);
}
if (effective.mode === 'interrupt' && lane.debounceTimer) {
clearTimeout(lane.debounceTimer);
lane.debounceTimer = undefined;
}
if (effective.mode === 'steer' || effective.mode === 'steer_backlog' || effective.mode === 'interrupt') {
@@ -125,6 +119,17 @@ export class LaneQueue {
});
}
// If nothing is running on this lane, execute immediately
if (!lane.active && !lane.debounceTimer && lane.queue.length === 0) {
lane.active = true;
try {
return await work();
} finally {
lane.active = false;
this.processNext(laneId);
}
}
if (lane.queue.length >= effective.cap) {
if (effective.overflow === 'drop_new') {
return Promise.reject(
@@ -168,12 +173,18 @@ export class LaneQueue {
});
}
/** Check if a lane currently has active work executing. */
/** Check if a lane is active or waiting to start due to debounce. */
isProcessing(laneId: string): boolean {
const lane = this.lanes.get(laneId);
return (lane?.active ?? false) || Boolean(lane?.debounceTimer);
}
/** Check if a lane currently has active work executing (not counting debounce delay). */
isActive(laneId: string): boolean {
const lane = this.lanes.get(laneId);
return lane?.active ?? false;
}
/** Get the number of pending (not yet started) items in a lane. */
queueLength(laneId: string): number {
return this.lanes.get(laneId)?.queue.length ?? 0;
+9
View File
@@ -95,6 +95,7 @@ export type EventType =
| 'tool_end'
| 'context_warning'
| 'attachment'
| 'run_state'
| 'done'
| 'error';
@@ -142,6 +143,14 @@ export interface AttachmentEventData {
filename?: string;
}
export type RunState = 'start' | 'cancel_requested' | 'cancelled' | 'complete' | 'error';
export interface RunStateEventData {
state: RunState;
timestamp: number;
message?: string;
}
export interface DoneEventData {
content: string;
}
+21
View File
@@ -715,6 +715,10 @@ async function sendMessage(client, overrideText) {
scrollToBottom();
// Create placeholder for assistant response
const statusLine = document.createElement('div');
statusLine.className = 'px-1 text-[11px] leading-none text-zinc-500 select-none hidden';
statusLine.textContent = 'Run status: queued';
_elements.messages.appendChild(statusLine);
const placeholder = document.createElement('div');
placeholder.className = 'rounded-lg px-3.5 py-2.5 text-sm leading-relaxed break-words whitespace-pre-wrap bg-zinc-900 border border-zinc-800 text-zinc-50 streaming-cursor';
placeholder.innerHTML = '<span class="text-zinc-500">Thinking...</span>';
@@ -759,6 +763,22 @@ async function sendMessage(client, overrideText) {
scrollToBottom();
});
stream.on('run_state', (data) => {
if (!data || !data.state) {
return;
}
const labels = {
start: 'Run status: working',
cancel_requested: 'Run status: cancellation requested',
cancelled: 'Run status: cancelled',
complete: 'Run status: complete',
error: `Run status: error${data.message ? ` (${data.message})` : ''}`,
};
statusLine.textContent = labels[data.state] || `Run status: ${data.state}`;
statusLine.classList.remove('hidden');
scrollToBottom();
});
const done = await stream.result;
const content = done?.content ?? done?.text ?? '(no response)';
const assistantMessage = createMessageEl('assistant', content, Date.now());
@@ -771,6 +791,7 @@ async function sendMessage(client, overrideText) {
_cancelling = false;
updateSendButton();
clearPendingAttachments();
statusLine.remove();
scrollToBottom();
}
}
+51
View File
@@ -185,4 +185,55 @@ describe('ChatPage wiring', () => {
expect(calls.some((entry) => entry.method === 'agent.cancel')).toBe(true);
});
it('renders run_state updates during streaming', async () => {
let resolveResult: ((value: { content: string }) => void) | undefined;
const handlers = new Map<string, Array<(data: any) => void>>();
const stream = {
on(event: string, cb: (data: any) => void) {
if (!handlers.has(event)) {
handlers.set(event, []);
}
handlers.get(event)?.push(cb);
},
emit(event: string, data: any) {
for (const cb of handlers.get(event) ?? []) {
cb(data);
}
},
result: new Promise<{ content: string }>((resolve) => {
resolveResult = resolve;
}),
};
const client = {
async call(method: string) {
if (method === 'sessions.list') {
return { sessions: [] };
}
return null;
},
stream() {
return stream;
},
};
await ChatPage.render(root, client);
const input = root.querySelector('#chat-input');
input.value = 'hello';
root.querySelector('#chat-send').dispatchEvent(new windowObj.Event('click', { bubbles: true }));
await Promise.resolve();
stream.emit('run_state', { state: 'start' });
await Promise.resolve();
const statusLine = Array.from(root.querySelectorAll('div'))
.find((el: any) => String(el.textContent ?? '').startsWith('Run status:'));
expect(statusLine).toBeTruthy();
expect(statusLine.classList.contains('hidden')).toBe(false);
resolveResult?.({ content: 'ok' });
await Promise.resolve();
});
});