fix: graceful ctrl+c shutdown and await session-end memory writes

This commit is contained in:
William Valentin
2026-02-18 11:30:47 -08:00
parent 55cde541ea
commit 21232748b9
6 changed files with 103 additions and 30 deletions
+15
View File
@@ -5423,6 +5423,21 @@
"docs/plans/state.json" "docs/plans/state.json"
], ],
"test_status": "pnpm test:run src/models/retry.test.ts src/models/router.test.ts src/backends/native/agent.test.ts + pnpm typecheck passing" "test_status": "pnpm test:run src/models/retry.test.ts src/models/router.test.ts src/backends/native/agent.test.ts + pnpm typecheck passing"
},
"graceful-shutdown-awaits-session-end-memory": {
"status": "completed",
"date": "2026-02-18",
"updated": "2026-02-18",
"summary": "Improved Ctrl+C/SIGTERM shutdown reliability: daemon now performs bounded graceful signal shutdown with force-exit fallback, gateway now awaits async session disconnect so session-end summaries/memory writes finish before process exit, and TUI signal cleanup now awaits lifecycle shutdown before closing storage.",
"files_modified": [
"src/daemon/services.ts",
"src/gateway/session-bridge.ts",
"src/gateway/server.ts",
"src/gateway/session-bridge.test.ts",
"src/cli/tui.ts",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/gateway/session-bridge.test.ts src/gateway/server.test.ts src/models/router.test.ts src/models/retry.test.ts + pnpm typecheck passing"
} }
}, },
"overall_progress": { "overall_progress": {
+28 -8
View File
@@ -227,15 +227,29 @@ export function registerTuiCommand(program: Command): void {
}, },
}); });
const cleanup = () => { let cleanupPromise: Promise<void> | null = null;
void lifecycle.shutdown(); const cleanup = async () => {
if (!cleanupPromise) {
cleanupPromise = (async () => {
await lifecycle.shutdown();
sessionStore.close(); sessionStore.close();
})();
}
return cleanupPromise;
}; };
process.on('SIGINT', () => { const signalHandler = (signal: NodeJS.Signals) => {
cleanup(); console.log(`\nReceived ${signal}; shutting down TUI...`);
process.exit(0); void cleanup()
.then(() => process.exit(0))
.catch((error) => {
console.error('TUI shutdown failed:', error);
process.exit(1);
}); });
};
process.on('SIGINT', signalHandler);
process.on('SIGTERM', signalHandler);
const transferSessionToTarget = (target: string): string => { const transferSessionToTarget = (target: string): string => {
const normalizedTarget = target.trim().toLowerCase(); const normalizedTarget = target.trim().toLowerCase();
@@ -274,7 +288,9 @@ export function registerTuiCommand(program: Command): void {
modelProviderConfigs, modelProviderConfigs,
contextThresholdPct: config.compaction.threshold_pct, contextThresholdPct: config.compaction.threshold_pct,
onTransfer: transferSessionToTarget, onTransfer: transferSessionToTarget,
onExit: cleanup, onExit: () => {
void cleanup();
},
}); });
} else { } else {
let switchingToFullscreen = false; let switchingToFullscreen = false;
@@ -316,12 +332,16 @@ export function registerTuiCommand(program: Command): void {
modelProviderConfigs, modelProviderConfigs,
contextThresholdPct: config.compaction.threshold_pct, contextThresholdPct: config.compaction.threshold_pct,
onTransfer: transferSessionToTarget, onTransfer: transferSessionToTarget,
onExit: cleanup, onExit: () => {
void cleanup();
},
}); });
return; return;
} }
} }
cleanup(); await cleanup();
process.off('SIGINT', signalHandler);
process.off('SIGTERM', signalHandler);
}); });
} }
+34 -2
View File
@@ -540,8 +540,40 @@ export async function startServices(deps: {
}); });
// Signal handlers // Signal handlers
const signalHandler = () => { let shutdownPromise: Promise<void> | null = null;
lifecycle.shutdown().then(() => process.exit(0)); let forceExitArmed = false;
const forceExitTimeoutMs = 15_000;
const signalHandler = (signal: NodeJS.Signals) => {
if (shutdownPromise) {
if (forceExitArmed) {
console.warn(`Second ${signal} received; forcing exit now.`);
process.exit(130);
}
forceExitArmed = true;
console.warn(`Shutdown already in progress. Send ${signal} again to force exit.`);
return;
}
console.log(`Received ${signal}; shutting down gracefully...`);
shutdownPromise = lifecycle.shutdown();
const forceTimer = setTimeout(() => {
forceExitArmed = true;
console.error(`Graceful shutdown timed out after ${forceExitTimeoutMs / 1000}s; forcing exit.`);
process.exit(130);
}, forceExitTimeoutMs);
forceTimer.unref();
shutdownPromise
.then(() => {
clearTimeout(forceTimer);
process.exit(0);
})
.catch((error) => {
clearTimeout(forceTimer);
console.error('Graceful shutdown failed:', error);
process.exit(1);
});
}; };
process.on('SIGINT', signalHandler); process.on('SIGINT', signalHandler);
+8 -3
View File
@@ -564,11 +564,14 @@ export class GatewayServer {
} }
} }
// Close all WebSocket connections first // Close all WebSocket connections first.
// Await disconnects so end-of-session summary/memory writes complete before process exit.
const disconnects: Array<Promise<void>> = [];
for (const [ws, connectionId] of this.connectionMap) { for (const [ws, connectionId] of this.connectionMap) {
this.sessionBridge.disconnect(connectionId); disconnects.push(this.sessionBridge.disconnect(connectionId));
ws.close(1001, 'Server shutting down'); ws.close(1001, 'Server shutting down');
} }
await Promise.allSettled(disconnects);
this.connectionMap.clear(); this.connectionMap.clear();
this.connectionStateMap.clear(); this.connectionStateMap.clear();
@@ -628,7 +631,9 @@ export class GatewayServer {
}); });
ws.on('close', () => { ws.on('close', () => {
this.sessionBridge.disconnect(connectionId); void this.sessionBridge.disconnect(connectionId).catch((error) => {
console.warn('Session disconnect failed:', error);
});
this.connectionMap.delete(ws); this.connectionMap.delete(ws);
this.connectionRateMap.delete(connectionId); this.connectionRateMap.delete(connectionId);
this.connectionStateMap.delete(connectionId); this.connectionStateMap.delete(connectionId);
+5 -6
View File
@@ -96,10 +96,10 @@ describe('SessionBridge', () => {
expect(bridge.getSessionId('conn-1')).toBe('ws:conn-1'); expect(bridge.getSessionId('conn-1')).toBe('ws:conn-1');
}); });
it('disconnect removes client but preserves session', () => { it('disconnect removes client but preserves session', async () => {
const bridge = createBridge(); const bridge = createBridge();
bridge.connect('conn-1'); bridge.connect('conn-1');
bridge.disconnect('conn-1'); await bridge.disconnect('conn-1');
expect(bridge.connectionCount).toBe(0); expect(bridge.connectionCount).toBe(0);
expect(bridge.getAgent('conn-1')).toBeUndefined(); expect(bridge.getAgent('conn-1')).toBeUndefined();
}); });
@@ -148,8 +148,7 @@ describe('SessionBridge', () => {
} as unknown as SessionBridgeConfig['config'], } as unknown as SessionBridgeConfig['config'],
}); });
bridge.connect('conn-end-summary'); bridge.connect('conn-end-summary');
bridge.disconnect('conn-end-summary'); await bridge.disconnect('conn-end-summary');
await new Promise(resolve => setTimeout(resolve, 0));
expect(memoryStore.write).toHaveBeenCalled(); expect(memoryStore.write).toHaveBeenCalled();
}); });
@@ -242,13 +241,13 @@ describe('SessionBridge', () => {
expect(sessions).toEqual([{ sessionId: 'ws:conn-1', connections: 2 }]); expect(sessions).toEqual([{ sessionId: 'ws:conn-1', connections: 2 }]);
}); });
it('shared sessions keep agent alive when one client disconnects', () => { it('shared sessions keep agent alive when one client disconnects', async () => {
const bridge = createBridge(); const bridge = createBridge();
bridge.connect('conn-1'); bridge.connect('conn-1');
bridge.connect('conn-2'); bridge.connect('conn-2');
bridge.switchSession('conn-2', 'ws:conn-1'); bridge.switchSession('conn-2', 'ws:conn-1');
bridge.disconnect('conn-1'); await bridge.disconnect('conn-1');
// conn-2 still has the session // conn-2 still has the session
expect(bridge.getAgent('conn-2')).toBeDefined(); expect(bridge.getAgent('conn-2')).toBeDefined();
expect(bridge.connectionCount).toBe(1); expect(bridge.connectionCount).toBe(1);
+6 -4
View File
@@ -53,7 +53,7 @@ export class SessionBridge {
} }
/** Remove a WS connection. Does NOT destroy the session (persists in SQLite). */ /** Remove a WS connection. Does NOT destroy the session (persists in SQLite). */
disconnect(connectionId: string): void { async disconnect(connectionId: string): Promise<void> {
const client = this.clients.get(connectionId); const client = this.clients.get(connectionId);
if (client) { if (client) {
// Only remove the agent if no other clients share the session // Only remove the agent if no other clients share the session
@@ -73,15 +73,17 @@ export class SessionBridge {
writeToMemory: summaryConfig.write_to_memory, writeToMemory: summaryConfig.write_to_memory,
memoryNamespace: summaryConfig.memory_namespace, memoryNamespace: summaryConfig.memory_namespace,
}; };
void summarizeSessionOnEnd({ try {
await summarizeSessionOnEnd({
agent, agent,
sessionId: client.sessionId, sessionId: client.sessionId,
history, history,
config: mappedConfig, config: mappedConfig,
memoryStore: this.config.memoryStore, memoryStore: this.config.memoryStore,
}).catch((error) => {
console.warn('Session end summary failed:', error);
}); });
} catch (error) {
console.warn('Session end summary failed:', error);
}
} }
this.agents.delete(client.sessionId); this.agents.delete(client.sessionId);
} }