feat(companion): add heartbeat failure-threshold controls
This commit is contained in:
@@ -1198,7 +1198,7 @@ Companion runtime helper:
|
|||||||
- shared `bootstrap()` helper (`register` + `getCapabilities`) for startup handshakes
|
- shared `bootstrap()` helper (`register` + `getCapabilities`) for startup handshakes
|
||||||
- shared `publishHeartbeat()` helper for periodic `node.status.set` updates with safe defaults
|
- shared `publishHeartbeat()` helper for periodic `node.status.set` updates with safe defaults
|
||||||
- optional `defaultSessionId` for canvas helper calls so `sessionId` can be omitted per call
|
- optional `defaultSessionId` for canvas helper calls so `sessionId` can be omitted per call
|
||||||
- `src/companion/heartbeatLoop.ts` provides `CompanionHeartbeatLoop` for periodic heartbeat scheduling (`publishHeartbeat`) with start/stop safety and error hooks.
|
- `src/companion/heartbeatLoop.ts` provides `CompanionHeartbeatLoop` for periodic heartbeat scheduling (`publishHeartbeat`) with start/stop safety, error hooks, and optional auto-stop after repeated failures.
|
||||||
|
|
||||||
## Canvas / A2UI Foundation
|
## Canvas / A2UI Foundation
|
||||||
|
|
||||||
|
|||||||
@@ -286,6 +286,19 @@
|
|||||||
],
|
],
|
||||||
"test_status": "pnpm test:run src/companion/platformClients.test.ts src/companion/platformClients.integration.test.ts + pnpm typecheck passing"
|
"test_status": "pnpm test:run src/companion/platformClients.test.ts src/companion/platformClients.integration.test.ts + pnpm typecheck passing"
|
||||||
},
|
},
|
||||||
|
"companion-heartbeat-loop-failure-threshold": {
|
||||||
|
"status": "completed",
|
||||||
|
"date": "2026-02-17",
|
||||||
|
"updated": "2026-02-17",
|
||||||
|
"summary": "Extended `CompanionHeartbeatLoop` with failure-threshold controls (`maxConsecutiveFailures`) and `onFailureLimitReached` callback support so companion runtimes can auto-stop noisy loops after repeated heartbeat failures.",
|
||||||
|
"files_modified": [
|
||||||
|
"src/companion/heartbeatLoop.ts",
|
||||||
|
"src/companion/heartbeatLoop.test.ts",
|
||||||
|
"README.md",
|
||||||
|
"docs/plans/state.json"
|
||||||
|
],
|
||||||
|
"test_status": "pnpm test:run src/companion/heartbeatLoop.test.ts src/companion/platformClients.test.ts + pnpm typecheck passing"
|
||||||
|
},
|
||||||
"browser-tools-activation-clarity": {
|
"browser-tools-activation-clarity": {
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"date": "2026-02-17",
|
"date": "2026-02-17",
|
||||||
|
|||||||
@@ -100,4 +100,34 @@ describe('CompanionHeartbeatLoop', () => {
|
|||||||
expect(publishHeartbeat).toHaveBeenCalledTimes(1);
|
expect(publishHeartbeat).toHaveBeenCalledTimes(1);
|
||||||
expect(loop.running).toBe(false);
|
expect(loop.running).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('stops when maxConsecutiveFailures threshold is reached', async () => {
|
||||||
|
const publishHeartbeat = vi
|
||||||
|
.fn<() => Promise<NodeStatusSetResult>>()
|
||||||
|
.mockRejectedValue(new Error('persistent-failure'));
|
||||||
|
const onError = vi.fn();
|
||||||
|
const onFailureLimitReached = vi.fn();
|
||||||
|
const loop = new CompanionHeartbeatLoop(
|
||||||
|
{ publishHeartbeat },
|
||||||
|
{
|
||||||
|
intervalMs: 300,
|
||||||
|
onError,
|
||||||
|
maxConsecutiveFailures: 2,
|
||||||
|
onFailureLimitReached,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
loop.start();
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(loop.running).toBe(true);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(300);
|
||||||
|
expect(loop.running).toBe(false);
|
||||||
|
expect(onError).toHaveBeenCalledTimes(2);
|
||||||
|
expect(onFailureLimitReached).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onFailureLimitReached).toHaveBeenCalledWith(expect.any(Error), 2);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1000);
|
||||||
|
expect(publishHeartbeat).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ export interface CompanionHeartbeatLoopOptions {
|
|||||||
intervalMs?: number;
|
intervalMs?: number;
|
||||||
buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
||||||
onError?: (error: Error) => void;
|
onError?: (error: Error) => void;
|
||||||
|
maxConsecutiveFailures?: number;
|
||||||
|
onFailureLimitReached?: (error: Error, consecutiveFailures: number) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -20,16 +22,21 @@ export class CompanionHeartbeatLoop {
|
|||||||
private readonly intervalMs: number;
|
private readonly intervalMs: number;
|
||||||
private readonly buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
private readonly buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
||||||
private readonly onError?: (error: Error) => void;
|
private readonly onError?: (error: Error) => void;
|
||||||
|
private readonly maxConsecutiveFailures: number;
|
||||||
|
private readonly onFailureLimitReached?: (error: Error, consecutiveFailures: number) => void;
|
||||||
|
|
||||||
private timer: NodeJS.Timeout | null = null;
|
private timer: NodeJS.Timeout | null = null;
|
||||||
private started = false;
|
private started = false;
|
||||||
private inFlight = false;
|
private inFlight = false;
|
||||||
|
private consecutiveFailures = 0;
|
||||||
|
|
||||||
constructor(publisher: HeartbeatPublisher, options: CompanionHeartbeatLoopOptions = {}) {
|
constructor(publisher: HeartbeatPublisher, options: CompanionHeartbeatLoopOptions = {}) {
|
||||||
this.publisher = publisher;
|
this.publisher = publisher;
|
||||||
this.intervalMs = options.intervalMs ?? 30_000;
|
this.intervalMs = options.intervalMs ?? 30_000;
|
||||||
this.buildHeartbeat = options.buildHeartbeat;
|
this.buildHeartbeat = options.buildHeartbeat;
|
||||||
this.onError = options.onError;
|
this.onError = options.onError;
|
||||||
|
this.maxConsecutiveFailures = options.maxConsecutiveFailures ?? Number.POSITIVE_INFINITY;
|
||||||
|
this.onFailureLimitReached = options.onFailureLimitReached;
|
||||||
}
|
}
|
||||||
|
|
||||||
get running(): boolean {
|
get running(): boolean {
|
||||||
@@ -41,6 +48,7 @@ export class CompanionHeartbeatLoop {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.started = true;
|
this.started = true;
|
||||||
|
this.consecutiveFailures = 0;
|
||||||
|
|
||||||
if (runImmediately) {
|
if (runImmediately) {
|
||||||
void this.tick();
|
void this.tick();
|
||||||
@@ -74,9 +82,16 @@ export class CompanionHeartbeatLoop {
|
|||||||
try {
|
try {
|
||||||
const payload = this.buildHeartbeat ? await this.buildHeartbeat() : undefined;
|
const payload = this.buildHeartbeat ? await this.buildHeartbeat() : undefined;
|
||||||
await this.publisher.publishHeartbeat(payload);
|
await this.publisher.publishHeartbeat(payload);
|
||||||
|
this.consecutiveFailures = 0;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const err = error instanceof Error ? error : new Error(String(error));
|
const err = error instanceof Error ? error : new Error(String(error));
|
||||||
|
this.consecutiveFailures += 1;
|
||||||
this.onError?.(err);
|
this.onError?.(err);
|
||||||
|
if (this.consecutiveFailures >= this.maxConsecutiveFailures) {
|
||||||
|
this.onFailureLimitReached?.(err, this.consecutiveFailures);
|
||||||
|
this.stop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
this.inFlight = false;
|
this.inFlight = false;
|
||||||
this.scheduleNext();
|
this.scheduleNext();
|
||||||
|
|||||||
Reference in New Issue
Block a user