feat(companion): add heartbeat failure-threshold controls
This commit is contained in:
@@ -100,4 +100,34 @@ describe('CompanionHeartbeatLoop', () => {
|
||||
expect(publishHeartbeat).toHaveBeenCalledTimes(1);
|
||||
expect(loop.running).toBe(false);
|
||||
});
|
||||
|
||||
it('stops when maxConsecutiveFailures threshold is reached', async () => {
|
||||
const publishHeartbeat = vi
|
||||
.fn<() => Promise<NodeStatusSetResult>>()
|
||||
.mockRejectedValue(new Error('persistent-failure'));
|
||||
const onError = vi.fn();
|
||||
const onFailureLimitReached = vi.fn();
|
||||
const loop = new CompanionHeartbeatLoop(
|
||||
{ publishHeartbeat },
|
||||
{
|
||||
intervalMs: 300,
|
||||
onError,
|
||||
maxConsecutiveFailures: 2,
|
||||
onFailureLimitReached,
|
||||
},
|
||||
);
|
||||
|
||||
loop.start();
|
||||
await Promise.resolve();
|
||||
expect(loop.running).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(300);
|
||||
expect(loop.running).toBe(false);
|
||||
expect(onError).toHaveBeenCalledTimes(2);
|
||||
expect(onFailureLimitReached).toHaveBeenCalledTimes(1);
|
||||
expect(onFailureLimitReached).toHaveBeenCalledWith(expect.any(Error), 2);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(publishHeartbeat).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,8 @@ export interface CompanionHeartbeatLoopOptions {
|
||||
intervalMs?: number;
|
||||
buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
||||
onError?: (error: Error) => void;
|
||||
maxConsecutiveFailures?: number;
|
||||
onFailureLimitReached?: (error: Error, consecutiveFailures: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -20,16 +22,21 @@ export class CompanionHeartbeatLoop {
|
||||
private readonly intervalMs: number;
|
||||
private readonly buildHeartbeat?: () => HeartbeatStatusInput | Promise<HeartbeatStatusInput>;
|
||||
private readonly onError?: (error: Error) => void;
|
||||
private readonly maxConsecutiveFailures: number;
|
||||
private readonly onFailureLimitReached?: (error: Error, consecutiveFailures: number) => void;
|
||||
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private started = false;
|
||||
private inFlight = false;
|
||||
private consecutiveFailures = 0;
|
||||
|
||||
constructor(publisher: HeartbeatPublisher, options: CompanionHeartbeatLoopOptions = {}) {
|
||||
this.publisher = publisher;
|
||||
this.intervalMs = options.intervalMs ?? 30_000;
|
||||
this.buildHeartbeat = options.buildHeartbeat;
|
||||
this.onError = options.onError;
|
||||
this.maxConsecutiveFailures = options.maxConsecutiveFailures ?? Number.POSITIVE_INFINITY;
|
||||
this.onFailureLimitReached = options.onFailureLimitReached;
|
||||
}
|
||||
|
||||
get running(): boolean {
|
||||
@@ -41,6 +48,7 @@ export class CompanionHeartbeatLoop {
|
||||
return;
|
||||
}
|
||||
this.started = true;
|
||||
this.consecutiveFailures = 0;
|
||||
|
||||
if (runImmediately) {
|
||||
void this.tick();
|
||||
@@ -74,9 +82,16 @@ export class CompanionHeartbeatLoop {
|
||||
try {
|
||||
const payload = this.buildHeartbeat ? await this.buildHeartbeat() : undefined;
|
||||
await this.publisher.publishHeartbeat(payload);
|
||||
this.consecutiveFailures = 0;
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.consecutiveFailures += 1;
|
||||
this.onError?.(err);
|
||||
if (this.consecutiveFailures >= this.maxConsecutiveFailures) {
|
||||
this.onFailureLimitReached?.(err, this.consecutiveFailures);
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
} finally {
|
||||
this.inFlight = false;
|
||||
this.scheduleNext();
|
||||
|
||||
Reference in New Issue
Block a user