test(heartbeat): verify failure alerts re-fire after cooldown window
This commit is contained in:
@@ -3497,7 +3497,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"overall_progress": {
|
"overall_progress": {
|
||||||
"total_test_count": 1866,
|
"total_test_count": 1867,
|
||||||
"all_tests_passing": true,
|
"all_tests_passing": true,
|
||||||
"p0_completion": "3/3 (100%)",
|
"p0_completion": "3/3 (100%)",
|
||||||
"p1_completion": "4/4 (100%)",
|
"p1_completion": "4/4 (100%)",
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
|
||||||
import { HeartbeatMonitor, parseInterval } from './heartbeat.js';
|
import { HeartbeatMonitor, parseInterval } from './heartbeat.js';
|
||||||
import type { HeartbeatDeps } from './heartbeat.js';
|
import type { HeartbeatDeps } from './heartbeat.js';
|
||||||
import type { HeartbeatConfig } from '../config/schema.js';
|
import type { HeartbeatConfig } from '../config/schema.js';
|
||||||
@@ -84,9 +84,15 @@ describe('parseInterval', () => {
|
|||||||
|
|
||||||
describe('HeartbeatMonitor', () => {
|
describe('HeartbeatMonitor', () => {
|
||||||
let monitor: HeartbeatMonitor;
|
let monitor: HeartbeatMonitor;
|
||||||
|
let nowSpy: ReturnType<typeof vi.spyOn> | undefined;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
nowSpy = undefined;
|
||||||
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
monitor?.stop();
|
monitor?.stop();
|
||||||
|
nowSpy?.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('start() does nothing when enabled: false', () => {
|
it('start() does nothing when enabled: false', () => {
|
||||||
@@ -256,6 +262,51 @@ describe('HeartbeatMonitor', () => {
|
|||||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('sends failure notification again after notify cooldown elapses', async () => {
|
||||||
|
const mockSend = vi.fn().mockResolvedValue(undefined);
|
||||||
|
const mockGet = vi.fn().mockReturnValue({ send: mockSend });
|
||||||
|
|
||||||
|
const deps = makeDeps({
|
||||||
|
config: makeConfig({
|
||||||
|
checks: ['model'],
|
||||||
|
failure_threshold: 1,
|
||||||
|
notify_cooldown: '1h',
|
||||||
|
notify: { channel: 'telegram', peer: '123' },
|
||||||
|
}),
|
||||||
|
modelRouter: undefined,
|
||||||
|
channelLookup: { get: mockGet },
|
||||||
|
});
|
||||||
|
monitor = new HeartbeatMonitor(deps);
|
||||||
|
|
||||||
|
let mockNow = 0;
|
||||||
|
nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => mockNow);
|
||||||
|
|
||||||
|
await monitor.runChecks();
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
mockNow = 1000;
|
||||||
|
Object.assign(deps, { modelRouter: { getTier: () => 'default' } });
|
||||||
|
await monitor.runChecks();
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
mockNow = 2000;
|
||||||
|
Object.assign(deps, { modelRouter: undefined });
|
||||||
|
await monitor.runChecks();
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
mockNow = 3600001;
|
||||||
|
Object.assign(deps, { modelRouter: { getTier: () => 'default' } });
|
||||||
|
await monitor.runChecks();
|
||||||
|
|
||||||
|
mockNow = 3600002;
|
||||||
|
Object.assign(deps, { modelRouter: undefined });
|
||||||
|
await monitor.runChecks();
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mockSend).toHaveBeenLastCalledWith('123', expect.objectContaining({
|
||||||
|
text: expect.stringContaining('FAILING'),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
it('recovery notification sent when checks pass after failures', async () => {
|
it('recovery notification sent when checks pass after failures', async () => {
|
||||||
const mockSend = vi.fn().mockResolvedValue(undefined);
|
const mockSend = vi.fn().mockResolvedValue(undefined);
|
||||||
const mockGet = vi.fn().mockReturnValue({ send: mockSend });
|
const mockGet = vi.fn().mockReturnValue({ send: mockSend });
|
||||||
|
|||||||
Reference in New Issue
Block a user