feat(heartbeat): add process memory and backup health checks

This commit is contained in:
William Valentin
2026-02-16 13:50:39 -08:00
parent 8684c3a07d
commit 07340ff0af
11 changed files with 282 additions and 8 deletions
+7
View File
@@ -1,2 +1,9 @@
export { runBackupSnapshot, backupInternals, type BackupRunOptions, type BackupResult } from './run.js';
export { BackupScheduler, type BackupSchedulerDeps } from './scheduler.js';
export {
getBackupHealthSnapshot,
initBackupHealth,
markBackupSuccess,
markBackupFailure,
type BackupHealthSnapshot,
} from './status.js';
+4
View File
@@ -3,6 +3,7 @@ import type { BackupConfig } from '../config/schema.js';
import type { OutboundMessage } from '../channels/types.js';
import { parseDuration } from '../session/index.js';
import { runBackupSnapshot } from './run.js';
import { initBackupHealth, markBackupFailure, markBackupSuccess } from './status.js';
interface ChannelLookup {
get(name: string): { send(peerId: string, message: OutboundMessage): Promise<void> } | undefined;
@@ -28,6 +29,7 @@ export class BackupScheduler {
}
start(): void {
initBackupHealth(this.deps.backupConfig.enabled);
if (!this.deps.backupConfig.enabled) {
return;
}
@@ -92,10 +94,12 @@ export class BackupScheduler {
backupConfig: this.deps.backupConfig,
});
console.log(`Backup completed: ${result.archivePath}${result.uploaded && result.remotePath ? ` -> ${result.remotePath}` : ''}`);
markBackupSuccess();
await this.handleSuccess();
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error(`Backup failed: ${message}`);
markBackupFailure(message);
await this.handleFailure(message);
} finally {
this.running = false;
+42
View File
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest';
import {
getBackupHealthSnapshot,
initBackupHealth,
markBackupFailure,
markBackupSuccess,
} from './status.js';
describe('backup status tracker', () => {
it('tracks failures and resets on success', () => {
initBackupHealth(true);
expect(getBackupHealthSnapshot()).toMatchObject({
enabled: true,
hasRun: false,
consecutiveFailures: 0,
});
markBackupFailure('upload failed', 123);
expect(getBackupHealthSnapshot()).toMatchObject({
enabled: true,
hasRun: true,
consecutiveFailures: 1,
lastFailureAt: 123,
lastError: 'upload failed',
});
markBackupFailure('upload failed again', 456);
expect(getBackupHealthSnapshot()).toMatchObject({
consecutiveFailures: 2,
lastFailureAt: 456,
lastError: 'upload failed again',
});
markBackupSuccess(789);
expect(getBackupHealthSnapshot()).toMatchObject({
hasRun: true,
consecutiveFailures: 0,
lastSuccessAt: 789,
lastError: undefined,
});
});
});
+46
View File
@@ -0,0 +1,46 @@
export interface BackupHealthSnapshot {
enabled: boolean;
hasRun: boolean;
consecutiveFailures: number;
lastSuccessAt?: number;
lastFailureAt?: number;
lastError?: string;
}
let snapshot: BackupHealthSnapshot = {
enabled: false,
hasRun: false,
consecutiveFailures: 0,
};
export function initBackupHealth(enabled: boolean): void {
snapshot = {
enabled,
hasRun: false,
consecutiveFailures: 0,
};
}
export function markBackupSuccess(now = Date.now()): void {
snapshot = {
...snapshot,
hasRun: true,
consecutiveFailures: 0,
lastSuccessAt: now,
lastError: undefined,
};
}
export function markBackupFailure(error: string, now = Date.now()): void {
snapshot = {
...snapshot,
hasRun: true,
consecutiveFailures: snapshot.consecutiveFailures + 1,
lastFailureAt: now,
lastError: error,
};
}
export function getBackupHealthSnapshot(): BackupHealthSnapshot {
return { ...snapshot };
}