feat(heartbeat): add process memory and backup health checks
This commit is contained in:
@@ -1,2 +1,9 @@
|
||||
export { runBackupSnapshot, backupInternals, type BackupRunOptions, type BackupResult } from './run.js';
|
||||
export { BackupScheduler, type BackupSchedulerDeps } from './scheduler.js';
|
||||
export {
|
||||
getBackupHealthSnapshot,
|
||||
initBackupHealth,
|
||||
markBackupSuccess,
|
||||
markBackupFailure,
|
||||
type BackupHealthSnapshot,
|
||||
} from './status.js';
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { BackupConfig } from '../config/schema.js';
|
||||
import type { OutboundMessage } from '../channels/types.js';
|
||||
import { parseDuration } from '../session/index.js';
|
||||
import { runBackupSnapshot } from './run.js';
|
||||
import { initBackupHealth, markBackupFailure, markBackupSuccess } from './status.js';
|
||||
|
||||
interface ChannelLookup {
|
||||
get(name: string): { send(peerId: string, message: OutboundMessage): Promise<void> } | undefined;
|
||||
@@ -28,6 +29,7 @@ export class BackupScheduler {
|
||||
}
|
||||
|
||||
start(): void {
|
||||
initBackupHealth(this.deps.backupConfig.enabled);
|
||||
if (!this.deps.backupConfig.enabled) {
|
||||
return;
|
||||
}
|
||||
@@ -92,10 +94,12 @@ export class BackupScheduler {
|
||||
backupConfig: this.deps.backupConfig,
|
||||
});
|
||||
console.log(`Backup completed: ${result.archivePath}${result.uploaded && result.remotePath ? ` -> ${result.remotePath}` : ''}`);
|
||||
markBackupSuccess();
|
||||
await this.handleSuccess();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`Backup failed: ${message}`);
|
||||
markBackupFailure(message);
|
||||
await this.handleFailure(message);
|
||||
} finally {
|
||||
this.running = false;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getBackupHealthSnapshot,
|
||||
initBackupHealth,
|
||||
markBackupFailure,
|
||||
markBackupSuccess,
|
||||
} from './status.js';
|
||||
|
||||
describe('backup status tracker', () => {
|
||||
it('tracks failures and resets on success', () => {
|
||||
initBackupHealth(true);
|
||||
expect(getBackupHealthSnapshot()).toMatchObject({
|
||||
enabled: true,
|
||||
hasRun: false,
|
||||
consecutiveFailures: 0,
|
||||
});
|
||||
|
||||
markBackupFailure('upload failed', 123);
|
||||
expect(getBackupHealthSnapshot()).toMatchObject({
|
||||
enabled: true,
|
||||
hasRun: true,
|
||||
consecutiveFailures: 1,
|
||||
lastFailureAt: 123,
|
||||
lastError: 'upload failed',
|
||||
});
|
||||
|
||||
markBackupFailure('upload failed again', 456);
|
||||
expect(getBackupHealthSnapshot()).toMatchObject({
|
||||
consecutiveFailures: 2,
|
||||
lastFailureAt: 456,
|
||||
lastError: 'upload failed again',
|
||||
});
|
||||
|
||||
markBackupSuccess(789);
|
||||
expect(getBackupHealthSnapshot()).toMatchObject({
|
||||
hasRun: true,
|
||||
consecutiveFailures: 0,
|
||||
lastSuccessAt: 789,
|
||||
lastError: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
export interface BackupHealthSnapshot {
|
||||
enabled: boolean;
|
||||
hasRun: boolean;
|
||||
consecutiveFailures: number;
|
||||
lastSuccessAt?: number;
|
||||
lastFailureAt?: number;
|
||||
lastError?: string;
|
||||
}
|
||||
|
||||
let snapshot: BackupHealthSnapshot = {
|
||||
enabled: false,
|
||||
hasRun: false,
|
||||
consecutiveFailures: 0,
|
||||
};
|
||||
|
||||
export function initBackupHealth(enabled: boolean): void {
|
||||
snapshot = {
|
||||
enabled,
|
||||
hasRun: false,
|
||||
consecutiveFailures: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function markBackupSuccess(now = Date.now()): void {
|
||||
snapshot = {
|
||||
...snapshot,
|
||||
hasRun: true,
|
||||
consecutiveFailures: 0,
|
||||
lastSuccessAt: now,
|
||||
lastError: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function markBackupFailure(error: string, now = Date.now()): void {
|
||||
snapshot = {
|
||||
...snapshot,
|
||||
hasRun: true,
|
||||
consecutiveFailures: snapshot.consecutiveFailures + 1,
|
||||
lastFailureAt: now,
|
||||
lastError: error,
|
||||
};
|
||||
}
|
||||
|
||||
export function getBackupHealthSnapshot(): BackupHealthSnapshot {
|
||||
return { ...snapshot };
|
||||
}
|
||||
Reference in New Issue
Block a user