feat(heartbeat): add provider error-rate spike check

src/automation/heartbeat.test.ts

@@ -13,6 +13,8 @@ function makeConfig(overrides?: Partial<HeartbeatConfig>): HeartbeatConfig {
     disk_threshold_mb: 100,
     process_memory_threshold_mb: 1500,
     backup_failure_threshold: 1,
+    provider_error_rate_threshold: 0.5,
+    provider_error_min_calls: 5,
     ...overrides,
   };
 }
@@ -37,6 +39,7 @@ function makeDeps(overrides?: Partial<HeartbeatDeps>): HeartbeatDeps {
       hasRun: false,
       consecutiveFailures: 0,
     }),
+    getModelCalls: () => [],
     ...overrides,
   };
 }
@@ -513,4 +516,41 @@ describe('HeartbeatMonitor', () => {
       expect(check.message).toContain('minio unavailable');
     });
   });
+
+  describe('provider_errors check', () => {
+    it('passes when no model calls are recorded', async () => {
+      const deps = makeDeps({
+        config: makeConfig({ checks: ['provider_errors'] }),
+        getModelCalls: () => [],
+      });
+      monitor = new HeartbeatMonitor(deps);
+
+      const result = await monitor.runChecks();
+      const check = result.checks.find((c) => c.name === 'provider_errors');
+      if (!check) {throw new Error('Expected provider_errors check result');}
+      expect(check.healthy).toBe(true);
+      expect(check.message).toContain('No model calls');
+    });
+
+    it('fails when a provider error rate breaches threshold', async () => {
+      const deps = makeDeps({
+        config: makeConfig({ checks: ['provider_errors'], provider_error_min_calls: 4, provider_error_rate_threshold: 0.5 }),
+        getModelCalls: () => [
+          { provider: 'openai', error: 'rate limited' },
+          { provider: 'openai', error: 'timeout' },
+          { provider: 'openai' },
+          { provider: 'openai' },
+          { provider: 'anthropic' },
+          { provider: 'anthropic' },
+        ],
+      });
+      monitor = new HeartbeatMonitor(deps);
+
+      const result = await monitor.runChecks();
+      const check = result.checks.find((c) => c.name === 'provider_errors');
+      if (!check) {throw new Error('Expected provider_errors check result');}
+      expect(check.healthy).toBe(false);
+      expect(check.message).toContain('openai');
+    });
+  });
 });

src/automation/heartbeat.ts

@@ -41,6 +41,7 @@ export interface HeartbeatDeps {
   channelLookup: ChannelLookup;
   processMemoryUsageMb?: () => number;
   backupHealthProvider?: () => BackupHealthSnapshot;
+  getModelCalls?: () => Array<{ provider: string; error?: string }>;
 }
 
 /**
@@ -135,6 +136,9 @@ export class HeartbeatMonitor {
           case 'backup':
             result = this.checkBackup(start);
             break;
+          case 'provider_errors':
+            result = this.checkProviderErrors(start);
+            break;
           default:
             result = { name: check, healthy: false, message: `Unknown check: ${check}`, durationMs: Date.now() - start };
         }
@@ -388,6 +392,62 @@ export class HeartbeatMonitor {
     }
   }
 
+  private checkProviderErrors(start: number): CheckResult {
+    try {
+      const calls = this.deps.getModelCalls ? this.deps.getModelCalls() : [];
+      if (calls.length === 0) {
+        return { name: 'provider_errors', healthy: true, message: 'No model calls recorded yet', durationMs: Date.now() - start };
+      }
+
+      const providers = new Map<string, { total: number; errors: number }>();
+      for (const call of calls) {
+        const current = providers.get(call.provider) ?? { total: 0, errors: 0 };
+        current.total += 1;
+        if (call.error) {
+          current.errors += 1;
+        }
+        providers.set(call.provider, current);
+      }
+
+      const minCalls = this.deps.config.provider_error_min_calls;
+      const threshold = this.deps.config.provider_error_rate_threshold;
+      const offenders: string[] = [];
+
+      for (const [provider, stats] of providers) {
+        if (stats.total < minCalls) {
+          continue;
+        }
+        const errorRate = stats.errors / stats.total;
+        if (errorRate >= threshold) {
+          offenders.push(`${provider} ${Math.round(errorRate * 100)}% (${stats.errors}/${stats.total})`);
+        }
+      }
+
+      if (offenders.length > 0) {
+        return {
+          name: 'provider_errors',
+          healthy: false,
+          message: `High provider error rate: ${offenders.join(', ')}`,
+          durationMs: Date.now() - start,
+        };
+      }
+
+      return {
+        name: 'provider_errors',
+        healthy: true,
+        message: `Provider error rates below threshold across ${providers.size} provider(s)`,
+        durationMs: Date.now() - start,
+      };
+    } catch (err) {
+      return {
+        name: 'provider_errors',
+        healthy: false,
+        message: err instanceof Error ? err.message : 'Failed to check provider error rates',
+        durationMs: Date.now() - start,
+      };
+    }
+  }
+
   // ── Notification ───────────────────────────────────────────────
 
   private async notify(text: string): Promise<void> {