#!/usr/bin/env node import { writeFile } from 'node:fs/promises'; import { parseArgs } from 'node:util'; import { queryAuditLogs } from '../src/audit/export.js'; import { evaluateBackendCanaryGate, renderBackendCanaryMarkdown, summarizeBackendCanary, type BackendCanaryGateThresholds, type BackendCanarySummaryOptions, type BackendRouteSource, type RoutedBackendName, } from '../src/audit/backendCanarySummary.js'; const DEFAULT_EVENT_TYPES = ['backend.route', 'backend.success', 'backend.fallback', 'session.message'] as const; function usage(): string { return [ 'Usage: node --import tsx/esm scripts/summarize-backend-canary.ts --audit [options]', '', 'Options:', ' --audit Path to audit.log (required)', ' --backend Target backend (default: pi_embedded)', ' --baseline Baseline backend (default: native)', ' --since Start time filter', ' --until End time filter', ' --session Restrict to session IDs', ' --channel Restrict to channels', ' --sender Restrict to senders', ' --source Restrict route sources (agent_override,default_external,native,forced_native_guard)', ' --format Output format (default: markdown)', ' --out Write output to file instead of stdout', '', 'Gate options (optional):', ' --gate-max-completion-drop-pp ', ' --gate-max-p50-latency-increase-ms ', ' --gate-max-p95-latency-increase-ms ', ' --gate-max-fallback-rate-pct ', ].join('\n'); } function parseTime(value: string | undefined, flag: string): number | undefined { if (!value) { return undefined; } if (/^\d+$/.test(value)) { const asNumber = Number(value); if (Number.isFinite(asNumber)) { return asNumber; } } const parsed = Date.parse(value); if (!Number.isFinite(parsed)) { throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`); } return parsed; } function parseCsv(value: string | undefined): string[] | undefined { if (!value) { return undefined; } const values = value .split(',') .map((item) => item.trim()) .filter((item) => item.length > 0); return values.length > 0 ? values : undefined; } function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined { if (!raw) { return undefined; } const parsed = Number(raw); if (!Number.isFinite(parsed)) { throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`); } return parsed; } function parseBackendName(raw: string | undefined, fallback: RoutedBackendName): RoutedBackendName { const value = (raw ?? fallback).trim() as RoutedBackendName; if ( value === 'native' || value === 'claude_code' || value === 'opencode' || value === 'codex' || value === 'gemini' || value === 'pi_embedded' ) { return value; } throw new Error(`Invalid backend "${value}".`); } function parseSources(raw: string | undefined): BackendRouteSource[] | undefined { const values = parseCsv(raw); if (!values) { return undefined; } const parsed: BackendRouteSource[] = []; for (const value of values) { if (value === 'agent_override' || value === 'default_external' || value === 'native' || value === 'forced_native_guard') { parsed.push(value); continue; } throw new Error(`Invalid source "${value}".`); } return parsed; } async function main(): Promise { const { values } = parseArgs({ options: { audit: { type: 'string' }, backend: { type: 'string' }, baseline: { type: 'string' }, since: { type: 'string' }, until: { type: 'string' }, session: { type: 'string' }, channel: { type: 'string' }, sender: { type: 'string' }, source: { type: 'string' }, format: { type: 'string' }, out: { type: 'string' }, 'gate-max-completion-drop-pp': { type: 'string' }, 'gate-max-p50-latency-increase-ms': { type: 'string' }, 'gate-max-p95-latency-increase-ms': { type: 'string' }, 'gate-max-fallback-rate-pct': { type: 'string' }, help: { type: 'boolean', short: 'h' }, }, strict: true, allowPositionals: false, }); if (values.help) { process.stdout.write(`${usage()}\n`); return; } if (!values.audit) { throw new Error('--audit is required.'); } const format = values.format ?? 'markdown'; if (format !== 'markdown' && format !== 'json') { throw new Error(`Invalid --format value "${format}".`); } const summaryOptions: BackendCanarySummaryOptions = { targetBackend: parseBackendName(values.backend, 'pi_embedded'), baselineBackend: parseBackendName(values.baseline, 'native'), sessionIds: parseCsv(values.session), channels: parseCsv(values.channel), senders: parseCsv(values.sender), routeSources: parseSources(values.source), }; const startTime = parseTime(values.since, '--since'); const endTime = parseTime(values.until, '--until'); const events = await queryAuditLogs(values.audit, { start_time: startTime, end_time: endTime, event_types: [...DEFAULT_EVENT_TYPES], }); const summary = summarizeBackendCanary(events, summaryOptions); const gateThresholds: BackendCanaryGateThresholds = { maxCompletionRateDropPp: parseOptionalNumber(values['gate-max-completion-drop-pp'], '--gate-max-completion-drop-pp'), maxP50LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p50-latency-increase-ms'], '--gate-max-p50-latency-increase-ms'), maxP95LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p95-latency-increase-ms'], '--gate-max-p95-latency-increase-ms'), maxFallbackRatePct: parseOptionalNumber(values['gate-max-fallback-rate-pct'], '--gate-max-fallback-rate-pct'), }; const hasGateThreshold = Object.values(gateThresholds).some((value) => typeof value === 'number'); const gateResult = hasGateThreshold ? evaluateBackendCanaryGate(summary, gateThresholds) : undefined; const output = format === 'json' ? JSON.stringify({ generated_at: new Date().toISOString(), event_count: events.length, filters: { since_ms: startTime, until_ms: endTime, }, options: summaryOptions, summary, gate: gateResult, }, null, 2) : renderBackendCanaryMarkdown(summary, summaryOptions, gateResult); if (values.out) { await writeFile(values.out, `${output}\n`, 'utf-8'); } else { process.stdout.write(`${output}\n`); } } main().catch((error) => { const message = error instanceof Error ? error.message : String(error); process.stderr.write(`${message}\n\n${usage()}\n`); process.exitCode = 1; });