flynn/scripts/summarize-backend-canary.ts

#!/usr/bin/env node

import { writeFile } from 'node:fs/promises';
import { parseArgs } from 'node:util';
import { queryAuditLogs } from '../src/audit/export.js';
import {
  evaluateBackendCanaryGate,
  renderBackendCanaryMarkdown,
  summarizeBackendCanary,
  type BackendCanaryGateThresholds,
  type BackendCanarySummaryOptions,
  type BackendRouteSource,
  type RoutedBackendName,
} from '../src/audit/backendCanarySummary.js';

const DEFAULT_EVENT_TYPES = ['backend.route', 'backend.success', 'backend.fallback', 'session.message'] as const;

function usage(): string {
  return [
    'Usage: node --import tsx/esm scripts/summarize-backend-canary.ts --audit <path> [options]',
    '',
    'Options:',
    '  --audit <path>                          Path to audit.log (required)',
    '  --backend <name>                        Target backend (default: pi_embedded)',
    '  --baseline <name>                       Baseline backend (default: native)',
    '  --since <ISO-8601|epoch_ms>             Start time filter',
    '  --until <ISO-8601|epoch_ms>             End time filter',
    '  --session <id[,id...]>                  Restrict to session IDs',
    '  --channel <name[,name...]>              Restrict to channels',
    '  --sender <id[,id...]>                   Restrict to senders',
    '  --source <name[,name...]>               Restrict route sources (agent_override,default_external,native,forced_native_guard)',
    '  --format <markdown|json>                Output format (default: markdown)',
    '  --out <path>                            Write output to file instead of stdout',
    '',
    'Gate options (optional):',
    '  --gate-min-target-routes <number>',
    '  --gate-min-baseline-routes <number>',
    '  --gate-min-target-attempts <number>',
    '  --gate-max-completion-drop-pp <number>',
    '  --gate-max-p50-latency-increase-ms <number>',
    '  --gate-max-p95-latency-increase-ms <number>',
    '  --gate-max-fallback-rate-pct <number>',
  ].join('\n');
}

function parseTime(value: string | undefined, flag: string): number | undefined {
  if (!value) {
    return undefined;
  }
  if (/^\d+$/.test(value)) {
    const asNumber = Number(value);
    if (Number.isFinite(asNumber)) {
      return asNumber;
    }
  }
  const parsed = Date.parse(value);
  if (!Number.isFinite(parsed)) {
    throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`);
  }
  return parsed;
}

function parseCsv(value: string | undefined): string[] | undefined {
  if (!value) {
    return undefined;
  }
  const values = value
    .split(',')
    .map((item) => item.trim())
    .filter((item) => item.length > 0);
  return values.length > 0 ? values : undefined;
}

function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined {
  if (!raw) {
    return undefined;
  }
  const parsed = Number(raw);
  if (!Number.isFinite(parsed)) {
    throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`);
  }
  return parsed;
}

function parseBackendName(raw: string | undefined, fallback: RoutedBackendName): RoutedBackendName {
  const value = (raw ?? fallback).trim() as RoutedBackendName;
  if (
    value === 'native'
    || value === 'claude_code'
    || value === 'opencode'
    || value === 'codex'
    || value === 'gemini'
    || value === 'pi_embedded'
  ) {
    return value;
  }
  throw new Error(`Invalid backend "${value}".`);
}

function parseSources(raw: string | undefined): BackendRouteSource[] | undefined {
  const values = parseCsv(raw);
  if (!values) {
    return undefined;
  }

  const parsed: BackendRouteSource[] = [];
  for (const value of values) {
    if (value === 'agent_override' || value === 'default_external' || value === 'native' || value === 'forced_native_guard') {
      parsed.push(value);
      continue;
    }
    throw new Error(`Invalid source "${value}".`);
  }

  return parsed;
}

async function main(): Promise<void> {
  const { values } = parseArgs({
    options: {
      audit: { type: 'string' },
      backend: { type: 'string' },
      baseline: { type: 'string' },
      since: { type: 'string' },
      until: { type: 'string' },
      session: { type: 'string' },
      channel: { type: 'string' },
      sender: { type: 'string' },
      source: { type: 'string' },
      format: { type: 'string' },
      out: { type: 'string' },
      'gate-min-target-routes': { type: 'string' },
      'gate-min-baseline-routes': { type: 'string' },
      'gate-min-target-attempts': { type: 'string' },
      'gate-max-completion-drop-pp': { type: 'string' },
      'gate-max-p50-latency-increase-ms': { type: 'string' },
      'gate-max-p95-latency-increase-ms': { type: 'string' },
      'gate-max-fallback-rate-pct': { type: 'string' },
      help: { type: 'boolean', short: 'h' },
    },
    strict: true,
    allowPositionals: false,
  });

  if (values.help) {
    process.stdout.write(`${usage()}\n`);
    return;
  }

  if (!values.audit) {
    throw new Error('--audit is required.');
  }

  const format = values.format ?? 'markdown';
  if (format !== 'markdown' && format !== 'json') {
    throw new Error(`Invalid --format value "${format}".`);
  }

  const summaryOptions: BackendCanarySummaryOptions = {
    targetBackend: parseBackendName(values.backend, 'pi_embedded'),
    baselineBackend: parseBackendName(values.baseline, 'native'),
    sessionIds: parseCsv(values.session),
    channels: parseCsv(values.channel),
    senders: parseCsv(values.sender),
    routeSources: parseSources(values.source),
  };

  const startTime = parseTime(values.since, '--since');
  const endTime = parseTime(values.until, '--until');

  const events = await queryAuditLogs(values.audit, {
    start_time: startTime,
    end_time: endTime,
    event_types: [...DEFAULT_EVENT_TYPES],
  });

  const summary = summarizeBackendCanary(events, summaryOptions);

  const gateThresholds: BackendCanaryGateThresholds = {
    minTargetRoutes: parseOptionalNumber(values['gate-min-target-routes'], '--gate-min-target-routes'),
    minBaselineRoutes: parseOptionalNumber(values['gate-min-baseline-routes'], '--gate-min-baseline-routes'),
    minTargetAttempts: parseOptionalNumber(values['gate-min-target-attempts'], '--gate-min-target-attempts'),
    maxCompletionRateDropPp: parseOptionalNumber(values['gate-max-completion-drop-pp'], '--gate-max-completion-drop-pp'),
    maxP50LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p50-latency-increase-ms'], '--gate-max-p50-latency-increase-ms'),
    maxP95LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p95-latency-increase-ms'], '--gate-max-p95-latency-increase-ms'),
    maxFallbackRatePct: parseOptionalNumber(values['gate-max-fallback-rate-pct'], '--gate-max-fallback-rate-pct'),
  };

  const hasGateThreshold = Object.values(gateThresholds).some((value) => typeof value === 'number');
  const gateResult = hasGateThreshold ? evaluateBackendCanaryGate(summary, gateThresholds) : undefined;

  const output = format === 'json'
    ? JSON.stringify({
      generated_at: new Date().toISOString(),
      event_count: events.length,
      filters: {
        since_ms: startTime,
        until_ms: endTime,
      },
      options: summaryOptions,
      summary,
      gate: gateResult,
    }, null, 2)
    : renderBackendCanaryMarkdown(summary, summaryOptions, gateResult);

  if (values.out) {
    await writeFile(values.out, `${output}\n`, 'utf-8');
  } else {
    process.stdout.write(`${output}\n`);
  }
}

main().catch((error) => {
  const message = error instanceof Error ? error.message : String(error);
  process.stderr.write(`${message}\n\n${usage()}\n`);
  process.exitCode = 1;
});