feat(audit): add pi canary summary analyzer and cli script
This commit is contained in:
William Valentin
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { queryAuditLogs } from '../src/audit/export.js';
|
||||
import {
|
||||
evaluateBackendCanaryGate,
|
||||
renderBackendCanaryMarkdown,
|
||||
summarizeBackendCanary,
|
||||
type BackendCanaryGateThresholds,
|
||||
type BackendCanarySummaryOptions,
|
||||
type BackendRouteSource,
|
||||
type RoutedBackendName,
|
||||
} from '../src/audit/backendCanarySummary.js';
|
||||
|
||||
const DEFAULT_EVENT_TYPES = ['backend.route', 'backend.success', 'backend.fallback', 'session.message'] as const;
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
'Usage: node --import tsx/esm scripts/summarize-backend-canary.ts --audit <path> [options]',
|
||||
'',
|
||||
'Options:',
|
||||
' --audit <path> Path to audit.log (required)',
|
||||
' --backend <name> Target backend (default: pi_embedded)',
|
||||
' --baseline <name> Baseline backend (default: native)',
|
||||
' --since <ISO-8601|epoch_ms> Start time filter',
|
||||
' --until <ISO-8601|epoch_ms> End time filter',
|
||||
' --session <id[,id...]> Restrict to session IDs',
|
||||
' --channel <name[,name...]> Restrict to channels',
|
||||
' --sender <id[,id...]> Restrict to senders',
|
||||
' --source <name[,name...]> Restrict route sources (agent_override,default_external,native,forced_native_guard)',
|
||||
' --format <markdown|json> Output format (default: markdown)',
|
||||
' --out <path> Write output to file instead of stdout',
|
||||
'',
|
||||
'Gate options (optional):',
|
||||
' --gate-max-completion-drop-pp <number>',
|
||||
' --gate-max-p50-latency-increase-ms <number>',
|
||||
' --gate-max-p95-latency-increase-ms <number>',
|
||||
' --gate-max-fallback-rate-pct <number>',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function parseTime(value: string | undefined, flag: string): number | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
if (/^\d+$/.test(value)) {
|
||||
const asNumber = Number(value);
|
||||
if (Number.isFinite(asNumber)) {
|
||||
return asNumber;
|
||||
}
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseCsv(value: string | undefined): string[] | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
const values = value
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
return values.length > 0 ? values : undefined;
|
||||
}
|
||||
|
||||
function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined {
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseBackendName(raw: string | undefined, fallback: RoutedBackendName): RoutedBackendName {
|
||||
const value = (raw ?? fallback).trim() as RoutedBackendName;
|
||||
if (
|
||||
value === 'native'
|
||||
|| value === 'claude_code'
|
||||
|| value === 'opencode'
|
||||
|| value === 'codex'
|
||||
|| value === 'gemini'
|
||||
|| value === 'pi_embedded'
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
throw new Error(`Invalid backend "${value}".`);
|
||||
}
|
||||
|
||||
function parseSources(raw: string | undefined): BackendRouteSource[] | undefined {
|
||||
const values = parseCsv(raw);
|
||||
if (!values) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed: BackendRouteSource[] = [];
|
||||
for (const value of values) {
|
||||
if (value === 'agent_override' || value === 'default_external' || value === 'native' || value === 'forced_native_guard') {
|
||||
parsed.push(value);
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Invalid source "${value}".`);
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
audit: { type: 'string' },
|
||||
backend: { type: 'string' },
|
||||
baseline: { type: 'string' },
|
||||
since: { type: 'string' },
|
||||
until: { type: 'string' },
|
||||
session: { type: 'string' },
|
||||
channel: { type: 'string' },
|
||||
sender: { type: 'string' },
|
||||
source: { type: 'string' },
|
||||
format: { type: 'string' },
|
||||
out: { type: 'string' },
|
||||
'gate-max-completion-drop-pp': { type: 'string' },
|
||||
'gate-max-p50-latency-increase-ms': { type: 'string' },
|
||||
'gate-max-p95-latency-increase-ms': { type: 'string' },
|
||||
'gate-max-fallback-rate-pct': { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h' },
|
||||
},
|
||||
strict: true,
|
||||
allowPositionals: false,
|
||||
});
|
||||
|
||||
if (values.help) {
|
||||
process.stdout.write(`${usage()}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!values.audit) {
|
||||
throw new Error('--audit is required.');
|
||||
}
|
||||
|
||||
const format = values.format ?? 'markdown';
|
||||
if (format !== 'markdown' && format !== 'json') {
|
||||
throw new Error(`Invalid --format value "${format}".`);
|
||||
}
|
||||
|
||||
const summaryOptions: BackendCanarySummaryOptions = {
|
||||
targetBackend: parseBackendName(values.backend, 'pi_embedded'),
|
||||
baselineBackend: parseBackendName(values.baseline, 'native'),
|
||||
sessionIds: parseCsv(values.session),
|
||||
channels: parseCsv(values.channel),
|
||||
senders: parseCsv(values.sender),
|
||||
routeSources: parseSources(values.source),
|
||||
};
|
||||
|
||||
const startTime = parseTime(values.since, '--since');
|
||||
const endTime = parseTime(values.until, '--until');
|
||||
|
||||
const events = await queryAuditLogs(values.audit, {
|
||||
start_time: startTime,
|
||||
end_time: endTime,
|
||||
event_types: [...DEFAULT_EVENT_TYPES],
|
||||
});
|
||||
|
||||
const summary = summarizeBackendCanary(events, summaryOptions);
|
||||
|
||||
const gateThresholds: BackendCanaryGateThresholds = {
|
||||
maxCompletionRateDropPp: parseOptionalNumber(values['gate-max-completion-drop-pp'], '--gate-max-completion-drop-pp'),
|
||||
maxP50LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p50-latency-increase-ms'], '--gate-max-p50-latency-increase-ms'),
|
||||
maxP95LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p95-latency-increase-ms'], '--gate-max-p95-latency-increase-ms'),
|
||||
maxFallbackRatePct: parseOptionalNumber(values['gate-max-fallback-rate-pct'], '--gate-max-fallback-rate-pct'),
|
||||
};
|
||||
|
||||
const hasGateThreshold = Object.values(gateThresholds).some((value) => typeof value === 'number');
|
||||
const gateResult = hasGateThreshold ? evaluateBackendCanaryGate(summary, gateThresholds) : undefined;
|
||||
|
||||
const output = format === 'json'
|
||||
? JSON.stringify({
|
||||
generated_at: new Date().toISOString(),
|
||||
event_count: events.length,
|
||||
filters: {
|
||||
since_ms: startTime,
|
||||
until_ms: endTime,
|
||||
},
|
||||
options: summaryOptions,
|
||||
summary,
|
||||
gate: gateResult,
|
||||
}, null, 2)
|
||||
: renderBackendCanaryMarkdown(summary, summaryOptions, gateResult);
|
||||
|
||||
if (values.out) {
|
||||
await writeFile(values.out, `${output}\n`, 'utf-8');
|
||||
} else {
|
||||
process.stdout.write(`${output}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(`${message}\n\n${usage()}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user