will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(audit): add backend-scoped phase0 live baseline capture

Browse Source
This commit is contained in:
William Valentin
2026-02-27 08:47:31 -08:00
parent a97cc9dc95
commit 68cdc2cf8b
20 changed files with 1055 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/capture-phase0-live-baseline.ts
+53 -1
View File
@@ -4,7 +4,10 @@ import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { parseArgs } from 'node:util';
import { queryAuditLogs } from '../src/audit/export.js';
import { capturePhase0LiveBaselineEvents } from '../src/audit/phase0LiveBaseline.js';
import {
capturePhase0LiveBaselineEvents,
type Phase0BackendTarget,
} from '../src/audit/phase0LiveBaseline.js';
import { findLatestGatewayCancelWindow } from '../src/audit/phase0GatewayWindow.js';
import {
renderPhase0BaselineMarkdown,
@@ -14,6 +17,14 @@ import {
} from '../src/audit/phase0BaselineSummary.js';
const DEFAULT_EVENT_TYPES = ['run.state', 'run.cancel', 'reaction.match', 'reaction.skip'] as const;
const BACKEND_TARGETS: readonly Phase0BackendTarget[] = [
'native',
'claude_code',
'opencode',
'codex',
'gemini',
'pi_embedded',
];
function usage(): string {
return [
@@ -25,6 +36,7 @@ function usage(): string {
' --until <ISO-8601|epoch_ms> End time filter',
' --channel <name[,name...]> Restrict sample to channels',
' --source <gateway|channel[,..]> Restrict sample to sources',
' --backend <native|pi_embedded|...[,..]> Restrict sample to selected backends (via backend.route timeline)',
' --exclude-session-substring <text[,..]> Exclude sessions containing any substring (default: probe)',
' --auto-gateway-cancel-window Auto-select latest gateway cancel/cancelled session window',
' --window-padding-ms <number> Milliseconds added before/after auto-selected window (default: 250)',
@@ -113,6 +125,22 @@ function parseOptionalNumber(raw: string | undefined, flag: string): number | un
return parsed;
}
function parseBackendTargets(raw: string | undefined): Phase0BackendTarget[] | undefined {
const values = parseCsv(raw);
if (!values) {
return undefined;
}
const parsed: Phase0BackendTarget[] = [];
for (const value of values) {
if (BACKEND_TARGETS.includes(value as Phase0BackendTarget)) {
parsed.push(value as Phase0BackendTarget);
continue;
}
throw new Error(`Invalid backend "${value}".`);
}
return parsed;
}
function isoDateTagNow(): string {
return new Date().toISOString().slice(0, 10);
}
@@ -130,6 +158,7 @@ async function main(): Promise<void> {
until: { type: 'string' },
channel: { type: 'string' },
source: { type: 'string' },
backend: { type: 'string' },
'exclude-session-substring': { type: 'string' },
'auto-gateway-cancel-window': { type: 'boolean' },
'window-padding-ms': { type: 'string' },
@@ -156,6 +185,7 @@ async function main(): Promise<void> {
const tag = values.tag ?? isoDateTagNow();
const channels = parseCsv(values.channel);
let sources = parseSources(values.source);
const backendTargets = parseBackendTargets(values.backend);
const excludeSessionSubstrings = parseCsv(values['exclude-session-substring']) ?? ['probe'];
const autoGatewayCancelWindow = Boolean(values['auto-gateway-cancel-window']);
const windowPaddingMs = parseOptionalNumber(values['window-padding-ms'], '--window-padding-ms');
@@ -190,8 +220,15 @@ async function main(): Promise<void> {
}
const isGatewayOnly = sources?.length === 1 && sources[0] === 'gateway';
const backendSuffix = backendTargets && backendTargets.length > 0
? backendTargets.length === 1
? `backend_${backendTargets[0]}`
: 'backend_scoped'
: undefined;
const defaultBaseName = isGatewayOnly
? `docs/plans/artifacts/phase0_baseline_live_gateway_${tag}`
: backendSuffix
? `docs/plans/artifacts/phase0_baseline_live_${backendSuffix}_${tag}`
: `docs/plans/artifacts/phase0_baseline_live_${tag}`;
const sampleOut = values['sample-out'] ?? `${defaultBaseName}.jsonl`;
const summaryJsonOut = values['summary-json-out'] ?? `${defaultBaseName}.json`;
@@ -205,6 +242,14 @@ async function main(): Promise<void> {
maxSkipReasons: parseOptionalNumber(values['max-skip-reasons'], '--max-skip-reasons') ?? 10,
};
const backendRouteEvents = backendTargets && backendTargets.length > 0
? await queryAuditLogs(auditPath, {
start_time: startTime,
end_time: endTime,
event_types: ['backend.route'],
})
: [];
const sourceEvents = await queryAuditLogs(auditPath, {
start_time: startTime,
end_time: endTime,
@@ -214,6 +259,8 @@ async function main(): Promise<void> {
const sampledEvents = capturePhase0LiveBaselineEvents(sourceEvents, {
channels,
sources,
backendTargets,
backendRouteEvents,
excludeSessionSubstrings,
anonymizeIdentifiers: !values['raw-identifiers'],
});
@@ -231,6 +278,7 @@ async function main(): Promise<void> {
until_ms: endTime,
channels,
sources,
backend_targets: backendTargets,
exclude_session_substrings: excludeSessionSubstrings,
anonymized_identifiers: !values['raw-identifiers'],
auto_gateway_cancel_window: autoWindow
@@ -239,6 +287,7 @@ async function main(): Promise<void> {
padding_ms: windowPaddingMs ?? 250,
}
: undefined,
backend_route_event_count: backendRouteEvents.length > 0 ? backendRouteEvents.length : undefined,
},
options: summaryOptions,
summary,
@@ -252,6 +301,9 @@ async function main(): Promise<void> {
if (autoWindow) {
process.stdout.write(`- auto gateway window: session=${autoWindow.session_id} start=${autoWindow.start_time_ms} end=${autoWindow.end_time_ms}\n`);
}
if (backendTargets && backendTargets.length > 0) {
process.stdout.write(`- backend targets: ${backendTargets.join(', ')} (route events: ${backendRouteEvents.length})\n`);
}
process.stdout.write(`- sample: ${sampleOut}\n`);
process.stdout.write(`- summary json: ${summaryJsonOut}\n`);
process.stdout.write(`- summary md: ${summaryMdOut}\n`);