William Valentin
a4794ddea8
Require non-negative integer maxSessions/maxChannels/maxSkipReasons in summary core and both phase0 summary/capture CLIs to prevent silent flooring of fractional values. Architecture/protocol diagrams reviewed; no flow or API shape changes required.
331 lines
12 KiB
JavaScript
331 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { dirname, resolve } from 'node:path';
|
|
import { parseArgs } from 'node:util';
|
|
import { queryAuditLogs } from '../src/audit/export.js';
|
|
import {
|
|
capturePhase0LiveBaselineEvents,
|
|
type Phase0BackendTarget,
|
|
} from '../src/audit/phase0LiveBaseline.js';
|
|
import { findLatestGatewayCancelWindow } from '../src/audit/phase0GatewayWindow.js';
|
|
import {
|
|
renderPhase0BaselineMarkdown,
|
|
summarizePhase0Baseline,
|
|
type AuditSource,
|
|
type Phase0BaselineSummaryOptions,
|
|
} from '../src/audit/phase0BaselineSummary.js';
|
|
|
|
const DEFAULT_EVENT_TYPES = ['run.state', 'run.cancel', 'reaction.match', 'reaction.skip'] as const;
|
|
const BACKEND_TARGETS: readonly Phase0BackendTarget[] = [
|
|
'native',
|
|
'claude_code',
|
|
'opencode',
|
|
'codex',
|
|
'gemini',
|
|
'pi_embedded',
|
|
];
|
|
|
|
function usage(): string {
|
|
return [
|
|
'Usage: node --import tsx/esm scripts/capture-phase0-live-baseline.ts [options]',
|
|
'',
|
|
'Options:',
|
|
' --audit <path> Source audit log path (default: ~/.local/share/flynn/audit.log)',
|
|
' --since <ISO-8601|epoch_ms> Start time filter',
|
|
' --until <ISO-8601|epoch_ms> End time filter',
|
|
' --channel <name[,name...]> Restrict sample to channels',
|
|
' --source <gateway|channel[,..]> Restrict sample to sources',
|
|
' --backend <native|pi_embedded|...[,..]> Restrict sample to selected backends (via backend.route timeline)',
|
|
' --exclude-session-substring <text[,..]> Exclude sessions containing any substring (default: probe)',
|
|
' --auto-gateway-cancel-window Auto-select latest gateway cancel/cancelled session window',
|
|
' --window-padding-ms <number> Milliseconds added before/after auto-selected window (default: 250)',
|
|
' --raw-identifiers Keep raw session/sender/request IDs (default: anonymized)',
|
|
' --tag <YYYY-MM-DD> Output file tag (default: current date UTC)',
|
|
' --sample-out <path> Output JSONL sample path override',
|
|
' --summary-json-out <path> Output summary JSON path override',
|
|
' --summary-md-out <path> Output summary markdown path override',
|
|
' --max-sessions <integer> Limit session rows in output (default: 20)',
|
|
' --max-channels <integer> Limit channel rows in output (default: 20)',
|
|
' --max-skip-reasons <integer> Limit skip reason rows in output (default: 10)',
|
|
].join('\n');
|
|
}
|
|
|
|
function expandHomePath(pathValue: string): string {
|
|
if (!pathValue.startsWith('~')) {
|
|
return pathValue;
|
|
}
|
|
const home = process.env.HOME;
|
|
if (!home) {
|
|
return pathValue;
|
|
}
|
|
return resolve(home, pathValue.slice(1));
|
|
}
|
|
|
|
function collapseHomePath(pathValue: string): string {
|
|
const home = process.env.HOME;
|
|
if (!home) {
|
|
return pathValue;
|
|
}
|
|
return pathValue.startsWith(home) ? `~${pathValue.slice(home.length)}` : pathValue;
|
|
}
|
|
|
|
function parseTime(value: string | undefined, flag: string): number | undefined {
|
|
if (!value) {
|
|
return undefined;
|
|
}
|
|
if (/^\d+$/.test(value)) {
|
|
const asNumber = Number(value);
|
|
if (Number.isFinite(asNumber)) {
|
|
return asNumber;
|
|
}
|
|
}
|
|
const parsed = Date.parse(value);
|
|
if (!Number.isFinite(parsed)) {
|
|
throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseCsv(value: string | undefined): string[] | undefined {
|
|
if (!value) {
|
|
return undefined;
|
|
}
|
|
const values = value
|
|
.split(',')
|
|
.map((item) => item.trim())
|
|
.filter((item) => item.length > 0);
|
|
return values.length > 0 ? values : undefined;
|
|
}
|
|
|
|
function parseSources(raw: string | undefined): AuditSource[] | undefined {
|
|
const values = parseCsv(raw);
|
|
if (!values) {
|
|
return undefined;
|
|
}
|
|
const parsed: AuditSource[] = [];
|
|
for (const value of values) {
|
|
if (value === 'gateway' || value === 'channel') {
|
|
parsed.push(value);
|
|
continue;
|
|
}
|
|
throw new Error(`Invalid source "${value}".`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined {
|
|
if (!raw) {
|
|
return undefined;
|
|
}
|
|
const parsed = Number(raw);
|
|
if (!Number.isFinite(parsed)) {
|
|
throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseOptionalInteger(raw: string | undefined, flag: string): number | undefined {
|
|
const parsed = parseOptionalNumber(raw, flag);
|
|
if (parsed === undefined) {
|
|
return undefined;
|
|
}
|
|
if (!Number.isInteger(parsed)) {
|
|
throw new Error(`Invalid ${flag} value "${raw}". Expected an integer.`);
|
|
}
|
|
if (parsed < 0) {
|
|
throw new Error(`${flag} must be greater than or equal to 0.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseBackendTargets(raw: string | undefined): Phase0BackendTarget[] | undefined {
|
|
const values = parseCsv(raw);
|
|
if (!values) {
|
|
return undefined;
|
|
}
|
|
const parsed: Phase0BackendTarget[] = [];
|
|
for (const value of values) {
|
|
if (BACKEND_TARGETS.includes(value as Phase0BackendTarget)) {
|
|
parsed.push(value as Phase0BackendTarget);
|
|
continue;
|
|
}
|
|
throw new Error(`Invalid backend "${value}".`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function isoDateTagNow(): string {
|
|
return new Date().toISOString().slice(0, 10);
|
|
}
|
|
|
|
async function writeTextFile(pathValue: string, contents: string): Promise<void> {
|
|
await mkdir(dirname(pathValue), { recursive: true });
|
|
await writeFile(pathValue, contents, 'utf8');
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const { values } = parseArgs({
|
|
options: {
|
|
audit: { type: 'string' },
|
|
since: { type: 'string' },
|
|
until: { type: 'string' },
|
|
channel: { type: 'string' },
|
|
source: { type: 'string' },
|
|
backend: { type: 'string' },
|
|
'exclude-session-substring': { type: 'string' },
|
|
'auto-gateway-cancel-window': { type: 'boolean' },
|
|
'window-padding-ms': { type: 'string' },
|
|
'raw-identifiers': { type: 'boolean' },
|
|
tag: { type: 'string' },
|
|
'sample-out': { type: 'string' },
|
|
'summary-json-out': { type: 'string' },
|
|
'summary-md-out': { type: 'string' },
|
|
'max-sessions': { type: 'string' },
|
|
'max-channels': { type: 'string' },
|
|
'max-skip-reasons': { type: 'string' },
|
|
help: { type: 'boolean', short: 'h' },
|
|
},
|
|
strict: true,
|
|
allowPositionals: false,
|
|
});
|
|
|
|
if (values.help) {
|
|
process.stdout.write(`${usage()}\n`);
|
|
return;
|
|
}
|
|
|
|
const auditPath = expandHomePath(values.audit ?? '~/.local/share/flynn/audit.log');
|
|
const tag = values.tag ?? isoDateTagNow();
|
|
const channels = parseCsv(values.channel);
|
|
let sources = parseSources(values.source);
|
|
const backendTargets = parseBackendTargets(values.backend);
|
|
const excludeSessionSubstrings = parseCsv(values['exclude-session-substring']) ?? ['probe'];
|
|
const autoGatewayCancelWindow = Boolean(values['auto-gateway-cancel-window']);
|
|
const windowPaddingMs = parseOptionalNumber(values['window-padding-ms'], '--window-padding-ms');
|
|
if (windowPaddingMs !== undefined && windowPaddingMs < 0) {
|
|
throw new Error('--window-padding-ms must be greater than or equal to 0.');
|
|
}
|
|
|
|
if (autoGatewayCancelWindow && (values.since || values.until)) {
|
|
throw new Error('--auto-gateway-cancel-window cannot be combined with --since/--until.');
|
|
}
|
|
if (autoGatewayCancelWindow && sources && !sources.includes('gateway')) {
|
|
throw new Error('--auto-gateway-cancel-window requires --source to include "gateway" (or omit --source).');
|
|
}
|
|
|
|
let startTime = parseTime(values.since, '--since');
|
|
let endTime = parseTime(values.until, '--until');
|
|
let autoWindow: ReturnType<typeof findLatestGatewayCancelWindow> = null;
|
|
|
|
if (autoGatewayCancelWindow) {
|
|
sources = sources ?? ['gateway'];
|
|
const autoWindowSourceEvents = await queryAuditLogs(auditPath, {
|
|
event_types: ['run.state', 'run.cancel'],
|
|
});
|
|
autoWindow = findLatestGatewayCancelWindow(autoWindowSourceEvents, {
|
|
padding_ms: windowPaddingMs ?? 250,
|
|
});
|
|
if (!autoWindow) {
|
|
throw new Error('No gateway cancel/cancelled session window found in audit log.');
|
|
}
|
|
startTime = autoWindow.start_time_ms;
|
|
endTime = autoWindow.end_time_ms;
|
|
}
|
|
|
|
const isGatewayOnly = sources?.length === 1 && sources[0] === 'gateway';
|
|
const backendSuffix = backendTargets && backendTargets.length > 0
|
|
? backendTargets.length === 1
|
|
? `backend_${backendTargets[0]}`
|
|
: 'backend_scoped'
|
|
: undefined;
|
|
const defaultBaseName = isGatewayOnly
|
|
? `docs/plans/artifacts/phase0_baseline_live_gateway_${tag}`
|
|
: backendSuffix
|
|
? `docs/plans/artifacts/phase0_baseline_live_${backendSuffix}_${tag}`
|
|
: `docs/plans/artifacts/phase0_baseline_live_${tag}`;
|
|
const sampleOut = values['sample-out'] ?? `${defaultBaseName}.jsonl`;
|
|
const summaryJsonOut = values['summary-json-out'] ?? `${defaultBaseName}.json`;
|
|
const summaryMdOut = values['summary-md-out'] ?? `${defaultBaseName}.md`;
|
|
|
|
const summaryOptions: Phase0BaselineSummaryOptions = {
|
|
channels,
|
|
sources,
|
|
maxSessions: parseOptionalInteger(values['max-sessions'], '--max-sessions') ?? 20,
|
|
maxChannels: parseOptionalInteger(values['max-channels'], '--max-channels') ?? 20,
|
|
maxSkipReasons: parseOptionalInteger(values['max-skip-reasons'], '--max-skip-reasons') ?? 10,
|
|
};
|
|
|
|
const backendRouteEvents = backendTargets && backendTargets.length > 0
|
|
? await queryAuditLogs(auditPath, {
|
|
start_time: startTime,
|
|
end_time: endTime,
|
|
event_types: ['backend.route'],
|
|
})
|
|
: [];
|
|
|
|
const sourceEvents = await queryAuditLogs(auditPath, {
|
|
start_time: startTime,
|
|
end_time: endTime,
|
|
event_types: [...DEFAULT_EVENT_TYPES],
|
|
});
|
|
|
|
const sampledEvents = capturePhase0LiveBaselineEvents(sourceEvents, {
|
|
channels,
|
|
sources,
|
|
backendTargets,
|
|
backendRouteEvents,
|
|
excludeSessionSubstrings,
|
|
anonymizeIdentifiers: !values['raw-identifiers'],
|
|
});
|
|
|
|
const summary = summarizePhase0Baseline(sampledEvents, summaryOptions);
|
|
const markdown = renderPhase0BaselineMarkdown(summary, summaryOptions);
|
|
const sampleJsonl = sampledEvents.map((entry) => JSON.stringify(entry)).join('\n');
|
|
const summaryJson = JSON.stringify({
|
|
generated_at: new Date().toISOString(),
|
|
source_audit_path: collapseHomePath(auditPath),
|
|
source_event_count: sourceEvents.length,
|
|
sampled_event_count: sampledEvents.length,
|
|
filters: {
|
|
since_ms: startTime,
|
|
until_ms: endTime,
|
|
channels,
|
|
sources,
|
|
backend_targets: backendTargets,
|
|
exclude_session_substrings: excludeSessionSubstrings,
|
|
anonymized_identifiers: !values['raw-identifiers'],
|
|
auto_gateway_cancel_window: autoWindow
|
|
? {
|
|
...autoWindow,
|
|
padding_ms: windowPaddingMs ?? 250,
|
|
}
|
|
: undefined,
|
|
backend_route_event_count: backendRouteEvents.length > 0 ? backendRouteEvents.length : undefined,
|
|
},
|
|
options: summaryOptions,
|
|
summary,
|
|
}, null, 2);
|
|
|
|
await writeTextFile(sampleOut, sampleJsonl.length > 0 ? `${sampleJsonl}\n` : '');
|
|
await writeTextFile(summaryJsonOut, `${summaryJson}\n`);
|
|
await writeTextFile(summaryMdOut, `${markdown}\n`);
|
|
|
|
process.stdout.write(`Captured ${sampledEvents.length} events from ${sourceEvents.length} source events.\n`);
|
|
if (autoWindow) {
|
|
process.stdout.write(`- auto gateway window: session=${autoWindow.session_id} start=${autoWindow.start_time_ms} end=${autoWindow.end_time_ms}\n`);
|
|
}
|
|
if (backendTargets && backendTargets.length > 0) {
|
|
process.stdout.write(`- backend targets: ${backendTargets.join(', ')} (route events: ${backendRouteEvents.length})\n`);
|
|
}
|
|
process.stdout.write(`- sample: ${sampleOut}\n`);
|
|
process.stdout.write(`- summary json: ${summaryJsonOut}\n`);
|
|
process.stdout.write(`- summary md: ${summaryMdOut}\n`);
|
|
}
|
|
|
|
main().catch((error) => {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
process.stderr.write(`${message}\n\n${usage()}\n`);
|
|
process.exitCode = 1;
|
|
});
|