feat(audit): automate gateway phase0 live-window capture

This commit is contained in:
William Valentin
2026-02-26 23:56:30 -08:00
parent 5a34e986bf
commit 826df1d35b
12 changed files with 310 additions and 21 deletions
+11 -1
View File
@@ -1629,7 +1629,17 @@ Live baseline artifacts (sample JSONL + JSON/Markdown summaries) can be captured
pnpm audit:phase0-baseline:live pnpm audit:phase0-baseline:live
``` ```
Gateway-origin windows can be captured separately (for example when validating cancel paths) by restricting source + time window: One-shot refresh for both channel + gateway live windows:
```bash
pnpm audit:phase0-baseline:live:refresh
```
Gateway-origin windows can be captured separately (for example when validating cancel paths):
```bash
pnpm audit:phase0-baseline:live:gateway
```
The gateway command auto-selects the most recent session window containing both `run.cancel` and `run.state=cancelled` (with configurable padding). You can still capture explicit windows by restricting source + time bounds:
```bash ```bash
node --import tsx/esm scripts/capture-phase0-live-baseline.ts \ node --import tsx/esm scripts/capture-phase0-live-baseline.ts \
--audit ~/.local/share/flynn/audit.log \ --audit ~/.local/share/flynn/audit.log \
+1 -1
View File
@@ -23,7 +23,7 @@ The gateway provides:
- **HTTP Server**: Serves static dashboard and handles webhook endpoints - **HTTP Server**: Serves static dashboard and handles webhook endpoints
- **Node Capability Negotiation**: Optional companion-node role/capability registration - **Node Capability Negotiation**: Optional companion-node role/capability registration
Operational note: onboarding (`flynn setup` / `flynn onboard`) now runs post-save live readiness checks (model/channel/memory/automation) and prints a guided first-success task flow. Companion CLI now also supports bootstrap-manifest export (`flynn companion --export-bootstrap <path|->`), release-bundle export (`--export-release-bundle <dir>` with optional `--signing-key`/`--signing-key-id` signature output), release-bundle verification (`--verify-release-bundle <dir>` with optional `--verify-signing-key`/`--verify-signing-key-id`/`--require-signature`), platform shell-template export (`--export-shell-template <dir>`), plus richer shell bootstrap flags for status/location/push (`--app-version`, `--latitude/--longitude`, `--push-token`, etc.) for desktop/mobile app packaging without changing JSON-RPC method/event shapes. Audit observability now includes live phase-0 baseline capture flows: `pnpm audit:phase0-baseline:live` for channel-origin windows and `scripts/capture-phase0-live-baseline.ts --source gateway --since ... --until ...` for gateway-origin windows. Operational note: onboarding (`flynn setup` / `flynn onboard`) now runs post-save live readiness checks (model/channel/memory/automation) and prints a guided first-success task flow. Companion CLI now also supports bootstrap-manifest export (`flynn companion --export-bootstrap <path|->`), release-bundle export (`--export-release-bundle <dir>` with optional `--signing-key`/`--signing-key-id` signature output), release-bundle verification (`--verify-release-bundle <dir>` with optional `--verify-signing-key`/`--verify-signing-key-id`/`--require-signature`), platform shell-template export (`--export-shell-template <dir>`), plus richer shell bootstrap flags for status/location/push (`--app-version`, `--latitude/--longitude`, `--push-token`, etc.) for desktop/mobile app packaging without changing JSON-RPC method/event shapes. Audit observability now includes live phase-0 baseline capture flows: `pnpm audit:phase0-baseline:live` for channel-origin windows, `pnpm audit:phase0-baseline:live:gateway` (auto-detected cancel window) for gateway-origin windows, and `pnpm audit:phase0-baseline:live:refresh` for one-shot refresh of both windows.
### Execution Model (Sessions + Per-Session Queue) ### Execution Model (Sessions + Per-Session Queue)
+2 -1
View File
@@ -167,7 +167,8 @@ Gateway streaming UX signals:
- `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync in CI. - `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync in CI.
- `flynn companion` can bootstrap status/location/push metadata on connect (`node.status.set` + optional `node.location.set`/`node.push_token.set`) so thin companion shells can register operational context in one launch. - `flynn companion` can bootstrap status/location/push metadata on connect (`node.status.set` + optional `node.location.set`/`node.push_token.set`) so thin companion shells can register operational context in one launch.
- `pnpm audit:phase0-baseline:live` captures anonymized channel-origin live run/reaction baseline artifacts from real audit logs. - `pnpm audit:phase0-baseline:live` captures anonymized channel-origin live run/reaction baseline artifacts from real audit logs.
- `scripts/capture-phase0-live-baseline.ts --source gateway --since ... --until ...` captures gateway-origin baseline windows (including cancel-path telemetry) as separate artifacts. - `pnpm audit:phase0-baseline:live:gateway` captures gateway-origin baseline windows by auto-selecting the latest cancel/cancelled session window (or use `scripts/capture-phase0-live-baseline.ts --source gateway --since ... --until ...` for explicit windows).
- `pnpm audit:phase0-baseline:live:refresh` runs both channel + gateway capture commands in one step for cadence refreshes.
- Canvas artifacts are persisted by the gateway so session UI surfaces can recover after daemon restarts. - Canvas artifacts are persisted by the gateway so session UI surfaces can recover after daemon restarts.
- TTS synthesis uses an ordered provider chain with health cooldown tracking; if all providers fail, replies degrade to text-only without dropping the response. - TTS synthesis uses an ordered provider chain with health cooldown tracking; if all providers fail, replies degrade to text-only without dropping the response.
- Talk mode accepts spoken/text `stop`/`cancel` while active and maps it onto the same `/stop` run-control cancellation path used for text sessions. - Talk mode accepts spoken/text `stop`/`cancel` while active and maps it onto the same `/stop` run-control cancellation path used for text sessions.
@@ -32,7 +32,8 @@ If you only want the protocol surface, see `docs/api/PROTOCOL.md`.
- CI workflow `.github/workflows/companion-release-bundle.yml` mirrors this pipeline for manual artifact generation/upload. - CI workflow `.github/workflows/companion-release-bundle.yml` mirrors this pipeline for manual artifact generation/upload.
- CI workflow `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync on pull requests. - CI workflow `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync on pull requests.
- Audit phase-0 live telemetry snapshots can be regenerated with `pnpm audit:phase0-baseline:live` (channel-origin anonymized sample JSONL + summary JSON/markdown artifacts). - Audit phase-0 live telemetry snapshots can be regenerated with `pnpm audit:phase0-baseline:live` (channel-origin anonymized sample JSONL + summary JSON/markdown artifacts).
- Gateway-origin phase-0 windows (including cancel-path samples) can be captured with `scripts/capture-phase0-live-baseline.ts --source gateway --since ... --until ...`. - Gateway-origin phase-0 windows (including cancel-path samples) can be captured with `pnpm audit:phase0-baseline:live:gateway` (auto-detect latest cancel window) or `scripts/capture-phase0-live-baseline.ts --source gateway --since ... --until ...` for explicit bounds.
- `pnpm audit:phase0-baseline:live:refresh` runs both capture paths to refresh channel + gateway artifacts in one command.
- Companion CLI supports one-shot shell bootstrap metadata for live sessions (`--app-version`/`--status-text`, `--latitude`/`--longitude`, `--push-token`) so desktop/mobile wrappers can initialize node status/location/push in a single launch flow. - Companion CLI supports one-shot shell bootstrap metadata for live sessions (`--app-version`/`--status-text`, `--latitude`/`--longitude`, `--push-token`) so desktop/mobile wrappers can initialize node status/location/push in a single launch flow.
- Canvas artifacts are persisted per session under the gateway data directory for UI recovery across restarts. - Canvas artifacts are persisted per session under the gateway data directory for UI recovery across restarts.
- TTS output is best-effort with ordered provider fallback + per-provider cooldown tracking; synthesis failures still fall back to text-only responses. - TTS output is best-effort with ordered provider fallback + per-provider cooldown tracking; synthesis failures still fall back to text-only responses.
@@ -203,7 +203,7 @@ Phase 0 is complete when:
2. A baseline summary artifact is generated and committed under `docs/plans/artifacts/`. 2. A baseline summary artifact is generated and committed under `docs/plans/artifacts/`.
3. No user-visible response behavior changed compared to pre-phase baseline. 3. No user-visible response behavior changed compared to pre-phase baseline.
Follow-up status (2026-02-27): live channel-session artifacts exist under `docs/plans/artifacts/phase0_baseline_live_2026-02-27.*` via `pnpm audit:phase0-baseline:live` (anonymized IDs), and a second gateway-origin live window (including `run.cancel` + `cancel_requested`/`cancelled`) exists under `docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.*`. Follow-up status (2026-02-27): live channel-session artifacts exist under `docs/plans/artifacts/phase0_baseline_live_2026-02-27.*` via `pnpm audit:phase0-baseline:live` (anonymized IDs), and a second gateway-origin live window (including `run.cancel` + `cancel_requested`/`cancelled`) exists under `docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.*`. Gateway window refreshes can now run via `pnpm audit:phase0-baseline:live:gateway` (auto-selected cancel window), and both windows can be refreshed together with `pnpm audit:phase0-baseline:live:refresh`.
## Subagent Model Assignment Plan ## Subagent Model Assignment Plan
@@ -1,5 +1,5 @@
{ {
"generated_at": "2026-02-27T07:49:58.821Z", "generated_at": "2026-02-27T07:55:30.862Z",
"source_audit_path": "~/.local/share/flynn/audit.log", "source_audit_path": "~/.local/share/flynn/audit.log",
"source_event_count": 94, "source_event_count": 94,
"sampled_event_count": 88, "sampled_event_count": 88,
@@ -1,18 +1,28 @@
{ {
"generated_at": "2026-02-27T07:47:41.346Z", "generated_at": "2026-02-27T07:55:31.178Z",
"source_audit_path": "~/.local/share/flynn/audit.log", "source_audit_path": "~/.local/share/flynn/audit.log",
"source_event_count": 6, "source_event_count": 6,
"sampled_event_count": 6, "sampled_event_count": 6,
"filters": { "filters": {
"since_ms": 1772178440693, "since_ms": 1772178441443,
"until_ms": 1772178442694, "until_ms": 1772178441944,
"sources": [ "sources": [
"gateway" "gateway"
], ],
"exclude_session_substrings": [ "exclude_session_substrings": [
"probe" "probe"
], ],
"anonymized_identifiers": true "anonymized_identifiers": true,
"auto_gateway_cancel_window": {
"session_id": "ws:phase0-gateway-window",
"start_time_ms": 1772178441443,
"end_time_ms": 1772178441944,
"event_count": 6,
"run_cancel_count": 1,
"cancel_requested_count": 1,
"cancelled_count": 1,
"padding_ms": 250
}
}, },
"options": { "options": {
"sources": [ "sources": [
+27 -4
View File
@@ -88,10 +88,12 @@
"status": "completed", "status": "completed",
"date": "2026-02-27", "date": "2026-02-27",
"updated": "2026-02-27", "updated": "2026-02-27",
"summary": "Added a dedicated live phase-0 baseline capture flow that reads audit logs, filters run/reaction telemetry, excludes probe sessions, anonymizes session/sender/request IDs, and writes sample + summary artifacts for operational refreshes across both channel-origin and gateway-origin windows.", "summary": "Added a dedicated live phase-0 baseline capture flow that reads audit logs, filters run/reaction telemetry, excludes probe sessions, anonymizes session/sender/request IDs, and writes sample + summary artifacts for operational refreshes across both channel-origin and gateway-origin windows. Gateway mode now supports auto-detection of the latest cancel/cancelled window.",
"files_modified": [ "files_modified": [
"src/audit/phase0LiveBaseline.ts", "src/audit/phase0LiveBaseline.ts",
"src/audit/phase0LiveBaseline.test.ts", "src/audit/phase0LiveBaseline.test.ts",
"src/audit/phase0GatewayWindow.ts",
"src/audit/phase0GatewayWindow.test.ts",
"scripts/capture-phase0-live-baseline.ts", "scripts/capture-phase0-live-baseline.ts",
"package.json", "package.json",
"README.md", "README.md",
@@ -107,7 +109,7 @@
"docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json", "docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json",
"docs/plans/state.json" "docs/plans/state.json"
], ],
"test_status": "pnpm audit:phase0-baseline:live + node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --since 1772178440693 --until 1772178442694 --sample-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.jsonl --summary-json-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json --summary-md-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.md + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing" "test_status": "pnpm audit:phase0-baseline:live:refresh + pnpm test:run src/audit/phase0GatewayWindow.test.ts src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
}, },
"phase0-live-baseline-gateway-window": { "phase0-live-baseline-gateway-window": {
"status": "completed", "status": "completed",
@@ -127,6 +129,27 @@
], ],
"test_status": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --since 1772178440693 --until 1772178442694 --sample-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.jsonl --summary-json-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json --summary-md-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.md + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing" "test_status": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --since 1772178440693 --until 1772178442694 --sample-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.jsonl --summary-json-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json --summary-md-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.md + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
}, },
"phase0-live-baseline-refresh-automation": {
"status": "completed",
"date": "2026-02-27",
"updated": "2026-02-27",
"summary": "Automated gateway live-window capture by adding auto-detection of the latest gateway cancel/cancelled window (`--auto-gateway-cancel-window`) plus a one-shot refresh command that regenerates both channel and gateway artifacts together (`pnpm audit:phase0-baseline:live:refresh`).",
"files_modified": [
"src/audit/phase0GatewayWindow.ts",
"src/audit/phase0GatewayWindow.test.ts",
"scripts/capture-phase0-live-baseline.ts",
"package.json",
"README.md",
"docs/api/PROTOCOL.md",
"docs/architecture/AGENT_DIAGRAM.md",
"docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md",
"docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md",
"docs/plans/artifacts/phase0_baseline_live_2026-02-27.json",
"docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json",
"docs/plans/state.json"
],
"test_status": "pnpm audit:phase0-baseline:live:refresh + pnpm test:run src/audit/phase0GatewayWindow.test.ts src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
},
"phase0-instrumentation-ticket-checklist": { "phase0-instrumentation-ticket-checklist": {
"status": "completed", "status": "completed",
"date": "2026-02-25", "date": "2026-02-25",
@@ -7276,7 +7299,7 @@
} }
}, },
"overall_progress": { "overall_progress": {
"total_test_count": 2585, "total_test_count": 2588,
"all_tests_passing": true, "all_tests_passing": true,
"p0_completion": "3/3 (100%)", "p0_completion": "3/3 (100%)",
"p1_completion": "4/4 (100%)", "p1_completion": "4/4 (100%)",
@@ -7313,7 +7336,7 @@
"deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters", "deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters",
"deeper_surfaces_phase0_ticket_04": "completed — added phase-0 baseline summary tooling for run outcomes, cancel latency, and reaction decisions with markdown/json CLI output", "deeper_surfaces_phase0_ticket_04": "completed — added phase-0 baseline summary tooling for run outcomes, cancel latency, and reaction decisions with markdown/json CLI output",
"deeper_surfaces_phase0_ticket_05": "completed — documented phase-0 telemetry fields/workflow, refreshed architecture/protocol docs, and generated anonymized live baseline artifacts for both channel-origin and gateway-origin traffic (including cancel-path coverage)", "deeper_surfaces_phase0_ticket_05": "completed — documented phase-0 telemetry fields/workflow, refreshed architecture/protocol docs, and generated anonymized live baseline artifacts for both channel-origin and gateway-origin traffic (including cancel-path coverage)",
"next_up": "Phase-0 baseline windows now cover channel and gateway sources; keep both artifact windows refreshed on cadence before additional run-control/reaction semantic changes.", "next_up": "Phase-0 baseline refresh flow is now automated for channel + gateway windows (`pnpm audit:phase0-baseline:live:refresh`); next step is scheduling this command on an operational cadence before additional run-control/reaction semantic changes.",
"pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default", "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
"pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)", "pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)",
"pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing", "pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing",
+2
View File
@@ -23,6 +23,8 @@
"audit:backend-canary": "node --import tsx/esm scripts/summarize-backend-canary.ts", "audit:backend-canary": "node --import tsx/esm scripts/summarize-backend-canary.ts",
"audit:phase0-baseline": "node --import tsx/esm scripts/summarize-phase0-baseline.ts", "audit:phase0-baseline": "node --import tsx/esm scripts/summarize-phase0-baseline.ts",
"audit:phase0-baseline:live": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source channel --exclude-session-substring probe --tag 2026-02-27", "audit:phase0-baseline:live": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source channel --exclude-session-substring probe --tag 2026-02-27",
"audit:phase0-baseline:live:gateway": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --auto-gateway-cancel-window --tag 2026-02-27",
"audit:phase0-baseline:live:refresh": "pnpm audit:phase0-baseline:live && pnpm audit:phase0-baseline:live:gateway",
"audit:backend-canary:probes": "node --import tsx/esm scripts/run-pi-canary-guard-probes.ts", "audit:backend-canary:probes": "node --import tsx/esm scripts/run-pi-canary-guard-probes.ts",
"companion:bundle": "node --import tsx/esm scripts/build-companion-release-bundle.ts", "companion:bundle": "node --import tsx/esm scripts/build-companion-release-bundle.ts",
"companion:reference-apps": "node --import tsx/esm scripts/export-companion-reference-apps.ts", "companion:reference-apps": "node --import tsx/esm scripts/export-companion-reference-apps.ts",
+54 -7
View File
@@ -5,6 +5,7 @@ import { dirname, resolve } from 'node:path';
import { parseArgs } from 'node:util'; import { parseArgs } from 'node:util';
import { queryAuditLogs } from '../src/audit/export.js'; import { queryAuditLogs } from '../src/audit/export.js';
import { capturePhase0LiveBaselineEvents } from '../src/audit/phase0LiveBaseline.js'; import { capturePhase0LiveBaselineEvents } from '../src/audit/phase0LiveBaseline.js';
import { findLatestGatewayCancelWindow } from '../src/audit/phase0GatewayWindow.js';
import { import {
renderPhase0BaselineMarkdown, renderPhase0BaselineMarkdown,
summarizePhase0Baseline, summarizePhase0Baseline,
@@ -25,6 +26,8 @@ function usage(): string {
' --channel <name[,name...]> Restrict sample to channels', ' --channel <name[,name...]> Restrict sample to channels',
' --source <gateway|channel[,..]> Restrict sample to sources', ' --source <gateway|channel[,..]> Restrict sample to sources',
' --exclude-session-substring <text[,..]> Exclude sessions containing any substring (default: probe)', ' --exclude-session-substring <text[,..]> Exclude sessions containing any substring (default: probe)',
' --auto-gateway-cancel-window Auto-select latest gateway cancel/cancelled session window',
' --window-padding-ms <number> Milliseconds added before/after auto-selected window (default: 250)',
' --raw-identifiers Keep raw session/sender/request IDs (default: anonymized)', ' --raw-identifiers Keep raw session/sender/request IDs (default: anonymized)',
' --tag <YYYY-MM-DD> Output file tag (default: current date UTC)', ' --tag <YYYY-MM-DD> Output file tag (default: current date UTC)',
' --sample-out <path> Output JSONL sample path override', ' --sample-out <path> Output JSONL sample path override',
@@ -128,6 +131,8 @@ async function main(): Promise<void> {
channel: { type: 'string' }, channel: { type: 'string' },
source: { type: 'string' }, source: { type: 'string' },
'exclude-session-substring': { type: 'string' }, 'exclude-session-substring': { type: 'string' },
'auto-gateway-cancel-window': { type: 'boolean' },
'window-padding-ms': { type: 'string' },
'raw-identifiers': { type: 'boolean' }, 'raw-identifiers': { type: 'boolean' },
tag: { type: 'string' }, tag: { type: 'string' },
'sample-out': { type: 'string' }, 'sample-out': { type: 'string' },
@@ -149,14 +154,48 @@ async function main(): Promise<void> {
const auditPath = expandHomePath(values.audit ?? '~/.local/share/flynn/audit.log'); const auditPath = expandHomePath(values.audit ?? '~/.local/share/flynn/audit.log');
const tag = values.tag ?? isoDateTagNow(); const tag = values.tag ?? isoDateTagNow();
const sampleOut = values['sample-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.jsonl`;
const summaryJsonOut = values['summary-json-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.json`;
const summaryMdOut = values['summary-md-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.md`;
const channels = parseCsv(values.channel); const channels = parseCsv(values.channel);
const sources = parseSources(values.source); let sources = parseSources(values.source);
const excludeSessionSubstrings = parseCsv(values['exclude-session-substring']) ?? ['probe']; const excludeSessionSubstrings = parseCsv(values['exclude-session-substring']) ?? ['probe'];
const startTime = parseTime(values.since, '--since'); const autoGatewayCancelWindow = Boolean(values['auto-gateway-cancel-window']);
const endTime = parseTime(values.until, '--until'); const windowPaddingMs = parseOptionalNumber(values['window-padding-ms'], '--window-padding-ms');
if (windowPaddingMs !== undefined && windowPaddingMs < 0) {
throw new Error('--window-padding-ms must be greater than or equal to 0.');
}
if (autoGatewayCancelWindow && (values.since || values.until)) {
throw new Error('--auto-gateway-cancel-window cannot be combined with --since/--until.');
}
if (autoGatewayCancelWindow && sources && !sources.includes('gateway')) {
throw new Error('--auto-gateway-cancel-window requires --source to include "gateway" (or omit --source).');
}
let startTime = parseTime(values.since, '--since');
let endTime = parseTime(values.until, '--until');
let autoWindow: ReturnType<typeof findLatestGatewayCancelWindow> = null;
if (autoGatewayCancelWindow) {
sources = sources ?? ['gateway'];
const autoWindowSourceEvents = await queryAuditLogs(auditPath, {
event_types: ['run.state', 'run.cancel'],
});
autoWindow = findLatestGatewayCancelWindow(autoWindowSourceEvents, {
padding_ms: windowPaddingMs ?? 250,
});
if (!autoWindow) {
throw new Error('No gateway cancel/cancelled session window found in audit log.');
}
startTime = autoWindow.start_time_ms;
endTime = autoWindow.end_time_ms;
}
const isGatewayOnly = sources?.length === 1 && sources[0] === 'gateway';
const defaultBaseName = isGatewayOnly
? `docs/plans/artifacts/phase0_baseline_live_gateway_${tag}`
: `docs/plans/artifacts/phase0_baseline_live_${tag}`;
const sampleOut = values['sample-out'] ?? `${defaultBaseName}.jsonl`;
const summaryJsonOut = values['summary-json-out'] ?? `${defaultBaseName}.json`;
const summaryMdOut = values['summary-md-out'] ?? `${defaultBaseName}.md`;
const summaryOptions: Phase0BaselineSummaryOptions = { const summaryOptions: Phase0BaselineSummaryOptions = {
channels, channels,
@@ -194,6 +233,12 @@ async function main(): Promise<void> {
sources, sources,
exclude_session_substrings: excludeSessionSubstrings, exclude_session_substrings: excludeSessionSubstrings,
anonymized_identifiers: !values['raw-identifiers'], anonymized_identifiers: !values['raw-identifiers'],
auto_gateway_cancel_window: autoWindow
? {
...autoWindow,
padding_ms: windowPaddingMs ?? 250,
}
: undefined,
}, },
options: summaryOptions, options: summaryOptions,
summary, summary,
@@ -204,6 +249,9 @@ async function main(): Promise<void> {
await writeTextFile(summaryMdOut, `${markdown}\n`); await writeTextFile(summaryMdOut, `${markdown}\n`);
process.stdout.write(`Captured ${sampledEvents.length} events from ${sourceEvents.length} source events.\n`); process.stdout.write(`Captured ${sampledEvents.length} events from ${sourceEvents.length} source events.\n`);
if (autoWindow) {
process.stdout.write(`- auto gateway window: session=${autoWindow.session_id} start=${autoWindow.start_time_ms} end=${autoWindow.end_time_ms}\n`);
}
process.stdout.write(`- sample: ${sampleOut}\n`); process.stdout.write(`- sample: ${sampleOut}\n`);
process.stdout.write(`- summary json: ${summaryJsonOut}\n`); process.stdout.write(`- summary json: ${summaryJsonOut}\n`);
process.stdout.write(`- summary md: ${summaryMdOut}\n`); process.stdout.write(`- summary md: ${summaryMdOut}\n`);
@@ -214,4 +262,3 @@ main().catch((error) => {
process.stderr.write(`${message}\n\n${usage()}\n`); process.stderr.write(`${message}\n\n${usage()}\n`);
process.exitCode = 1; process.exitCode = 1;
}); });
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from 'vitest';
import type { AuditEvent } from './types.js';
import { findLatestGatewayCancelWindow } from './phase0GatewayWindow.js';
function event(
timestamp: number,
eventType: AuditEvent['event_type'],
payload: Record<string, unknown>,
): AuditEvent {
return {
timestamp,
level: 'info',
event_type: eventType,
event: payload,
};
}
describe('findLatestGatewayCancelWindow', () => {
it('returns the latest gateway session containing run.cancel and cancelled states', () => {
const events: AuditEvent[] = [
event(100, 'run.state', { session_id: 'old', source: 'gateway', state: 'start' }),
event(110, 'run.cancel', { session_id: 'old', source: 'gateway', acknowledged: true }),
event(115, 'run.state', { session_id: 'old', source: 'gateway', state: 'cancel_requested' }),
event(120, 'run.state', { session_id: 'old', source: 'gateway', state: 'cancelled' }),
event(200, 'run.state', { session_id: 'new', source: 'gateway', state: 'start' }),
event(210, 'run.cancel', { session_id: 'new', source: 'gateway', acknowledged: true }),
event(220, 'run.state', { session_id: 'new', source: 'gateway', state: 'cancelled' }),
event(300, 'run.state', { session_id: 'channel', source: 'channel', state: 'cancelled' }),
];
const window = findLatestGatewayCancelWindow(events);
expect(window).toEqual({
session_id: 'new',
start_time_ms: 200,
end_time_ms: 220,
event_count: 3,
run_cancel_count: 1,
cancel_requested_count: 0,
cancelled_count: 1,
});
});
it('applies padding and ignores malformed/missing payload fields', () => {
const events: AuditEvent[] = [
event(1000, 'run.state', { session_id: 's1', source: 'gateway', state: 'start' }),
event(1010, 'run.cancel', { session_id: 's1', source: 'gateway' }),
event(1020, 'run.state', { session_id: 's1', source: 'gateway', state: 'cancel_requested' }),
event(1030, 'run.state', { session_id: 's1', source: 'gateway', state: 'cancelled' }),
event(1040, 'run.cancel', { source: 'gateway' }),
event(1050, 'run.state', { session_id: 's2', state: 'cancelled' }),
];
const window = findLatestGatewayCancelWindow(events, { padding_ms: 25 });
expect(window).toEqual({
session_id: 's1',
start_time_ms: 975,
end_time_ms: 1055,
event_count: 4,
run_cancel_count: 1,
cancel_requested_count: 1,
cancelled_count: 1,
});
});
it('returns null when no gateway cancel+cancelled window exists', () => {
const events: AuditEvent[] = [
event(1, 'run.state', { session_id: 's1', source: 'gateway', state: 'start' }),
event(2, 'run.state', { session_id: 's1', source: 'gateway', state: 'complete' }),
event(3, 'run.cancel', { session_id: 's2', source: 'channel' }),
];
expect(findLatestGatewayCancelWindow(events)).toBeNull();
});
});
+121
View File
@@ -0,0 +1,121 @@
import type { AuditEvent } from './types.js';
export interface GatewayCancelWindowSummary {
session_id: string;
start_time_ms: number;
end_time_ms: number;
event_count: number;
run_cancel_count: number;
cancel_requested_count: number;
cancelled_count: number;
}
export interface FindGatewayCancelWindowOptions {
padding_ms?: number;
}
interface SessionWindowAccumulator {
session_id: string;
min_ts: number;
max_ts: number;
event_count: number;
run_cancel_count: number;
cancel_requested_count: number;
cancelled_count: number;
}
function toPayload(value: unknown): Record<string, unknown> {
return (value && typeof value === 'object') ? value as Record<string, unknown> : {};
}
function readString(value: unknown): string | undefined {
return typeof value === 'string' ? value : undefined;
}
function isGatewayEvent(payload: Record<string, unknown>): boolean {
return readString(payload.source) === 'gateway';
}
export function findLatestGatewayCancelWindow(
events: AuditEvent[],
options: FindGatewayCancelWindowOptions = {},
): GatewayCancelWindowSummary | null {
const bySession = new Map<string, SessionWindowAccumulator>();
for (const event of events) {
if (event.event_type !== 'run.state' && event.event_type !== 'run.cancel') {
continue;
}
const payload = toPayload(event.event);
if (!isGatewayEvent(payload)) {
continue;
}
const sessionId = readString(payload.session_id);
if (!sessionId) {
continue;
}
const acc = bySession.get(sessionId) ?? {
session_id: sessionId,
min_ts: event.timestamp,
max_ts: event.timestamp,
event_count: 0,
run_cancel_count: 0,
cancel_requested_count: 0,
cancelled_count: 0,
};
acc.event_count += 1;
acc.min_ts = Math.min(acc.min_ts, event.timestamp);
acc.max_ts = Math.max(acc.max_ts, event.timestamp);
if (event.event_type === 'run.cancel') {
acc.run_cancel_count += 1;
} else {
const state = readString(payload.state);
if (state === 'cancel_requested') {
acc.cancel_requested_count += 1;
} else if (state === 'cancelled') {
acc.cancelled_count += 1;
}
}
bySession.set(sessionId, acc);
}
const candidates = [...bySession.values()]
.filter((row) => row.run_cancel_count > 0 && row.cancelled_count > 0)
.sort((a, b) => {
const tsDelta = b.max_ts - a.max_ts;
if (tsDelta !== 0) {
return tsDelta;
}
const cancelDelta = b.run_cancel_count - a.run_cancel_count;
if (cancelDelta !== 0) {
return cancelDelta;
}
return a.session_id.localeCompare(b.session_id);
});
const latest = candidates[0];
if (!latest) {
return null;
}
const padRaw = options.padding_ms ?? 0;
const paddingMs = Number.isFinite(padRaw) && padRaw > 0
? Math.floor(padRaw)
: 0;
return {
session_id: latest.session_id,
start_time_ms: Math.max(0, latest.min_ts - paddingMs),
end_time_ms: latest.max_ts + paddingMs,
event_count: latest.event_count,
run_cancel_count: latest.run_cancel_count,
cancel_requested_count: latest.cancel_requested_count,
cancelled_count: latest.cancelled_count,
};
}