feat(audit): automate gateway phase0 live-window capture
This commit is contained in:
William Valentin
@@ -203,7 +203,7 @@ Phase 0 is complete when:
|
||||
2. A baseline summary artifact is generated and committed under `docs/plans/artifacts/`.
|
||||
3. No user-visible response behavior changed compared to pre-phase baseline.
|
||||
|
||||
Follow-up status (2026-02-27): live channel-session artifacts exist under `docs/plans/artifacts/phase0_baseline_live_2026-02-27.*` via `pnpm audit:phase0-baseline:live` (anonymized IDs), and a second gateway-origin live window (including `run.cancel` + `cancel_requested`/`cancelled`) exists under `docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.*`.
|
||||
Follow-up status (2026-02-27): live channel-session artifacts exist under `docs/plans/artifacts/phase0_baseline_live_2026-02-27.*` via `pnpm audit:phase0-baseline:live` (anonymized IDs), and a second gateway-origin live window (including `run.cancel` + `cancel_requested`/`cancelled`) exists under `docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.*`. Gateway window refreshes can now run via `pnpm audit:phase0-baseline:live:gateway` (auto-selected cancel window), and both windows can be refreshed together with `pnpm audit:phase0-baseline:live:refresh`.
|
||||
|
||||
## Subagent Model Assignment Plan
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"generated_at": "2026-02-27T07:49:58.821Z",
|
||||
"generated_at": "2026-02-27T07:55:30.862Z",
|
||||
"source_audit_path": "~/.local/share/flynn/audit.log",
|
||||
"source_event_count": 94,
|
||||
"sampled_event_count": 88,
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
{
|
||||
"generated_at": "2026-02-27T07:47:41.346Z",
|
||||
"generated_at": "2026-02-27T07:55:31.178Z",
|
||||
"source_audit_path": "~/.local/share/flynn/audit.log",
|
||||
"source_event_count": 6,
|
||||
"sampled_event_count": 6,
|
||||
"filters": {
|
||||
"since_ms": 1772178440693,
|
||||
"until_ms": 1772178442694,
|
||||
"since_ms": 1772178441443,
|
||||
"until_ms": 1772178441944,
|
||||
"sources": [
|
||||
"gateway"
|
||||
],
|
||||
"exclude_session_substrings": [
|
||||
"probe"
|
||||
],
|
||||
"anonymized_identifiers": true
|
||||
"anonymized_identifiers": true,
|
||||
"auto_gateway_cancel_window": {
|
||||
"session_id": "ws:phase0-gateway-window",
|
||||
"start_time_ms": 1772178441443,
|
||||
"end_time_ms": 1772178441944,
|
||||
"event_count": 6,
|
||||
"run_cancel_count": 1,
|
||||
"cancel_requested_count": 1,
|
||||
"cancelled_count": 1,
|
||||
"padding_ms": 250
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"sources": [
|
||||
|
||||
+27
-4
@@ -88,10 +88,12 @@
|
||||
"status": "completed",
|
||||
"date": "2026-02-27",
|
||||
"updated": "2026-02-27",
|
||||
"summary": "Added a dedicated live phase-0 baseline capture flow that reads audit logs, filters run/reaction telemetry, excludes probe sessions, anonymizes session/sender/request IDs, and writes sample + summary artifacts for operational refreshes across both channel-origin and gateway-origin windows.",
|
||||
"summary": "Added a dedicated live phase-0 baseline capture flow that reads audit logs, filters run/reaction telemetry, excludes probe sessions, anonymizes session/sender/request IDs, and writes sample + summary artifacts for operational refreshes across both channel-origin and gateway-origin windows. Gateway mode now supports auto-detection of the latest cancel/cancelled window.",
|
||||
"files_modified": [
|
||||
"src/audit/phase0LiveBaseline.ts",
|
||||
"src/audit/phase0LiveBaseline.test.ts",
|
||||
"src/audit/phase0GatewayWindow.ts",
|
||||
"src/audit/phase0GatewayWindow.test.ts",
|
||||
"scripts/capture-phase0-live-baseline.ts",
|
||||
"package.json",
|
||||
"README.md",
|
||||
@@ -107,7 +109,7 @@
|
||||
"docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json",
|
||||
"docs/plans/state.json"
|
||||
],
|
||||
"test_status": "pnpm audit:phase0-baseline:live + node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --since 1772178440693 --until 1772178442694 --sample-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.jsonl --summary-json-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json --summary-md-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.md + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
||||
"test_status": "pnpm audit:phase0-baseline:live:refresh + pnpm test:run src/audit/phase0GatewayWindow.test.ts src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
||||
},
|
||||
"phase0-live-baseline-gateway-window": {
|
||||
"status": "completed",
|
||||
@@ -127,6 +129,27 @@
|
||||
],
|
||||
"test_status": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source gateway --since 1772178440693 --until 1772178442694 --sample-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.jsonl --summary-json-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json --summary-md-out docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.md + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
||||
},
|
||||
"phase0-live-baseline-refresh-automation": {
|
||||
"status": "completed",
|
||||
"date": "2026-02-27",
|
||||
"updated": "2026-02-27",
|
||||
"summary": "Automated gateway live-window capture by adding auto-detection of the latest gateway cancel/cancelled window (`--auto-gateway-cancel-window`) plus a one-shot refresh command that regenerates both channel and gateway artifacts together (`pnpm audit:phase0-baseline:live:refresh`).",
|
||||
"files_modified": [
|
||||
"src/audit/phase0GatewayWindow.ts",
|
||||
"src/audit/phase0GatewayWindow.test.ts",
|
||||
"scripts/capture-phase0-live-baseline.ts",
|
||||
"package.json",
|
||||
"README.md",
|
||||
"docs/api/PROTOCOL.md",
|
||||
"docs/architecture/AGENT_DIAGRAM.md",
|
||||
"docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md",
|
||||
"docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md",
|
||||
"docs/plans/artifacts/phase0_baseline_live_2026-02-27.json",
|
||||
"docs/plans/artifacts/phase0_baseline_live_gateway_2026-02-27.json",
|
||||
"docs/plans/state.json"
|
||||
],
|
||||
"test_status": "pnpm audit:phase0-baseline:live:refresh + pnpm test:run src/audit/phase0GatewayWindow.test.ts src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
||||
},
|
||||
"phase0-instrumentation-ticket-checklist": {
|
||||
"status": "completed",
|
||||
"date": "2026-02-25",
|
||||
@@ -7276,7 +7299,7 @@
|
||||
}
|
||||
},
|
||||
"overall_progress": {
|
||||
"total_test_count": 2585,
|
||||
"total_test_count": 2588,
|
||||
"all_tests_passing": true,
|
||||
"p0_completion": "3/3 (100%)",
|
||||
"p1_completion": "4/4 (100%)",
|
||||
@@ -7313,7 +7336,7 @@
|
||||
"deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters",
|
||||
"deeper_surfaces_phase0_ticket_04": "completed — added phase-0 baseline summary tooling for run outcomes, cancel latency, and reaction decisions with markdown/json CLI output",
|
||||
"deeper_surfaces_phase0_ticket_05": "completed — documented phase-0 telemetry fields/workflow, refreshed architecture/protocol docs, and generated anonymized live baseline artifacts for both channel-origin and gateway-origin traffic (including cancel-path coverage)",
|
||||
"next_up": "Phase-0 baseline windows now cover channel and gateway sources; keep both artifact windows refreshed on cadence before additional run-control/reaction semantic changes.",
|
||||
"next_up": "Phase-0 baseline refresh flow is now automated for channel + gateway windows (`pnpm audit:phase0-baseline:live:refresh`); next step is scheduling this command on an operational cadence before additional run-control/reaction semantic changes.",
|
||||
"pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
|
||||
"pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)",
|
||||
"pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing",
|
||||
|
||||
Reference in New Issue
Block a user