From 4b07a1f16612d2e521feb3ec32084237c06e70cc Mon Sep 17 00:00:00 2001 From: William Valentin Date: Thu, 26 Feb 2026 23:41:13 -0800 Subject: [PATCH] feat(audit): replace probe baseline workflow with live anonymized capture --- README.md | 5 + docs/api/PROTOCOL.md | 2 +- docs/architecture/AGENT_DIAGRAM.md | 1 + .../GATEWAY_SESSIONS_AND_QUEUE.md | 1 + ...phase0-instrumentation-ticket-checklist.md | 2 + .../phase0_baseline_live_2026-02-27.json | 371 ++++++++++++++++++ .../phase0_baseline_live_2026-02-27.jsonl | 88 +++++ .../phase0_baseline_live_2026-02-27.md | 65 +++ docs/plans/state.json | 40 +- package.json | 1 + scripts/capture-phase0-live-baseline.ts | 217 ++++++++++ src/audit/phase0LiveBaseline.test.ts | 79 ++++ src/audit/phase0LiveBaseline.ts | 106 +++++ 13 files changed, 968 insertions(+), 10 deletions(-) create mode 100644 docs/plans/artifacts/phase0_baseline_live_2026-02-27.json create mode 100644 docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl create mode 100644 docs/plans/artifacts/phase0_baseline_live_2026-02-27.md create mode 100644 scripts/capture-phase0-live-baseline.ts create mode 100644 src/audit/phase0LiveBaseline.test.ts create mode 100644 src/audit/phase0LiveBaseline.ts diff --git a/README.md b/README.md index ec770fa..266cc4a 100644 --- a/README.md +++ b/README.md @@ -1624,6 +1624,11 @@ Baseline summaries can be generated from audit logs: pnpm audit:phase0-baseline --audit ~/.local/share/flynn/audit.log --since 2026-02-25T00:00:00Z --format markdown ``` +Live baseline artifacts (sample JSONL + JSON/Markdown summaries) can be captured with anonymized identifiers: +```bash +pnpm audit:phase0-baseline:live +``` + ## Gateway Lock Single-client mode for the WebSocket gateway. When enabled, only one WebSocket connection is allowed at a time. Additional connections are rejected with close code `4003`. diff --git a/docs/api/PROTOCOL.md b/docs/api/PROTOCOL.md index 2b0387d..f80316c 100644 --- a/docs/api/PROTOCOL.md +++ b/docs/api/PROTOCOL.md @@ -23,7 +23,7 @@ The gateway provides: - **HTTP Server**: Serves static dashboard and handles webhook endpoints - **Node Capability Negotiation**: Optional companion-node role/capability registration -Operational note: onboarding (`flynn setup` / `flynn onboard`) now runs post-save live readiness checks (model/channel/memory/automation) and prints a guided first-success task flow. Companion CLI now also supports bootstrap-manifest export (`flynn companion --export-bootstrap `), release-bundle export (`--export-release-bundle ` with optional `--signing-key`/`--signing-key-id` signature output), release-bundle verification (`--verify-release-bundle ` with optional `--verify-signing-key`/`--verify-signing-key-id`/`--require-signature`), platform shell-template export (`--export-shell-template `), plus richer shell bootstrap flags for status/location/push (`--app-version`, `--latitude/--longitude`, `--push-token`, etc.) for desktop/mobile app packaging without changing JSON-RPC method/event shapes. +Operational note: onboarding (`flynn setup` / `flynn onboard`) now runs post-save live readiness checks (model/channel/memory/automation) and prints a guided first-success task flow. Companion CLI now also supports bootstrap-manifest export (`flynn companion --export-bootstrap `), release-bundle export (`--export-release-bundle ` with optional `--signing-key`/`--signing-key-id` signature output), release-bundle verification (`--verify-release-bundle ` with optional `--verify-signing-key`/`--verify-signing-key-id`/`--require-signature`), platform shell-template export (`--export-shell-template `), plus richer shell bootstrap flags for status/location/push (`--app-version`, `--latitude/--longitude`, `--push-token`, etc.) for desktop/mobile app packaging without changing JSON-RPC method/event shapes. Audit observability now includes a live phase-0 baseline capture flow (`pnpm audit:phase0-baseline:live`) that emits anonymized run/reaction sample artifacts. ### Execution Model (Sessions + Per-Session Queue) diff --git a/docs/architecture/AGENT_DIAGRAM.md b/docs/architecture/AGENT_DIAGRAM.md index 1f1bcbb..2d45474 100644 --- a/docs/architecture/AGENT_DIAGRAM.md +++ b/docs/architecture/AGENT_DIAGRAM.md @@ -166,6 +166,7 @@ Gateway streaming UX signals: - `.github/workflows/companion-release-bundle.yml` provides CI artifact generation for companion release bundles using the same build-and-verify pipeline. - `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync in CI. - `flynn companion` can bootstrap status/location/push metadata on connect (`node.status.set` + optional `node.location.set`/`node.push_token.set`) so thin companion shells can register operational context in one launch. +- `pnpm audit:phase0-baseline:live` captures anonymized live run/reaction baseline artifacts from real audit logs to replace probe-only telemetry samples. - Canvas artifacts are persisted by the gateway so session UI surfaces can recover after daemon restarts. - TTS synthesis uses an ordered provider chain with health cooldown tracking; if all providers fail, replies degrade to text-only without dropping the response. - Talk mode accepts spoken/text `stop`/`cancel` while active and maps it onto the same `/stop` run-control cancellation path used for text sessions. diff --git a/docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md b/docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md index f9b221e..6956805 100644 --- a/docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md +++ b/docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md @@ -31,6 +31,7 @@ If you only want the protocol surface, see `docs/api/PROTOCOL.md`. - Companion reference-app sync can be enforced with `pnpm companion:reference-apps:check` (regenerate + diff fail on drift). - CI workflow `.github/workflows/companion-release-bundle.yml` mirrors this pipeline for manual artifact generation/upload. - CI workflow `.github/workflows/companion-reference-apps-check.yml` enforces reference-app generator sync on pull requests. +- Audit phase-0 live telemetry snapshots can be regenerated with `pnpm audit:phase0-baseline:live` (anonymized sample JSONL + summary JSON/markdown artifacts). - Companion CLI supports one-shot shell bootstrap metadata for live sessions (`--app-version`/`--status-text`, `--latitude`/`--longitude`, `--push-token`) so desktop/mobile wrappers can initialize node status/location/push in a single launch flow. - Canvas artifacts are persisted per session under the gateway data directory for UI recovery across restarts. - TTS output is best-effort with ordered provider fallback + per-provider cooldown tracking; synthesis failures still fall back to text-only responses. diff --git a/docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md b/docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md index 78c39eb..1a95bf9 100644 --- a/docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md +++ b/docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md @@ -203,6 +203,8 @@ Phase 0 is complete when: 2. A baseline summary artifact is generated and committed under `docs/plans/artifacts/`. 3. No user-visible response behavior changed compared to pre-phase baseline. +Follow-up status (2026-02-27): live channel-session artifacts now exist under `docs/plans/artifacts/phase0_baseline_live_2026-02-27.*` via `pnpm audit:phase0-baseline:live` (anonymized IDs); gateway-origin live samples remain a future slice. + ## Subagent Model Assignment Plan 1. `claude-haiku-4.5`: diff --git a/docs/plans/artifacts/phase0_baseline_live_2026-02-27.json b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.json new file mode 100644 index 0000000..10c7871 --- /dev/null +++ b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.json @@ -0,0 +1,371 @@ +{ + "generated_at": "2026-02-27T07:39:16.384Z", + "source_audit_path": "~/.local/share/flynn/audit.log", + "source_event_count": 88, + "sampled_event_count": 88, + "filters": { + "sources": [ + "channel" + ], + "exclude_session_substrings": [ + "probe" + ], + "anonymized_identifiers": true + }, + "options": { + "sources": [ + "channel" + ], + "maxSessions": 20, + "maxChannels": 20, + "maxSkipReasons": 10 + }, + "summary": { + "event_counts": { + "run_state": 55, + "run_cancel": 0, + "reaction_match": 0, + "reaction_skip": 33 + }, + "run_outcomes": { + "overall": { + "total_outcomes": 23, + "complete": 23, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 32, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + }, + "by_channel": [ + { + "key": "gmail", + "stats": { + "total_outcomes": 22, + "complete": 22, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 22, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "cron", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 10, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + } + ], + "by_session": [ + { + "key": "session_2f2f1e414e81", + "stats": { + "total_outcomes": 4, + "complete": 4, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 4, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_f4d8ddc04194", + "stats": { + "total_outcomes": 3, + "complete": 3, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 3, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_eabc3c2a91b9", + "stats": { + "total_outcomes": 2, + "complete": 2, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 2, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_33469de5a1ee", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_4d9e843358a3", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_58a64b6f2c91", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_7d3c3ff67d4f", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_8849a4464275", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_8b51db8cde21", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_9067cf5e3558", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_a4b91821c664", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_a83fde4c8fdb", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_cb9a69d8a362", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_e0a2a17b7329", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_ea839415979e", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_f6304f25e43b", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_fd6536fa5ff4", + "stats": { + "total_outcomes": 1, + "complete": 1, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": 100, + "cancel_rate_pct": 0, + "error_rate_pct": 0 + } + }, + { + "key": "session_2b07a8d38406", + "stats": { + "total_outcomes": 0, + "complete": 0, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": null, + "cancel_rate_pct": null, + "error_rate_pct": null + } + }, + { + "key": "session_2d8872945bf8", + "stats": { + "total_outcomes": 0, + "complete": 0, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": null, + "cancel_rate_pct": null, + "error_rate_pct": null + } + }, + { + "key": "session_31b6400467ce", + "stats": { + "total_outcomes": 0, + "complete": 0, + "cancelled": 0, + "error": 0, + "cancel_requested": 0, + "start": 1, + "completion_rate_pct": null, + "cancel_rate_pct": null, + "error_rate_pct": null + } + } + ] + }, + "cancel_latency_ms": null, + "reactions": { + "matched": 0, + "skipped": 33, + "total": 33, + "match_rate_pct": 0, + "skip_rate_pct": 100, + "skip_reasons": [ + { + "reason": "no_rules", + "count": 33, + "pct": 100 + } + ] + } + } +} diff --git a/docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl new file mode 100644 index 0000000..275f9e0 --- /dev/null +++ b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl @@ -0,0 +1,88 @@ +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2d8872945bf8","channel":"cron","sender":"sender_4787722f90c7","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772082000080} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2d8872945bf8","channel":"cron","sender":"sender_4787722f90c7","source":"channel","state":"start","request_id":"request_eabae852ec40"},"timestamp":1772082000082} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_ffcee254d546","channel":"cron","sender":"sender_75c445c6fdad","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772089200084} +{"level":"info","event_type":"run.state","event":{"session_id":"session_ffcee254d546","channel":"cron","sender":"sender_75c445c6fdad","source":"channel","state":"start","request_id":"request_f99421283d74"},"timestamp":1772089200086} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_49b700741e03","channel":"cron","sender":"sender_ecc7d1bae06e","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772096400007} +{"level":"info","event_type":"run.state","event":{"session_id":"session_49b700741e03","channel":"cron","sender":"sender_ecc7d1bae06e","source":"channel","state":"start","request_id":"request_33cb5e8b6843"},"timestamp":1772096400008} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772107655159} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"start","request_id":"request_7da150aff098"},"timestamp":1772107655160} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"complete","request_id":"request_7da150aff098","duration_ms":3324},"timestamp":1772107658484} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_4cd8ba5e6df5","channel":"cron","sender":"sender_247a7c21dbdd","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772110800018} +{"level":"info","event_type":"run.state","event":{"session_id":"session_4cd8ba5e6df5","channel":"cron","sender":"sender_247a7c21dbdd","source":"channel","state":"start","request_id":"request_60b338f30d46"},"timestamp":1772110800019} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772114255688} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"start","request_id":"request_55013bd2ec5f"},"timestamp":1772114255688} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"complete","request_id":"request_55013bd2ec5f","duration_ms":3006},"timestamp":1772114258694} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772119955933} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"start","request_id":"request_a01be9a4284b"},"timestamp":1772119955933} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"complete","request_id":"request_a01be9a4284b","duration_ms":2379},"timestamp":1772119958312} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772120856043} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"start","request_id":"request_49c2900b17a3"},"timestamp":1772120856043} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2f2f1e414e81","channel":"gmail","sender":"sender_323cedc3233a","source":"channel","state":"complete","request_id":"request_49c2900b17a3","duration_ms":4223},"timestamp":1772120860266} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_a83fde4c8fdb","channel":"cron","sender":"sender_99a8e5949abe","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772121600016} +{"level":"info","event_type":"run.state","event":{"session_id":"session_a83fde4c8fdb","channel":"cron","sender":"sender_99a8e5949abe","source":"channel","state":"start","request_id":"request_011dde9a88a3"},"timestamp":1772121600017} +{"level":"info","event_type":"run.state","event":{"session_id":"session_a83fde4c8fdb","channel":"cron","sender":"sender_99a8e5949abe","source":"channel","state":"complete","request_id":"request_011dde9a88a3","duration_ms":41131},"timestamp":1772121641148} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_cb9a69d8a362","channel":"gmail","sender":"sender_48feae1a0ad8","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772122956201} +{"level":"info","event_type":"run.state","event":{"session_id":"session_cb9a69d8a362","channel":"gmail","sender":"sender_48feae1a0ad8","source":"channel","state":"start","request_id":"request_59fd88029c97"},"timestamp":1772122956201} +{"level":"info","event_type":"run.state","event":{"session_id":"session_cb9a69d8a362","channel":"gmail","sender":"sender_48feae1a0ad8","source":"channel","state":"complete","request_id":"request_59fd88029c97","duration_ms":2854},"timestamp":1772122959055} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_3c43a0cc0a62","channel":"cron","sender":"sender_a2f138926e17","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772125200025} +{"level":"info","event_type":"run.state","event":{"session_id":"session_3c43a0cc0a62","channel":"cron","sender":"sender_a2f138926e17","source":"channel","state":"start","request_id":"request_eabfc26524d0"},"timestamp":1772125200026} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_a4b91821c664","channel":"gmail","sender":"sender_9eff7c852e06","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772129742056} +{"level":"info","event_type":"run.state","event":{"session_id":"session_a4b91821c664","channel":"gmail","sender":"sender_9eff7c852e06","source":"channel","state":"start","request_id":"request_7734a5dbc98e"},"timestamp":1772129742057} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772129742379} +{"level":"info","event_type":"run.state","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","state":"start","request_id":"request_81d00f26b8cb"},"timestamp":1772129742379} +{"level":"info","event_type":"run.state","event":{"session_id":"session_a4b91821c664","channel":"gmail","sender":"sender_9eff7c852e06","source":"channel","state":"complete","request_id":"request_7734a5dbc98e","duration_ms":5944},"timestamp":1772129748001} +{"level":"info","event_type":"run.state","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","state":"complete","request_id":"request_81d00f26b8cb","duration_ms":8017},"timestamp":1772129750396} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_58a64b6f2c91","channel":"gmail","sender":"sender_4222a55cdd53","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772131242612} +{"level":"info","event_type":"run.state","event":{"session_id":"session_58a64b6f2c91","channel":"gmail","sender":"sender_4222a55cdd53","source":"channel","state":"start","request_id":"request_e59d82ef75e8"},"timestamp":1772131242612} +{"level":"info","event_type":"run.state","event":{"session_id":"session_58a64b6f2c91","channel":"gmail","sender":"sender_4222a55cdd53","source":"channel","state":"complete","request_id":"request_e59d82ef75e8","duration_ms":5232},"timestamp":1772131247844} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_e0a2a17b7329","channel":"gmail","sender":"sender_5c8cb7bfc88d","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772131542650} +{"level":"info","event_type":"run.state","event":{"session_id":"session_e0a2a17b7329","channel":"gmail","sender":"sender_5c8cb7bfc88d","source":"channel","state":"start","request_id":"request_ed877aec1e58"},"timestamp":1772131542651} +{"level":"info","event_type":"run.state","event":{"session_id":"session_e0a2a17b7329","channel":"gmail","sender":"sender_5c8cb7bfc88d","source":"channel","state":"complete","request_id":"request_ed877aec1e58","duration_ms":6190},"timestamp":1772131548841} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_f6304f25e43b","channel":"gmail","sender":"sender_311c7608cc58","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772132142832} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f6304f25e43b","channel":"gmail","sender":"sender_311c7608cc58","source":"channel","state":"start","request_id":"request_8fdb3054a74d"},"timestamp":1772132142833} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772132142976} +{"level":"info","event_type":"run.state","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","state":"start","request_id":"request_487012d053f7"},"timestamp":1772132142976} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f6304f25e43b","channel":"gmail","sender":"sender_311c7608cc58","source":"channel","state":"complete","request_id":"request_8fdb3054a74d","duration_ms":3727},"timestamp":1772132146560} +{"level":"info","event_type":"run.state","event":{"session_id":"session_eabc3c2a91b9","channel":"gmail","sender":"sender_4fe02519d59e","source":"channel","state":"complete","request_id":"request_487012d053f7","duration_ms":4848},"timestamp":1772132147824} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_ea839415979e","channel":"gmail","sender":"sender_63a36881e696","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772133342779} +{"level":"info","event_type":"run.state","event":{"session_id":"session_ea839415979e","channel":"gmail","sender":"sender_63a36881e696","source":"channel","state":"start","request_id":"request_46feddcf35ba"},"timestamp":1772133342779} +{"level":"info","event_type":"run.state","event":{"session_id":"session_ea839415979e","channel":"gmail","sender":"sender_63a36881e696","source":"channel","state":"complete","request_id":"request_46feddcf35ba","duration_ms":2091},"timestamp":1772133344870} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_4d9e843358a3","channel":"gmail","sender":"sender_597782907690","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772135679506} +{"level":"info","event_type":"run.state","event":{"session_id":"session_4d9e843358a3","channel":"gmail","sender":"sender_597782907690","source":"channel","state":"start","request_id":"request_a43658b8d10f"},"timestamp":1772135679510} +{"level":"info","event_type":"run.state","event":{"session_id":"session_4d9e843358a3","channel":"gmail","sender":"sender_597782907690","source":"channel","state":"complete","request_id":"request_a43658b8d10f","duration_ms":7829},"timestamp":1772135687339} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_84ba9e30a4aa","channel":"telegram","sender":"sender_403740748465","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772135865178} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_7d3c3ff67d4f","channel":"gmail","sender":"sender_1625f89b7500","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772136364511} +{"level":"info","event_type":"run.state","event":{"session_id":"session_7d3c3ff67d4f","channel":"gmail","sender":"sender_1625f89b7500","source":"channel","state":"start","request_id":"request_882bdf0a0b51"},"timestamp":1772136364512} +{"level":"info","event_type":"run.state","event":{"session_id":"session_7d3c3ff67d4f","channel":"gmail","sender":"sender_1625f89b7500","source":"channel","state":"complete","request_id":"request_882bdf0a0b51","duration_ms":5046},"timestamp":1772136369558} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_494cb3b392af","channel":"cron","sender":"sender_dfc2df9eb18e","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772139600015} +{"level":"info","event_type":"run.state","event":{"session_id":"session_494cb3b392af","channel":"cron","sender":"sender_dfc2df9eb18e","source":"channel","state":"start","request_id":"request_ea48f4337dc3"},"timestamp":1772139600017} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772142493741} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"start","request_id":"request_b67bb7dcfb3e"},"timestamp":1772142493741} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"complete","request_id":"request_b67bb7dcfb3e","duration_ms":6271},"timestamp":1772142500012} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_8849a4464275","channel":"gmail","sender":"sender_5aa2798b691a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772143093516} +{"level":"info","event_type":"run.state","event":{"session_id":"session_8849a4464275","channel":"gmail","sender":"sender_5aa2798b691a","source":"channel","state":"start","request_id":"request_f0b319de0e0f"},"timestamp":1772143093516} +{"level":"info","event_type":"run.state","event":{"session_id":"session_8849a4464275","channel":"gmail","sender":"sender_5aa2798b691a","source":"channel","state":"complete","request_id":"request_f0b319de0e0f","duration_ms":3148},"timestamp":1772143096664} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_9067cf5e3558","channel":"gmail","sender":"sender_5375920e43c6","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772144293783} +{"level":"info","event_type":"run.state","event":{"session_id":"session_9067cf5e3558","channel":"gmail","sender":"sender_5375920e43c6","source":"channel","state":"start","request_id":"request_802086e4ecb7"},"timestamp":1772144293784} +{"level":"info","event_type":"run.state","event":{"session_id":"session_9067cf5e3558","channel":"gmail","sender":"sender_5375920e43c6","source":"channel","state":"complete","request_id":"request_802086e4ecb7","duration_ms":3535},"timestamp":1772144297319} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_33469de5a1ee","channel":"gmail","sender":"sender_23f2c718f92a","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772147893922} +{"level":"info","event_type":"run.state","event":{"session_id":"session_33469de5a1ee","channel":"gmail","sender":"sender_23f2c718f92a","source":"channel","state":"start","request_id":"request_e1ee7948be6e"},"timestamp":1772147893923} +{"level":"info","event_type":"run.state","event":{"session_id":"session_33469de5a1ee","channel":"gmail","sender":"sender_23f2c718f92a","source":"channel","state":"complete","request_id":"request_e1ee7948be6e","duration_ms":4090},"timestamp":1772147898013} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772151794148} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"start","request_id":"request_dc04b15aeb0d"},"timestamp":1772151794148} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772151794297} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"start","request_id":"request_f786a5385b23"},"timestamp":1772151794297} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"complete","request_id":"request_dc04b15aeb0d","duration_ms":8516},"timestamp":1772151802664} +{"level":"info","event_type":"run.state","event":{"session_id":"session_f4d8ddc04194","channel":"gmail","sender":"sender_c8a436a5eb54","source":"channel","state":"complete","request_id":"request_f786a5385b23","duration_ms":9850},"timestamp":1772151804147} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_2b07a8d38406","channel":"cron","sender":"sender_2a0d0e4fce24","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772154000024} +{"level":"info","event_type":"run.state","event":{"session_id":"session_2b07a8d38406","channel":"cron","sender":"sender_2a0d0e4fce24","source":"channel","state":"start","request_id":"request_136e87e1b4ce"},"timestamp":1772154000024} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_fd6536fa5ff4","channel":"gmail","sender":"sender_fcf96878ddcb","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772158394778} +{"level":"info","event_type":"run.state","event":{"session_id":"session_fd6536fa5ff4","channel":"gmail","sender":"sender_fcf96878ddcb","source":"channel","state":"start","request_id":"request_ff70daf25a96"},"timestamp":1772158394779} +{"level":"info","event_type":"run.state","event":{"session_id":"session_fd6536fa5ff4","channel":"gmail","sender":"sender_fcf96878ddcb","source":"channel","state":"complete","request_id":"request_ff70daf25a96","duration_ms":5690},"timestamp":1772158400469} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_8b51db8cde21","channel":"gmail","sender":"sender_c9788a77f027","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772163195186} +{"level":"info","event_type":"run.state","event":{"session_id":"session_8b51db8cde21","channel":"gmail","sender":"sender_c9788a77f027","source":"channel","state":"start","request_id":"request_9054b78eda1d"},"timestamp":1772163195187} +{"level":"info","event_type":"run.state","event":{"session_id":"session_8b51db8cde21","channel":"gmail","sender":"sender_c9788a77f027","source":"channel","state":"complete","request_id":"request_9054b78eda1d","duration_ms":2356},"timestamp":1772163197543} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_683372f346c3","channel":"cron","sender":"sender_485b96f48f25","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772168400037} +{"level":"info","event_type":"run.state","event":{"session_id":"session_683372f346c3","channel":"cron","sender":"sender_485b96f48f25","source":"channel","state":"start","request_id":"request_caa9ec775af8"},"timestamp":1772168400038} +{"level":"debug","event_type":"reaction.skip","event":{"session_id":"session_31b6400467ce","channel":"cron","sender":"sender_8255de70c756","source":"channel","reason":"no_rules","candidate_count":0},"timestamp":1772175600039} +{"level":"info","event_type":"run.state","event":{"session_id":"session_31b6400467ce","channel":"cron","sender":"sender_8255de70c756","source":"channel","state":"start","request_id":"request_40d7589e715b"},"timestamp":1772175600040} diff --git a/docs/plans/artifacts/phase0_baseline_live_2026-02-27.md b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.md new file mode 100644 index 0000000..ca5228b --- /dev/null +++ b/docs/plans/artifacts/phase0_baseline_live_2026-02-27.md @@ -0,0 +1,65 @@ +# Phase 0 Baseline Telemetry Summary + +- Run state events: 55 +- Run cancel events: 0 +- Reaction matches: 0 +- Reaction skips: 33 + +- Sources: channel + +## Run Outcomes (Overall) + +- Total outcomes: 23 +- Complete: 23 (100.00%) +- Cancelled: 0 (0.00%) +- Errors: 0 (0.00%) +- Cancel requested: 0 +- Starts: 32 + +## Run Outcomes by Channel + +| Channel | Outcomes | Complete | Cancelled | Error | Complete % | Cancel % | Error % | Cancel Req | Starts | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| gmail | 22 | 22 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 22 | +| cron | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 10 | + +## Run Outcomes by Session + +| Session | Outcomes | Complete | Cancelled | Error | Complete % | Cancel % | Error % | Cancel Req | Starts | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| session_2f2f1e414e81 | 4 | 4 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 4 | +| session_f4d8ddc04194 | 3 | 3 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 3 | +| session_eabc3c2a91b9 | 2 | 2 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 2 | +| session_33469de5a1ee | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_4d9e843358a3 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_58a64b6f2c91 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_7d3c3ff67d4f | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_8849a4464275 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_8b51db8cde21 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_9067cf5e3558 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_a4b91821c664 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_a83fde4c8fdb | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_cb9a69d8a362 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_e0a2a17b7329 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_ea839415979e | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_f6304f25e43b | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_fd6536fa5ff4 | 1 | 1 | 0 | 0 | 100.00% | 0.00% | 0.00% | 0 | 1 | +| session_2b07a8d38406 | 0 | 0 | 0 | 0 | n/a | n/a | n/a | 0 | 1 | +| session_2d8872945bf8 | 0 | 0 | 0 | 0 | n/a | n/a | n/a | 0 | 1 | +| session_31b6400467ce | 0 | 0 | 0 | 0 | n/a | n/a | n/a | 0 | 1 | + +## Cancel Latency + +- No cancel latency samples. + +## Reaction Decisions + +- Matched: 0 (0.00%) +- Skipped: 33 (100.00%) + +### Skip Reasons + +| Reason | Count | Percent | +| --- | ---: | ---: | +| no_rules | 33 | 100.00% | + diff --git a/docs/plans/state.json b/docs/plans/state.json index cb5690e..01278f1 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -66,20 +66,42 @@ "phase0-ticket-0.5-docs-diagram-state-sync": { "status": "completed", "date": "2026-02-25", - "updated": "2026-02-25", - "summary": "Updated protocol/docs/diagrams for phase-0 telemetry fields, documented baseline workflow, and generated phase-0 baseline artifacts using a probe log with representative channel + gateway run/reaction events.", + "updated": "2026-02-27", + "summary": "Updated protocol/docs/diagrams for phase-0 telemetry fields, documented baseline workflow, and replaced the original probe-only baseline workflow with anonymized live channel-session audit artifacts (`phase0_baseline_live_2026-02-27.*`).", "files_modified": [ "README.md", "docs/api/PROTOCOL.md", "docs/architecture/AGENT_DIAGRAM.md", "docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md", "docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md", - "docs/plans/artifacts/phase0_baseline_probe_2026-02-25.jsonl", - "docs/plans/artifacts/phase0_baseline_2026-02-25.md", - "docs/plans/artifacts/phase0_baseline_2026-02-25.json", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.md", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.json", "docs/plans/state.json" ], - "test_status": "pnpm audit:phase0-baseline --audit docs/plans/artifacts/phase0_baseline_probe_2026-02-25.jsonl --format markdown/json (probe log with representative events)" + "test_status": "pnpm audit:phase0-baseline:live + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing" + }, + "phase0-live-baseline-capture-tooling": { + "status": "completed", + "date": "2026-02-27", + "updated": "2026-02-27", + "summary": "Added a dedicated live phase-0 baseline capture flow that reads audit logs, filters run/reaction telemetry, excludes probe sessions, anonymizes session/sender/request IDs, and writes sample + summary artifacts for operational refreshes.", + "files_modified": [ + "src/audit/phase0LiveBaseline.ts", + "src/audit/phase0LiveBaseline.test.ts", + "scripts/capture-phase0-live-baseline.ts", + "package.json", + "README.md", + "docs/api/PROTOCOL.md", + "docs/architecture/AGENT_DIAGRAM.md", + "docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md", + "docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.jsonl", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.md", + "docs/plans/artifacts/phase0_baseline_live_2026-02-27.json", + "docs/plans/state.json" + ], + "test_status": "pnpm audit:phase0-baseline:live + pnpm test:run src/audit/phase0LiveBaseline.test.ts src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing" }, "phase0-instrumentation-ticket-checklist": { "status": "completed", @@ -6786,7 +6808,7 @@ "test_status": "docs only" }, "personal-assistant-productization-plan-2026-02-26": { - "status": "in_progress", + "status": "completed", "date": "2026-02-26", "updated": "2026-02-27", "summary": "Rebaselined Flynn's OpenClaw-style personal-assistant gaps and defined an execution-ready 8-10 week roadmap. Phase 3 browser reliability, Phase 1 companion reconnect/handoff reliability, Phase 2 voice daily-driver reliability (talk controls + TTS provider fallback/health + interruption-safe voice cancel semantics), and Phase 4 onboarding first-success funnel improvements are now shipped.", @@ -7266,8 +7288,8 @@ "deeper_surfaces_phase0_ticket_02": "completed — gateway + daemon routing emit run lifecycle/cancel telemetry and reaction match/skip audit events with filter summaries and cancellation latency, plus focused tests", "deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters", "deeper_surfaces_phase0_ticket_04": "completed — added phase-0 baseline summary tooling for run outcomes, cancel latency, and reaction decisions with markdown/json CLI output", - "deeper_surfaces_phase0_ticket_05": "completed — documented phase-0 telemetry fields/workflow, refreshed architecture/protocol docs, and generated baseline artifacts from a probe log with representative channel + gateway events", - "next_up": "Replace probe baseline artifacts with live audit samples once gateway/channel sessions emit real run/reaction events", + "deeper_surfaces_phase0_ticket_05": "completed — documented phase-0 telemetry fields/workflow, refreshed architecture/protocol docs, and generated anonymized live baseline artifacts from real channel audit traffic (probe-only artifact workflow superseded)", + "next_up": "Capture a gateway-origin live phase-0 baseline sample (including run.cancel/cancelled paths) and append as a second live artifact window alongside the channel sample", "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default", "pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)", "pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing", diff --git a/package.json b/package.json index 18c0310..e4c9639 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "config:profiles:check": "node scripts/generate-config-profiles.mjs --check", "audit:backend-canary": "node --import tsx/esm scripts/summarize-backend-canary.ts", "audit:phase0-baseline": "node --import tsx/esm scripts/summarize-phase0-baseline.ts", + "audit:phase0-baseline:live": "node --import tsx/esm scripts/capture-phase0-live-baseline.ts --audit ~/.local/share/flynn/audit.log --source channel --exclude-session-substring probe --tag 2026-02-27", "audit:backend-canary:probes": "node --import tsx/esm scripts/run-pi-canary-guard-probes.ts", "companion:bundle": "node --import tsx/esm scripts/build-companion-release-bundle.ts", "companion:reference-apps": "node --import tsx/esm scripts/export-companion-reference-apps.ts", diff --git a/scripts/capture-phase0-live-baseline.ts b/scripts/capture-phase0-live-baseline.ts new file mode 100644 index 0000000..ccd55e5 --- /dev/null +++ b/scripts/capture-phase0-live-baseline.ts @@ -0,0 +1,217 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; +import { queryAuditLogs } from '../src/audit/export.js'; +import { capturePhase0LiveBaselineEvents } from '../src/audit/phase0LiveBaseline.js'; +import { + renderPhase0BaselineMarkdown, + summarizePhase0Baseline, + type AuditSource, + type Phase0BaselineSummaryOptions, +} from '../src/audit/phase0BaselineSummary.js'; + +const DEFAULT_EVENT_TYPES = ['run.state', 'run.cancel', 'reaction.match', 'reaction.skip'] as const; + +function usage(): string { + return [ + 'Usage: node --import tsx/esm scripts/capture-phase0-live-baseline.ts [options]', + '', + 'Options:', + ' --audit Source audit log path (default: ~/.local/share/flynn/audit.log)', + ' --since Start time filter', + ' --until End time filter', + ' --channel Restrict sample to channels', + ' --source Restrict sample to sources', + ' --exclude-session-substring Exclude sessions containing any substring (default: probe)', + ' --raw-identifiers Keep raw session/sender/request IDs (default: anonymized)', + ' --tag Output file tag (default: current date UTC)', + ' --sample-out Output JSONL sample path override', + ' --summary-json-out Output summary JSON path override', + ' --summary-md-out Output summary markdown path override', + ' --max-sessions Limit session rows in output (default: 20)', + ' --max-channels Limit channel rows in output (default: 20)', + ' --max-skip-reasons Limit skip reason rows in output (default: 10)', + ].join('\n'); +} + +function expandHomePath(pathValue: string): string { + if (!pathValue.startsWith('~')) { + return pathValue; + } + const home = process.env.HOME; + if (!home) { + return pathValue; + } + return resolve(home, pathValue.slice(1)); +} + +function collapseHomePath(pathValue: string): string { + const home = process.env.HOME; + if (!home) { + return pathValue; + } + return pathValue.startsWith(home) ? `~${pathValue.slice(home.length)}` : pathValue; +} + +function parseTime(value: string | undefined, flag: string): number | undefined { + if (!value) { + return undefined; + } + if (/^\d+$/.test(value)) { + const asNumber = Number(value); + if (Number.isFinite(asNumber)) { + return asNumber; + } + } + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`); + } + return parsed; +} + +function parseCsv(value: string | undefined): string[] | undefined { + if (!value) { + return undefined; + } + const values = value + .split(',') + .map((item) => item.trim()) + .filter((item) => item.length > 0); + return values.length > 0 ? values : undefined; +} + +function parseSources(raw: string | undefined): AuditSource[] | undefined { + const values = parseCsv(raw); + if (!values) { + return undefined; + } + const parsed: AuditSource[] = []; + for (const value of values) { + if (value === 'gateway' || value === 'channel') { + parsed.push(value); + continue; + } + throw new Error(`Invalid source "${value}".`); + } + return parsed; +} + +function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined { + if (!raw) { + return undefined; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`); + } + return parsed; +} + +function isoDateTagNow(): string { + return new Date().toISOString().slice(0, 10); +} + +async function writeTextFile(pathValue: string, contents: string): Promise { + await mkdir(dirname(pathValue), { recursive: true }); + await writeFile(pathValue, contents, 'utf8'); +} + +async function main(): Promise { + const { values } = parseArgs({ + options: { + audit: { type: 'string' }, + since: { type: 'string' }, + until: { type: 'string' }, + channel: { type: 'string' }, + source: { type: 'string' }, + 'exclude-session-substring': { type: 'string' }, + 'raw-identifiers': { type: 'boolean' }, + tag: { type: 'string' }, + 'sample-out': { type: 'string' }, + 'summary-json-out': { type: 'string' }, + 'summary-md-out': { type: 'string' }, + 'max-sessions': { type: 'string' }, + 'max-channels': { type: 'string' }, + 'max-skip-reasons': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + strict: true, + allowPositionals: false, + }); + + if (values.help) { + process.stdout.write(`${usage()}\n`); + return; + } + + const auditPath = expandHomePath(values.audit ?? '~/.local/share/flynn/audit.log'); + const tag = values.tag ?? isoDateTagNow(); + const sampleOut = values['sample-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.jsonl`; + const summaryJsonOut = values['summary-json-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.json`; + const summaryMdOut = values['summary-md-out'] ?? `docs/plans/artifacts/phase0_baseline_live_${tag}.md`; + const channels = parseCsv(values.channel); + const sources = parseSources(values.source); + const excludeSessionSubstrings = parseCsv(values['exclude-session-substring']) ?? ['probe']; + const startTime = parseTime(values.since, '--since'); + const endTime = parseTime(values.until, '--until'); + + const summaryOptions: Phase0BaselineSummaryOptions = { + channels, + sources, + maxSessions: parseOptionalNumber(values['max-sessions'], '--max-sessions') ?? 20, + maxChannels: parseOptionalNumber(values['max-channels'], '--max-channels') ?? 20, + maxSkipReasons: parseOptionalNumber(values['max-skip-reasons'], '--max-skip-reasons') ?? 10, + }; + + const sourceEvents = await queryAuditLogs(auditPath, { + start_time: startTime, + end_time: endTime, + event_types: [...DEFAULT_EVENT_TYPES], + }); + + const sampledEvents = capturePhase0LiveBaselineEvents(sourceEvents, { + channels, + sources, + excludeSessionSubstrings, + anonymizeIdentifiers: !values['raw-identifiers'], + }); + + const summary = summarizePhase0Baseline(sampledEvents, summaryOptions); + const markdown = renderPhase0BaselineMarkdown(summary, summaryOptions); + const sampleJsonl = sampledEvents.map((entry) => JSON.stringify(entry)).join('\n'); + const summaryJson = JSON.stringify({ + generated_at: new Date().toISOString(), + source_audit_path: collapseHomePath(auditPath), + source_event_count: sourceEvents.length, + sampled_event_count: sampledEvents.length, + filters: { + since_ms: startTime, + until_ms: endTime, + channels, + sources, + exclude_session_substrings: excludeSessionSubstrings, + anonymized_identifiers: !values['raw-identifiers'], + }, + options: summaryOptions, + summary, + }, null, 2); + + await writeTextFile(sampleOut, sampleJsonl.length > 0 ? `${sampleJsonl}\n` : ''); + await writeTextFile(summaryJsonOut, `${summaryJson}\n`); + await writeTextFile(summaryMdOut, `${markdown}\n`); + + process.stdout.write(`Captured ${sampledEvents.length} events from ${sourceEvents.length} source events.\n`); + process.stdout.write(`- sample: ${sampleOut}\n`); + process.stdout.write(`- summary json: ${summaryJsonOut}\n`); + process.stdout.write(`- summary md: ${summaryMdOut}\n`); +} + +main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n\n${usage()}\n`); + process.exitCode = 1; +}); + diff --git a/src/audit/phase0LiveBaseline.test.ts b/src/audit/phase0LiveBaseline.test.ts new file mode 100644 index 0000000..6dc5dc3 --- /dev/null +++ b/src/audit/phase0LiveBaseline.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; +import type { AuditEvent } from './types.js'; +import { capturePhase0LiveBaselineEvents } from './phase0LiveBaseline.js'; + +function event( + timestamp: number, + eventType: AuditEvent['event_type'], + payload: Record, +): AuditEvent { + return { + timestamp, + level: 'info', + event_type: eventType, + event: payload, + }; +} + +describe('capturePhase0LiveBaselineEvents', () => { + it('filters to phase-0 event types and applies channel/source/session filters', () => { + const events: AuditEvent[] = [ + event(3, 'session.message', { session_id: 's0', channel: 'cron', sender: 'bot', source: 'channel' }), + event(2, 'reaction.skip', { session_id: 'probe-1', channel: 'cron', sender: 'bot', source: 'channel' }), + event(1, 'run.state', { session_id: 's1', channel: 'cron', sender: 'bot', source: 'channel', state: 'start' }), + event(4, 'run.state', { session_id: 's2', channel: 'telegram', sender: 'u1', source: 'channel', state: 'start' }), + event(5, 'run.cancel', { session_id: 's3', channel: 'cron', sender: 'bot', source: 'gateway', latency_ms: 120 }), + ]; + + const filtered = capturePhase0LiveBaselineEvents(events, { + channels: ['cron'], + sources: ['channel'], + excludeSessionSubstrings: ['probe'], + anonymizeIdentifiers: false, + }); + + expect(filtered).toHaveLength(1); + expect(filtered[0].event_type).toBe('run.state'); + expect(filtered[0].timestamp).toBe(1); + expect(filtered[0].event.session_id).toBe('s1'); + }); + + it('anonymizes session/sender/request/lane identifiers deterministically', () => { + const events: AuditEvent[] = [ + event(1, 'run.state', { + session_id: 'gmail:user@example.com', + sender: 'user@example.com', + request_id: 'req-1', + lane_id: 'lane-1', + channel: 'gmail', + source: 'channel', + state: 'start', + }), + event(2, 'run.state', { + session_id: 'gmail:user@example.com', + sender: 'user@example.com', + request_id: 'req-2', + lane_id: 'lane-2', + channel: 'gmail', + source: 'channel', + state: 'complete', + }), + ]; + + const anonymized = capturePhase0LiveBaselineEvents(events, { + anonymizeIdentifiers: true, + }); + + const first = anonymized[0].event; + const second = anonymized[1].event; + expect(first.session_id).toMatch(/^session_[0-9a-f]{12}$/); + expect(first.sender).toMatch(/^sender_[0-9a-f]{12}$/); + expect(first.request_id).toMatch(/^request_[0-9a-f]{12}$/); + expect(first.lane_id).toMatch(/^lane_[0-9a-f]{12}$/); + expect(first.session_id).toBe(second.session_id); + expect(first.sender).toBe(second.sender); + expect(first.request_id).not.toBe(second.request_id); + expect(first.lane_id).not.toBe(second.lane_id); + }); +}); + diff --git a/src/audit/phase0LiveBaseline.ts b/src/audit/phase0LiveBaseline.ts new file mode 100644 index 0000000..e49b80d --- /dev/null +++ b/src/audit/phase0LiveBaseline.ts @@ -0,0 +1,106 @@ +import { createHash } from 'node:crypto'; +import type { AuditEvent, AuditEventType } from './types.js'; +import type { AuditSource } from './phase0BaselineSummary.js'; + +const PHASE0_BASELINE_EVENT_TYPES: readonly AuditEventType[] = [ + 'run.state', + 'run.cancel', + 'reaction.match', + 'reaction.skip', +]; + +export interface CapturePhase0LiveBaselineOptions { + channels?: string[]; + sources?: AuditSource[]; + excludeSessionSubstrings?: string[]; + anonymizeIdentifiers?: boolean; +} + +function readStringField(payload: Record, key: string): string | undefined { + const value = payload[key]; + return typeof value === 'string' ? value : undefined; +} + +function toPayload(value: unknown): Record { + return (value && typeof value === 'object') + ? { ...(value as Record) } + : {}; +} + +function hashIdentifier(prefix: string, value: string): string { + const digest = createHash('sha256').update(value).digest('hex').slice(0, 12); + return `${prefix}_${digest}`; +} + +function anonymizePayloadIdentifiers(payload: Record): Record { + const next = { ...payload }; + const sessionId = readStringField(next, 'session_id'); + const sender = readStringField(next, 'sender'); + const requestId = readStringField(next, 'request_id'); + const laneId = readStringField(next, 'lane_id'); + + if (sessionId) { + next.session_id = hashIdentifier('session', sessionId); + } + if (sender) { + next.sender = hashIdentifier('sender', sender); + } + if (requestId) { + next.request_id = hashIdentifier('request', requestId); + } + if (laneId) { + next.lane_id = hashIdentifier('lane', laneId); + } + + return next; +} + +export function capturePhase0LiveBaselineEvents( + events: AuditEvent[], + options: CapturePhase0LiveBaselineOptions = {}, +): AuditEvent[] { + const channelFilter = new Set((options.channels ?? []).filter((value) => value.length > 0)); + const sourceFilter = new Set(options.sources ?? []); + const excludeSessionSubstrings = (options.excludeSessionSubstrings ?? []) + .map((value) => value.trim().toLowerCase()) + .filter((value) => value.length > 0); + const anonymizeIdentifiers = options.anonymizeIdentifiers ?? true; + + const filtered: AuditEvent[] = []; + + for (const event of events) { + if (!PHASE0_BASELINE_EVENT_TYPES.includes(event.event_type)) { + continue; + } + + const payload = toPayload(event.event); + const channel = readStringField(payload, 'channel'); + const source = readStringField(payload, 'source'); + const sessionId = readStringField(payload, 'session_id'); + + if (channelFilter.size > 0 && (!channel || !channelFilter.has(channel))) { + continue; + } + if (sourceFilter.size > 0 && (!source || !sourceFilter.has(source as AuditSource))) { + continue; + } + if ( + sessionId + && excludeSessionSubstrings.some((needle) => sessionId.toLowerCase().includes(needle)) + ) { + continue; + } + + const nextPayload = anonymizeIdentifiers + ? anonymizePayloadIdentifiers(payload) + : payload; + + filtered.push({ + ...event, + event: nextPayload, + }); + } + + return filtered.sort((a, b) => a.timestamp - b.timestamp); +} +