will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(eval): record guard-coverage preprobe window and updated gates

Browse Source
This commit is contained in:
William Valentin
2026-02-23 22:38:18 -08:00
parent c5b310c852
commit 7b80c1e7a4
9 changed files with 291 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/plans/state.json
+5 -3
View File
@@ -7,7 +7,7 @@
"status": "in_progress",
"date": "2026-02-24",
"updated": "2026-02-24",
"summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample gate thresholds, and normalized Pi-specific fallback categories. Window A (2026-02-24) remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails minimum-sample/baseline requirements, so expansion remains blocked.",
"summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample thresholds, and guard-coverage thresholds for controlled probe windows. Window A remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails sample/baseline minimums; Window C pre-probe baseline confirms guard-coverage evidence is still missing, so expansion remains blocked.",
"files_modified": [
"src/audit/backendCanarySummary.ts",
"src/audit/backendCanarySummary.test.ts",
@@ -20,12 +20,14 @@
"docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json",
"docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md",
"docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.md",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.json",
"docs/architecture/AGENT_DIAGRAM.md",
"docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md",
"docs/api/PROTOCOL.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample gates (Window A + Window B artifacts regenerated) passing; pnpm lint unchanged warnings-only baseline"
"test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample + guard-coverage gates (Window A/B regenerated, Window C pre-probe generated) passing; pnpm lint unchanged warnings-only baseline"
},
"pi-embedded-backend-canary-spike": {
"status": "completed",
@@ -6486,7 +6488,7 @@
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas",
"pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
"pi_embedded_evaluation_phase": "in progress — minimum-sample gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; no cohort expansion"
"pi_embedded_evaluation_phase": "in progress — minimum-sample and guard-coverage gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; Window C pre-probe shows 0 guard hits across pi_no_tools_mode/capability_query/attachments_present; no cohort expansion"
},
"soul_md_and_cron_create": {
"date": "2026-02-11",