docs(eval): close pi canary phase with rollback decision and probe evidence

README.md

@@ -380,6 +380,9 @@ pnpm audit:backend-canary \
 For controlled guardrail probes, also require:
 `--gate-min-guard-pi-no-tools-count 1 --gate-min-guard-capability-query-count 1 --gate-min-guard-attachments-present-count 1`
 
+To generate a local synthetic guard probe log through the router path:
+`pnpm audit:backend-canary:probes`
+
 Phase-2 evaluation checklist and decision template: `docs/plans/pi_embedded_evaluation.md`.
 
 When `args` is non-empty:

docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json

@@ -0,0 +1,105 @@
+{
+  "generated_at": "2026-02-24T06:44:04.629Z",
+  "event_count": 11,
+  "filters": {},
+  "options": {
+    "targetBackend": "pi_embedded",
+    "baselineBackend": "native",
+    "sessionIds": [
+      "telegram:8367012007"
+    ]
+  },
+  "summary": {
+    "route_stats": {
+      "total": 4,
+      "by_backend": {
+        "pi_embedded": 1,
+        "native": 3
+      },
+      "by_source": {
+        "default_external": 1,
+        "forced_native_guard": 3
+      },
+      "forced_native_guards": {
+        "pi_no_tools_mode": 1,
+        "capability_query": 1,
+        "attachments_present": 1
+      }
+    },
+    "target": {
+      "backend": "pi_embedded",
+      "routes": 1,
+      "completed_turns": 1,
+      "incomplete_turns": 0,
+      "completion_rate_pct": 100,
+      "e2e_latency_ms": {
+        "count": 1,
+        "avg_ms": 0,
+        "p50_ms": 0,
+        "p95_ms": 0,
+        "min_ms": 0,
+        "max_ms": 0
+      }
+    },
+    "baseline": {
+      "backend": "native",
+      "routes": 3,
+      "completed_turns": 3,
+      "incomplete_turns": 0,
+      "completion_rate_pct": 100,
+      "e2e_latency_ms": {
+        "count": 3,
+        "avg_ms": 0,
+        "p50_ms": 0,
+        "p95_ms": 0,
+        "min_ms": 0,
+        "max_ms": 0
+      }
+    },
+    "target_external_attempts": {
+      "attempts": 1,
+      "successes": 1,
+      "fallbacks": 0,
+      "unresolved_attempts": 0,
+      "success_rate_pct": 100,
+      "attempt_latency_ms": {
+        "count": 1,
+        "avg_ms": 0,
+        "p50_ms": 0,
+        "p95_ms": 0,
+        "min_ms": 0,
+        "max_ms": 0
+      }
+    },
+    "comparison": {
+      "completion_rate_delta_pp": 0,
+      "p50_latency_delta_ms": 0,
+      "p95_latency_delta_ms": 0
+    },
+    "fallback_categories": [],
+    "fallback_top_reasons": []
+  },
+  "gate": {
+    "pass": true,
+    "criteria": [
+      {
+        "criterion": "Minimum pi_no_tools_mode guard hits",
+        "pass": true,
+        "actual": "1",
+        "threshold": ">= 1"
+      },
+      {
+        "criterion": "Minimum capability_query guard hits",
+        "pass": true,
+        "actual": "1",
+        "threshold": ">= 1"
+      },
+      {
+        "criterion": "Minimum attachments_present guard hits",
+        "pass": true,
+        "actual": "1",
+        "threshold": ">= 1"
+      }
+    ]
+  }
+}

docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md

@@ -0,0 +1,55 @@
+# Pi Embedded Canary Summary
+
+- Target backend: `pi_embedded`
+- Baseline backend: `native`
+- Routes analyzed: 4
+
+## Route Distribution
+
+| Backend | Routes |
+| --- | ---: |
+| native | 3 |
+| pi_embedded | 1 |
+
+### Forced Native Guards
+
+| Guard reason | Count |
+| --- | ---: |
+| attachments_present | 1 |
+| capability_query | 1 |
+| pi_no_tools_mode | 1 |
+
+## Reliability
+
+| Metric | Target | Baseline | Delta |
+| --- | ---: | ---: | ---: |
+| Turn completion rate | 100.00% | 100.00% | 0.00pp |
+| External success rate | 100.00% | n/a | n/a |
+| External attempts | 1 | n/a | n/a |
+| External fallbacks | 0 | n/a | n/a |
+
+## Latency
+
+- Target end-to-end: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
+- Baseline end-to-end: count=3, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
+- P50 delta (target - baseline): 0ms
+- P95 delta (target - baseline): 0ms
+- Target external attempt: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
+
+## Fallback Taxonomy
+
+| Category | Count | Percent |
+| --- | ---: | ---: |
+| _none_ | 0 | 0.00% |
+
+## Top Fallback Reasons
+
+- none
+
+## Gate Evaluation
+
+- Gate result: PASS
+- [x] Minimum pi_no_tools_mode guard hits: actual=1, threshold=>= 1
+- [x] Minimum capability_query guard hits: actual=1, threshold=>= 1
+- [x] Minimum attachments_present guard hits: actual=1, threshold=>= 1
+

docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl

@@ -0,0 +1,15 @@
+{"timestamp":1771915435243,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":23,"attachments_count":0}}
+{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"pi_embedded","source":"default_external"}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"user","content_length":23}}
+{"timestamp":1771915435244,"level":"info","event_type":"backend.success","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","backend":"pi_embedded","duration_ms":0,"response_length":26}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}}
+{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":42,"attachments_count":0}}
+{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"pi_no_tools_mode"}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":62}}
+{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":25,"attachments_count":0}}
+{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"capability_query"}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":47}}
+{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":35,"attachments_count":1}}
+{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"attachments_present"}}
+{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":57}}

docs/plans/pi_embedded_evaluation.md

@@ -1,6 +1,6 @@
 # Pi Embedded Canary Evaluation (Phase 2)
 
-Status: in progress  
+Status: completed  
 Owner: Flynn maintainers  
 Started: 2026-02-24
 
@@ -93,6 +93,26 @@ pnpm audit:backend-canary \
   --format markdown
 ```
 
+Generate a local router guard-probe log (synthetic inbound turns through `createMessageRouter`):
+
+```bash
+pnpm audit:backend-canary:probes
+```
+
+Then evaluate guard coverage against the probe log:
+
+```bash
+pnpm audit:backend-canary \
+  --audit docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl \
+  --backend pi_embedded \
+  --baseline native \
+  --session telegram:8367012007 \
+  --gate-min-guard-pi-no-tools-count 1 \
+  --gate-min-guard-capability-query-count 1 \
+  --gate-min-guard-attachments-present-count 1 \
+  --format markdown
+```
+
 ## Evaluation Log
 
 ### Window A
@@ -154,26 +174,41 @@ pnpm audit:backend-canary \
 | P95 latency delta | +5695ms (fail) | gate <= +700ms |
 | Fallback rate | 25.00% (fail) | 2 fallbacks / 8 attempts; gate <= 5.00% |
 
+### Window D (Guard Coverage Controlled Probes)
+
+- Dates: February 24, 2026 (local synthetic probe run)
+- Route volume: 4 total routes (`pi_embedded`: 1, `native`: 3)
+- Summary artifacts:
+  - `docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl`
+  - `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md`
+  - `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json`
+
+| Check | Result | Notes |
+| --- | --- | --- |
+| Minimum `pi_no_tools_mode` guard hits | 1 (pass) | gate >= 1 |
+| Minimum `capability_query` guard hits | 1 (pass) | gate >= 1 |
+| Minimum `attachments_present` guard hits | 1 (pass) | gate >= 1 |
+
 ## Tool Compatibility Findings
 
 Track all tool-adjacent/risky prompts that were force-routed to native (`no_tools_mode`) and any misses.
 
 | Class | Observed behavior | Action |
 | --- | --- | --- |
-| Tool-adjacent prompts | Not observed in Window A/B (`forced_native_guard` count 0). | Run controlled probe turns that should trigger `pi_no_tools_mode`. |
+| Tool-adjacent prompts | Verified in controlled probes: `pi_no_tools_mode` guard hit = 1. | Keep this in regression probes for future backend iterations. |
-| Capability-query prompts | Not observed in Window A/B (`guard_reason=capability_query` count 0). | Run explicit capability-query probe prompts and confirm forced-native routing. |
+| Capability-query prompts | Verified in controlled probes: `capability_query` guard hit = 1. | Keep this in regression probes for future backend iterations. |
-| Attachments-present turns | Not observed in Window A/B (`guard_reason=attachments_present` count 0). | Run attachment-bearing probe turns and confirm forced-native routing. |
+| Attachments-present turns | Verified in controlled probes: `attachments_present` guard hit = 1. | Keep this in regression probes for future backend iterations. |
 
 ## Decision Record
 
 - Decision date: February 24, 2026
-- Decision: `hold` (no cohort expansion yet)
+- Decision: `rollback` (end canary routing for now)
-- Rationale: Window A fails 3/4 numeric gates (p50 delta, p95 delta, fallback rate) with only 10 total routed turns, including two concrete fallback failure modes:
+- Rationale: Window A fails core numeric gates (p50 delta, p95 delta, fallback rate) with two concrete fallback failure modes:
-  and Window C pre-probe baseline confirms missing guard-coverage evidence (`pi_no_tools_mode`, `capability_query`, `attachments_present` all at 0).
   - `pi_module_interface`
   - `empty_assistant_text`
-  Window B shows fallback recovery (0%) in a post-fallback slice but fails minimum sample thresholds and has no native baseline routes for delta-gate evaluation.
+  Window B shows fallback recovery (0%) but fails minimum sample/baseline gates, so it does not overturn Window A.
-- Next cohort/config delta: none until a baseline-balanced window meets minimum sample thresholds and guardrail coverage probes are completed.
+  Window D confirms guardrail compatibility behavior in controlled probes, but guard compatibility alone is insufficient to justify expansion.
+- Next cohort/config delta: route canary users back to native path (remove/disable `pi_embedded` canary routing) until latency/fallback defects are remediated and a fresh canary is re-approved.
 
 ## Diagram/Protocol Impact Review
 

docs/plans/state.json

@@ -4,14 +4,15 @@
   "description": "Tracks the status of all Flynn plans and implementation phases",
   "plans": {
     "pi-embedded-backend-canary-evaluation-phase": {
-      "status": "in_progress",
+      "status": "completed",
       "date": "2026-02-24",
       "updated": "2026-02-24",
-      "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample thresholds, and guard-coverage thresholds for controlled probe windows. Window A remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails sample/baseline minimums; Window C pre-probe baseline confirms guard-coverage evidence is still missing, so expansion remains blocked.",
+      "summary": "Completed formal Pi embedded canary evaluation with audit-log summaries, minimum-sample thresholds, and controlled guard-coverage probes. Decision: `rollback` canary routing for now (do not expand) due to failed latency/fallback gates in Window A despite verified guard behavior in controlled probes.",
       "files_modified": [
         "src/audit/backendCanarySummary.ts",
         "src/audit/backendCanarySummary.test.ts",
         "scripts/summarize-backend-canary.ts",
+        "scripts/run-pi-canary-guard-probes.ts",
         "package.json",
         "README.md",
         "docs/plans/pi_embedded_evaluation.md",
@@ -22,12 +23,15 @@
         "docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json",
         "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.md",
         "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.json",
+        "docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl",
+        "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md",
+        "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json",
         "docs/architecture/AGENT_DIAGRAM.md",
         "docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md",
         "docs/api/PROTOCOL.md",
         "docs/plans/state.json"
       ],
-      "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample + guard-coverage gates (Window A/B regenerated, Window C pre-probe generated) passing; pnpm lint unchanged warnings-only baseline"
+      "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log (Window A/B/C-preprobe) + pnpm audit:backend-canary:probes + pnpm audit:backend-canary against generated probe log (Window C-postprobe) passing; pnpm lint unchanged warnings-only baseline"
     },
     "pi-embedded-backend-canary-spike": {
       "status": "completed",
@@ -6456,7 +6460,7 @@
     }
   },
   "overall_progress": {
-    "total_test_count": 1992,
+    "total_test_count": 1996,
     "all_tests_passing": true,
     "p0_completion": "3/3 (100%)",
     "p1_completion": "4/4 (100%)",
@@ -6488,7 +6492,7 @@
     "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
     "next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas",
     "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
-    "pi_embedded_evaluation_phase": "in progress — minimum-sample and guard-coverage gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; Window C pre-probe shows 0 guard hits across pi_no_tools_mode/capability_query/attachments_present; no cohort expansion"
+    "pi_embedded_evaluation_phase": "completed — final decision rollback: Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)"
   },
   "soul_md_and_cron_create": {
     "date": "2026-02-11",