docs(eval): enforce min-sample gates in canary artifacts and decision log

README.md

@@ -371,6 +371,9 @@ pnpm audit:backend-canary \
   --backend pi_embedded \
   --baseline native \
   --session telegram:8367012007 \
+  --gate-min-target-routes 8 \
+  --gate-min-baseline-routes 2 \
+  --gate-min-target-attempts 8 \
   --format markdown
 ```
 

docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-02-24T06:28:12.014Z",
+  "generated_at": "2026-02-24T06:33:31.111Z",
   "event_count": 45,
   "filters": {},
   "options": {
@@ -73,12 +73,12 @@
     },
     "fallback_categories": [
       {
-        "category": "loaded pi module does not expose a supported session factory (expected one of: c",
+        "category": "empty_assistant_text",
         "count": 1,
         "pct": 50
       },
       {
-        "category": "pi agent runtime produced no assistant text",
+        "category": "pi_module_interface",
         "count": 1,
         "pct": 50
       }
@@ -97,6 +97,24 @@
   "gate": {
     "pass": false,
     "criteria": [
+      {
+        "criterion": "Minimum target routes",
+        "pass": true,
+        "actual": "8",
+        "threshold": ">= 8"
+      },
+      {
+        "criterion": "Minimum baseline routes",
+        "pass": true,
+        "actual": "2",
+        "threshold": ">= 2"
+      },
+      {
+        "criterion": "Minimum target external attempts",
+        "pass": true,
+        "actual": "8",
+        "threshold": ">= 8"
+      },
       {
         "criterion": "Completion rate delta (target - baseline)",
         "pass": true,

docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.md

@@ -32,8 +32,8 @@
 
 | Category | Count | Percent |
 | --- | ---: | ---: |
-| loaded pi module does not expose a supported session factory (expected one of: c | 1 | 50.00% |
+| empty_assistant_text | 1 | 50.00% |
-| pi agent runtime produced no assistant text | 1 | 50.00% |
+| pi_module_interface | 1 | 50.00% |
 
 ## Top Fallback Reasons
 
@@ -43,6 +43,9 @@
 ## Gate Evaluation
 
 - Gate result: HOLD
+- [x] Minimum target routes: actual=8, threshold=>= 8
+- [x] Minimum baseline routes: actual=2, threshold=>= 2
+- [x] Minimum target external attempts: actual=8, threshold=>= 8
 - [x] Completion rate delta (target - baseline): actual=0.00pp, threshold=>= -2.00pp
 - [ ] P50 latency delta (target - baseline): actual=259ms, threshold=<= 250ms
 - [ ] P95 latency delta (target - baseline): actual=5695ms, threshold=<= 700ms

docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-02-24T06:30:22.439Z",
+  "generated_at": "2026-02-24T06:33:37.931Z",
   "event_count": 24,
   "filters": {
     "since_ms": 1771913640000
@@ -71,6 +71,24 @@
   "gate": {
     "pass": false,
     "criteria": [
+      {
+        "criterion": "Minimum target routes",
+        "pass": false,
+        "actual": "6",
+        "threshold": ">= 8"
+      },
+      {
+        "criterion": "Minimum baseline routes",
+        "pass": false,
+        "actual": "0",
+        "threshold": ">= 2"
+      },
+      {
+        "criterion": "Minimum target external attempts",
+        "pass": false,
+        "actual": "6",
+        "threshold": ">= 8"
+      },
       {
         "criterion": "Completion rate delta (target - baseline)",
         "pass": false,

docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md

@@ -40,6 +40,9 @@
 ## Gate Evaluation
 
 - Gate result: HOLD
+- [ ] Minimum target routes: actual=6, threshold=>= 8
+- [ ] Minimum baseline routes: actual=0, threshold=>= 2
+- [ ] Minimum target external attempts: actual=6, threshold=>= 8
 - [ ] Completion rate delta (target - baseline): actual=n/a, threshold=>= -2.00pp
 - [ ] P50 latency delta (target - baseline): actual=n/a, threshold=<= 250ms
 - [ ] P95 latency delta (target - baseline): actual=n/a, threshold=<= 700ms

docs/plans/pi_embedded_evaluation.md

@@ -21,6 +21,9 @@ Use the same thresholds for every evaluation window.
 
 | Metric | Gate |
 | --- | --- |
+| Minimum target routes | >= 8 |
+| Minimum baseline routes | >= 2 |
+| Minimum target external attempts | >= 8 |
 | Completion rate delta (target - baseline) | >= -2.00pp |
 | P50 latency delta (target - baseline) | <= +250ms |
 | P95 latency delta (target - baseline) | <= +700ms |
@@ -42,6 +45,9 @@ pnpm audit:backend-canary \
   --backend pi_embedded \
   --baseline native \
   --session telegram:8367012007 \
+  --gate-min-target-routes 8 \
+  --gate-min-baseline-routes 2 \
+  --gate-min-target-attempts 8 \
   --format markdown
 ```
 
@@ -55,6 +61,9 @@ pnpm audit:backend-canary \
   --session telegram:8367012007 \
   --format json \
   --out docs/plans/artifacts/pi_embedded_eval_latest.json \
+  --gate-min-target-routes 8 \
+  --gate-min-baseline-routes 2 \
+  --gate-min-target-attempts 8 \
   --gate-max-completion-drop-pp 2 \
   --gate-max-p50-latency-increase-ms 250 \
   --gate-max-p95-latency-increase-ms 700 \
@@ -73,6 +82,9 @@ pnpm audit:backend-canary \
 
 | Check | Result | Notes |
 | --- | --- | --- |
+| Minimum target routes | 8 (pass) | gate >= 8 |
+| Minimum baseline routes | 2 (pass) | gate >= 2 |
+| Minimum target external attempts | 8 (pass) | gate >= 8 |
 | Completion rate delta | 0.00pp (pass) | target 100.00% vs baseline 100.00% |
 | P50 latency delta | +259ms (fail) | gate <= +250ms |
 | P95 latency delta | +5695ms (fail) | gate <= +700ms |
@@ -89,6 +101,9 @@ pnpm audit:backend-canary \
 
 | Check | Result | Notes |
 | --- | --- | --- |
+| Minimum target routes | 6 (fail) | gate >= 8 |
+| Minimum baseline routes | 0 (fail) | gate >= 2 |
+| Minimum target external attempts | 6 (fail) | gate >= 8 |
 | Completion rate delta | n/a (insufficient baseline) | no native-routed turns in this slice |
 | P50 latency delta | n/a (insufficient baseline) | no native-routed turns in this slice |
 | P95 latency delta | n/a (insufficient baseline) | no native-routed turns in this slice |
@@ -101,19 +116,19 @@ Track all tool-adjacent/risky prompts that were force-routed to native (`no_tool
 
 | Class | Observed behavior | Action |
 | --- | --- | --- |
-| Tool-adjacent prompts | Not observed in Window A (`forced_native_guard` count 0). | Collect dedicated tool-adjacent prompts in Window B to validate `no_tools_mode` behavior. |
+| Tool-adjacent prompts | Not observed in Window A/B (`forced_native_guard` count 0). | Run controlled probe turns that should trigger `pi_no_tools_mode`. |
-| Capability-query prompts | Not observed in Window A (`guard_reason=capability_query` count 0). | Add explicit capability-query probes in Window B. |
+| Capability-query prompts | Not observed in Window A/B (`guard_reason=capability_query` count 0). | Run explicit capability-query probe prompts and confirm forced-native routing. |
-| Attachments-present turns | Not observed in Window A (`guard_reason=attachments_present` count 0). | Add attachment turns in Window B. |
+| Attachments-present turns | Not observed in Window A/B (`guard_reason=attachments_present` count 0). | Run attachment-bearing probe turns and confirm forced-native routing. |
 
 ## Decision Record
 
 - Decision date: February 24, 2026
 - Decision: `hold` (no cohort expansion yet)
 - Rationale: Window A fails 3/4 numeric gates (p50 delta, p95 delta, fallback rate) with only 10 total routed turns, including two concrete fallback failure modes:
-  - module session factory mismatch
+  - `pi_module_interface`
-  - no assistant text returned from Pi runtime
+  - `empty_assistant_text`
-  Window B shows fallback recovery (0%) in a post-fallback slice but cannot evaluate delta gates because it contains no baseline native routes.
+  Window B shows fallback recovery (0%) in a post-fallback slice but fails minimum sample thresholds and has no native baseline routes for delta-gate evaluation.
-- Next cohort/config delta: none until an additional baseline-balanced window confirms delta gates and guardrail coverage probes are completed.
+- Next cohort/config delta: none until a baseline-balanced window meets minimum sample thresholds and guardrail coverage probes are completed.
 
 ## Diagram/Protocol Impact Review
 

docs/plans/state.json

@@ -7,7 +7,7 @@
       "status": "in_progress",
       "date": "2026-02-24",
       "updated": "2026-02-24",
-      "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization and fixed rollout gates. Window A (2026-02-24) produced `hold` due to fallback rate and latency deltas above thresholds. Window B (post-fallback slice) shows fallback recovery (0%) but remains baseline-insufficient for delta-gate pass/fail, so expansion is still blocked.",
+      "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample gate thresholds, and normalized Pi-specific fallback categories. Window A (2026-02-24) remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails minimum-sample/baseline requirements, so expansion remains blocked.",
       "files_modified": [
         "src/audit/backendCanarySummary.ts",
         "src/audit/backendCanarySummary.test.ts",
@@ -25,7 +25,7 @@
         "docs/api/PROTOCOL.md",
         "docs/plans/state.json"
       ],
-      "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log (Window A + Window B artifacts generated) passing; pnpm lint unchanged warnings-only baseline"
+      "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample gates (Window A + Window B artifacts regenerated) passing; pnpm lint unchanged warnings-only baseline"
     },
     "pi-embedded-backend-canary-spike": {
       "status": "completed",
@@ -6486,7 +6486,7 @@
     "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
     "next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas",
     "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
-    "pi_embedded_evaluation_phase": "in progress — Window A gate HOLD (p50 +259ms, p95 +5695ms, fallback 25%); Window B post-fallback slice has 0% fallback but insufficient native baseline for delta gates; no cohort expansion"
+    "pi_embedded_evaluation_phase": "in progress — minimum-sample gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; no cohort expansion"
   },
   "soul_md_and_cron_create": {
     "date": "2026-02-11",