diff --git a/README.md b/README.md index 6a6a67f..ff38d14 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,9 @@ pnpm audit:backend-canary \ --backend pi_embedded \ --baseline native \ --session telegram:8367012007 \ + --gate-min-target-routes 8 \ + --gate-min-baseline-routes 2 \ + --gate-min-target-attempts 8 \ --format markdown ``` diff --git a/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json b/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json index 5f25605..4f394bf 100644 --- a/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json +++ b/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-02-24T06:28:12.014Z", + "generated_at": "2026-02-24T06:33:31.111Z", "event_count": 45, "filters": {}, "options": { @@ -73,12 +73,12 @@ }, "fallback_categories": [ { - "category": "loaded pi module does not expose a supported session factory (expected one of: c", + "category": "empty_assistant_text", "count": 1, "pct": 50 }, { - "category": "pi agent runtime produced no assistant text", + "category": "pi_module_interface", "count": 1, "pct": 50 } @@ -97,6 +97,24 @@ "gate": { "pass": false, "criteria": [ + { + "criterion": "Minimum target routes", + "pass": true, + "actual": "8", + "threshold": ">= 8" + }, + { + "criterion": "Minimum baseline routes", + "pass": true, + "actual": "2", + "threshold": ">= 2" + }, + { + "criterion": "Minimum target external attempts", + "pass": true, + "actual": "8", + "threshold": ">= 8" + }, { "criterion": "Completion rate delta (target - baseline)", "pass": true, diff --git a/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.md b/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.md index e7af055..8cae403 100644 --- a/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.md +++ b/docs/plans/artifacts/pi_embedded_eval_window_a_2026-02-24.md @@ -32,8 +32,8 @@ | Category | Count | Percent | | --- | ---: | ---: | -| loaded pi module does not expose a supported session factory (expected one of: c | 1 | 50.00% | -| pi agent runtime produced no assistant text | 1 | 50.00% | +| empty_assistant_text | 1 | 50.00% | +| pi_module_interface | 1 | 50.00% | ## Top Fallback Reasons @@ -43,6 +43,9 @@ ## Gate Evaluation - Gate result: HOLD +- [x] Minimum target routes: actual=8, threshold=>= 8 +- [x] Minimum baseline routes: actual=2, threshold=>= 2 +- [x] Minimum target external attempts: actual=8, threshold=>= 8 - [x] Completion rate delta (target - baseline): actual=0.00pp, threshold=>= -2.00pp - [ ] P50 latency delta (target - baseline): actual=259ms, threshold=<= 250ms - [ ] P95 latency delta (target - baseline): actual=5695ms, threshold=<= 700ms diff --git a/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json b/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json index ae64c7e..ca4205d 100644 --- a/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json +++ b/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-02-24T06:30:22.439Z", + "generated_at": "2026-02-24T06:33:37.931Z", "event_count": 24, "filters": { "since_ms": 1771913640000 @@ -71,6 +71,24 @@ "gate": { "pass": false, "criteria": [ + { + "criterion": "Minimum target routes", + "pass": false, + "actual": "6", + "threshold": ">= 8" + }, + { + "criterion": "Minimum baseline routes", + "pass": false, + "actual": "0", + "threshold": ">= 2" + }, + { + "criterion": "Minimum target external attempts", + "pass": false, + "actual": "6", + "threshold": ">= 8" + }, { "criterion": "Completion rate delta (target - baseline)", "pass": false, diff --git a/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md b/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md index 4087bf6..0377817 100644 --- a/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md +++ b/docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.md @@ -40,6 +40,9 @@ ## Gate Evaluation - Gate result: HOLD +- [ ] Minimum target routes: actual=6, threshold=>= 8 +- [ ] Minimum baseline routes: actual=0, threshold=>= 2 +- [ ] Minimum target external attempts: actual=6, threshold=>= 8 - [ ] Completion rate delta (target - baseline): actual=n/a, threshold=>= -2.00pp - [ ] P50 latency delta (target - baseline): actual=n/a, threshold=<= 250ms - [ ] P95 latency delta (target - baseline): actual=n/a, threshold=<= 700ms diff --git a/docs/plans/pi_embedded_evaluation.md b/docs/plans/pi_embedded_evaluation.md index afd2ac5..42d9539 100644 --- a/docs/plans/pi_embedded_evaluation.md +++ b/docs/plans/pi_embedded_evaluation.md @@ -21,6 +21,9 @@ Use the same thresholds for every evaluation window. | Metric | Gate | | --- | --- | +| Minimum target routes | >= 8 | +| Minimum baseline routes | >= 2 | +| Minimum target external attempts | >= 8 | | Completion rate delta (target - baseline) | >= -2.00pp | | P50 latency delta (target - baseline) | <= +250ms | | P95 latency delta (target - baseline) | <= +700ms | @@ -42,6 +45,9 @@ pnpm audit:backend-canary \ --backend pi_embedded \ --baseline native \ --session telegram:8367012007 \ + --gate-min-target-routes 8 \ + --gate-min-baseline-routes 2 \ + --gate-min-target-attempts 8 \ --format markdown ``` @@ -55,6 +61,9 @@ pnpm audit:backend-canary \ --session telegram:8367012007 \ --format json \ --out docs/plans/artifacts/pi_embedded_eval_latest.json \ + --gate-min-target-routes 8 \ + --gate-min-baseline-routes 2 \ + --gate-min-target-attempts 8 \ --gate-max-completion-drop-pp 2 \ --gate-max-p50-latency-increase-ms 250 \ --gate-max-p95-latency-increase-ms 700 \ @@ -73,6 +82,9 @@ pnpm audit:backend-canary \ | Check | Result | Notes | | --- | --- | --- | +| Minimum target routes | 8 (pass) | gate >= 8 | +| Minimum baseline routes | 2 (pass) | gate >= 2 | +| Minimum target external attempts | 8 (pass) | gate >= 8 | | Completion rate delta | 0.00pp (pass) | target 100.00% vs baseline 100.00% | | P50 latency delta | +259ms (fail) | gate <= +250ms | | P95 latency delta | +5695ms (fail) | gate <= +700ms | @@ -89,6 +101,9 @@ pnpm audit:backend-canary \ | Check | Result | Notes | | --- | --- | --- | +| Minimum target routes | 6 (fail) | gate >= 8 | +| Minimum baseline routes | 0 (fail) | gate >= 2 | +| Minimum target external attempts | 6 (fail) | gate >= 8 | | Completion rate delta | n/a (insufficient baseline) | no native-routed turns in this slice | | P50 latency delta | n/a (insufficient baseline) | no native-routed turns in this slice | | P95 latency delta | n/a (insufficient baseline) | no native-routed turns in this slice | @@ -101,19 +116,19 @@ Track all tool-adjacent/risky prompts that were force-routed to native (`no_tool | Class | Observed behavior | Action | | --- | --- | --- | -| Tool-adjacent prompts | Not observed in Window A (`forced_native_guard` count 0). | Collect dedicated tool-adjacent prompts in Window B to validate `no_tools_mode` behavior. | -| Capability-query prompts | Not observed in Window A (`guard_reason=capability_query` count 0). | Add explicit capability-query probes in Window B. | -| Attachments-present turns | Not observed in Window A (`guard_reason=attachments_present` count 0). | Add attachment turns in Window B. | +| Tool-adjacent prompts | Not observed in Window A/B (`forced_native_guard` count 0). | Run controlled probe turns that should trigger `pi_no_tools_mode`. | +| Capability-query prompts | Not observed in Window A/B (`guard_reason=capability_query` count 0). | Run explicit capability-query probe prompts and confirm forced-native routing. | +| Attachments-present turns | Not observed in Window A/B (`guard_reason=attachments_present` count 0). | Run attachment-bearing probe turns and confirm forced-native routing. | ## Decision Record - Decision date: February 24, 2026 - Decision: `hold` (no cohort expansion yet) - Rationale: Window A fails 3/4 numeric gates (p50 delta, p95 delta, fallback rate) with only 10 total routed turns, including two concrete fallback failure modes: - - module session factory mismatch - - no assistant text returned from Pi runtime - Window B shows fallback recovery (0%) in a post-fallback slice but cannot evaluate delta gates because it contains no baseline native routes. -- Next cohort/config delta: none until an additional baseline-balanced window confirms delta gates and guardrail coverage probes are completed. + - `pi_module_interface` + - `empty_assistant_text` + Window B shows fallback recovery (0%) in a post-fallback slice but fails minimum sample thresholds and has no native baseline routes for delta-gate evaluation. +- Next cohort/config delta: none until a baseline-balanced window meets minimum sample thresholds and guardrail coverage probes are completed. ## Diagram/Protocol Impact Review diff --git a/docs/plans/state.json b/docs/plans/state.json index c0f9607..d0ef946 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -7,7 +7,7 @@ "status": "in_progress", "date": "2026-02-24", "updated": "2026-02-24", - "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization and fixed rollout gates. Window A (2026-02-24) produced `hold` due to fallback rate and latency deltas above thresholds. Window B (post-fallback slice) shows fallback recovery (0%) but remains baseline-insufficient for delta-gate pass/fail, so expansion is still blocked.", + "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample gate thresholds, and normalized Pi-specific fallback categories. Window A (2026-02-24) remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails minimum-sample/baseline requirements, so expansion remains blocked.", "files_modified": [ "src/audit/backendCanarySummary.ts", "src/audit/backendCanarySummary.test.ts", @@ -25,7 +25,7 @@ "docs/api/PROTOCOL.md", "docs/plans/state.json" ], - "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log (Window A + Window B artifacts generated) passing; pnpm lint unchanged warnings-only baseline" + "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample gates (Window A + Window B artifacts regenerated) passing; pnpm lint unchanged warnings-only baseline" }, "pi-embedded-backend-canary-spike": { "status": "completed", @@ -6486,7 +6486,7 @@ "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", "next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas", "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default", - "pi_embedded_evaluation_phase": "in progress — Window A gate HOLD (p50 +259ms, p95 +5695ms, fallback 25%); Window B post-fallback slice has 0% fallback but insufficient native baseline for delta gates; no cohort expansion" + "pi_embedded_evaluation_phase": "in progress — minimum-sample gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; no cohort expansion" }, "soul_md_and_cron_create": { "date": "2026-02-11",