will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(eval): close pi canary phase with rollback decision and probe evidence

Browse Source
This commit is contained in:
William Valentin
2026-02-23 22:44:30 -08:00
parent 959216ac5c
commit 8b1f2d9a67
6 changed files with 231 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json
+105
View File
@@ -0,0 +1,105 @@
{
"generated_at": "2026-02-24T06:44:04.629Z",
"event_count": 11,
"filters": {},
"options": {
"targetBackend": "pi_embedded",
"baselineBackend": "native",
"sessionIds": [
"telegram:8367012007"
]
},
"summary": {
"route_stats": {
"total": 4,
"by_backend": {
"pi_embedded": 1,
"native": 3
},
"by_source": {
"default_external": 1,
"forced_native_guard": 3
},
"forced_native_guards": {
"pi_no_tools_mode": 1,
"capability_query": 1,
"attachments_present": 1
}
},
"target": {
"backend": "pi_embedded",
"routes": 1,
"completed_turns": 1,
"incomplete_turns": 0,
"completion_rate_pct": 100,
"e2e_latency_ms": {
"count": 1,
"avg_ms": 0,
"p50_ms": 0,
"p95_ms": 0,
"min_ms": 0,
"max_ms": 0
}
},
"baseline": {
"backend": "native",
"routes": 3,
"completed_turns": 3,
"incomplete_turns": 0,
"completion_rate_pct": 100,
"e2e_latency_ms": {
"count": 3,
"avg_ms": 0,
"p50_ms": 0,
"p95_ms": 0,
"min_ms": 0,
"max_ms": 0
}
},
"target_external_attempts": {
"attempts": 1,
"successes": 1,
"fallbacks": 0,
"unresolved_attempts": 0,
"success_rate_pct": 100,
"attempt_latency_ms": {
"count": 1,
"avg_ms": 0,
"p50_ms": 0,
"p95_ms": 0,
"min_ms": 0,
"max_ms": 0
}
},
"comparison": {
"completion_rate_delta_pp": 0,
"p50_latency_delta_ms": 0,
"p95_latency_delta_ms": 0
},
"fallback_categories": [],
"fallback_top_reasons": []
},
"gate": {
"pass": true,
"criteria": [
{
"criterion": "Minimum pi_no_tools_mode guard hits",
"pass": true,
"actual": "1",
"threshold": ">= 1"
},
{
"criterion": "Minimum capability_query guard hits",
"pass": true,
"actual": "1",
"threshold": ">= 1"
},
{
"criterion": "Minimum attachments_present guard hits",
"pass": true,
"actual": "1",
"threshold": ">= 1"
}
]
}
}
docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md
+55
View File
@@ -0,0 +1,55 @@
# Pi Embedded Canary Summary
- Target backend: `pi_embedded`
- Baseline backend: `native`
- Routes analyzed: 4
## Route Distribution
| Backend | Routes |
| --- | ---: |
| native | 3 |
| pi_embedded | 1 |
### Forced Native Guards
| Guard reason | Count |
| --- | ---: |
| attachments_present | 1 |
| capability_query | 1 |
| pi_no_tools_mode | 1 |
## Reliability
| Metric | Target | Baseline | Delta |
| --- | ---: | ---: | ---: |
| Turn completion rate | 100.00% | 100.00% | 0.00pp |
| External success rate | 100.00% | n/a | n/a |
| External attempts | 1 | n/a | n/a |
| External fallbacks | 0 | n/a | n/a |
## Latency
- Target end-to-end: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
- Baseline end-to-end: count=3, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
- P50 delta (target - baseline): 0ms
- P95 delta (target - baseline): 0ms
- Target external attempt: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms
## Fallback Taxonomy
| Category | Count | Percent |
| --- | ---: | ---: |
| _none_ | 0 | 0.00% |
## Top Fallback Reasons
- none
## Gate Evaluation
- Gate result: PASS
- [x] Minimum pi_no_tools_mode guard hits: actual=1, threshold=>= 1
- [x] Minimum capability_query guard hits: actual=1, threshold=>= 1
- [x] Minimum attachments_present guard hits: actual=1, threshold=>= 1
docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl
+15
View File
@@ -0,0 +1,15 @@
{"timestamp":1771915435243,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":23,"attachments_count":0}}
{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"pi_embedded","source":"default_external"}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"user","content_length":23}}
{"timestamp":1771915435244,"level":"info","event_type":"backend.success","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","backend":"pi_embedded","duration_ms":0,"response_length":26}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}}
{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":42,"attachments_count":0}}
{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"pi_no_tools_mode"}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":62}}
{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":25,"attachments_count":0}}
{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"capability_query"}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":47}}
{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":35,"attachments_count":1}}
{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"attachments_present"}}
{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":57}}
docs/plans/pi_embedded_evaluation.md
+44 -9
View File
@@ -1,6 +1,6 @@
# Pi Embedded Canary Evaluation (Phase 2)
Status: in progress
Status: completed
Owner: Flynn maintainers
Started: 2026-02-24
@@ -93,6 +93,26 @@ pnpm audit:backend-canary \
--format markdown
```
Generate a local router guard-probe log (synthetic inbound turns through `createMessageRouter`):
```bash
pnpm audit:backend-canary:probes
```
Then evaluate guard coverage against the probe log:
```bash
pnpm audit:backend-canary \
--audit docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl \
--backend pi_embedded \
--baseline native \
--session telegram:8367012007 \
--gate-min-guard-pi-no-tools-count 1 \
--gate-min-guard-capability-query-count 1 \
--gate-min-guard-attachments-present-count 1 \
--format markdown
```
## Evaluation Log
### Window A
@@ -154,26 +174,41 @@ pnpm audit:backend-canary \
| P95 latency delta | +5695ms (fail) | gate <= +700ms |
| Fallback rate | 25.00% (fail) | 2 fallbacks / 8 attempts; gate <= 5.00% |
### Window D (Guard Coverage Controlled Probes)
- Dates: February 24, 2026 (local synthetic probe run)
- Route volume: 4 total routes (`pi_embedded`: 1, `native`: 3)
- Summary artifacts:
- `docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl`
- `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md`
- `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json`
| Check | Result | Notes |
| --- | --- | --- |
| Minimum `pi_no_tools_mode` guard hits | 1 (pass) | gate >= 1 |
| Minimum `capability_query` guard hits | 1 (pass) | gate >= 1 |
| Minimum `attachments_present` guard hits | 1 (pass) | gate >= 1 |
## Tool Compatibility Findings
Track all tool-adjacent/risky prompts that were force-routed to native (`no_tools_mode`) and any misses.
| Class | Observed behavior | Action |
| --- | --- | --- |
| Tool-adjacent prompts | Not observed in Window A/B (`forced_native_guard` count 0). | Run controlled probe turns that should trigger `pi_no_tools_mode`. |
| Capability-query prompts | Not observed in Window A/B (`guard_reason=capability_query` count 0). | Run explicit capability-query probe prompts and confirm forced-native routing. |
| Attachments-present turns | Not observed in Window A/B (`guard_reason=attachments_present` count 0). | Run attachment-bearing probe turns and confirm forced-native routing. |
| Tool-adjacent prompts | Verified in controlled probes: `pi_no_tools_mode` guard hit = 1. | Keep this in regression probes for future backend iterations. |
| Capability-query prompts | Verified in controlled probes: `capability_query` guard hit = 1. | Keep this in regression probes for future backend iterations. |
| Attachments-present turns | Verified in controlled probes: `attachments_present` guard hit = 1. | Keep this in regression probes for future backend iterations. |
## Decision Record
- Decision date: February 24, 2026
- Decision: `hold` (no cohort expansion yet)
- Rationale: Window A fails 3/4 numeric gates (p50 delta, p95 delta, fallback rate) with only 10 total routed turns, including two concrete fallback failure modes:
and Window C pre-probe baseline confirms missing guard-coverage evidence (`pi_no_tools_mode`, `capability_query`, `attachments_present` all at 0).
- Decision: `rollback` (end canary routing for now)
- Rationale: Window A fails core numeric gates (p50 delta, p95 delta, fallback rate) with two concrete fallback failure modes:
- `pi_module_interface`
- `empty_assistant_text`
Window B shows fallback recovery (0%) in a post-fallback slice but fails minimum sample thresholds and has no native baseline routes for delta-gate evaluation.
- Next cohort/config delta: none until a baseline-balanced window meets minimum sample thresholds and guardrail coverage probes are completed.
Window B shows fallback recovery (0%) but fails minimum sample/baseline gates, so it does not overturn Window A.
Window D confirms guardrail compatibility behavior in controlled probes, but guard compatibility alone is insufficient to justify expansion.
- Next cohort/config delta: route canary users back to native path (remove/disable `pi_embedded` canary routing) until latency/fallback defects are remediated and a fresh canary is re-approved.
## Diagram/Protocol Impact Review
docs/plans/state.json
+9 -5
View File
@@ -4,14 +4,15 @@
"description": "Tracks the status of all Flynn plans and implementation phases",
"plans": {
"pi-embedded-backend-canary-evaluation-phase": {
"status": "in_progress",
"status": "completed",
"date": "2026-02-24",
"updated": "2026-02-24",
"summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample thresholds, and guard-coverage thresholds for controlled probe windows. Window A remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails sample/baseline minimums; Window C pre-probe baseline confirms guard-coverage evidence is still missing, so expansion remains blocked.",
"summary": "Completed formal Pi embedded canary evaluation with audit-log summaries, minimum-sample thresholds, and controlled guard-coverage probes. Decision: `rollback` canary routing for now (do not expand) due to failed latency/fallback gates in Window A despite verified guard behavior in controlled probes.",
"files_modified": [
"src/audit/backendCanarySummary.ts",
"src/audit/backendCanarySummary.test.ts",
"scripts/summarize-backend-canary.ts",
"scripts/run-pi-canary-guard-probes.ts",
"package.json",
"README.md",
"docs/plans/pi_embedded_evaluation.md",
@@ -22,12 +23,15 @@
"docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.md",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.json",
"docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md",
"docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json",
"docs/architecture/AGENT_DIAGRAM.md",
"docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md",
"docs/api/PROTOCOL.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample + guard-coverage gates (Window A/B regenerated, Window C pre-probe generated) passing; pnpm lint unchanged warnings-only baseline"
"test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log (Window A/B/C-preprobe) + pnpm audit:backend-canary:probes + pnpm audit:backend-canary against generated probe log (Window C-postprobe) passing; pnpm lint unchanged warnings-only baseline"
},
"pi-embedded-backend-canary-spike": {
"status": "completed",
@@ -6456,7 +6460,7 @@
}
},
"overall_progress": {
"total_test_count": 1992,
"total_test_count": 1996,
"all_tests_passing": true,
"p0_completion": "3/3 (100%)",
"p1_completion": "4/4 (100%)",
@@ -6488,7 +6492,7 @@
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas",
"pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
"pi_embedded_evaluation_phase": "in progress — minimum-sample and guard-coverage gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; Window C pre-probe shows 0 guard hits across pi_no_tools_mode/capability_query/attachments_present; no cohort expansion"
"pi_embedded_evaluation_phase": "completed — final decision rollback: Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)"
},
"soul_md_and_cron_create": {
"date": "2026-02-11",