diff --git a/README.md b/README.md index ccaf849..ada8350 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,9 @@ pnpm audit:backend-canary \ For controlled guardrail probes, also require: `--gate-min-guard-pi-no-tools-count 1 --gate-min-guard-capability-query-count 1 --gate-min-guard-attachments-present-count 1` +To generate a local synthetic guard probe log through the router path: +`pnpm audit:backend-canary:probes` + Phase-2 evaluation checklist and decision template: `docs/plans/pi_embedded_evaluation.md`. When `args` is non-empty: diff --git a/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json b/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json new file mode 100644 index 0000000..dd8356b --- /dev/null +++ b/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json @@ -0,0 +1,105 @@ +{ + "generated_at": "2026-02-24T06:44:04.629Z", + "event_count": 11, + "filters": {}, + "options": { + "targetBackend": "pi_embedded", + "baselineBackend": "native", + "sessionIds": [ + "telegram:8367012007" + ] + }, + "summary": { + "route_stats": { + "total": 4, + "by_backend": { + "pi_embedded": 1, + "native": 3 + }, + "by_source": { + "default_external": 1, + "forced_native_guard": 3 + }, + "forced_native_guards": { + "pi_no_tools_mode": 1, + "capability_query": 1, + "attachments_present": 1 + } + }, + "target": { + "backend": "pi_embedded", + "routes": 1, + "completed_turns": 1, + "incomplete_turns": 0, + "completion_rate_pct": 100, + "e2e_latency_ms": { + "count": 1, + "avg_ms": 0, + "p50_ms": 0, + "p95_ms": 0, + "min_ms": 0, + "max_ms": 0 + } + }, + "baseline": { + "backend": "native", + "routes": 3, + "completed_turns": 3, + "incomplete_turns": 0, + "completion_rate_pct": 100, + "e2e_latency_ms": { + "count": 3, + "avg_ms": 0, + "p50_ms": 0, + "p95_ms": 0, + "min_ms": 0, + "max_ms": 0 + } + }, + "target_external_attempts": { + "attempts": 1, + "successes": 1, + "fallbacks": 0, + "unresolved_attempts": 0, + "success_rate_pct": 100, + "attempt_latency_ms": { + "count": 1, + "avg_ms": 0, + "p50_ms": 0, + "p95_ms": 0, + "min_ms": 0, + "max_ms": 0 + } + }, + "comparison": { + "completion_rate_delta_pp": 0, + "p50_latency_delta_ms": 0, + "p95_latency_delta_ms": 0 + }, + "fallback_categories": [], + "fallback_top_reasons": [] + }, + "gate": { + "pass": true, + "criteria": [ + { + "criterion": "Minimum pi_no_tools_mode guard hits", + "pass": true, + "actual": "1", + "threshold": ">= 1" + }, + { + "criterion": "Minimum capability_query guard hits", + "pass": true, + "actual": "1", + "threshold": ">= 1" + }, + { + "criterion": "Minimum attachments_present guard hits", + "pass": true, + "actual": "1", + "threshold": ">= 1" + } + ] + } +} diff --git a/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md b/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md new file mode 100644 index 0000000..aa4c5ba --- /dev/null +++ b/docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md @@ -0,0 +1,55 @@ +# Pi Embedded Canary Summary + +- Target backend: `pi_embedded` +- Baseline backend: `native` +- Routes analyzed: 4 + +## Route Distribution + +| Backend | Routes | +| --- | ---: | +| native | 3 | +| pi_embedded | 1 | + +### Forced Native Guards + +| Guard reason | Count | +| --- | ---: | +| attachments_present | 1 | +| capability_query | 1 | +| pi_no_tools_mode | 1 | + +## Reliability + +| Metric | Target | Baseline | Delta | +| --- | ---: | ---: | ---: | +| Turn completion rate | 100.00% | 100.00% | 0.00pp | +| External success rate | 100.00% | n/a | n/a | +| External attempts | 1 | n/a | n/a | +| External fallbacks | 0 | n/a | n/a | + +## Latency + +- Target end-to-end: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms +- Baseline end-to-end: count=3, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms +- P50 delta (target - baseline): 0ms +- P95 delta (target - baseline): 0ms +- Target external attempt: count=1, avg=0ms, p50=0ms, p95=0ms, min=0ms, max=0ms + +## Fallback Taxonomy + +| Category | Count | Percent | +| --- | ---: | ---: | +| _none_ | 0 | 0.00% | + +## Top Fallback Reasons + +- none + +## Gate Evaluation + +- Gate result: PASS +- [x] Minimum pi_no_tools_mode guard hits: actual=1, threshold=>= 1 +- [x] Minimum capability_query guard hits: actual=1, threshold=>= 1 +- [x] Minimum attachments_present guard hits: actual=1, threshold=>= 1 + diff --git a/docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl b/docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl new file mode 100644 index 0000000..ce9cc5f --- /dev/null +++ b/docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl @@ -0,0 +1,15 @@ +{"timestamp":1771915435243,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":23,"attachments_count":0}} +{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"pi_embedded","source":"default_external"}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"user","content_length":23}} +{"timestamp":1771915435244,"level":"info","event_type":"backend.success","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","backend":"pi_embedded","duration_ms":0,"response_length":26}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":26}} +{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":42,"attachments_count":0}} +{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"pi_no_tools_mode"}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":62}} +{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":25,"attachments_count":0}} +{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"capability_query"}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":47}} +{"timestamp":1771915435244,"level":"info","event_type":"user.action","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","source":"channel","action_type":"message","content_length":35,"attachments_count":1}} +{"timestamp":1771915435244,"level":"info","event_type":"backend.route","event":{"session_id":"telegram:8367012007","channel":"telegram","sender":"8367012007","selected_backend":"native","source":"forced_native_guard","guard_reason":"attachments_present"}} +{"timestamp":1771915435244,"level":"debug","event_type":"session.message","event":{"session_id":"telegram:8367012007","role":"assistant","content_length":57}} diff --git a/docs/plans/pi_embedded_evaluation.md b/docs/plans/pi_embedded_evaluation.md index b978056..45f4350 100644 --- a/docs/plans/pi_embedded_evaluation.md +++ b/docs/plans/pi_embedded_evaluation.md @@ -1,6 +1,6 @@ # Pi Embedded Canary Evaluation (Phase 2) -Status: in progress +Status: completed Owner: Flynn maintainers Started: 2026-02-24 @@ -93,6 +93,26 @@ pnpm audit:backend-canary \ --format markdown ``` +Generate a local router guard-probe log (synthetic inbound turns through `createMessageRouter`): + +```bash +pnpm audit:backend-canary:probes +``` + +Then evaluate guard coverage against the probe log: + +```bash +pnpm audit:backend-canary \ + --audit docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl \ + --backend pi_embedded \ + --baseline native \ + --session telegram:8367012007 \ + --gate-min-guard-pi-no-tools-count 1 \ + --gate-min-guard-capability-query-count 1 \ + --gate-min-guard-attachments-present-count 1 \ + --format markdown +``` + ## Evaluation Log ### Window A @@ -154,26 +174,41 @@ pnpm audit:backend-canary \ | P95 latency delta | +5695ms (fail) | gate <= +700ms | | Fallback rate | 25.00% (fail) | 2 fallbacks / 8 attempts; gate <= 5.00% | +### Window D (Guard Coverage Controlled Probes) + +- Dates: February 24, 2026 (local synthetic probe run) +- Route volume: 4 total routes (`pi_embedded`: 1, `native`: 3) +- Summary artifacts: + - `docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl` + - `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md` + - `docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json` + +| Check | Result | Notes | +| --- | --- | --- | +| Minimum `pi_no_tools_mode` guard hits | 1 (pass) | gate >= 1 | +| Minimum `capability_query` guard hits | 1 (pass) | gate >= 1 | +| Minimum `attachments_present` guard hits | 1 (pass) | gate >= 1 | + ## Tool Compatibility Findings Track all tool-adjacent/risky prompts that were force-routed to native (`no_tools_mode`) and any misses. | Class | Observed behavior | Action | | --- | --- | --- | -| Tool-adjacent prompts | Not observed in Window A/B (`forced_native_guard` count 0). | Run controlled probe turns that should trigger `pi_no_tools_mode`. | -| Capability-query prompts | Not observed in Window A/B (`guard_reason=capability_query` count 0). | Run explicit capability-query probe prompts and confirm forced-native routing. | -| Attachments-present turns | Not observed in Window A/B (`guard_reason=attachments_present` count 0). | Run attachment-bearing probe turns and confirm forced-native routing. | +| Tool-adjacent prompts | Verified in controlled probes: `pi_no_tools_mode` guard hit = 1. | Keep this in regression probes for future backend iterations. | +| Capability-query prompts | Verified in controlled probes: `capability_query` guard hit = 1. | Keep this in regression probes for future backend iterations. | +| Attachments-present turns | Verified in controlled probes: `attachments_present` guard hit = 1. | Keep this in regression probes for future backend iterations. | ## Decision Record - Decision date: February 24, 2026 -- Decision: `hold` (no cohort expansion yet) -- Rationale: Window A fails 3/4 numeric gates (p50 delta, p95 delta, fallback rate) with only 10 total routed turns, including two concrete fallback failure modes: - and Window C pre-probe baseline confirms missing guard-coverage evidence (`pi_no_tools_mode`, `capability_query`, `attachments_present` all at 0). +- Decision: `rollback` (end canary routing for now) +- Rationale: Window A fails core numeric gates (p50 delta, p95 delta, fallback rate) with two concrete fallback failure modes: - `pi_module_interface` - `empty_assistant_text` - Window B shows fallback recovery (0%) in a post-fallback slice but fails minimum sample thresholds and has no native baseline routes for delta-gate evaluation. -- Next cohort/config delta: none until a baseline-balanced window meets minimum sample thresholds and guardrail coverage probes are completed. + Window B shows fallback recovery (0%) but fails minimum sample/baseline gates, so it does not overturn Window A. + Window D confirms guardrail compatibility behavior in controlled probes, but guard compatibility alone is insufficient to justify expansion. +- Next cohort/config delta: route canary users back to native path (remove/disable `pi_embedded` canary routing) until latency/fallback defects are remediated and a fresh canary is re-approved. ## Diagram/Protocol Impact Review diff --git a/docs/plans/state.json b/docs/plans/state.json index 758f005..8afb5e3 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -4,14 +4,15 @@ "description": "Tracks the status of all Flynn plans and implementation phases", "plans": { "pi-embedded-backend-canary-evaluation-phase": { - "status": "in_progress", + "status": "completed", "date": "2026-02-24", "updated": "2026-02-24", - "summary": "Formal Pi embedded canary evaluation is active with audit-log summarization, minimum-sample thresholds, and guard-coverage thresholds for controlled probe windows. Window A remains `hold` due to latency/fallback failures; Window B shows fallback recovery (0%) but fails sample/baseline minimums; Window C pre-probe baseline confirms guard-coverage evidence is still missing, so expansion remains blocked.", + "summary": "Completed formal Pi embedded canary evaluation with audit-log summaries, minimum-sample thresholds, and controlled guard-coverage probes. Decision: `rollback` canary routing for now (do not expand) due to failed latency/fallback gates in Window A despite verified guard behavior in controlled probes.", "files_modified": [ "src/audit/backendCanarySummary.ts", "src/audit/backendCanarySummary.test.ts", "scripts/summarize-backend-canary.ts", + "scripts/run-pi-canary-guard-probes.ts", "package.json", "README.md", "docs/plans/pi_embedded_evaluation.md", @@ -22,12 +23,15 @@ "docs/plans/artifacts/pi_embedded_eval_window_b_2026-02-24_post_fallbacks.json", "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.md", "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_preprobe.json", + "docs/plans/artifacts/pi_embedded_eval_window_c_guard_probes.jsonl", + "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.md", + "docs/plans/artifacts/pi_embedded_eval_window_c_2026-02-24_guard_postprobe.json", "docs/architecture/AGENT_DIAGRAM.md", "docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md", "docs/api/PROTOCOL.md", "docs/plans/state.json" ], - "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log with minimum-sample + guard-coverage gates (Window A/B regenerated, Window C pre-probe generated) passing; pnpm lint unchanged warnings-only baseline" + "test_status": "pnpm test:run src/audit/backendCanarySummary.test.ts + pnpm typecheck + pnpm audit:backend-canary against ~/.local/share/flynn/audit.log (Window A/B/C-preprobe) + pnpm audit:backend-canary:probes + pnpm audit:backend-canary against generated probe log (Window C-postprobe) passing; pnpm lint unchanged warnings-only baseline" }, "pi-embedded-backend-canary-spike": { "status": "completed", @@ -6456,7 +6460,7 @@ } }, "overall_progress": { - "total_test_count": 1992, + "total_test_count": 1996, "all_tests_passing": true, "p0_completion": "3/3 (100%)", "p1_completion": "4/4 (100%)", @@ -6488,7 +6492,7 @@ "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", "next_up": "Track OpenClaw evolution regularly for inspiration and feature ideas", "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default", - "pi_embedded_evaluation_phase": "in progress — minimum-sample and guard-coverage gates enforced; Window A HOLD (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B has 0% fallback but fails sample/baseline gates; Window C pre-probe shows 0 guard hits across pi_no_tools_mode/capability_query/attachments_present; no cohort expansion" + "pi_embedded_evaluation_phase": "completed — final decision rollback: Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)" }, "soul_md_and_cron_create": { "date": "2026-02-11",