William Valentin
231 lines
6.7 KiB
JSON
231 lines
6.7 KiB
JSON
{
|
|
"generated_at": "2026-02-27T18:40:12.019Z",
|
|
"artifacts_dir": "/home/will/lab/flynn/docs/plans/artifacts",
|
|
"backends": [
|
|
"pi_embedded",
|
|
"native"
|
|
],
|
|
"candidate_tag": "2026-02-27-184011",
|
|
"report_tag": "2026-02-27-184011",
|
|
"max_age_hours": 36,
|
|
"thresholds": {
|
|
"requireBaselineHistory": false,
|
|
"minCandidateSampledEvents": 10,
|
|
"maxSampledEventsDropPct": 80,
|
|
"maxRunOutcomesDropPct": 80,
|
|
"maxCompletionRateDropPp": 35,
|
|
"maxCancelRateIncreasePp": 25,
|
|
"maxErrorRateIncreasePp": 25,
|
|
"maxCancelLatencyP95IncreaseMs": 6000
|
|
},
|
|
"overall_pass": true,
|
|
"reports": {
|
|
"summary_json_out": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_drift_2026-02-27-184011.json",
|
|
"summary_md_out": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_drift_2026-02-27-184011.md"
|
|
},
|
|
"results": [
|
|
{
|
|
"backend": "pi_embedded",
|
|
"pass": true,
|
|
"candidate": {
|
|
"tag": "2026-02-27-184011",
|
|
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_pi_embedded_2026-02-27-184011.json",
|
|
"generated_at": "2026-02-27T18:40:11.816Z"
|
|
},
|
|
"baseline": {
|
|
"tag": "2026-02-27-175943",
|
|
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_pi_embedded_2026-02-27-175943.json",
|
|
"generated_at": "2026-02-27T17:59:43.514Z"
|
|
},
|
|
"comparison": {
|
|
"baseline": {
|
|
"source_event_count": 115,
|
|
"sampled_event_count": 59,
|
|
"run_total_outcomes": 26,
|
|
"completion_rate_pct": 100,
|
|
"cancel_rate_pct": 0,
|
|
"error_rate_pct": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pct": 0,
|
|
"reaction_skip_rate_pct": 100
|
|
},
|
|
"candidate": {
|
|
"source_event_count": 115,
|
|
"sampled_event_count": 59,
|
|
"run_total_outcomes": 26,
|
|
"completion_rate_pct": 100,
|
|
"cancel_rate_pct": 0,
|
|
"error_rate_pct": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pct": 0,
|
|
"reaction_skip_rate_pct": 100
|
|
},
|
|
"deltas": {
|
|
"sampled_event_count_pct": 0,
|
|
"run_total_outcomes_pct": 0,
|
|
"completion_rate_pp": 0,
|
|
"cancel_rate_pp": 0,
|
|
"error_rate_pp": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pp": 0,
|
|
"reaction_skip_rate_pp": 0
|
|
}
|
|
},
|
|
"freshness": {
|
|
"enabled": true,
|
|
"pass": true,
|
|
"actual_age_hours": 0,
|
|
"threshold_hours": 36
|
|
},
|
|
"drift_gate": {
|
|
"pass": true,
|
|
"criteria": [
|
|
{
|
|
"criterion": "candidate_sampled_events",
|
|
"pass": true,
|
|
"actual": "59",
|
|
"threshold": ">= 10"
|
|
},
|
|
{
|
|
"criterion": "sampled_events_drop_pct",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 80"
|
|
},
|
|
{
|
|
"criterion": "run_outcomes_drop_pct",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 80"
|
|
},
|
|
{
|
|
"criterion": "completion_rate_drop_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 35"
|
|
},
|
|
{
|
|
"criterion": "cancel_rate_increase_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 25"
|
|
},
|
|
{
|
|
"criterion": "error_rate_increase_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 25"
|
|
},
|
|
{
|
|
"criterion": "cancel_latency_p95_increase_ms",
|
|
"pass": true,
|
|
"actual": "n/a",
|
|
"threshold": "<= 6000"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"backend": "native",
|
|
"pass": true,
|
|
"candidate": {
|
|
"tag": "2026-02-27-184011",
|
|
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_native_2026-02-27-184011.json",
|
|
"generated_at": "2026-02-27T18:40:11.931Z"
|
|
},
|
|
"baseline": {
|
|
"tag": "2026-02-27-175943",
|
|
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_native_2026-02-27-175943.json",
|
|
"generated_at": "2026-02-27T17:59:43.624Z"
|
|
},
|
|
"comparison": {
|
|
"baseline": {
|
|
"source_event_count": 115,
|
|
"sampled_event_count": 15,
|
|
"run_total_outcomes": 2,
|
|
"completion_rate_pct": 100,
|
|
"cancel_rate_pct": 0,
|
|
"error_rate_pct": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pct": 0,
|
|
"reaction_skip_rate_pct": 100
|
|
},
|
|
"candidate": {
|
|
"source_event_count": 115,
|
|
"sampled_event_count": 15,
|
|
"run_total_outcomes": 2,
|
|
"completion_rate_pct": 100,
|
|
"cancel_rate_pct": 0,
|
|
"error_rate_pct": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pct": 0,
|
|
"reaction_skip_rate_pct": 100
|
|
},
|
|
"deltas": {
|
|
"sampled_event_count_pct": 0,
|
|
"run_total_outcomes_pct": 0,
|
|
"completion_rate_pp": 0,
|
|
"cancel_rate_pp": 0,
|
|
"error_rate_pp": 0,
|
|
"cancel_latency_p95_ms": null,
|
|
"reaction_match_rate_pp": 0,
|
|
"reaction_skip_rate_pp": 0
|
|
}
|
|
},
|
|
"freshness": {
|
|
"enabled": true,
|
|
"pass": true,
|
|
"actual_age_hours": 0,
|
|
"threshold_hours": 36
|
|
},
|
|
"drift_gate": {
|
|
"pass": true,
|
|
"criteria": [
|
|
{
|
|
"criterion": "candidate_sampled_events",
|
|
"pass": true,
|
|
"actual": "15",
|
|
"threshold": ">= 10"
|
|
},
|
|
{
|
|
"criterion": "sampled_events_drop_pct",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 80"
|
|
},
|
|
{
|
|
"criterion": "run_outcomes_drop_pct",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 80"
|
|
},
|
|
{
|
|
"criterion": "completion_rate_drop_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 35"
|
|
},
|
|
{
|
|
"criterion": "cancel_rate_increase_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 25"
|
|
},
|
|
{
|
|
"criterion": "error_rate_increase_pp",
|
|
"pass": true,
|
|
"actual": "0",
|
|
"threshold": "<= 25"
|
|
},
|
|
{
|
|
"criterion": "cancel_latency_p95_increase_ms",
|
|
"pass": true,
|
|
"actual": "n/a",
|
|
"threshold": "<= 6000"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|