will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(audit): add rolling timestamped phase0 refresh cadence

Browse Source
This commit is contained in:
William Valentin
2026-02-27 10:00:45 -08:00
parent 55f1a3dd7b
commit 149adb1c85
21 changed files with 1821 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/plans/artifacts/phase0_baseline_live_backend_drift_2026-02-27-175943.json
+230
View File
@@ -0,0 +1,230 @@
{
"generated_at": "2026-02-27T17:59:43.707Z",
"artifacts_dir": "/home/will/lab/flynn/docs/plans/artifacts",
"backends": [
"pi_embedded",
"native"
],
"candidate_tag": "2026-02-27-175943",
"report_tag": "2026-02-27-175943",
"max_age_hours": 36,
"thresholds": {
"requireBaselineHistory": false,
"minCandidateSampledEvents": 10,
"maxSampledEventsDropPct": 80,
"maxRunOutcomesDropPct": 80,
"maxCompletionRateDropPp": 35,
"maxCancelRateIncreasePp": 25,
"maxErrorRateIncreasePp": 25,
"maxCancelLatencyP95IncreaseMs": 6000
},
"overall_pass": true,
"reports": {
"summary_json_out": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_drift_2026-02-27-175943.json",
"summary_md_out": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_drift_2026-02-27-175943.md"
},
"results": [
{
"backend": "pi_embedded",
"pass": true,
"candidate": {
"tag": "2026-02-27-175943",
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_pi_embedded_2026-02-27-175943.json",
"generated_at": "2026-02-27T17:59:43.514Z"
},
"baseline": {
"tag": "2026-02-27",
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_pi_embedded_2026-02-27.json",
"generated_at": "2026-02-27T17:36:02.214Z"
},
"comparison": {
"baseline": {
"source_event_count": 115,
"sampled_event_count": 59,
"run_total_outcomes": 26,
"completion_rate_pct": 100,
"cancel_rate_pct": 0,
"error_rate_pct": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pct": 0,
"reaction_skip_rate_pct": 100
},
"candidate": {
"source_event_count": 115,
"sampled_event_count": 59,
"run_total_outcomes": 26,
"completion_rate_pct": 100,
"cancel_rate_pct": 0,
"error_rate_pct": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pct": 0,
"reaction_skip_rate_pct": 100
},
"deltas": {
"sampled_event_count_pct": 0,
"run_total_outcomes_pct": 0,
"completion_rate_pp": 0,
"cancel_rate_pp": 0,
"error_rate_pp": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pp": 0,
"reaction_skip_rate_pp": 0
}
},
"freshness": {
"enabled": true,
"pass": true,
"actual_age_hours": 0,
"threshold_hours": 36
},
"drift_gate": {
"pass": true,
"criteria": [
{
"criterion": "candidate_sampled_events",
"pass": true,
"actual": "59",
"threshold": ">= 10"
},
{
"criterion": "sampled_events_drop_pct",
"pass": true,
"actual": "0",
"threshold": "<= 80"
},
{
"criterion": "run_outcomes_drop_pct",
"pass": true,
"actual": "0",
"threshold": "<= 80"
},
{
"criterion": "completion_rate_drop_pp",
"pass": true,
"actual": "0",
"threshold": "<= 35"
},
{
"criterion": "cancel_rate_increase_pp",
"pass": true,
"actual": "0",
"threshold": "<= 25"
},
{
"criterion": "error_rate_increase_pp",
"pass": true,
"actual": "0",
"threshold": "<= 25"
},
{
"criterion": "cancel_latency_p95_increase_ms",
"pass": true,
"actual": "n/a",
"threshold": "<= 6000"
}
]
}
},
{
"backend": "native",
"pass": true,
"candidate": {
"tag": "2026-02-27-175943",
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_native_2026-02-27-175943.json",
"generated_at": "2026-02-27T17:59:43.624Z"
},
"baseline": {
"tag": "2026-02-27",
"path": "/home/will/lab/flynn/docs/plans/artifacts/phase0_baseline_live_backend_native_2026-02-27.json",
"generated_at": "2026-02-27T17:36:02.514Z"
},
"comparison": {
"baseline": {
"source_event_count": 115,
"sampled_event_count": 15,
"run_total_outcomes": 2,
"completion_rate_pct": 100,
"cancel_rate_pct": 0,
"error_rate_pct": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pct": 0,
"reaction_skip_rate_pct": 100
},
"candidate": {
"source_event_count": 115,
"sampled_event_count": 15,
"run_total_outcomes": 2,
"completion_rate_pct": 100,
"cancel_rate_pct": 0,
"error_rate_pct": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pct": 0,
"reaction_skip_rate_pct": 100
},
"deltas": {
"sampled_event_count_pct": 0,
"run_total_outcomes_pct": 0,
"completion_rate_pp": 0,
"cancel_rate_pp": 0,
"error_rate_pp": 0,
"cancel_latency_p95_ms": null,
"reaction_match_rate_pp": 0,
"reaction_skip_rate_pp": 0
}
},
"freshness": {
"enabled": true,
"pass": true,
"actual_age_hours": 0,
"threshold_hours": 36
},
"drift_gate": {
"pass": true,
"criteria": [
{
"criterion": "candidate_sampled_events",
"pass": true,
"actual": "15",
"threshold": ">= 10"
},
{
"criterion": "sampled_events_drop_pct",
"pass": true,
"actual": "0",
"threshold": "<= 80"
},
{
"criterion": "run_outcomes_drop_pct",
"pass": true,
"actual": "0",
"threshold": "<= 80"
},
{
"criterion": "completion_rate_drop_pp",
"pass": true,
"actual": "0",
"threshold": "<= 35"
},
{
"criterion": "cancel_rate_increase_pp",
"pass": true,
"actual": "0",
"threshold": "<= 25"
},
{
"criterion": "error_rate_increase_pp",
"pass": true,
"actual": "0",
"threshold": "<= 25"
},
{
"criterion": "cancel_latency_p95_increase_ms",
"pass": true,
"actual": "n/a",
"threshold": "<= 6000"
}
]
}
}
]
}