will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(audit): add sample-size gate thresholds for canary evaluation

Browse Source
This commit is contained in:
William Valentin
2026-02-23 22:34:50 -08:00
parent 2d31f85c75
commit 34d1562ce8
3 changed files with 164 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/summarize-backend-canary.ts
+9
View File
@@ -33,6 +33,9 @@ function usage(): string {
' --out <path> Write output to file instead of stdout',
'',
'Gate options (optional):',
' --gate-min-target-routes <number>',
' --gate-min-baseline-routes <number>',
' --gate-min-target-attempts <number>',
' --gate-max-completion-drop-pp <number>',
' --gate-max-p50-latency-increase-ms <number>',
' --gate-max-p95-latency-increase-ms <number>',
@@ -126,6 +129,9 @@ async function main(): Promise<void> {
source: { type: 'string' },
format: { type: 'string' },
out: { type: 'string' },
'gate-min-target-routes': { type: 'string' },
'gate-min-baseline-routes': { type: 'string' },
'gate-min-target-attempts': { type: 'string' },
'gate-max-completion-drop-pp': { type: 'string' },
'gate-max-p50-latency-increase-ms': { type: 'string' },
'gate-max-p95-latency-increase-ms': { type: 'string' },
@@ -171,6 +177,9 @@ async function main(): Promise<void> {
const summary = summarizeBackendCanary(events, summaryOptions);
const gateThresholds: BackendCanaryGateThresholds = {
minTargetRoutes: parseOptionalNumber(values['gate-min-target-routes'], '--gate-min-target-routes'),
minBaselineRoutes: parseOptionalNumber(values['gate-min-baseline-routes'], '--gate-min-baseline-routes'),
minTargetAttempts: parseOptionalNumber(values['gate-min-target-attempts'], '--gate-min-target-attempts'),
maxCompletionRateDropPp: parseOptionalNumber(values['gate-max-completion-drop-pp'], '--gate-max-completion-drop-pp'),
maxP50LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p50-latency-increase-ms'], '--gate-max-p50-latency-increase-ms'),
maxP95LatencyIncreaseMs: parseOptionalNumber(values['gate-max-p95-latency-increase-ms'], '--gate-max-p95-latency-increase-ms'),