fix(audit): require integer drift minimum sample thresholds
Enforce non-negative integer minCandidateSampledEvents/minBaselineSampledEvents in drift gate evaluation and CLI parsing; add regression coverage. Architecture/protocol diagrams reviewed and no updates were needed for this validation-only change.
This commit is contained in:
@@ -507,6 +507,19 @@
|
|||||||
],
|
],
|
||||||
"test_status": "pnpm test:run src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
"test_status": "pnpm test:run src/audit/phase0BaselineSummary.test.ts + pnpm typecheck passing"
|
||||||
},
|
},
|
||||||
|
"phase0-live-baseline-drift-min-threshold-integer-validation": {
|
||||||
|
"status": "completed",
|
||||||
|
"date": "2026-02-27",
|
||||||
|
"updated": "2026-02-27",
|
||||||
|
"summary": "Hardened phase-0 backend drift gating by requiring `minCandidateSampledEvents` and `minBaselineSampledEvents` to be non-negative integers in both gate evaluation and CLI parsing, preventing fractional count thresholds.",
|
||||||
|
"files_modified": [
|
||||||
|
"src/audit/phase0BaselineDrift.ts",
|
||||||
|
"src/audit/phase0BaselineDrift.test.ts",
|
||||||
|
"scripts/check-phase0-baseline-backend-drift.ts",
|
||||||
|
"docs/plans/state.json"
|
||||||
|
],
|
||||||
|
"test_status": "pnpm test:run src/audit/phase0BaselineDrift.test.ts + pnpm typecheck passing"
|
||||||
|
},
|
||||||
"phase0-instrumentation-ticket-checklist": {
|
"phase0-instrumentation-ticket-checklist": {
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"date": "2026-02-25",
|
"date": "2026-02-25",
|
||||||
|
|||||||
@@ -69,8 +69,8 @@ function usage(): string {
|
|||||||
' --out <path> Write output to file instead of stdout',
|
' --out <path> Write output to file instead of stdout',
|
||||||
'',
|
'',
|
||||||
'Drift thresholds (optional):',
|
'Drift thresholds (optional):',
|
||||||
' --min-candidate-sampled-events <number>',
|
' --min-candidate-sampled-events <integer>',
|
||||||
' --min-baseline-sampled-events <number>',
|
' --min-baseline-sampled-events <integer>',
|
||||||
' --max-sampled-events-drop-pct <number>',
|
' --max-sampled-events-drop-pct <number>',
|
||||||
' --max-run-outcomes-drop-pct <number>',
|
' --max-run-outcomes-drop-pct <number>',
|
||||||
' --max-completion-rate-drop-pp <number>',
|
' --max-completion-rate-drop-pp <number>',
|
||||||
@@ -106,6 +106,20 @@ function parseOptionalNumber(raw: string | undefined, flag: string): number | un
|
|||||||
return parsed;
|
return parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseOptionalInteger(raw: string | undefined, flag: string): number | undefined {
|
||||||
|
const parsed = parseOptionalNumber(raw, flag);
|
||||||
|
if (parsed === undefined) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(parsed)) {
|
||||||
|
throw new Error(`Invalid ${flag} value "${raw}". Expected an integer.`);
|
||||||
|
}
|
||||||
|
if (parsed < 0) {
|
||||||
|
throw new Error(`${flag} must be greater than or equal to 0.`);
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
function parseBackends(raw: string | undefined): Phase0BackendTarget[] {
|
function parseBackends(raw: string | undefined): Phase0BackendTarget[] {
|
||||||
const values = parseCsv(raw) ?? ['pi_embedded', 'native'];
|
const values = parseCsv(raw) ?? ['pi_embedded', 'native'];
|
||||||
const parsed: Phase0BackendTarget[] = [];
|
const parsed: Phase0BackendTarget[] = [];
|
||||||
@@ -162,8 +176,8 @@ async function writeOutput(pathValue: string, output: string): Promise<void> {
|
|||||||
function buildThresholds(values: Record<string, string | boolean | undefined>): Phase0BaselineDriftGateThresholds {
|
function buildThresholds(values: Record<string, string | boolean | undefined>): Phase0BaselineDriftGateThresholds {
|
||||||
return {
|
return {
|
||||||
requireBaselineHistory: Boolean(values['require-baseline-history']),
|
requireBaselineHistory: Boolean(values['require-baseline-history']),
|
||||||
minCandidateSampledEvents: parseOptionalNumber(values['min-candidate-sampled-events'] as string | undefined, '--min-candidate-sampled-events'),
|
minCandidateSampledEvents: parseOptionalInteger(values['min-candidate-sampled-events'] as string | undefined, '--min-candidate-sampled-events'),
|
||||||
minBaselineSampledEvents: parseOptionalNumber(values['min-baseline-sampled-events'] as string | undefined, '--min-baseline-sampled-events'),
|
minBaselineSampledEvents: parseOptionalInteger(values['min-baseline-sampled-events'] as string | undefined, '--min-baseline-sampled-events'),
|
||||||
maxSampledEventsDropPct: parseOptionalNumber(values['max-sampled-events-drop-pct'] as string | undefined, '--max-sampled-events-drop-pct'),
|
maxSampledEventsDropPct: parseOptionalNumber(values['max-sampled-events-drop-pct'] as string | undefined, '--max-sampled-events-drop-pct'),
|
||||||
maxRunOutcomesDropPct: parseOptionalNumber(values['max-run-outcomes-drop-pct'] as string | undefined, '--max-run-outcomes-drop-pct'),
|
maxRunOutcomesDropPct: parseOptionalNumber(values['max-run-outcomes-drop-pct'] as string | undefined, '--max-run-outcomes-drop-pct'),
|
||||||
maxCompletionRateDropPp: parseOptionalNumber(values['max-completion-rate-drop-pp'] as string | undefined, '--max-completion-rate-drop-pp'),
|
maxCompletionRateDropPp: parseOptionalNumber(values['max-completion-rate-drop-pp'] as string | undefined, '--max-completion-rate-drop-pp'),
|
||||||
|
|||||||
@@ -382,4 +382,49 @@ describe('phase0BaselineDrift', () => {
|
|||||||
minCandidateSampledEvents: -5,
|
minCandidateSampledEvents: -5,
|
||||||
})).toThrow('minCandidateSampledEvents');
|
})).toThrow('minCandidateSampledEvents');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects non-integer sampled-event minimum thresholds', () => {
|
||||||
|
const comparison = comparePhase0BaselineDrift({
|
||||||
|
sampled_event_count: 10,
|
||||||
|
summary: {
|
||||||
|
event_counts: {
|
||||||
|
run_state: 0,
|
||||||
|
run_cancel: 0,
|
||||||
|
reaction_match: 0,
|
||||||
|
reaction_skip: 0,
|
||||||
|
},
|
||||||
|
run_outcomes: {
|
||||||
|
overall: {
|
||||||
|
total_outcomes: 5,
|
||||||
|
complete: 4,
|
||||||
|
cancelled: 1,
|
||||||
|
error: 0,
|
||||||
|
cancel_requested: 0,
|
||||||
|
start: 5,
|
||||||
|
completion_rate_pct: 80,
|
||||||
|
cancel_rate_pct: 20,
|
||||||
|
error_rate_pct: 0,
|
||||||
|
},
|
||||||
|
by_channel: [],
|
||||||
|
by_session: [],
|
||||||
|
},
|
||||||
|
cancel_latency_ms: null,
|
||||||
|
reactions: {
|
||||||
|
matched: 0,
|
||||||
|
skipped: 0,
|
||||||
|
total: 0,
|
||||||
|
match_rate_pct: null,
|
||||||
|
skip_rate_pct: null,
|
||||||
|
skip_reasons: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(() => evaluatePhase0BaselineDriftGate(comparison, {
|
||||||
|
minCandidateSampledEvents: 10.5,
|
||||||
|
})).toThrow('minCandidateSampledEvents');
|
||||||
|
expect(() => evaluatePhase0BaselineDriftGate(comparison, {
|
||||||
|
minBaselineSampledEvents: 8.2,
|
||||||
|
})).toThrow('minBaselineSampledEvents');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -80,6 +80,17 @@ function readThreshold(value: unknown, thresholdName: string): number | undefine
|
|||||||
return parsed;
|
return parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readIntegerThreshold(value: unknown, thresholdName: string): number | undefined {
|
||||||
|
const parsed = readThreshold(value, thresholdName);
|
||||||
|
if (parsed === undefined) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(parsed)) {
|
||||||
|
throw new Error(`${thresholdName} must be an integer greater than or equal to 0.`);
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
function toPctDelta(baseline: number, candidate: number): number | null {
|
function toPctDelta(baseline: number, candidate: number): number | null {
|
||||||
if (!Number.isFinite(baseline) || baseline <= 0 || !Number.isFinite(candidate)) {
|
if (!Number.isFinite(baseline) || baseline <= 0 || !Number.isFinite(candidate)) {
|
||||||
return null;
|
return null;
|
||||||
@@ -166,7 +177,7 @@ export function evaluatePhase0BaselineDriftGate(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const minCandidateSampledEvents = readThreshold(thresholds.minCandidateSampledEvents, 'minCandidateSampledEvents');
|
const minCandidateSampledEvents = readIntegerThreshold(thresholds.minCandidateSampledEvents, 'minCandidateSampledEvents');
|
||||||
if (typeof minCandidateSampledEvents === 'number') {
|
if (typeof minCandidateSampledEvents === 'number') {
|
||||||
criteria.push({
|
criteria.push({
|
||||||
criterion: 'candidate_sampled_events',
|
criterion: 'candidate_sampled_events',
|
||||||
@@ -176,7 +187,7 @@ export function evaluatePhase0BaselineDriftGate(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const minBaselineSampledEvents = readThreshold(thresholds.minBaselineSampledEvents, 'minBaselineSampledEvents');
|
const minBaselineSampledEvents = readIntegerThreshold(thresholds.minBaselineSampledEvents, 'minBaselineSampledEvents');
|
||||||
if (typeof minBaselineSampledEvents === 'number') {
|
if (typeof minBaselineSampledEvents === 'number') {
|
||||||
if (!baseline) {
|
if (!baseline) {
|
||||||
criteria.push({
|
criteria.push({
|
||||||
|
|||||||
Reference in New Issue
Block a user