will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(skills): add rollout promotion contract and sync planning state

Browse Source
This commit is contained in:
William Valentin
2026-02-13 08:51:19 -08:00
parent 46099664f0
commit 8f644d5e25
8 changed files with 464 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/skills.test.ts
+151
View File
@@ -35,6 +35,7 @@ import {
recommendShellRunnerRolloutPhase,
sanitizeSkillInstallerAuditReason,
summarizeShellRunnerAuditWindow,
toShellRunnerPromotionContract,
resolveSkillInstallerCommandRunner,
runSkillExecuteAction,
runSkillInstallAction,
@@ -753,6 +754,70 @@ describe('skills CLI helpers', () => {
expect(policy.blockers).toContain('failures increased by 1 vs previous window');
});
it('builds machine-readable promotion contract with gate status and blockers', () => {
const contract = toShellRunnerPromotionContract({
generatedAt: '2026-02-13T00:00:00.000Z',
days: 7,
recommendation: 'guarded_review',
guardrails: { blockers: ['skills.installation_execution must be enabled'] },
summary: {
command_result_total: 4,
command_result_failed: 1,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 3,
unhashed_command_count: 1,
},
trend: {
current: {
command_result_total: 4,
command_result_failed: 1,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 3,
unhashed_command_count: 1,
},
previous: {
command_result_total: 4,
command_result_failed: 0,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 4,
unhashed_command_count: 0,
},
deltas: {
failures: 1,
allowlist_blocks: 0,
hash_coverage_pct: -25,
},
},
promotionPolicy: {
eligible: false,
recommendation: 'not_eligible',
cadence_days: 7,
reviewed_window_days: 7,
success_rate: 0.75,
minimum_success_rate: 0.9,
failures_delta: 1,
allowlist_blocks_delta: 0,
hash_coverage_delta_pct: -25,
blockers: ['success rate 75.00% below minimum 90.00%'],
},
governance: {
owner: 'skills-team',
review_cadence_days: 7,
promotion_min_success_rate: 0.9,
},
});
expect(contract.schema).toBe('skills.rollout.promotion_contract.v1');
expect(contract.gate.status).toBe('fail');
expect(contract.gate.exit_code).toBe(1);
expect(contract.gate.blockers).toContain('skills.installation_execution must be enabled');
expect(contract.gate.blockers).toContain('success rate 75.00% below minimum 90.00%');
expect(contract.summary.hash_coverage_pct).toBe(75);
});
it('marks promotion policy eligible when thresholds and trends are healthy', () => {
const policy = evaluateShellRunnerPromotionPolicy({
trend: {
@@ -2161,6 +2226,92 @@ describe('skills CLI helpers', () => {
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status emits dedicated promotion contract JSON with exit code', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
const auditPath = join(root, 'audit.log');
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeFileSync(auditPath, '', 'utf-8');
writeSkillsCliConfig(configPath, {
managedDir,
bundledDir,
workspaceDir,
installationExecution: 'enabled',
allowShellRunner: true,
shellRunnerAllowlist: ['npm install*'],
shellRunnerGovernanceOwner: 'skills-team',
auditPath,
});
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'rollout-status', '--contract', '-c', configPath], { from: 'user' });
const payload = JSON.parse(String(logSpy.mock.calls[0]?.[0]));
expect(payload.schema).toBe('skills.rollout.promotion_contract.v1');
expect(payload.gate.status).toBe('fail');
expect(payload.gate.exit_code).toBe(1);
expect(payload.governance.owner).toBe('skills-team');
expect(process.exitCode).toBe(1);
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status writes dedicated promotion contract to output file', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
const auditPath = join(root, 'audit.log');
const outputPath = join(root, 'rollout-contract.json');
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeFileSync(auditPath, '', 'utf-8');
writeSkillsCliConfig(configPath, {
managedDir,
bundledDir,
workspaceDir,
installationExecution: 'enabled',
allowShellRunner: true,
shellRunnerAllowlist: ['npm install*'],
shellRunnerGovernanceOwner: 'skills-team',
auditPath,
});
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'rollout-status', '--contract', '--out', outputPath, '-c', configPath], {
from: 'user',
});
expect(existsSync(outputPath)).toBe(true);
const payload = JSON.parse(readFileSync(outputPath, 'utf-8'));
expect(payload.schema).toBe('skills.rollout.promotion_contract.v1');
expect(payload.gate).toBeDefined();
expect(payload.summary).toBeDefined();
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status includes trend deltas across adjacent windows', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
src/cli/skills.ts
+99 -2
View File
@@ -132,6 +132,32 @@ export interface ShellRunnerPromotionPolicyStatus {
blockers: string[];
}
export interface ShellRunnerPromotionContract {
schema: 'skills.rollout.promotion_contract.v1';
generated_at: string;
window_days: number;
gate: {
status: 'pass' | 'fail';
exit_code: 0 | 1;
reason: 'promotion_eligible' | 'promotion_not_eligible';
blockers: string[];
};
recommendation: ShellRunnerRolloutRecommendation;
governance: {
owner: string | null;
review_cadence_days: number;
promotion_min_success_rate: number;
};
summary: {
command_result_total: number;
command_result_failed: number;
allowlist_blocked: number;
hash_coverage_pct: number;
};
promotion_policy: ShellRunnerPromotionPolicyStatus;
trend: ShellRunnerAuditTrendSnapshot['deltas'];
}
export type ShellRunnerRolloutRecommendation = 'locked' | 'guarded_observe' | 'guarded_review' | 'expand_candidate';
export function evaluateShellRunnerRolloutGuardrails(
@@ -321,6 +347,50 @@ export function recommendShellRunnerRolloutPhase(
return 'expand_candidate';
}
export function toShellRunnerPromotionContract(args: {
generatedAt: string;
days: number;
recommendation: ShellRunnerRolloutRecommendation;
guardrails: ShellRunnerRolloutGuardrailStatus;
summary: ShellRunnerAuditWindowSummary;
trend: ShellRunnerAuditTrendSnapshot;
promotionPolicy: ShellRunnerPromotionPolicyStatus;
governance: {
owner: string | null;
review_cadence_days: number;
promotion_min_success_rate: number;
};
}): ShellRunnerPromotionContract {
const blockers = [...args.guardrails.blockers, ...args.promotionPolicy.blockers];
const eligible = args.promotionPolicy.eligible && blockers.length === 0;
return {
schema: 'skills.rollout.promotion_contract.v1',
generated_at: args.generatedAt,
window_days: args.days,
gate: {
status: eligible ? 'pass' : 'fail',
exit_code: eligible ? 0 : 1,
reason: eligible ? 'promotion_eligible' : 'promotion_not_eligible',
blockers,
},
recommendation: args.recommendation,
governance: {
owner: args.governance.owner,
review_cadence_days: args.governance.review_cadence_days,
promotion_min_success_rate: args.governance.promotion_min_success_rate,
},
summary: {
command_result_total: args.summary.command_result_total,
command_result_failed: args.summary.command_result_failed,
allowlist_blocked: args.summary.allowlist_blocked,
hash_coverage_pct: calculateShellRunnerHashCoveragePercent(args.summary),
},
promotion_policy: args.promotionPolicy,
trend: args.trend.deltas,
};
}
function expandHomePath(pathValue: string): string {
if (pathValue.startsWith('~/')) {
return resolve(homedir(), pathValue.slice(2));
@@ -1337,10 +1407,11 @@ export function registerSkillsCommand(program: Command): void {
.command('rollout-status')
.description('Show shell runner rollout guardrails and audit review summary')
.option('--days <n>', 'Look back N days in audit logs (default: 7)', '7')
.option('--contract', 'Output dedicated machine-readable promotion contract JSON')
.option('--out <path>', 'Write rollout JSON payload to file')
.option('--json', 'Output as JSON')
.option('-c, --config <path>', 'Config file path')
.action(async (opts: { days?: string; out?: string; json?: boolean; config?: string }) => {
.action(async (opts: { days?: string; contract?: boolean; out?: string; json?: boolean; config?: string }) => {
const loaded = loadConfigSafe(opts.config);
if (loaded.error || !loaded.config) {
console.error(loaded.error ?? 'Failed to load config');
@@ -1380,7 +1451,9 @@ export function registerSkillsCommand(program: Command): void {
promotion_min_success_rate: governance.promotion_min_success_rate,
},
});
const generatedAt = new Date(nowMs).toISOString();
const rolloutPayload = {
generated_at: generatedAt,
days: parsedDays,
guardrails,
summary: trend.current,
@@ -1393,9 +1466,33 @@ export function registerSkillsCommand(program: Command): void {
promotion_min_success_rate: governance.promotion_min_success_rate,
},
};
const promotionContract = toShellRunnerPromotionContract({
generatedAt,
days: parsedDays,
recommendation,
guardrails,
summary: trend.current,
trend,
promotionPolicy,
governance: {
owner: governance.owner ?? null,
review_cadence_days: governance.review_cadence_days,
promotion_min_success_rate: governance.promotion_min_success_rate,
},
});
if (opts.out) {
writeFileSync(expandHomePath(opts.out), JSON.stringify(rolloutPayload, null, 2), 'utf-8');
writeFileSync(
expandHomePath(opts.out),
JSON.stringify(opts.contract ? promotionContract : rolloutPayload, null, 2),
'utf-8',
);
}
if (opts.contract) {
console.log(JSON.stringify(promotionContract, null, 2));
process.exitCode = promotionContract.gate.exit_code;
return;
}
if (opts.json) {