feat(skills): add rollout promotion contract and sync planning state

This commit is contained in:
William Valentin
2026-02-13 08:51:19 -08:00
parent 46099664f0
commit 8f644d5e25
8 changed files with 464 additions and 56 deletions
+151
View File
@@ -35,6 +35,7 @@ import {
recommendShellRunnerRolloutPhase,
sanitizeSkillInstallerAuditReason,
summarizeShellRunnerAuditWindow,
toShellRunnerPromotionContract,
resolveSkillInstallerCommandRunner,
runSkillExecuteAction,
runSkillInstallAction,
@@ -753,6 +754,70 @@ describe('skills CLI helpers', () => {
expect(policy.blockers).toContain('failures increased by 1 vs previous window');
});
it('builds machine-readable promotion contract with gate status and blockers', () => {
const contract = toShellRunnerPromotionContract({
generatedAt: '2026-02-13T00:00:00.000Z',
days: 7,
recommendation: 'guarded_review',
guardrails: { blockers: ['skills.installation_execution must be enabled'] },
summary: {
command_result_total: 4,
command_result_failed: 1,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 3,
unhashed_command_count: 1,
},
trend: {
current: {
command_result_total: 4,
command_result_failed: 1,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 3,
unhashed_command_count: 1,
},
previous: {
command_result_total: 4,
command_result_failed: 0,
allowlist_blocked: 0,
execution_blocked: 0,
hashed_command_count: 4,
unhashed_command_count: 0,
},
deltas: {
failures: 1,
allowlist_blocks: 0,
hash_coverage_pct: -25,
},
},
promotionPolicy: {
eligible: false,
recommendation: 'not_eligible',
cadence_days: 7,
reviewed_window_days: 7,
success_rate: 0.75,
minimum_success_rate: 0.9,
failures_delta: 1,
allowlist_blocks_delta: 0,
hash_coverage_delta_pct: -25,
blockers: ['success rate 75.00% below minimum 90.00%'],
},
governance: {
owner: 'skills-team',
review_cadence_days: 7,
promotion_min_success_rate: 0.9,
},
});
expect(contract.schema).toBe('skills.rollout.promotion_contract.v1');
expect(contract.gate.status).toBe('fail');
expect(contract.gate.exit_code).toBe(1);
expect(contract.gate.blockers).toContain('skills.installation_execution must be enabled');
expect(contract.gate.blockers).toContain('success rate 75.00% below minimum 90.00%');
expect(contract.summary.hash_coverage_pct).toBe(75);
});
it('marks promotion policy eligible when thresholds and trends are healthy', () => {
const policy = evaluateShellRunnerPromotionPolicy({
trend: {
@@ -2161,6 +2226,92 @@ describe('skills CLI helpers', () => {
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status emits dedicated promotion contract JSON with exit code', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
const auditPath = join(root, 'audit.log');
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeFileSync(auditPath, '', 'utf-8');
writeSkillsCliConfig(configPath, {
managedDir,
bundledDir,
workspaceDir,
installationExecution: 'enabled',
allowShellRunner: true,
shellRunnerAllowlist: ['npm install*'],
shellRunnerGovernanceOwner: 'skills-team',
auditPath,
});
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'rollout-status', '--contract', '-c', configPath], { from: 'user' });
const payload = JSON.parse(String(logSpy.mock.calls[0]?.[0]));
expect(payload.schema).toBe('skills.rollout.promotion_contract.v1');
expect(payload.gate.status).toBe('fail');
expect(payload.gate.exit_code).toBe(1);
expect(payload.governance.owner).toBe('skills-team');
expect(process.exitCode).toBe(1);
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status writes dedicated promotion contract to output file', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
const auditPath = join(root, 'audit.log');
const outputPath = join(root, 'rollout-contract.json');
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeFileSync(auditPath, '', 'utf-8');
writeSkillsCliConfig(configPath, {
managedDir,
bundledDir,
workspaceDir,
installationExecution: 'enabled',
allowShellRunner: true,
shellRunnerAllowlist: ['npm install*'],
shellRunnerGovernanceOwner: 'skills-team',
auditPath,
});
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'rollout-status', '--contract', '--out', outputPath, '-c', configPath], {
from: 'user',
});
expect(existsSync(outputPath)).toBe(true);
const payload = JSON.parse(readFileSync(outputPath, 'utf-8'));
expect(payload.schema).toBe('skills.rollout.promotion_contract.v1');
expect(payload.gate).toBeDefined();
expect(payload.summary).toBeDefined();
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills rollout-status includes trend deltas across adjacent windows', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');