will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(skills): add edge-case receipt tests for partial/missing runner results

Browse Source
This commit is contained in:
William Valentin
2026-02-12 21:12:15 -08:00
parent 8c4ac2cb00
commit d3647567ee
2 changed files with 223 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/plans/state.json
+10 -2
View File
@@ -1495,6 +1495,14 @@
"src/cli/skills.test.ts"
],
"test_status": "pnpm typecheck + pnpm test:run src/cli/skills.test.ts + pnpm test:run + pnpm lint (warnings only, 0 errors) + pnpm build passing"
},
"installer_receipt_edge_case_tests": {
"status": "completed",
"description": "Added edge-case coverage for execution receipt mapping: partial/empty runner results, deterministic fallback reasons, and commander-path JSON fallback statuses for install/execute flows",
"files_modified": [
"src/cli/skills.test.ts"
],
"test_status": "pnpm typecheck + pnpm test:run src/cli/skills.test.ts + pnpm test:run + pnpm lint (warnings only, 0 errors) + pnpm build passing"
}
}
}
@@ -1523,7 +1531,7 @@
},
"overall_progress": {
"total_test_count": 1555,
"total_test_count": 1560,
"all_tests_passing": true,
"p0_completion": "3/3 (100%)",
"p1_completion": "4/4 (100%)",
@@ -1543,7 +1551,7 @@
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 2/2 (100%) — component registry, confidence routing. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "Skills infrastructure Phase 3: add edge-case receipt tests for partial/missing runner results and deterministic fallback reasons across install/execute flows"
"next_up": "Skills infrastructure follow-up: define policy/autonomy model for safely enabling real installer execution beyond the current execution-disabled default"
},
"soul_md_and_cron_create": {
"date": "2026-02-11",
src/cli/skills.test.ts
+213
View File
@@ -520,6 +520,65 @@ describe('skills CLI helpers', () => {
]);
});
it('fills missing commands with runner_no_result when runner only returns partial results', () => {
const attempted = [
{ installer_type: 'brew', command: 'brew install jq' },
{ installer_type: 'node', command: 'pnpm add -g zx' },
];
const results = mergeInstallerExecutionResults(
attempted,
{ confirmed: true, execution_enabled: true, reason: 'execution_enabled' },
[{ command: 'brew install jq', status: 'succeeded', reason: 'ok' }],
);
expect(results).toEqual([
{
installer_type: 'brew',
command: 'brew install jq',
status: 'succeeded',
reason: 'ok',
},
{
installer_type: 'node',
command: 'pnpm add -g zx',
status: 'failed',
reason: 'runner_no_result',
},
]);
});
it('applies deterministic fallback reasons when runner omits reason fields', () => {
const attempted = [
{ installer_type: 'brew', command: 'brew install jq' },
{ installer_type: 'node', command: 'pnpm add -g zx' },
];
const results = mergeInstallerExecutionResults(
attempted,
{ confirmed: true, execution_enabled: true, reason: 'execution_enabled' },
[
{ command: 'brew install jq', status: 'succeeded' },
{ command: 'pnpm add -g zx', status: 'failed' },
],
);
expect(results).toEqual([
{
installer_type: 'brew',
command: 'brew install jq',
status: 'succeeded',
reason: 'runner_reported_success',
},
{
installer_type: 'node',
command: 'pnpm add -g zx',
status: 'failed',
reason: 'runner_reported_failure',
},
]);
});
it('summarizes refresh counts across status and tiers', () => {
const summary = summarizeSkillsRefresh([
buildSkill({ manifest: { name: 'a', description: 'a', version: '1.0.0', tier: 'bundled' } }),
@@ -961,6 +1020,104 @@ describe('skills CLI helpers', () => {
rmSync(root, { recursive: true, force: true });
});
it('skills install JSON uses execution_disabled fallback when --execute is omitted', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const sourceDir = join(root, 'source-skill');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
mkdirSync(sourceDir, { recursive: true });
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
writeFileSync(join(sourceDir, 'SKILL.md'), '# Install Skill\nInstructions');
writeFileSync(
join(sourceDir, 'manifest.json'),
JSON.stringify({
name: 'cli-install-no-exec',
description: 'CLI install no execute',
version: '1.0.0',
installers: [{ type: 'download', url: 'https://example.com/cli-install-no-exec.tgz' }],
}),
'utf-8',
);
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'install', sourceDir, '--json', '--confirm', '-c', configPath], { from: 'user' });
const payload = JSON.parse(String(logSpy.mock.calls[logSpy.mock.calls.length - 1]?.[0]));
expect(payload.execution.execution_enabled).toBe(false);
expect(payload.execution.reason).toBe('execution_disabled');
expect(payload.execution.results).toEqual([
{
installer_type: 'download',
command: 'download https://example.com/cli-install-no-exec.tgz -> <default destination>',
status: 'skipped',
reason: 'execution_disabled',
},
]);
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills install JSON uses confirmation_required fallback when --confirm is omitted', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const sourceDir = join(root, 'source-skill');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
mkdirSync(sourceDir, { recursive: true });
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
writeFileSync(join(sourceDir, 'SKILL.md'), '# Install Skill\nInstructions');
writeFileSync(
join(sourceDir, 'manifest.json'),
JSON.stringify({
name: 'cli-install-no-confirm',
description: 'CLI install no confirm',
version: '1.0.0',
installers: [{ type: 'download', url: 'https://example.com/cli-install-no-confirm.tgz' }],
}),
'utf-8',
);
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'install', sourceDir, '--json', '-c', configPath], { from: 'user' });
const payload = JSON.parse(String(logSpy.mock.calls[logSpy.mock.calls.length - 1]?.[0]));
expect(payload.execution.execution_enabled).toBe(false);
expect(payload.execution.reason).toBe('confirmation_required');
expect(payload.execution.results).toEqual([
{
installer_type: 'download',
command: 'download https://example.com/cli-install-no-confirm.tgz -> <default destination>',
status: 'blocked',
reason: 'confirmation_required',
},
]);
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills install rejects --execute without --confirm', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
@@ -976,14 +1133,17 @@ describe('skills CLI helpers', () => {
registerSkillsCommand(program);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'install', '/tmp/any-skill', '--execute', '-c', configPath], { from: 'user' });
expect(errorSpy).toHaveBeenCalledWith('`--execute` requires `--confirm`. No installer commands were run.');
expect(logSpy).not.toHaveBeenCalled();
expect(process.exitCode).toBe(1);
errorSpy.mockRestore();
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
@@ -1031,6 +1191,56 @@ describe('skills CLI helpers', () => {
rmSync(root, { recursive: true, force: true });
});
it('skills execute JSON uses execution_disabled fallback when --execute is omitted', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
const skillDir = join(managedDir, 'cli-exec-no-exec');
mkdirSync(skillDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
writeFileSync(join(skillDir, 'SKILL.md'), '# Execute Skill\nInstructions');
writeFileSync(
join(skillDir, 'manifest.json'),
JSON.stringify({
name: 'cli-exec-no-exec',
description: 'CLI execute no execute',
version: '1.0.0',
installers: [{ type: 'download', url: 'https://example.com/cli-exec-no-exec.tgz' }],
}),
'utf-8',
);
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'execute', 'cli-exec-no-exec', '--json', '--confirm', '-c', configPath], {
from: 'user',
});
const payload = JSON.parse(String(logSpy.mock.calls[0]?.[0]));
expect(payload.execution_enabled).toBe(false);
expect(payload.reason).toBe('execution_disabled');
expect(payload.results).toEqual([
{
installer_type: 'download',
command: 'download https://example.com/cli-exec-no-exec.tgz -> <default destination>',
status: 'skipped',
reason: 'execution_disabled',
},
]);
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills execute rejects --execute without --confirm', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
@@ -1057,14 +1267,17 @@ describe('skills CLI helpers', () => {
registerSkillsCommand(program);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(['skills', 'execute', 'cli-exec-skill', '--execute', '-c', configPath], { from: 'user' });
expect(errorSpy).toHaveBeenCalledWith('`--execute` requires `--confirm`. No installer commands were run.');
expect(logSpy).not.toHaveBeenCalled();
expect(process.exitCode).toBe(1);
errorSpy.mockRestore();
logSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});