feat(skills): hash installer audit commands for shell runner telemetry

src/cli/skills.test.ts

@@ -25,6 +25,10 @@ import {
   runInstallerCommandsWithPolicy,
   noOpSkillInstallerCommandRunner,
   createShellSkillInstallerCommandRunner,
+  checkCommandAgainstAllowlist,
+  emitShellRunnerAuditEvents,
+  hashSkillInstallerAuditCommand,
+  sanitizeSkillInstallerAuditReason,
   resolveSkillInstallerCommandRunner,
   runSkillExecuteAction,
   runSkillInstallAction,
@@ -48,7 +52,18 @@ function buildSkill(overrides: Partial<Skill>): Skill {
   };
 }
 
-function writeSkillsCliConfig(configPath: string, opts: { managedDir: string; bundledDir: string; workspaceDir: string }): void {
+function writeSkillsCliConfig(
+  configPath: string,
+  opts: {
+    managedDir: string;
+    bundledDir: string;
+    workspaceDir: string;
+    installationExecution?: 'disabled' | 'enabled';
+    allowShellRunner?: boolean;
+    shellRunnerAllowlist?: string[];
+  },
+): void {
+  const allowlist = opts.shellRunnerAllowlist ?? [];
   writeFileSync(
     configPath,
     [
@@ -60,6 +75,9 @@ function writeSkillsCliConfig(configPath: string, opts: { managedDir: string; bu
       `  managed_dir: ${opts.managedDir}`,
       `  bundled_dir: ${opts.bundledDir}`,
       `  workspace_dir: ${opts.workspaceDir}`,
+      `  installation_execution: ${opts.installationExecution ?? 'disabled'}`,
+      `  allow_shell_runner: ${opts.allowShellRunner ?? false}`,
+      `  shell_runner_allowlist: [${allowlist.map((item) => `'${item}'`).join(', ')}]`,
     ].join('\n'),
     'utf-8',
   );
@@ -319,6 +337,25 @@ describe('skills CLI helpers', () => {
     expect(output).toContain('[brew] succeeded brew install jq (runner_reported_success)');
   });
 
+  it('renders policy guidance when execution is blocked by config policy', () => {
+    const output = renderSkillInstallerExecutionStub({
+      skill: { name: 'exec-stub', tier: 'bundled', version: '1.0.0' },
+      execution: 'stub',
+      mode: 'install',
+      confirmed: true,
+      execution_enabled: false,
+      executed: [],
+      reason: 'execution_policy_disabled',
+      attempted: [{ installer_type: 'brew', command: 'brew install jq' }],
+      results: [{ installer_type: 'brew', command: 'brew install jq', status: 'skipped', reason: 'execution_policy_disabled' }],
+      wouldRun: ['brew install jq'],
+      skipped: [],
+    });
+
+    expect(output).toContain('Execution policy blocked installer commands.');
+    expect(output).toContain('skills.installation_execution: enabled');
+  });
+
   it('derives execution stub view from preflight data', () => {
     const preflight = {
       sourcePath: '/tmp/source-skill',
@@ -396,6 +433,147 @@ describe('skills CLI helpers', () => {
     expect(policy.reason).toBe('execution_enabled');
   });
 
+  it('keeps execution disabled when config policy is disabled', () => {
+    const policy = evaluateInstallerExecutionPolicy({
+      mode: 'install',
+      confirmed: true,
+      executionRequested: true,
+      configPolicyEnabled: false,
+    });
+
+    expect(policy.confirmed).toBe(true);
+    expect(policy.execution_enabled).toBe(false);
+    expect(policy.reason).toBe('execution_policy_disabled');
+  });
+
+  it('matches shell command allowlist patterns with wildcard support', () => {
+    expect(checkCommandAgainstAllowlist('npm install -g zx', ['npm install*'])).toBe(true);
+    expect(checkCommandAgainstAllowlist('node -e "process.exit(0)"', ['node -e*'])).toBe(true);
+    expect(checkCommandAgainstAllowlist('rm -rf /tmp/demo', ['npm install*'])).toBe(false);
+    expect(checkCommandAgainstAllowlist('echo hi', [])).toBe(false);
+  });
+
+  it('emits audit denied events for allowlist-blocked shell commands', () => {
+    const logger = {
+      skillsInstallerExecutionBlocked: vi.fn(),
+      skillsInstallerCommandResult: vi.fn(),
+    };
+
+    emitShellRunnerAuditEvents({
+      skillName: 'audit-skill',
+      phase: 'install',
+      executionRequested: true,
+      executionEnabled: true,
+      reason: 'execution_enabled',
+      results: [
+        {
+          installer_type: 'download',
+          command: 'download https://example.com/pkg.tgz -> <default destination>',
+          status: 'failed',
+          reason: 'allowlist_blocked',
+        },
+      ],
+      logger,
+    });
+
+    expect(logger.skillsInstallerCommandResult).toHaveBeenCalledWith(
+      expect.objectContaining({
+        skill_name: 'audit-skill',
+        phase: 'install',
+        status: 'failed',
+        reason: 'allowlist_blocked',
+      }),
+    );
+    expect(logger.skillsInstallerCommandResult).toHaveBeenCalledWith(
+      expect.objectContaining({
+        command: hashSkillInstallerAuditCommand('download https://example.com/pkg.tgz -> <default destination>'),
+      }),
+    );
+    expect(logger.skillsInstallerExecutionBlocked).not.toHaveBeenCalled();
+  });
+
+  it('hashes audit command values deterministically for non-sensitive commands', () => {
+    const command = 'download https://example.com/tool.tgz -> <default destination>';
+    expect(hashSkillInstallerAuditCommand(command)).toBe(hashSkillInstallerAuditCommand(command));
+    expect(hashSkillInstallerAuditCommand(command)).toMatch(/^sha256:[a-f0-9]{64}$/);
+  });
+
+  it('sanitizes spawn_error reason values for audit events', () => {
+    expect(sanitizeSkillInstallerAuditReason('spawn_error:ENOENT some sensitive detail')).toBe('spawn_error');
+    expect(sanitizeSkillInstallerAuditReason('allowlist_blocked')).toBe('allowlist_blocked');
+  });
+
+  it('emits hashed command values for both successful and failed audit command results', () => {
+    const logger = {
+      skillsInstallerExecutionBlocked: vi.fn(),
+      skillsInstallerCommandResult: vi.fn(),
+    };
+
+    emitShellRunnerAuditEvents({
+      skillName: 'audit-skill',
+      phase: 'execute',
+      executionRequested: true,
+      executionEnabled: true,
+      reason: 'execution_enabled',
+      results: [
+        {
+          installer_type: 'node',
+          command: 'node -e "process.exit(0)"',
+          status: 'succeeded',
+          reason: 'runner_reported_success',
+        },
+        {
+          installer_type: 'node',
+          command: 'node -e "process.exit(7)"',
+          status: 'failed',
+          reason: 'exit_code_7',
+        },
+      ],
+      logger,
+    });
+
+    expect(logger.skillsInstallerCommandResult).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        command: hashSkillInstallerAuditCommand('node -e "process.exit(0)"'),
+        status: 'succeeded',
+      }),
+    );
+    expect(logger.skillsInstallerCommandResult).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({
+        command: hashSkillInstallerAuditCommand('node -e "process.exit(7)"'),
+        status: 'failed',
+      }),
+    );
+  });
+
+  it('emits audit denied event when shell execution is policy-blocked', () => {
+    const logger = {
+      skillsInstallerExecutionBlocked: vi.fn(),
+      skillsInstallerCommandResult: vi.fn(),
+    };
+
+    emitShellRunnerAuditEvents({
+      skillName: 'audit-skill',
+      phase: 'execute',
+      executionRequested: true,
+      executionEnabled: false,
+      reason: 'execution_policy_disabled',
+      results: [],
+      logger,
+    });
+
+    expect(logger.skillsInstallerExecutionBlocked).toHaveBeenCalledWith(
+      expect.objectContaining({
+        skill_name: 'audit-skill',
+        phase: 'execute',
+        reason: 'execution_policy_disabled',
+      }),
+    );
+    expect(logger.skillsInstallerCommandResult).not.toHaveBeenCalled();
+  });
+
   it('resolves installer runner mode and validates invalid values', () => {
     const noop = resolveSkillInstallerCommandRunner();
     expect('error' in noop).toBe(false);
@@ -472,6 +650,20 @@ describe('skills CLI helpers', () => {
     ]);
   });
 
+  it('shell command runner blocks commands outside allowlist', () => {
+    const runner = createShellSkillInstallerCommandRunner(['npm install*']);
+
+    const results = runner.run(['node -e "process.exit(0)"']);
+
+    expect(results).toEqual([
+      {
+        command: 'node -e "process.exit(0)"',
+        status: 'failed',
+        reason: 'allowlist_blocked',
+      },
+    ]);
+  });
+
   it('maps runner command results into structured per-step statuses', () => {
     const attempted = [
       { installer_type: 'brew', command: 'brew install jq' },
@@ -958,7 +1150,7 @@ describe('skills CLI helpers', () => {
     mkdirSync(managedDir, { recursive: true });
     mkdirSync(bundledDir, { recursive: true });
     mkdirSync(workspaceDir, { recursive: true });
-    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, installationExecution: 'enabled' });
 
     const program = new Command();
     registerSkillsCommand(program);
@@ -976,6 +1168,94 @@ describe('skills CLI helpers', () => {
     rmSync(root, { recursive: true, force: true });
   });
 
+  it('skills install rejects shell runner when config disallows it', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
+    const configPath = join(root, 'config.yaml');
+    const managedDir = join(root, 'managed');
+    const bundledDir = join(root, 'bundled');
+    const workspaceDir = join(root, 'workspace');
+    mkdirSync(managedDir, { recursive: true });
+    mkdirSync(bundledDir, { recursive: true });
+    mkdirSync(workspaceDir, { recursive: true });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, allowShellRunner: false });
+
+    const program = new Command();
+    registerSkillsCommand(program);
+
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    process.exitCode = undefined;
+
+    await program.parseAsync(['skills', 'install', '/tmp/any-skill', '--runner', 'shell', '-c', configPath], { from: 'user' });
+
+    expect(errorSpy).toHaveBeenCalledWith("Runner 'shell' is disabled by config. Set skills.allow_shell_runner: true to enable it.");
+    expect(logSpy).not.toHaveBeenCalled();
+    expect(process.exitCode).toBe(1);
+
+    errorSpy.mockRestore();
+    logSpy.mockRestore();
+    process.exitCode = undefined;
+    rmSync(root, { recursive: true, force: true });
+  });
+
+  it('skills install marks shell execution as allowlist_blocked when command is not allowed', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
+    const configPath = join(root, 'config.yaml');
+    const sourceDir = join(root, 'source-skill');
+    const managedDir = join(root, 'managed');
+    const bundledDir = join(root, 'bundled');
+    const workspaceDir = join(root, 'workspace');
+    mkdirSync(sourceDir, { recursive: true });
+    mkdirSync(managedDir, { recursive: true });
+    mkdirSync(bundledDir, { recursive: true });
+    mkdirSync(workspaceDir, { recursive: true });
+    writeSkillsCliConfig(configPath, {
+      managedDir,
+      bundledDir,
+      workspaceDir,
+      installationExecution: 'enabled',
+      allowShellRunner: true,
+      shellRunnerAllowlist: ['echo*'],
+    });
+    writeFileSync(join(sourceDir, 'SKILL.md'), '# Install Skill\nInstructions');
+    writeFileSync(
+      join(sourceDir, 'manifest.json'),
+      JSON.stringify({
+        name: 'cli-install-allowlist-blocked',
+        description: 'CLI install allowlist blocked',
+        version: '1.0.0',
+        installers: [{ type: 'download', url: 'https://example.com/cli-install-allowlist-blocked.tgz' }],
+      }),
+      'utf-8',
+    );
+
+    const program = new Command();
+    registerSkillsCommand(program);
+
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    process.exitCode = undefined;
+
+    await program.parseAsync(
+      ['skills', 'install', sourceDir, '--json', '--execute', '--confirm', '--runner', 'shell', '-c', configPath],
+      { from: 'user' },
+    );
+
+    const payload = JSON.parse(String(logSpy.mock.calls[logSpy.mock.calls.length - 1]?.[0]));
+    expect(payload.execution.execution_enabled).toBe(true);
+    expect(payload.execution.results).toEqual([
+      {
+        installer_type: 'download',
+        command: 'download https://example.com/cli-install-allowlist-blocked.tgz -> <default destination>',
+        status: 'failed',
+        reason: 'allowlist_blocked',
+      },
+    ]);
+
+    logSpy.mockRestore();
+    process.exitCode = undefined;
+    rmSync(root, { recursive: true, force: true });
+  });
+
   it('skills install parses execute flags and emits execution-enabled JSON receipt', async () => {
     const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
     const configPath = join(root, 'config.yaml');
@@ -987,7 +1267,7 @@ describe('skills CLI helpers', () => {
     mkdirSync(managedDir, { recursive: true });
     mkdirSync(bundledDir, { recursive: true });
     mkdirSync(workspaceDir, { recursive: true });
-    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, installationExecution: 'enabled' });
     writeFileSync(join(sourceDir, 'SKILL.md'), '# Install Skill\nInstructions');
     writeFileSync(
       join(sourceDir, 'manifest.json'),
@@ -1158,7 +1438,7 @@ describe('skills CLI helpers', () => {
     mkdirSync(skillDir, { recursive: true });
     mkdirSync(bundledDir, { recursive: true });
     mkdirSync(workspaceDir, { recursive: true });
-    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, installationExecution: 'enabled' });
     writeFileSync(join(skillDir, 'SKILL.md'), '# Execute Skill\nInstructions');
     writeFileSync(
       join(skillDir, 'manifest.json'),
@@ -1191,6 +1471,68 @@ describe('skills CLI helpers', () => {
     rmSync(root, { recursive: true, force: true });
   });
 
+  it('skills execute keeps execution policy-disabled even with --execute --confirm', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
+    const configPath = join(root, 'config.yaml');
+    const managedDir = join(root, 'managed');
+    const bundledDir = join(root, 'bundled');
+    const workspaceDir = join(root, 'workspace');
+    const skillDir = join(managedDir, 'cli-exec-policy-disabled');
+    mkdirSync(skillDir, { recursive: true });
+    mkdirSync(bundledDir, { recursive: true });
+    mkdirSync(workspaceDir, { recursive: true });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, installationExecution: 'disabled' });
+    writeFileSync(join(skillDir, 'SKILL.md'), '# Execute Skill\nInstructions');
+    writeFileSync(
+      join(skillDir, 'manifest.json'),
+      JSON.stringify({
+        name: 'cli-exec-policy-disabled',
+        description: 'CLI execute policy disabled',
+        version: '1.0.0',
+        installers: [{ type: 'download', url: 'https://example.com/cli-exec-policy-disabled.tgz' }],
+      }),
+      'utf-8',
+    );
+
+    const program = new Command();
+    registerSkillsCommand(program);
+
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    process.exitCode = undefined;
+
+    await program.parseAsync(
+      [
+        'skills',
+        'execute',
+        'cli-exec-policy-disabled',
+        '--json',
+        '--execute',
+        '--confirm',
+        '--runner',
+        'noop',
+        '-c',
+        configPath,
+      ],
+      { from: 'user' },
+    );
+
+    const payload = JSON.parse(String(logSpy.mock.calls[0]?.[0]));
+    expect(payload.execution_enabled).toBe(false);
+    expect(payload.reason).toBe('execution_policy_disabled');
+    expect(payload.results).toEqual([
+      {
+        installer_type: 'download',
+        command: 'download https://example.com/cli-exec-policy-disabled.tgz -> <default destination>',
+        status: 'skipped',
+        reason: 'execution_policy_disabled',
+      },
+    ]);
+
+    logSpy.mockRestore();
+    process.exitCode = undefined;
+    rmSync(root, { recursive: true, force: true });
+  });
+
   it('skills execute JSON uses execution_disabled fallback when --execute is omitted', async () => {
     const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
     const configPath = join(root, 'config.yaml');
@@ -1281,4 +1623,113 @@ describe('skills CLI helpers', () => {
     process.exitCode = undefined;
     rmSync(root, { recursive: true, force: true });
   });
+
+  it('skills execute rejects shell runner when config disallows it', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
+    const configPath = join(root, 'config.yaml');
+    const managedDir = join(root, 'managed');
+    const bundledDir = join(root, 'bundled');
+    const workspaceDir = join(root, 'workspace');
+    const skillDir = join(managedDir, 'cli-exec-skill');
+    mkdirSync(skillDir, { recursive: true });
+    mkdirSync(bundledDir, { recursive: true });
+    mkdirSync(workspaceDir, { recursive: true });
+    writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, allowShellRunner: false });
+    writeFileSync(join(skillDir, 'SKILL.md'), '# Execute Skill\nInstructions');
+    writeFileSync(
+      join(skillDir, 'manifest.json'),
+      JSON.stringify({
+        name: 'cli-exec-skill',
+        description: 'CLI execute parse',
+        version: '1.0.0',
+      }),
+      'utf-8',
+    );
+
+    const program = new Command();
+    registerSkillsCommand(program);
+
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    process.exitCode = undefined;
+
+    await program.parseAsync(['skills', 'execute', 'cli-exec-skill', '--runner', 'shell', '-c', configPath], { from: 'user' });
+
+    expect(errorSpy).toHaveBeenCalledWith("Runner 'shell' is disabled by config. Set skills.allow_shell_runner: true to enable it.");
+    expect(logSpy).not.toHaveBeenCalled();
+    expect(process.exitCode).toBe(1);
+
+    errorSpy.mockRestore();
+    logSpy.mockRestore();
+    process.exitCode = undefined;
+    rmSync(root, { recursive: true, force: true });
+  });
+
+  it('skills execute marks shell execution as allowlist_blocked when command is not allowed', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
+    const configPath = join(root, 'config.yaml');
+    const managedDir = join(root, 'managed');
+    const bundledDir = join(root, 'bundled');
+    const workspaceDir = join(root, 'workspace');
+    const skillDir = join(managedDir, 'cli-exec-allowlist-blocked');
+    mkdirSync(skillDir, { recursive: true });
+    mkdirSync(bundledDir, { recursive: true });
+    mkdirSync(workspaceDir, { recursive: true });
+    writeSkillsCliConfig(configPath, {
+      managedDir,
+      bundledDir,
+      workspaceDir,
+      installationExecution: 'enabled',
+      allowShellRunner: true,
+      shellRunnerAllowlist: ['echo*'],
+    });
+    writeFileSync(join(skillDir, 'SKILL.md'), '# Execute Skill\nInstructions');
+    writeFileSync(
+      join(skillDir, 'manifest.json'),
+      JSON.stringify({
+        name: 'cli-exec-allowlist-blocked',
+        description: 'CLI execute allowlist blocked',
+        version: '1.0.0',
+        installers: [{ type: 'download', url: 'https://example.com/cli-exec-allowlist-blocked.tgz' }],
+      }),
+      'utf-8',
+    );
+
+    const program = new Command();
+    registerSkillsCommand(program);
+
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    process.exitCode = undefined;
+
+    await program.parseAsync(
+      [
+        'skills',
+        'execute',
+        'cli-exec-allowlist-blocked',
+        '--json',
+        '--execute',
+        '--confirm',
+        '--runner',
+        'shell',
+        '-c',
+        configPath,
+      ],
+      { from: 'user' },
+    );
+
+    const payload = JSON.parse(String(logSpy.mock.calls[0]?.[0]));
+    expect(payload.execution_enabled).toBe(true);
+    expect(payload.results).toEqual([
+      {
+        installer_type: 'download',
+        command: 'download https://example.com/cli-exec-allowlist-blocked.tgz -> <default destination>',
+        status: 'failed',
+        reason: 'allowlist_blocked',
+      },
+    ]);
+
+    logSpy.mockRestore();
+    process.exitCode = undefined;
+    rmSync(root, { recursive: true, force: true });
+  });
 });