feat(skills): add registry doctor diagnostics and docs

This commit is contained in:
William Valentin
2026-02-16 00:53:25 -08:00
parent 23609a03a4
commit ae36248da8
11 changed files with 298 additions and 23 deletions
+32 -1
View File
@@ -93,6 +93,36 @@ flynn completion zsh --install # Install zsh completions to ~/.zshrc
flynn completion fish --install # Install fish completions to config
```
## Skills Registry
Flynn supports registry-backed skill discovery and install-by-id.
Registry source configuration:
```yaml
skills:
registry_source: ~/.config/flynn/skills-registry.json
```
You can also set `FLYNN_SKILLS_REGISTRY_SOURCE` instead of `skills.registry_source`.
Common commands:
```bash
# Discover registry entries
flynn skills registry list
flynn skills registry list --search todo --publisher acme --json
flynn skills registry show todoist
# Install by registry id
flynn skills install --registry-id todoist
# For non-local registry sources (git/archive), explicit confirmation is required
flynn skills install --registry-id todoist --confirm
```
Registry metadata (`publisher`, `homepage`, `sha256`) is treated as declared and unverified. Skill scanner checks still run before installation succeeds.
## Configuration
Config location: `~/.config/flynn/config.yaml` (or set `FLYNN_CONFIG`)
@@ -849,7 +879,7 @@ See `docs/deployment/NIX.md` for the flake (package + dev shell + optional NixOS
## Doctor Diagnostics
`flynn doctor` runs 10 health checks to validate your setup:
`flynn doctor` runs health checks to validate your setup:
```
$ flynn doctor
@@ -885,6 +915,7 @@ Results: 8 passed, 0 failed, 0 warnings, 1 skipped
| Telegram bot configured | Bot token is present and reasonable length |
| MCP servers configured | Lists configured MCP tool servers |
| Skills loaded | Discovers and loads skill packages |
| Skills registry | Verifies registry source is configured and catalog is reachable/parsible |
Exit code is `1` if any check fails, `0` otherwise. Checks that depend on a valid config are skipped when config is invalid.
+3
View File
@@ -114,6 +114,9 @@ hooks:
# context_level: normal # minimal | normal | detailed | debug
# skills:
# # Registry catalog source for discovery and install-by-id:
# # local path or HTTPS URL. Can also be set via FLYNN_SKILLS_REGISTRY_SOURCE.
# registry_source: ~/.config/flynn/skills-registry.json
# # Global installer execution policy.
# # disabled: never run installer commands (default)
# # enabled: allow command execution only with --execute --confirm
@@ -95,11 +95,11 @@ Tests:
Checklist:
- [ ] Update `README.md` skills section with registry usage.
- [ ] Update `docs/security/SAFE_PERSONAL_AGENT.md` with registry trust model.
- [ ] Add doctor diagnostics:
- [ ] registry source reachable/parsible
- [ ] clear warning when registry disabled/unconfigured
- [x] Update `README.md` skills section with registry usage.
- [x] Update `docs/security/SAFE_PERSONAL_AGENT.md` with registry trust model.
- [x] Add doctor diagnostics:
- [x] registry source reachable/parsible
- [x] clear warning when registry disabled/unconfigured
Acceptance:
@@ -107,17 +107,17 @@ Acceptance:
Tests:
- [ ] Doctor tests for registry health reporting.
- [x] Doctor tests for registry health reporting.
## Security Guardrails
- [ ] Registry metadata is never treated as trusted code.
- [ ] Skill scanner remains mandatory before skill becomes available.
- [ ] Prompt injection and symlink/binary checks still gate registry-installed skills.
- [ ] Secrets are never accepted from registry metadata.
- [x] Registry metadata is never treated as trusted code.
- [x] Skill scanner remains mandatory before skill becomes available.
- [x] Prompt injection and symlink/binary checks still gate registry-installed skills.
- [x] Secrets are never accepted from registry metadata.
## Final Validation
- [ ] `pnpm typecheck`
- [x] `pnpm typecheck`
- [ ] `pnpm test:run`
- [ ] Update `docs/plans/state.json` to `completed` with summary + test status once all phases land.
+10 -3
View File
@@ -216,7 +216,7 @@
"status": "in_progress",
"date": "2026-02-16",
"updated": "2026-02-16",
"summary": "Completed Phase 3 install-by-registry-id flow: added `flynn skills install --registry-id` with registry lookup and source-type resolution (local/git/archive), non-local confirmation guard, temp materialization path, and dedicated registry install audit events while preserving existing scanner/install safety gates.",
"summary": "Completed Phase 4 docs + runtime visibility for ClawHub registry: added config-backed registry source (`skills.registry_source`), doctor diagnostics for registry configured/reachable/parsable state, README usage guidance, and security trust-model documentation. Registry install flow now also falls back to config registry source.",
"files_created": [
"docs/plans/2026-02-16-clawhub-registry-checklist.md",
"src/skills/registrySource.ts",
@@ -228,10 +228,17 @@
"src/cli/skills.test.ts",
"src/audit/types.ts",
"src/audit/logger.ts",
"src/cli/doctor.ts",
"src/cli/doctor.test.ts",
"src/config/schema.ts",
"src/config/schema.test.ts",
"README.md",
"docs/security/SAFE_PERSONAL_AGENT.md",
"config/default.yaml",
"docs/plans/2026-02-16-clawhub-registry-checklist.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/skills/registrySource.test.ts src/cli/skills.test.ts + pnpm typecheck passing"
"test_status": "pnpm test:run src/skills/registrySource.test.ts src/cli/skills.test.ts src/cli/doctor.test.ts src/config/schema.test.ts + pnpm typecheck passing; full pnpm test:run currently fails on unrelated src/channels/whatsapp/adapter.test.ts assertions (13 failures)"
},
"credential-system-v2-api-and-oauth": {
"file": "2026-02-15-credential-system-v2-api-and-oauth-checklist.md",
@@ -2772,7 +2779,7 @@
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "ClawHub registry Phase 4: docs + runtime visibility (README/security docs and doctor registry diagnostics)"
"next_up": "Stabilize unrelated WhatsApp adapter test failures in full suite, then run full validation and close ClawHub registry milestone"
},
"soul_md_and_cron_create": {
"date": "2026-02-11",
+12
View File
@@ -69,6 +69,18 @@ Skills without `permissions` still load, but:
- If a skill is activated (via routing) and it has no `permissions` block, **it has no tool access**.
- This is deliberate: skills should be auditable capability packages.
## Registry Trust Model (ClawHub / Community Catalogs)
Registry catalogs are discovery metadata, not trusted code.
- Flynn supports registry discovery and install-by-id via `flynn skills registry *` and `flynn skills install --registry-id`.
- Registry metadata fields such as `publisher`, `homepage`, and `sha256` are treated as **declared/unverified**.
- Non-local registry sources require explicit operator confirmation (`--confirm`) during install.
- Resolved sources (local/git/archive) are still routed through the same skill scanner and installer safety gates.
- Registry-driven installs emit dedicated audit events (`skills.registry_install`) including registry id/source and outcome.
Operationally: treat a registry as a candidate index. Trust is established by your own review and scanner outcomes, not by catalog claims alone.
## Runtime Enforcement
Enforcement happens in two places:
+66
View File
@@ -315,6 +315,72 @@ skills:
expect(skillsCheck?.detail).toContain('1 skill(s), 1 available, 0 unavailable');
});
it('reports WARN for skills registry when unconfigured', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'registry-unconfigured.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const registryCheck = results.find(r => r.label === 'Skills registry');
expect(registryCheck?.status).toBe('warn');
expect(registryCheck?.detail).toContain('unconfigured');
});
it('reports PASS for skills registry when source is parsable', async () => {
mkdirSync(testDir, { recursive: true });
const registryPath = join(testDir, 'registry.json');
writeFileSync(registryPath, JSON.stringify({ skills: [] }), 'utf-8');
const configPath = join(testDir, 'registry-pass.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
skills:
registry_source: "${registryPath}"
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const registryCheck = results.find(r => r.label === 'Skills registry');
expect(registryCheck?.status).toBe('pass');
expect(registryCheck?.detail).toContain('loaded 0 entries');
});
it('reports FAIL for skills registry when source cannot be loaded', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'registry-fail.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
skills:
registry_source: "${join(testDir, 'missing-registry.json')}"
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const registryCheck = results.find(r => r.label === 'Skills registry');
expect(registryCheck?.status).toBe('fail');
});
it('reports FAIL when OpenAI auth_mode=api_key has no available key sources', async () => {
const originalHome = process.env.HOME;
const originalKey = process.env.OPENAI_API_KEY;
+56
View File
@@ -422,6 +422,61 @@ const checkSkills: Check = async (ctx) => {
}
};
function toRegistrySource(value: string): { type: 'file'; path: string } | { type: 'url'; url: string } | null {
const trimmed = value.trim();
if (!trimmed) {
return null;
}
if (trimmed.startsWith('https://')) {
return { type: 'url', url: trimmed };
}
if (trimmed.startsWith('http://')) {
return null;
}
return { type: 'file', path: trimmed };
}
const checkSkillsRegistry: Check = async (ctx) => {
if (!ctx.config) {
return { status: 'skip', label: 'Skills registry', detail: '(config invalid)' };
}
const configured = ctx.config.skills.registry_source?.trim() || process.env.FLYNN_SKILLS_REGISTRY_SOURCE?.trim() || '';
if (!configured) {
return {
status: 'warn',
label: 'Skills registry',
detail: 'registry discovery unconfigured (set skills.registry_source or FLYNN_SKILLS_REGISTRY_SOURCE)',
};
}
const source = toRegistrySource(configured);
if (!source) {
return {
status: 'fail',
label: 'Skills registry',
detail: `invalid registry source '${configured}' (use local path or https:// URL)`,
};
}
try {
const { loadSkillRegistryCatalog } = await import('../skills/index.js');
const catalog = await loadSkillRegistryCatalog(source);
const sourceLabel = source.type === 'file' ? source.path : source.url;
return {
status: 'pass',
label: 'Skills registry',
detail: `loaded ${catalog.skills.length} entr${catalog.skills.length === 1 ? 'y' : 'ies'} from ${sourceLabel}`,
};
} catch (err) {
return {
status: 'fail',
label: 'Skills registry',
detail: err instanceof Error ? err.message : String(err),
};
}
};
const checkTailscale: Check = async (ctx) => {
if (!ctx.config?.server?.tailscale?.serve) {
return { status: 'skip', label: 'Tailscale Serve', detail: '(not enabled)' };
@@ -533,6 +588,7 @@ const allChecks: Check[] = [
checkGmail,
checkMcpServers,
checkSkills,
checkSkillsRegistry,
checkTailscale,
];
+68
View File
@@ -80,6 +80,7 @@ function writeSkillsCliConfig(
allowShellRunner?: boolean;
shellRunnerAllowlist?: string[];
shellRunnerGovernanceOwner?: string;
registrySource?: string;
auditEnabled?: boolean;
auditPath?: string;
},
@@ -105,6 +106,7 @@ function writeSkillsCliConfig(
` installation_execution: ${opts.installationExecution ?? 'disabled'}`,
` allow_shell_runner: ${opts.allowShellRunner ?? false}`,
` shell_runner_allowlist: [${allowlist.map((item) => `'${item}'`).join(', ')}]`,
...(opts.registrySource ? [` registry_source: '${opts.registrySource}'`] : []),
...governanceOwnerLines,
...auditLines,
].join('\n'),
@@ -293,6 +295,10 @@ describe('skills CLI helpers', () => {
expect(noneTrust.status).toBe('none_declared');
expect(resolveSkillRegistrySource('./registry.json').source).toEqual({ type: 'file', path: './registry.json' });
expect(resolveSkillRegistrySource(undefined, './configured-registry.json').source).toEqual({
type: 'file',
path: './configured-registry.json',
});
expect(resolveSkillRegistrySource('https://registry.example/catalog.json').source).toEqual({
type: 'url',
url: 'https://registry.example/catalog.json',
@@ -1968,6 +1974,68 @@ describe('skills CLI helpers', () => {
rmSync(root, { recursive: true, force: true });
});
it('skills install supports --registry-id using config skills.registry_source fallback', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
const registryPath = join(root, 'registry.json');
const sourceSkillDir = join(root, 'registry-skills', 'todoist-config');
const managedDir = join(root, 'managed');
const bundledDir = join(root, 'bundled');
const workspaceDir = join(root, 'workspace');
mkdirSync(sourceSkillDir, { recursive: true });
mkdirSync(managedDir, { recursive: true });
mkdirSync(bundledDir, { recursive: true });
mkdirSync(workspaceDir, { recursive: true });
writeFileSync(join(sourceSkillDir, 'SKILL.md'), '# Todoist Config Skill\nInstructions');
writeFileSync(
join(sourceSkillDir, 'manifest.json'),
JSON.stringify({
name: 'todoist-config',
description: 'Todoist integration config fallback',
version: '1.0.0',
}),
'utf-8',
);
writeFileSync(
registryPath,
JSON.stringify({
skills: [
{
id: 'todoist-config',
name: 'Todoist Config',
version: '1.0.0',
source: './registry-skills/todoist-config',
summary: 'Task manager integration via config fallback',
},
],
}),
'utf-8',
);
writeSkillsCliConfig(configPath, { managedDir, bundledDir, workspaceDir, registrySource: registryPath });
const program = new Command();
registerSkillsCommand(program);
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
process.exitCode = undefined;
await program.parseAsync(
['skills', 'install', '--registry-id', 'todoist-config', '-c', configPath],
{ from: 'user' },
);
expect(errorSpy).not.toHaveBeenCalled();
expect(logSpy).toHaveBeenCalledWith("Installed skill 'todoist-config' (1.0.0).");
expect(existsSync(join(managedDir, 'todoist-config', 'SKILL.md'))).toBe(true);
expect(process.exitCode).toBeUndefined();
logSpy.mockRestore();
errorSpy.mockRestore();
process.exitCode = undefined;
rmSync(root, { recursive: true, force: true });
});
it('skills install requires --confirm for remote registry sources', async () => {
const root = mkdtempSync(join(tmpdir(), 'flynn-skills-cli-'));
const configPath = join(root, 'config.yaml');
+20 -8
View File
@@ -239,11 +239,12 @@ export function renderSkillRegistryEntry(entry: SkillRegistryEntry): string {
export function resolveSkillRegistrySource(
sourceArg?: string,
configSource?: string,
): { source?: SkillRegistrySource; error?: string } {
const raw = sourceArg?.trim() || process.env[SKILL_REGISTRY_SOURCE_ENV]?.trim();
const raw = sourceArg?.trim() || configSource?.trim() || process.env[SKILL_REGISTRY_SOURCE_ENV]?.trim();
if (!raw) {
return {
error: `Registry source is required. Pass --source <path-or-https-url> or set ${SKILL_REGISTRY_SOURCE_ENV}.`,
error: `Registry source is required. Pass --source <path-or-https-url>, set skills.registry_source, or set ${SKILL_REGISTRY_SOURCE_ENV}.`,
};
}
@@ -261,8 +262,9 @@ export function resolveSkillRegistrySource(
export async function loadRegistrySkillLookup(
registryId: string,
sourceArg?: string,
configSource?: string,
): Promise<{ lookup?: RegistrySkillLookup; error?: string }> {
const sourceResult = resolveSkillRegistrySource(sourceArg);
const sourceResult = resolveSkillRegistrySource(sourceArg, configSource);
if (sourceResult.error || !sourceResult.source) {
return { error: sourceResult.error ?? 'Failed to resolve registry source.' };
}
@@ -1746,8 +1748,11 @@ export function registerSkillsCommand(program: Command): void {
.option('--search <term>', 'Filter by id/name/summary/publisher')
.option('--publisher <name>', 'Filter by exact publisher name (case-insensitive)')
.option('--json', 'Output as JSON')
.action(async (opts: { source?: string; search?: string; publisher?: string; json?: boolean }) => {
const sourceResult = resolveSkillRegistrySource(opts.source);
.option('-c, --config <path>', 'Config file path')
.action(async (opts: { source?: string; search?: string; publisher?: string; json?: boolean; config?: string }) => {
const loaded = loadConfigSafe(opts.config);
const configSource = loaded.config?.skills.registry_source;
const sourceResult = resolveSkillRegistrySource(opts.source, configSource);
if (sourceResult.error || !sourceResult.source) {
console.error(sourceResult.error ?? 'Failed to resolve registry source.');
process.exitCode = 1;
@@ -1781,8 +1786,11 @@ export function registerSkillsCommand(program: Command): void {
.description('Show details for a registry entry')
.option('--source <path-or-url>', 'Registry catalog source (local file path or HTTPS URL)')
.option('--json', 'Output as JSON')
.action(async (id: string, opts: { source?: string; json?: boolean }) => {
const sourceResult = resolveSkillRegistrySource(opts.source);
.option('-c, --config <path>', 'Config file path')
.action(async (id: string, opts: { source?: string; json?: boolean; config?: string }) => {
const loaded = loadConfigSafe(opts.config);
const configSource = loaded.config?.skills.registry_source;
const sourceResult = resolveSkillRegistrySource(opts.source, configSource);
if (sourceResult.error || !sourceResult.source) {
console.error(sourceResult.error ?? 'Failed to resolve registry source.');
process.exitCode = 1;
@@ -1903,7 +1911,11 @@ export function registerSkillsCommand(program: Command): void {
| undefined;
if (hasRegistryId) {
const lookupResult = await loadRegistrySkillLookup(opts.registryId ?? '', opts.registrySource);
const lookupResult = await loadRegistrySkillLookup(
opts.registryId ?? '',
opts.registrySource,
loaded.config.skills.registry_source,
);
if (lookupResult.error || !lookupResult.lookup) {
console.error(lookupResult.error ?? 'Failed to resolve registry skill.');
process.exitCode = 1;
+18
View File
@@ -664,3 +664,21 @@ describe('configSchema — agents truthfulness/autonomy', () => {
})).toThrow();
});
});
describe('configSchema — skills registry source', () => {
const minimalConfig = {
telegram: { bot_token: 'test', allowed_chat_ids: [1] },
models: { default: { provider: 'anthropic', model: 'claude-3' } },
};
it('accepts skills.registry_source when provided', () => {
const result = configSchema.parse({
...minimalConfig,
skills: {
registry_source: 'https://registry.example/catalog.json',
},
});
expect(result.skills.registry_source).toBe('https://registry.example/catalog.json');
});
});
+2
View File
@@ -134,6 +134,8 @@ const skillsShellRunnerGovernanceSchema = z.object({
}).default({});
const skillsSchema = z.object({
/** Registry catalog source for `flynn skills registry` and install-by-id (file path or HTTPS URL). */
registry_source: z.string().optional(),
/** Directory for user-created workspace skills. */
workspace_dir: z.string().optional(),
/** Directory for managed (installed) skills. Defaults to ~/.flynn/workspace/skills. */