From 4391c6e5b3c5b5311a9037c8027e1881ce4f0c25 Mon Sep 17 00:00:00 2001 From: William Valentin Date: Mon, 16 Feb 2026 00:10:41 -0800 Subject: [PATCH] feat(skills): add ClawHub registry catalog parser and loader --- .../2026-02-16-clawhub-registry-checklist.md | 20 +-- docs/plans/state.json | 16 +- src/skills/index.ts | 2 + src/skills/registrySource.test.ts | 148 ++++++++++++++++++ src/skills/registrySource.ts | 125 +++++++++++++++ 5 files changed, 298 insertions(+), 13 deletions(-) create mode 100644 src/skills/registrySource.test.ts create mode 100644 src/skills/registrySource.ts diff --git a/docs/plans/2026-02-16-clawhub-registry-checklist.md b/docs/plans/2026-02-16-clawhub-registry-checklist.md index b921b94..95e7bc9 100644 --- a/docs/plans/2026-02-16-clawhub-registry-checklist.md +++ b/docs/plans/2026-02-16-clawhub-registry-checklist.md @@ -27,22 +27,22 @@ Checklist: -- [ ] Add registry types + parser module (`src/skills/registrySource.ts` or equivalent). -- [ ] Support one source shape: - - [ ] local JSON file path (for deterministic tests and offline use) - - [ ] optional HTTPS URL source (fetch + timeout + parse) -- [ ] Validate required fields for each skill entry: - - [ ] `id`, `name`, `version`, `source`, `summary` - - [ ] optional trust metadata (`publisher`, `homepage`, `sha256`) -- [ ] Reject malformed registry entries with actionable errors. +- [x] Add registry types + parser module (`src/skills/registrySource.ts` or equivalent). +- [x] Support one source shape: + - [x] local JSON file path (for deterministic tests and offline use) + - [x] optional HTTPS URL source (fetch + timeout + parse) +- [x] Validate required fields for each skill entry: + - [x] `id`, `name`, `version`, `source`, `summary` + - [x] optional trust metadata (`publisher`, `homepage`, `sha256`) +- [x] Reject malformed registry entries with actionable errors. Acceptance: -- `flynn skills` internals can load a normalized registry catalog. +- [x] `flynn skills` internals can load a normalized registry catalog. Tests: -- [ ] Unit tests for parser/validation edge cases. +- [x] Unit tests for parser/validation edge cases. --- diff --git a/docs/plans/state.json b/docs/plans/state.json index 71fc5c6..3a0fab4 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -213,10 +213,20 @@ }, "clawhub-registry": { "file": "2026-02-16-clawhub-registry-checklist.md", - "status": "planned", + "status": "in_progress", "date": "2026-02-16", "updated": "2026-02-16", - "summary": "Scoped the next OpenClaw-gap milestone as a phased ClawHub-style registry implementation: catalog source + validation, CLI discovery, install-by-id via existing scanner pipeline, and docs/doctor visibility." + "summary": "Started ClawHub registry milestone with Phase 1 complete: added validated registry catalog loader/parser with local file + HTTPS source support and tests. Next step is CLI registry discovery UX.", + "files_created": [ + "docs/plans/2026-02-16-clawhub-registry-checklist.md", + "src/skills/registrySource.ts", + "src/skills/registrySource.test.ts" + ], + "files_modified": [ + "src/skills/index.ts", + "docs/plans/state.json" + ], + "test_status": "pnpm test:run src/skills/registrySource.test.ts + pnpm typecheck passing" }, "credential-system-v2-api-and-oauth": { "file": "2026-02-15-credential-system-v2-api-and-oauth-checklist.md", @@ -2757,7 +2767,7 @@ "gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram", "native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback", "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", - "next_up": "Implement ClawHub registry milestone checklist (Phase 1: registry source/types and validation)" + "next_up": "ClawHub registry Phase 2: add CLI discovery commands (registry list/show with JSON + filtering)" }, "soul_md_and_cron_create": { "date": "2026-02-11", diff --git a/src/skills/index.ts b/src/skills/index.ts index 7c1672a..c46c79a 100644 --- a/src/skills/index.ts +++ b/src/skills/index.ts @@ -14,5 +14,7 @@ export { buildInstallerPlan } from './planner.js'; export type { InstallerPlan, InstallerPlanStep, InstallerPlanSkip, InstallerPlanningOptions } from './planner.js'; export { SkillRegistry } from './registry.js'; export { SkillInstaller } from './installer.js'; +export { loadSkillRegistryCatalog, parseSkillRegistryCatalog } from './registrySource.js'; +export type { SkillRegistryCatalog, SkillRegistryEntry, SkillRegistrySource, SkillRegistryLoadOptions } from './registrySource.js'; export { SkillsWatcher } from './watcher.js'; export type { SkillsWatcherConfig, SkillsWatcherEvent } from './watcher.js'; diff --git a/src/skills/registrySource.test.ts b/src/skills/registrySource.test.ts new file mode 100644 index 0000000..31c1129 --- /dev/null +++ b/src/skills/registrySource.test.ts @@ -0,0 +1,148 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { describe, expect, it } from 'vitest'; +import { loadSkillRegistryCatalog, parseSkillRegistryCatalog } from './registrySource.js'; + +describe('parseSkillRegistryCatalog', () => { + it('parses a valid registry catalog', () => { + const catalog = parseSkillRegistryCatalog({ + skills: [ + { + id: 'todoist', + name: 'Todoist', + version: '1.2.3', + source: 'https://example.com/skills/todoist.git', + summary: 'Task manager integration', + publisher: 'acme', + homepage: 'https://example.com/todoist', + }, + ], + }); + + expect(catalog.skills).toHaveLength(1); + expect(catalog.skills[0]?.id).toBe('todoist'); + expect(catalog.skills[0]?.publisher).toBe('acme'); + }); + + it('rejects missing required fields', () => { + expect(() => + parseSkillRegistryCatalog({ + skills: [ + { + id: 'missing-summary', + name: 'Broken', + version: '0.1.0', + source: 'https://example.com/skills/broken.git', + }, + ], + }), + ).toThrow(/summary/); + }); + + it('rejects invalid ids', () => { + expect(() => + parseSkillRegistryCatalog({ + skills: [ + { + id: 'Not Valid!', + name: 'Broken', + version: '0.1.0', + source: 'https://example.com/skills/broken.git', + summary: 'Broken', + }, + ], + }), + ).toThrow(/id/); + }); +}); + +describe('loadSkillRegistryCatalog', () => { + it('loads catalog from a local file path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'flynn-registry-test-')); + const filePath = join(dir, 'registry.json'); + + writeFileSync( + filePath, + JSON.stringify({ + skills: [ + { + id: 'notes', + name: 'Notes', + version: '0.0.1', + source: './skills/notes', + summary: 'Local notes skill', + }, + ], + }), + 'utf8', + ); + + try { + const catalog = await loadSkillRegistryCatalog({ type: 'file', path: filePath }); + expect(catalog.skills).toHaveLength(1); + expect(catalog.skills[0]?.source).toBe('./skills/notes'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('loads catalog from HTTPS URL with injected fetch', async () => { + const catalog = await loadSkillRegistryCatalog( + { type: 'url', url: 'https://registry.example.com/catalog.json' }, + { + fetchImpl: async () => + ({ + ok: true, + status: 200, + json: async () => ({ + skills: [ + { + id: 'calendar', + name: 'Calendar', + version: '2.0.0', + source: 'https://example.com/skills/calendar.git', + summary: 'Calendar integration', + }, + ], + }), + }) as Response, + }, + ); + + expect(catalog.skills).toHaveLength(1); + expect(catalog.skills[0]?.id).toBe('calendar'); + }); + + it('rejects non-HTTPS URL sources', async () => { + await expect( + loadSkillRegistryCatalog( + { type: 'url', url: 'http://registry.example.com/catalog.json' }, + { + fetchImpl: async () => + ({ + ok: true, + status: 200, + json: async () => ({ skills: [] }), + }) as Response, + }, + ), + ).rejects.toThrow(/https/); + }); + + it('propagates HTTP fetch failures with status', async () => { + await expect( + loadSkillRegistryCatalog( + { type: 'url', url: 'https://registry.example.com/catalog.json' }, + { + fetchImpl: async () => + ({ + ok: false, + status: 502, + json: async () => ({}), + }) as Response, + }, + ), + ).rejects.toThrow(/502/); + }); +}); diff --git a/src/skills/registrySource.ts b/src/skills/registrySource.ts new file mode 100644 index 0000000..01bdcce --- /dev/null +++ b/src/skills/registrySource.ts @@ -0,0 +1,125 @@ +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +export interface SkillRegistryEntry { + id: string; + name: string; + version: string; + source: string; + summary: string; + publisher?: string; + homepage?: string; + sha256?: string; +} + +export interface SkillRegistryCatalog { + skills: SkillRegistryEntry[]; +} + +export type SkillRegistrySource = + | { type: 'file'; path: string } + | { type: 'url'; url: string }; + +export interface SkillRegistryLoadOptions { + timeoutMs?: number; + fetchImpl?: typeof fetch; +} + +const DEFAULT_TIMEOUT_MS = 10_000; +const SKILL_ID_RE = /^[a-z0-9][a-z0-9._-]{0,127}$/; + +function describePath(ctx: string, key: string): string { + return `${ctx}.${key}`; +} + +function readRequiredString(value: unknown, ctx: string, key: string): string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Invalid registry entry: expected non-empty string at ${describePath(ctx, key)}`); + } + return value.trim(); +} + +function readOptionalString(value: unknown, ctx: string, key: string): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Invalid registry entry: expected non-empty string at ${describePath(ctx, key)}`); + } + return value.trim(); +} + +function parseSkillEntry(raw: unknown, index: number): SkillRegistryEntry { + const ctx = `skills[${index}]`; + if (!raw || typeof raw !== 'object') { + throw new Error(`Invalid registry entry: expected object at ${ctx}`); + } + + const obj = raw as Record; + const id = readRequiredString(obj.id, ctx, 'id'); + if (!SKILL_ID_RE.test(id)) { + throw new Error(`Invalid registry entry: ${ctx}.id must match ${SKILL_ID_RE.source}`); + } + + const source = readRequiredString(obj.source, ctx, 'source'); + if (!source.includes('/') && !source.includes(':') && !source.startsWith('.')) { + throw new Error(`Invalid registry entry: ${ctx}.source must be a path or URL`); + } + + return { + id, + name: readRequiredString(obj.name, ctx, 'name'), + version: readRequiredString(obj.version, ctx, 'version'), + source, + summary: readRequiredString(obj.summary, ctx, 'summary'), + publisher: readOptionalString(obj.publisher, ctx, 'publisher'), + homepage: readOptionalString(obj.homepage, ctx, 'homepage'), + sha256: readOptionalString(obj.sha256, ctx, 'sha256'), + }; +} + +export function parseSkillRegistryCatalog(raw: unknown): SkillRegistryCatalog { + if (!raw || typeof raw !== 'object') { + throw new Error('Invalid registry catalog: expected top-level object'); + } + + const obj = raw as Record; + if (!Array.isArray(obj.skills)) { + throw new Error('Invalid registry catalog: expected skills array'); + } + + return { + skills: obj.skills.map((entry, index) => parseSkillEntry(entry, index)), + }; +} + +async function fetchRegistryJson(url: string, opts: SkillRegistryLoadOptions): Promise { + if (!url.startsWith('https://')) { + throw new Error(`Registry URL must use https:// (${url})`); + } + + const fetchImpl = opts.fetchImpl ?? fetch; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const signal = AbortSignal.timeout(timeoutMs); + const response = await fetchImpl(url, { signal }); + + if (!response.ok) { + throw new Error(`Registry fetch failed: HTTP ${response.status}`); + } + + return await response.json(); +} + +export async function loadSkillRegistryCatalog( + source: SkillRegistrySource, + opts: SkillRegistryLoadOptions = {}, +): Promise { + if (source.type === 'file') { + const absPath = resolve(source.path); + const raw = readFileSync(absPath, 'utf8'); + return parseSkillRegistryCatalog(JSON.parse(raw) as unknown); + } + + const json = await fetchRegistryJson(source.url, opts); + return parseSkillRegistryCatalog(json); +}