diff --git a/README.md b/README.md index bbab16d..3c63e63 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,7 @@ SearXNG configuration: web_search: provider: searxng endpoint: "http://searxng:8080" + fallback_endpoint: "https://searxng.homelab.local" # Optional backup endpoint max_results: 5 ``` @@ -466,7 +467,8 @@ web_search: |-------|----------|-------------| | `provider` | no | `brave` or `searxng` (default: `brave`) | | `api_key` | yes (Brave) | Brave Search API key | -| `endpoint` | yes (SearXNG) | Base URL for self-hosted SearXNG | +| `endpoint` | yes (SearXNG primary) | Primary base URL for self-hosted SearXNG | +| `fallback_endpoint` | no (SearXNG) | Backup SearXNG base URL used if primary endpoint fails | | `max_results` | no | Default result count for `web.search*` tools (default: `5`, max: `20`) | Brave-specific query fields (supported by `web.search`, and by `web.search.news` except `safeSearch`): @@ -482,6 +484,12 @@ Optional local Brave Search container (for MCP-based workflows): docker compose --profile search up -d brave-search ``` +Optional local SearXNG container (for self-hosted web search): + +```bash +docker compose --profile search up -d searxng +``` + Note: Flynn's built-in `web.search*` tools call Brave's HTTP API directly and use `web_search.api_key`; they do not route through the `brave-search` MCP container. ### Text-to-Speech (TTS) Reply Audio diff --git a/config/default.yaml b/config/default.yaml index 7c655dc..65cb98d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -255,9 +255,12 @@ models: # api_key: "${BRAVE_API_KEY}" # max_results: 5 # # endpoint: "http://searxng:8080" # Used when provider=searxng +# # fallback_endpoint: "https://searxng.homelab.local" # Optional backup endpoint for provider=searxng # # Optional local Brave Search MCP container (for MCP workflows only): # docker compose --profile search up -d brave-search +# Optional local SearXNG container: +# docker compose --profile search up -d searxng # Built-in web.search* tools call Brave/SearXNG HTTP endpoints directly. hooks: diff --git a/docker-compose.yml b/docker-compose.yml index bc4ea09..4fb4e42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,19 @@ services: - BRAVE_MCP_HOST=0.0.0.0 - BRAVE_MCP_PORT=8000 + # Optional local dependency: SearXNG metasearch instance. + # Start with: docker compose --profile search up -d searxng + searxng: + image: searxng/searxng:latest + container_name: searxng + restart: unless-stopped + profiles: ["search"] + ports: + - "18803:8080" + environment: + - BASE_URL=http://localhost:18803/ + - INSTANCE_NAME=Flynn Local SearXNG + volumes: flynn-data: whisper-models: diff --git a/docs/api/PROTOCOL.md b/docs/api/PROTOCOL.md index 989bc6e..53c7b65 100644 --- a/docs/api/PROTOCOL.md +++ b/docs/api/PROTOCOL.md @@ -384,7 +384,7 @@ Return status for user-level local LLM backend daemons (for example `ollama.serv #### `system.dockerDependencies` -Return status for docker-compose managed dependencies discovered from `docker-compose.yml` (excluding Flynn's own `flynn` service). Includes profile-scoped services (for example `whisper-server`, `brave-search`) when profiles are defined. +Return status for docker-compose managed dependencies discovered from `docker-compose.yml` (excluding Flynn's own `flynn` service). Includes profile-scoped services (for example `whisper-server`, `brave-search`, `searxng`) when profiles are defined. **Request:** ```json @@ -421,6 +421,17 @@ Return status for docker-compose managed dependencies discovered from `docker-co "statusText": "Up 2 minutes", "containerName": "brave-search", "availableActions": ["restart", "stop", "update"] + }, + { + "id": "searxng", + "name": "SearXNG", + "service": "searxng", + "configured": true, + "state": "running", + "health": "none", + "statusText": "Up 2 minutes", + "containerName": "searxng", + "availableActions": ["restart", "stop", "update"] } ] } diff --git a/docs/plans/state.json b/docs/plans/state.json index 3019fa4..304d017 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -3,6 +3,27 @@ "updated_at": "2026-02-23", "description": "Tracks the status of all Flynn plans and implementation phases", "plans": { + "local-searxng-compose-with-fallback-endpoint": { + "status": "completed", + "date": "2026-02-23", + "updated": "2026-02-23", + "summary": "Added an optional local `searxng` Docker Compose dependency (search profile) and extended `web_search` config/tooling with `fallback_endpoint` so Flynn can query a local SearXNG first and automatically fail over to a homelab backup endpoint when the primary is unavailable.", + "files_modified": [ + "docker-compose.yml", + "src/tools/builtin/web-search.ts", + "src/tools/builtin/web-search.test.ts", + "src/config/schema.ts", + "src/daemon/tools.ts", + "src/gateway/handlers/services.ts", + "src/gateway/handlers/dockerDependencies.ts", + "src/gateway/handlers/dockerDependencies.test.ts", + "README.md", + "config/default.yaml", + "docs/api/PROTOCOL.md", + "docs/plans/state.json" + ], + "test_status": "pnpm test:run src/tools/builtin/web-search.test.ts src/gateway/handlers/services.test.ts src/config/schema.test.ts src/gateway/handlers/dockerDependencies.test.ts + pnpm typecheck passing" + }, "dashboard-observability-graphs-and-service-logs": { "status": "completed", "date": "2026-02-23", diff --git a/src/config/schema.ts b/src/config/schema.ts index 0a55792..c44d950 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -776,6 +776,7 @@ const webSearchSchema = z.object({ provider: z.enum(['brave', 'searxng']).default('brave'), api_key: z.string().optional(), endpoint: z.string().optional(), + fallback_endpoint: z.string().optional(), max_results: z.number().min(1).max(20).default(5), }).default({}); diff --git a/src/daemon/tools.ts b/src/daemon/tools.ts index fe14c2f..969f176 100644 --- a/src/daemon/tools.ts +++ b/src/daemon/tools.ts @@ -31,6 +31,7 @@ export function initTools(deps: ToolsDeps): ToolsResult { provider: config.web_search.provider, apiKey: config.web_search.api_key, endpoint: config.web_search.endpoint, + fallbackEndpoint: config.web_search.fallback_endpoint, maxResults: config.web_search.max_results, })) { toolRegistry.register(tool); diff --git a/src/gateway/handlers/dockerDependencies.test.ts b/src/gateway/handlers/dockerDependencies.test.ts index 755baf4..3c29fe8 100644 --- a/src/gateway/handlers/dockerDependencies.test.ts +++ b/src/gateway/handlers/dockerDependencies.test.ts @@ -7,6 +7,7 @@ function createConfig(params?: { audioEnabled?: boolean; webSearchProvider?: 'brave' | 'searxng'; webSearchApiKey?: string; + webSearchEndpoint?: string; }): Config { return { audio: { @@ -19,7 +20,7 @@ function createConfig(params?: { web_search: { provider: params?.webSearchProvider ?? 'brave', api_key: params?.webSearchApiKey, - endpoint: undefined, + endpoint: params?.webSearchEndpoint, max_results: 5, }, } as unknown as Config; @@ -201,6 +202,46 @@ describe('listDockerDependencyStatuses', () => { expect(statuses[0]?.id).toBe('whisper'); expect(statuses[0]?.configured).toBe(false); }); + + it('labels searxng dependency and marks configured when searxng endpoint is set', async () => { + const runner = async (args: string[]) => { + if (args.join(' ') === 'config --profiles') { + return { stdout: 'search\n', stderr: '' }; + } + if (args.join(' ') === '--profile search config --services') { + return { stdout: 'flynn\nsearxng\n', stderr: '' }; + } + if (args.join(' ') === '--profile search ps --all --format json') { + return { stdout: '[]', stderr: '' }; + } + if (args.join(' ') === '--profile search ps searxng --format json') { + return { + stdout: JSON.stringify([{ + Name: 'searxng', + Service: 'searxng', + State: 'running', + Status: 'Up 5 minutes', + }]), + stderr: '', + }; + } + throw new Error(`Unexpected args: ${args.join(' ')}`); + }; + + const statuses = await listDockerDependencyStatuses( + createConfig({ webSearchProvider: 'searxng', webSearchEndpoint: 'http://searxng:8080' }), + runner, + ); + expect(statuses).toHaveLength(1); + expect(statuses[0]).toMatchObject({ + id: 'searxng', + name: 'SearXNG', + service: 'searxng', + configured: true, + state: 'running', + statusText: 'Up 5 minutes', + }); + }); }); describe('controlDockerDependency', () => { diff --git a/src/gateway/handlers/dockerDependencies.ts b/src/gateway/handlers/dockerDependencies.ts index 88d8a7e..3acca01 100644 --- a/src/gateway/handlers/dockerDependencies.ts +++ b/src/gateway/handlers/dockerDependencies.ts @@ -55,6 +55,7 @@ const FLYNN_SERVICE = 'flynn'; const WHISPER_SERVICE = 'whisper-server'; const WHISPER_DEPENDENCY_ID = 'whisper'; const BRAVE_SEARCH_SERVICE = 'brave-search'; +const SEARXNG_SERVICE = 'searxng'; function runCompose(args: string[], timeout: number): Promise { return execFile('docker', ['compose', '-f', COMPOSE_FILE, ...args], { @@ -212,6 +213,12 @@ function isBraveSearchConfigured(config: Config): boolean { return typeof apiKey === 'string' && apiKey.trim().length > 0; } +function isSearxngConfigured(config: Config): boolean { + if (config.web_search.provider !== 'searxng') {return false;} + const endpoint = config.web_search.endpoint; + return typeof endpoint === 'string' && endpoint.trim().length > 0; +} + function toDependencyId(service: string): DockerDependencyId { if (service === WHISPER_SERVICE) { return WHISPER_DEPENDENCY_ID; @@ -247,6 +254,14 @@ function describeDependency(service: string, config: Config): DockerDependencyDe configured: isBraveSearchConfigured(config), }; } + if (service === SEARXNG_SERVICE) { + return { + id: SEARXNG_SERVICE, + name: 'SearXNG', + service, + configured: isSearxngConfigured(config), + }; + } return { id: toDependencyId(service), name: toDependencyName(service), diff --git a/src/gateway/handlers/services.ts b/src/gateway/handlers/services.ts index 6daef29..58d48f8 100644 --- a/src/gateway/handlers/services.ts +++ b/src/gateway/handlers/services.ts @@ -97,6 +97,7 @@ export function discoverServices( metadata: { provider: config.web_search?.provider ?? 'brave', endpoint: config.web_search?.endpoint, + fallback_endpoint: config.web_search?.fallback_endpoint, max_results: config.web_search?.max_results, }, }); diff --git a/src/tools/builtin/web-search.test.ts b/src/tools/builtin/web-search.test.ts index 392f290..abb3d45 100644 --- a/src/tools/builtin/web-search.test.ts +++ b/src/tools/builtin/web-search.test.ts @@ -69,6 +69,12 @@ const searxngConfig: WebSearchConfig = { endpoint: 'http://searxng:8080', }; +const searxngWithFallbackConfig: WebSearchConfig = { + provider: 'searxng', + endpoint: 'http://searxng:8080', + fallbackEndpoint: 'https://searxng.homelab.local', +}; + // ═════════════════════════════════════════════════════════════════════════════ // Tests // ═════════════════════════════════════════════════════════════════════════════ @@ -216,6 +222,47 @@ describe('web.search', () => { expect(result.error).toBe('SearXNG endpoint not configured'); expect(mockFetch).not.toHaveBeenCalled(); }); + + it('falls back to backup endpoint when primary endpoint fails', async () => { + mockFetch + .mockRejectedValueOnce(new Error('connect ECONNREFUSED')) + .mockResolvedValueOnce(mockJsonResponse(searxngResults)); + + const tool = createWebSearchTool(searxngWithFallbackConfig); + const result = await tool.execute({ query: 'test query' }); + + expect(result.success).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch.mock.calls[0][0]).toContain('http://searxng:8080/search'); + expect(mockFetch.mock.calls[1][0]).toContain('https://searxng.homelab.local/search'); + }); + + it('returns combined error when primary and fallback both fail', async () => { + mockFetch + .mockRejectedValueOnce(new Error('primary timeout')) + .mockRejectedValueOnce(new Error('fallback timeout')); + + const tool = createWebSearchTool(searxngWithFallbackConfig); + const result = await tool.execute({ query: 'test query' }); + + expect(result.success).toBe(false); + expect(result.error).toContain('SearXNG primary failed'); + expect(result.error).toContain('fallback failed'); + }); + + it('allows fallback-only endpoint configuration', async () => { + mockFetch.mockResolvedValue(mockJsonResponse(searxngResults)); + + const tool = createWebSearchTool({ + provider: 'searxng', + fallbackEndpoint: 'https://searxng.homelab.local', + }); + + const result = await tool.execute({ query: 'test query' }); + expect(result.success).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch.mock.calls[0][0]).toContain('https://searxng.homelab.local/search'); + }); }); // ── Count parameter ───────────────────────────────────────────────────── diff --git a/src/tools/builtin/web-search.ts b/src/tools/builtin/web-search.ts index 48141d9..666fa6a 100644 --- a/src/tools/builtin/web-search.ts +++ b/src/tools/builtin/web-search.ts @@ -7,6 +7,8 @@ export interface WebSearchConfig { apiKey?: string; /** Required for SearXNG (e.g. 'http://searxng:8080'). */ endpoint?: string; + /** Optional fallback SearXNG endpoint (for example a homelab instance). */ + fallbackEndpoint?: string; /** Maximum number of results to return (default: 5). */ maxResults?: number; } @@ -191,6 +193,66 @@ async function searchSearxng( })); } +function normalizeEndpoint(endpoint: string | undefined): string | null { + if (typeof endpoint !== 'string') { + return null; + } + const trimmed = endpoint.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function resolveSearxngEndpoints(config: WebSearchConfig): string[] { + const endpoints: string[] = []; + const primary = normalizeEndpoint(config.endpoint); + const fallback = normalizeEndpoint(config.fallbackEndpoint); + + if (primary) { + endpoints.push(primary); + } + if (fallback && fallback !== primary) { + endpoints.push(fallback); + } + + return endpoints; +} + +function normalizeErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function searchSearxngWithFallback( + query: string, + count: number, + config: WebSearchConfig, + signal?: AbortSignal, +): Promise { + const endpoints = resolveSearxngEndpoints(config); + if (endpoints.length === 0) { + throw new Error('SearXNG endpoint not configured'); + } + + if (endpoints.length === 1) { + return searchSearxng(query, count, endpoints[0] as string, signal); + } + + const [primary, fallback] = endpoints; + try { + return await searchSearxng(query, count, primary as string, signal); + } catch (primaryError) { + if (signal?.aborted) { + throw primaryError; + } + + try { + return await searchSearxng(query, count, fallback as string, signal); + } catch (fallbackError) { + throw new Error( + `SearXNG primary failed: ${normalizeErrorMessage(primaryError)}; fallback failed: ${normalizeErrorMessage(fallbackError)}`, + ); + } + } +} + function normalizeToolError(error: unknown): ToolResult { if (error instanceof Error && (error.name === 'AbortError' || error.message.toLowerCase().includes('aborted'))) { return { success: false, output: '', error: 'Operation aborted' }; @@ -274,14 +336,7 @@ export function createWebSearchTool(config: WebSearchConfig): Tool { results = await searchBraveWeb(braveArgs, config.apiKey, context?.signal); } else { // SearXNG provider - if (!config.endpoint) { - return { - success: false, - output: '', - error: 'SearXNG endpoint not configured', - }; - } - results = await searchSearxng(args.query, count, config.endpoint, context?.signal); + results = await searchSearxngWithFallback(args.query, count, config, context?.signal); } if (results.length === 0) {