Improve local service checks and web search tooling

2026-02-22 22:12:44 -08:00
parent b477705806
commit f27aabae3b
12 changed files with 228 additions and 11 deletions
@@ -459,6 +459,7 @@ SearXNG configuration:
 web_search:
  provider: searxng
  endpoint: "http://searxng:8080"
+  fallback_endpoint: "https://searxng.homelab.local" # Optional backup endpoint
  max_results: 5
 ```

@@ -466,7 +467,8 @@ web_search:
 |-------|----------|-------------|
 | `provider` | no | `brave` or `searxng` (default: `brave`) |
 | `api_key` | yes (Brave) | Brave Search API key |
-| `endpoint` | yes (SearXNG) | Base URL for self-hosted SearXNG |
+| `endpoint` | yes (SearXNG primary) | Primary base URL for self-hosted SearXNG |
+| `fallback_endpoint` | no (SearXNG) | Backup SearXNG base URL used if primary endpoint fails |
 | `max_results` | no | Default result count for `web.search*` tools (default: `5`, max: `20`) |

 Brave-specific query fields (supported by `web.search`, and by `web.search.news` except `safeSearch`):
@@ -482,6 +484,12 @@ Optional local Brave Search container (for MCP-based workflows):
 docker compose --profile search up -d brave-search
 ```

+Optional local SearXNG container (for self-hosted web search):
+
+```bash
+docker compose --profile search up -d searxng
+```
+
 Note: Flynn's built-in `web.search*` tools call Brave's HTTP API directly and use `web_search.api_key`; they do not route through the `brave-search` MCP container.

 ### Text-to-Speech (TTS) Reply Audio
@@ -255,9 +255,12 @@ models:
 #   api_key: "${BRAVE_API_KEY}"
 #   max_results: 5
 #   # endpoint: "http://searxng:8080"     # Used when provider=searxng
+#   # fallback_endpoint: "https://searxng.homelab.local"  # Optional backup endpoint for provider=searxng
 #
 # Optional local Brave Search MCP container (for MCP workflows only):
 #   docker compose --profile search up -d brave-search
+# Optional local SearXNG container:
+#   docker compose --profile search up -d searxng
 # Built-in web.search* tools call Brave/SearXNG HTTP endpoints directly.

 hooks:
@@ -77,6 +77,19 @@ services:
      - BRAVE_MCP_HOST=0.0.0.0
      - BRAVE_MCP_PORT=8000

+  # Optional local dependency: SearXNG metasearch instance.
+  # Start with: docker compose --profile search up -d searxng
+  searxng:
+    image: searxng/searxng:latest
+    container_name: searxng
+    restart: unless-stopped
+    profiles: ["search"]
+    ports:
+      - "18803:8080"
+    environment:
+      - BASE_URL=http://localhost:18803/
+      - INSTANCE_NAME=Flynn Local SearXNG
+
 volumes:
  flynn-data:
  whisper-models:
@@ -384,7 +384,7 @@ Return status for user-level local LLM backend daemons (for example `ollama.serv

 #### `system.dockerDependencies`

-Return status for docker-compose managed dependencies discovered from `docker-compose.yml` (excluding Flynn's own `flynn` service). Includes profile-scoped services (for example `whisper-server`, `brave-search`) when profiles are defined.
+Return status for docker-compose managed dependencies discovered from `docker-compose.yml` (excluding Flynn's own `flynn` service). Includes profile-scoped services (for example `whisper-server`, `brave-search`, `searxng`) when profiles are defined.

 **Request:**
 ```json
@@ -421,6 +421,17 @@ Return status for docker-compose managed dependencies discovered from `docker-co
        "statusText": "Up 2 minutes",
        "containerName": "brave-search",
        "availableActions": ["restart", "stop", "update"]
+      },
+      {
+        "id": "searxng",
+        "name": "SearXNG",
+        "service": "searxng",
+        "configured": true,
+        "state": "running",
+        "health": "none",
+        "statusText": "Up 2 minutes",
+        "containerName": "searxng",
+        "availableActions": ["restart", "stop", "update"]
      }
    ]
  }
@@ -3,6 +3,27 @@
  "updated_at": "2026-02-23",
  "description": "Tracks the status of all Flynn plans and implementation phases",
  "plans": {
+    "local-searxng-compose-with-fallback-endpoint": {
+      "status": "completed",
+      "date": "2026-02-23",
+      "updated": "2026-02-23",
+      "summary": "Added an optional local `searxng` Docker Compose dependency (search profile) and extended `web_search` config/tooling with `fallback_endpoint` so Flynn can query a local SearXNG first and automatically fail over to a homelab backup endpoint when the primary is unavailable.",
+      "files_modified": [
+        "docker-compose.yml",
+        "src/tools/builtin/web-search.ts",
+        "src/tools/builtin/web-search.test.ts",
+        "src/config/schema.ts",
+        "src/daemon/tools.ts",
+        "src/gateway/handlers/services.ts",
+        "src/gateway/handlers/dockerDependencies.ts",
+        "src/gateway/handlers/dockerDependencies.test.ts",
+        "README.md",
+        "config/default.yaml",
+        "docs/api/PROTOCOL.md",
+        "docs/plans/state.json"
+      ],
+      "test_status": "pnpm test:run src/tools/builtin/web-search.test.ts src/gateway/handlers/services.test.ts src/config/schema.test.ts src/gateway/handlers/dockerDependencies.test.ts + pnpm typecheck passing"
+    },
    "dashboard-observability-graphs-and-service-logs": {
      "status": "completed",
      "date": "2026-02-23",
@@ -776,6 +776,7 @@ const webSearchSchema = z.object({
  provider: z.enum(['brave', 'searxng']).default('brave'),
  api_key: z.string().optional(),
  endpoint: z.string().optional(),
+  fallback_endpoint: z.string().optional(),
  max_results: z.number().min(1).max(20).default(5),
 }).default({});

@@ -31,6 +31,7 @@ export function initTools(deps: ToolsDeps): ToolsResult {
      provider: config.web_search.provider,
      apiKey: config.web_search.api_key,
      endpoint: config.web_search.endpoint,
+      fallbackEndpoint: config.web_search.fallback_endpoint,
      maxResults: config.web_search.max_results,
    })) {
      toolRegistry.register(tool);
@@ -7,6 +7,7 @@ function createConfig(params?: {
  audioEnabled?: boolean;
  webSearchProvider?: 'brave' | 'searxng';
  webSearchApiKey?: string;
+  webSearchEndpoint?: string;
 }): Config {
  return {
    audio: {
@@ -19,7 +20,7 @@ function createConfig(params?: {
    web_search: {
      provider: params?.webSearchProvider ?? 'brave',
      api_key: params?.webSearchApiKey,
-      endpoint: undefined,
+      endpoint: params?.webSearchEndpoint,
      max_results: 5,
    },
  } as unknown as Config;
@@ -201,6 +202,46 @@ describe('listDockerDependencyStatuses', () => {
    expect(statuses[0]?.id).toBe('whisper');
    expect(statuses[0]?.configured).toBe(false);
  });
+
+  it('labels searxng dependency and marks configured when searxng endpoint is set', async () => {
+    const runner = async (args: string[]) => {
+      if (args.join(' ') === 'config --profiles') {
+        return { stdout: 'search\n', stderr: '' };
+      }
+      if (args.join(' ') === '--profile search config --services') {
+        return { stdout: 'flynn\nsearxng\n', stderr: '' };
+      }
+      if (args.join(' ') === '--profile search ps --all --format json') {
+        return { stdout: '[]', stderr: '' };
+      }
+      if (args.join(' ') === '--profile search ps searxng --format json') {
+        return {
+          stdout: JSON.stringify([{
+            Name: 'searxng',
+            Service: 'searxng',
+            State: 'running',
+            Status: 'Up 5 minutes',
+          }]),
+          stderr: '',
+        };
+      }
+      throw new Error(`Unexpected args: ${args.join(' ')}`);
+    };
+
+    const statuses = await listDockerDependencyStatuses(
+      createConfig({ webSearchProvider: 'searxng', webSearchEndpoint: 'http://searxng:8080' }),
+      runner,
+    );
+    expect(statuses).toHaveLength(1);
+    expect(statuses[0]).toMatchObject({
+      id: 'searxng',
+      name: 'SearXNG',
+      service: 'searxng',
+      configured: true,
+      state: 'running',
+      statusText: 'Up 5 minutes',
+    });
+  });
 });

 describe('controlDockerDependency', () => {
@@ -55,6 +55,7 @@ const FLYNN_SERVICE = 'flynn';
 const WHISPER_SERVICE = 'whisper-server';
 const WHISPER_DEPENDENCY_ID = 'whisper';
 const BRAVE_SEARCH_SERVICE = 'brave-search';
+const SEARXNG_SERVICE = 'searxng';

 function runCompose(args: string[], timeout: number): Promise<DockerComposeResult> {
  return execFile('docker', ['compose', '-f', COMPOSE_FILE, ...args], {
@@ -212,6 +213,12 @@ function isBraveSearchConfigured(config: Config): boolean {
  return typeof apiKey === 'string' && apiKey.trim().length > 0;
 }

+function isSearxngConfigured(config: Config): boolean {
+  if (config.web_search.provider !== 'searxng') {return false;}
+  const endpoint = config.web_search.endpoint;
+  return typeof endpoint === 'string' && endpoint.trim().length > 0;
+}
+
 function toDependencyId(service: string): DockerDependencyId {
  if (service === WHISPER_SERVICE) {
    return WHISPER_DEPENDENCY_ID;
@@ -247,6 +254,14 @@ function describeDependency(service: string, config: Config): DockerDependencyDe
      configured: isBraveSearchConfigured(config),
    };
  }
+  if (service === SEARXNG_SERVICE) {
+    return {
+      id: SEARXNG_SERVICE,
+      name: 'SearXNG',
+      service,
+      configured: isSearxngConfigured(config),
+    };
+  }
  return {
    id: toDependencyId(service),
    name: toDependencyName(service),
@@ -97,6 +97,7 @@ export function discoverServices(
    metadata: {
      provider: config.web_search?.provider ?? 'brave',
      endpoint: config.web_search?.endpoint,
+      fallback_endpoint: config.web_search?.fallback_endpoint,
      max_results: config.web_search?.max_results,
    },
  });
@@ -69,6 +69,12 @@ const searxngConfig: WebSearchConfig = {
  endpoint: 'http://searxng:8080',
 };

+const searxngWithFallbackConfig: WebSearchConfig = {
+  provider: 'searxng',
+  endpoint: 'http://searxng:8080',
+  fallbackEndpoint: 'https://searxng.homelab.local',
+};
+
 // ═════════════════════════════════════════════════════════════════════════════
 // Tests
 // ═════════════════════════════════════════════════════════════════════════════
@@ -216,6 +222,47 @@ describe('web.search', () => {
      expect(result.error).toBe('SearXNG endpoint not configured');
      expect(mockFetch).not.toHaveBeenCalled();
    });
+
+    it('falls back to backup endpoint when primary endpoint fails', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('connect ECONNREFUSED'))
+        .mockResolvedValueOnce(mockJsonResponse(searxngResults));
+
+      const tool = createWebSearchTool(searxngWithFallbackConfig);
+      const result = await tool.execute({ query: 'test query' });
+
+      expect(result.success).toBe(true);
+      expect(mockFetch).toHaveBeenCalledTimes(2);
+      expect(mockFetch.mock.calls[0][0]).toContain('http://searxng:8080/search');
+      expect(mockFetch.mock.calls[1][0]).toContain('https://searxng.homelab.local/search');
+    });
+
+    it('returns combined error when primary and fallback both fail', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('primary timeout'))
+        .mockRejectedValueOnce(new Error('fallback timeout'));
+
+      const tool = createWebSearchTool(searxngWithFallbackConfig);
+      const result = await tool.execute({ query: 'test query' });
+
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('SearXNG primary failed');
+      expect(result.error).toContain('fallback failed');
+    });
+
+    it('allows fallback-only endpoint configuration', async () => {
+      mockFetch.mockResolvedValue(mockJsonResponse(searxngResults));
+
+      const tool = createWebSearchTool({
+        provider: 'searxng',
+        fallbackEndpoint: 'https://searxng.homelab.local',
+      });
+
+      const result = await tool.execute({ query: 'test query' });
+      expect(result.success).toBe(true);
+      expect(mockFetch).toHaveBeenCalledTimes(1);
+      expect(mockFetch.mock.calls[0][0]).toContain('https://searxng.homelab.local/search');
+    });
  });

  // ── Count parameter ─────────────────────────────────────────────────────
@@ -7,6 +7,8 @@ export interface WebSearchConfig {
  apiKey?: string;
  /** Required for SearXNG (e.g. 'http://searxng:8080'). */
  endpoint?: string;
+  /** Optional fallback SearXNG endpoint (for example a homelab instance). */
+  fallbackEndpoint?: string;
  /** Maximum number of results to return (default: 5). */
  maxResults?: number;
 }
@@ -191,6 +193,66 @@ async function searchSearxng(
  }));
 }

+function normalizeEndpoint(endpoint: string | undefined): string | null {
+  if (typeof endpoint !== 'string') {
+    return null;
+  }
+  const trimmed = endpoint.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function resolveSearxngEndpoints(config: WebSearchConfig): string[] {
+  const endpoints: string[] = [];
+  const primary = normalizeEndpoint(config.endpoint);
+  const fallback = normalizeEndpoint(config.fallbackEndpoint);
+
+  if (primary) {
+    endpoints.push(primary);
+  }
+  if (fallback && fallback !== primary) {
+    endpoints.push(fallback);
+  }
+
+  return endpoints;
+}
+
+function normalizeErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+async function searchSearxngWithFallback(
+  query: string,
+  count: number,
+  config: WebSearchConfig,
+  signal?: AbortSignal,
+): Promise<SearchResult[]> {
+  const endpoints = resolveSearxngEndpoints(config);
+  if (endpoints.length === 0) {
+    throw new Error('SearXNG endpoint not configured');
+  }
+
+  if (endpoints.length === 1) {
+    return searchSearxng(query, count, endpoints[0] as string, signal);
+  }
+
+  const [primary, fallback] = endpoints;
+  try {
+    return await searchSearxng(query, count, primary as string, signal);
+  } catch (primaryError) {
+    if (signal?.aborted) {
+      throw primaryError;
+    }
+
+    try {
+      return await searchSearxng(query, count, fallback as string, signal);
+    } catch (fallbackError) {
+      throw new Error(
+        `SearXNG primary failed: ${normalizeErrorMessage(primaryError)}; fallback failed: ${normalizeErrorMessage(fallbackError)}`,
+      );
+    }
+  }
+}
+
 function normalizeToolError(error: unknown): ToolResult {
  if (error instanceof Error && (error.name === 'AbortError' || error.message.toLowerCase().includes('aborted'))) {
    return { success: false, output: '', error: 'Operation aborted' };
@@ -274,14 +336,7 @@ export function createWebSearchTool(config: WebSearchConfig): Tool {
          results = await searchBraveWeb(braveArgs, config.apiKey, context?.signal);
        } else {
          // SearXNG provider
-          if (!config.endpoint) {
-            return {
-              success: false,
-              output: '',
-              error: 'SearXNG endpoint not configured',
-            };
-          }
-          results = await searchSearxng(args.query, count, config.endpoint, context?.signal);
+          results = await searchSearxngWithFallback(args.query, count, config, context?.signal);
        }

        if (results.length === 0) {