fix: provider-aware model routing with fallback visibility

- Extract createClientFromConfig() to dispatch on provider field instead
  of hardcoding all tiers as AnthropicClient
- Add fallback/fallbackReason metadata to ChatResponse and ChatStreamEvent
  so callers know when a fallback model was used
- Enhance doctor check to report full model stack and warn on missing
  API keys for cloud providers
- Log fallback warnings in NativeAgent and display them in TUI
- Support tier names and local_providers entries in fallback_chain
- Add 8 tests for createClientFromConfig covering all provider types
This commit is contained in:
William Valentin
2026-02-06 09:58:56 -08:00
parent c9b1c607d5
commit e4b7f96d33
9 changed files with 334 additions and 56 deletions
+125
View File
@@ -0,0 +1,125 @@
import { describe, it, expect, afterEach } from 'vitest';
import { runChecks, type CheckResult, type DoctorContext } from './doctor.js';
import { writeFileSync, mkdirSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
describe('doctor checks', () => {
const testDir = join(tmpdir(), 'flynn-test-doctor');
afterEach(() => {
try { rmSync(testDir, { recursive: true }); } catch {}
});
it('reports PASS when config file exists and is valid', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'config.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const configExists = results.find(r => r.label.includes('Config file'));
expect(configExists?.status).toBe('pass');
const configParses = results.find(r => r.label.includes('parses'));
expect(configParses?.status).toBe('pass');
const configValidates = results.find(r => r.label.includes('validates'));
expect(configValidates?.status).toBe('pass');
});
it('reports FAIL when config file does not exist', async () => {
const ctx: DoctorContext = { configPath: '/nonexistent/config.yaml', dataDir: testDir };
const results = await runChecks(ctx);
const configExists = results.find(r => r.label.includes('Config file'));
expect(configExists?.status).toBe('fail');
});
it('reports FAIL on invalid YAML', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'bad.yaml');
writeFileSync(configPath, '{{{{bad yaml');
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const configParses = results.find(r => r.label.includes('parses'));
expect(configParses?.status).toBe('fail');
});
it('reports FAIL on schema validation failure', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'invalid.yaml');
writeFileSync(configPath, `
telegram:
bot_token: ""
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const configValidates = results.find(r => r.label.includes('validates'));
expect(configValidates?.status).toBe('fail');
});
it('reports PASS for writable data directory', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'config.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const dataDir = results.find(r => r.label.includes('Data directory'));
expect(dataDir?.status).toBe('pass');
});
it('reports PASS for accessible session DB', async () => {
mkdirSync(testDir, { recursive: true });
const configPath = join(testDir, 'config.yaml');
writeFileSync(configPath, `
telegram:
bot_token: "test-token"
allowed_chat_ids: [123]
models:
default:
provider: anthropic
model: claude-sonnet
`);
const ctx: DoctorContext = { configPath, dataDir: testDir };
const results = await runChecks(ctx);
const sessionDb = results.find(r => r.label.includes('Session DB'));
expect(sessionDb?.status).toBe('pass');
});
it('skips downstream checks when config is invalid', async () => {
const ctx: DoctorContext = { configPath: '/nonexistent/config.yaml', dataDir: testDir };
const results = await runChecks(ctx);
const modelCheck = results.find(r => r.label.includes('Model connectivity'));
expect(modelCheck?.status).toBe('skip');
const telegramCheck = results.find(r => r.label.includes('Telegram'));
expect(telegramCheck?.status).toBe('skip');
});
});
+21 -3
View File
@@ -116,12 +116,30 @@ const checkModelConnectivity: Check = async (ctx) => {
if (!ctx.config) {
return { status: 'skip', label: 'Model connectivity', detail: '(config invalid)' };
}
// Skip actual API call in doctor — just verify config looks complete
const model = ctx.config.models.default;
const models = ctx.config.models;
const model = models.default;
if (!model.model) {
return { status: 'fail', label: 'Model connectivity', detail: 'no default model configured' };
}
return { status: 'pass', label: 'Model connectivity', detail: `(${model.provider}: ${model.model})` };
// Check if API key is present for providers that need one
const needsKey = ['anthropic', 'openai', 'gemini'];
if (needsKey.includes(model.provider) && !model.api_key && !model.auth_token) {
const envVar = model.provider === 'anthropic' ? 'ANTHROPIC_API_KEY' : model.provider === 'openai' ? 'OPENAI_API_KEY' : undefined;
const hasEnv = envVar && process.env[envVar];
if (!hasEnv) {
return { status: 'warn', label: 'Model connectivity', detail: `${model.provider}/${model.model} — no API key or auth token found` };
}
}
// Build a summary of the model stack
const parts = [`default: ${model.provider}/${model.model}`];
if (models.fast) parts.push(`fast: ${models.fast.provider}/${models.fast.model}`);
if (models.complex) parts.push(`complex: ${models.complex.provider}/${models.complex.model}`);
if (models.local) parts.push(`local: ${models.local.provider}/${models.local.model}`);
parts.push(`fallback: [${models.fallback_chain.join(', ')}]`);
return { status: 'pass', label: 'Model connectivity', detail: parts.join(', ') };
};
const checkTelegram: Check = async (ctx) => {