fix: provider-aware model routing with fallback visibility
- Extract createClientFromConfig() to dispatch on provider field instead of hardcoding all tiers as AnthropicClient - Add fallback/fallbackReason metadata to ChatResponse and ChatStreamEvent so callers know when a fallback model was used - Enhance doctor check to report full model stack and warn on missing API keys for cloud providers - Log fallback warnings in NativeAgent and display them in TUI - Support tier names and local_providers entries in fallback_chain - Add 8 tests for createClientFromConfig covering all provider types
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { runChecks, type CheckResult, type DoctorContext } from './doctor.js';
|
||||
import { writeFileSync, mkdirSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
describe('doctor checks', () => {
|
||||
const testDir = join(tmpdir(), 'flynn-test-doctor');
|
||||
|
||||
afterEach(() => {
|
||||
try { rmSync(testDir, { recursive: true }); } catch {}
|
||||
});
|
||||
|
||||
it('reports PASS when config file exists and is valid', async () => {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
const configPath = join(testDir, 'config.yaml');
|
||||
writeFileSync(configPath, `
|
||||
telegram:
|
||||
bot_token: "test-token"
|
||||
allowed_chat_ids: [123]
|
||||
models:
|
||||
default:
|
||||
provider: anthropic
|
||||
model: claude-sonnet
|
||||
`);
|
||||
|
||||
const ctx: DoctorContext = { configPath, dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const configExists = results.find(r => r.label.includes('Config file'));
|
||||
expect(configExists?.status).toBe('pass');
|
||||
|
||||
const configParses = results.find(r => r.label.includes('parses'));
|
||||
expect(configParses?.status).toBe('pass');
|
||||
|
||||
const configValidates = results.find(r => r.label.includes('validates'));
|
||||
expect(configValidates?.status).toBe('pass');
|
||||
});
|
||||
|
||||
it('reports FAIL when config file does not exist', async () => {
|
||||
const ctx: DoctorContext = { configPath: '/nonexistent/config.yaml', dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const configExists = results.find(r => r.label.includes('Config file'));
|
||||
expect(configExists?.status).toBe('fail');
|
||||
});
|
||||
|
||||
it('reports FAIL on invalid YAML', async () => {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
const configPath = join(testDir, 'bad.yaml');
|
||||
writeFileSync(configPath, '{{{{bad yaml');
|
||||
|
||||
const ctx: DoctorContext = { configPath, dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const configParses = results.find(r => r.label.includes('parses'));
|
||||
expect(configParses?.status).toBe('fail');
|
||||
});
|
||||
|
||||
it('reports FAIL on schema validation failure', async () => {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
const configPath = join(testDir, 'invalid.yaml');
|
||||
writeFileSync(configPath, `
|
||||
telegram:
|
||||
bot_token: ""
|
||||
`);
|
||||
|
||||
const ctx: DoctorContext = { configPath, dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const configValidates = results.find(r => r.label.includes('validates'));
|
||||
expect(configValidates?.status).toBe('fail');
|
||||
});
|
||||
|
||||
it('reports PASS for writable data directory', async () => {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
const configPath = join(testDir, 'config.yaml');
|
||||
writeFileSync(configPath, `
|
||||
telegram:
|
||||
bot_token: "test-token"
|
||||
allowed_chat_ids: [123]
|
||||
models:
|
||||
default:
|
||||
provider: anthropic
|
||||
model: claude-sonnet
|
||||
`);
|
||||
|
||||
const ctx: DoctorContext = { configPath, dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const dataDir = results.find(r => r.label.includes('Data directory'));
|
||||
expect(dataDir?.status).toBe('pass');
|
||||
});
|
||||
|
||||
it('reports PASS for accessible session DB', async () => {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
const configPath = join(testDir, 'config.yaml');
|
||||
writeFileSync(configPath, `
|
||||
telegram:
|
||||
bot_token: "test-token"
|
||||
allowed_chat_ids: [123]
|
||||
models:
|
||||
default:
|
||||
provider: anthropic
|
||||
model: claude-sonnet
|
||||
`);
|
||||
|
||||
const ctx: DoctorContext = { configPath, dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const sessionDb = results.find(r => r.label.includes('Session DB'));
|
||||
expect(sessionDb?.status).toBe('pass');
|
||||
});
|
||||
|
||||
it('skips downstream checks when config is invalid', async () => {
|
||||
const ctx: DoctorContext = { configPath: '/nonexistent/config.yaml', dataDir: testDir };
|
||||
const results = await runChecks(ctx);
|
||||
|
||||
const modelCheck = results.find(r => r.label.includes('Model connectivity'));
|
||||
expect(modelCheck?.status).toBe('skip');
|
||||
|
||||
const telegramCheck = results.find(r => r.label.includes('Telegram'));
|
||||
expect(telegramCheck?.status).toBe('skip');
|
||||
});
|
||||
});
|
||||
+21
-3
@@ -116,12 +116,30 @@ const checkModelConnectivity: Check = async (ctx) => {
|
||||
if (!ctx.config) {
|
||||
return { status: 'skip', label: 'Model connectivity', detail: '(config invalid)' };
|
||||
}
|
||||
// Skip actual API call in doctor — just verify config looks complete
|
||||
const model = ctx.config.models.default;
|
||||
const models = ctx.config.models;
|
||||
const model = models.default;
|
||||
if (!model.model) {
|
||||
return { status: 'fail', label: 'Model connectivity', detail: 'no default model configured' };
|
||||
}
|
||||
return { status: 'pass', label: 'Model connectivity', detail: `(${model.provider}: ${model.model})` };
|
||||
|
||||
// Check if API key is present for providers that need one
|
||||
const needsKey = ['anthropic', 'openai', 'gemini'];
|
||||
if (needsKey.includes(model.provider) && !model.api_key && !model.auth_token) {
|
||||
const envVar = model.provider === 'anthropic' ? 'ANTHROPIC_API_KEY' : model.provider === 'openai' ? 'OPENAI_API_KEY' : undefined;
|
||||
const hasEnv = envVar && process.env[envVar];
|
||||
if (!hasEnv) {
|
||||
return { status: 'warn', label: 'Model connectivity', detail: `${model.provider}/${model.model} — no API key or auth token found` };
|
||||
}
|
||||
}
|
||||
|
||||
// Build a summary of the model stack
|
||||
const parts = [`default: ${model.provider}/${model.model}`];
|
||||
if (models.fast) parts.push(`fast: ${models.fast.provider}/${models.fast.model}`);
|
||||
if (models.complex) parts.push(`complex: ${models.complex.provider}/${models.complex.model}`);
|
||||
if (models.local) parts.push(`local: ${models.local.provider}/${models.local.model}`);
|
||||
parts.push(`fallback: [${models.fallback_chain.join(', ')}]`);
|
||||
|
||||
return { status: 'pass', label: 'Model connectivity', detail: parts.join(', ') };
|
||||
};
|
||||
|
||||
const checkTelegram: Check = async (ctx) => {
|
||||
|
||||
Reference in New Issue
Block a user