feat(councils): add preflight, schema-driven outputs, and artifact reporting

2026-02-22 15:56:30 -08:00
parent dafe9b4d3d
commit 44c7409a20
18 changed files with 1686 additions and 29 deletions
@@ -0,0 +1,347 @@
+import type { AgentConfigRegistry } from '../agents/registry.js';
+import {
+  loadStoredAnthropicAuth,
+  loadStoredAnthropicAuthToken,
+  loadStoredOpenAIApiKey,
+  loadStoredOpenAIAuth,
+  loadStoredZaiAuth,
+} from '../auth/index.js';
+import type { Config, ModelConfig, ModelProvider } from '../config/index.js';
+import type { ModelTier } from '../models/router.js';
+import type { ChatResponseFormat } from '../models/types.js';
+
+interface DelegateRunner {
+  delegate(request: {
+    tier: ModelTier;
+    systemPrompt: string;
+    message: string;
+    maxTokens?: number;
+    responseFormat?: ChatResponseFormat;
+  }): Promise<{
+    content: string;
+    usage: { inputTokens: number; outputTokens: number };
+    tier: ModelTier;
+  }>;
+}
+
+interface CredentialState {
+  openaiOAuth: boolean;
+  openaiApiKeyStored: boolean;
+  anthropicApiKeyStored: boolean;
+  anthropicAuthTokenStored: boolean;
+  zaiStored: boolean;
+}
+
+export interface CouncilPreflightOptions {
+  config: Config;
+  registry: AgentConfigRegistry;
+  delegateRunner: DelegateRunner;
+  activeTier?: ModelTier;
+  includeLiveProbe?: boolean;
+  credentialState?: CredentialState;
+}
+
+interface TierConfigResolution {
+  modelConfig: ModelConfig;
+  fellBackToDefault: boolean;
+}
+
+interface AuthResolution {
+  mode: string;
+  source: string;
+  note?: string;
+}
+
+const DEFAULT_ENDPOINTS: Partial<Record<ModelProvider, string>> = {
+  openai: 'https://api.openai.com/v1',
+  openrouter: 'https://openrouter.ai/api/v1',
+  vercel: 'https://ai-gateway.vercel.sh/v1',
+  zhipuai: 'https://api.z.ai/api/paas/v4',
+  xai: 'https://api.x.ai/v1',
+  minimax: 'https://api.minimax.io/v1',
+  moonshot: 'https://api.moonshot.cn/v1',
+  ollama: 'http://localhost:11434',
+  llamacpp: 'http://localhost:8080',
+};
+
+function getCredentialStateFromSystem(): CredentialState {
+  return {
+    openaiOAuth: Boolean(loadStoredOpenAIAuth()),
+    openaiApiKeyStored: Boolean(loadStoredOpenAIApiKey()),
+    anthropicApiKeyStored: Boolean(loadStoredAnthropicAuth()),
+    anthropicAuthTokenStored: Boolean(loadStoredAnthropicAuthToken()),
+    zaiStored: Boolean(loadStoredZaiAuth()),
+  };
+}
+
+function getEffectiveAuthMode(cfg: ModelConfig): 'auto' | 'api_key' | 'oauth' {
+  if (cfg.auth_mode) {
+    return cfg.auth_mode;
+  }
+  if (cfg.use_oauth) {
+    return 'oauth';
+  }
+  return 'auto';
+}
+
+function resolveTierConfig(config: Config, tier: ModelTier): TierConfigResolution {
+  if (tier === 'default') {
+    return { modelConfig: config.models.default, fellBackToDefault: false };
+  }
+  const direct = config.models[tier];
+  if (direct) {
+    return { modelConfig: direct, fellBackToDefault: false };
+  }
+  return { modelConfig: config.models.default, fellBackToDefault: true };
+}
+
+function firstTruthySource(sources: Array<[boolean, string]>): string {
+  for (const [present, label] of sources) {
+    if (present) {
+      return label;
+    }
+  }
+  return 'missing';
+}
+
+function resolveAuth(cfg: ModelConfig, credentialState: CredentialState): AuthResolution {
+  if (cfg.provider === 'zhipuai') {
+    const source = firstTruthySource([
+      [Boolean(cfg.api_key?.trim()), 'config.api_key'],
+      [Boolean(cfg.auth_token?.trim()), 'config.auth_token'],
+      [Boolean(process.env.ZAI_API_KEY?.trim()), 'env:ZAI_API_KEY'],
+      [Boolean(process.env.ZHIPUAI_API_KEY?.trim()), 'env:ZHIPUAI_API_KEY'],
+      [Boolean(process.env.ZHIPUAI_AUTH_TOKEN?.trim()), 'env:ZHIPUAI_AUTH_TOKEN'],
+      [credentialState.zaiStored, 'auth.json:zai'],
+    ]);
+    const note = cfg.use_oauth || cfg.auth_mode === 'oauth'
+      ? 'use_oauth/auth_mode=oauth is ignored for zhipuai'
+      : undefined;
+    return {
+      mode: 'bearer_credential',
+      source,
+      note,
+    };
+  }
+
+  if (cfg.provider === 'openai') {
+    const authMode = getEffectiveAuthMode(cfg);
+    const apiSource = firstTruthySource([
+      [Boolean(cfg.api_keys && cfg.api_keys.length > 0), 'config.api_keys'],
+      [Boolean(cfg.api_key?.trim()), 'config.api_key'],
+      [Boolean(process.env.OPENAI_API_KEY?.trim()), 'env:OPENAI_API_KEY'],
+      [credentialState.openaiApiKeyStored, 'auth.json:openai.api_key'],
+    ]);
+    const oauthSource = credentialState.openaiOAuth ? 'auth.json:openai.oauth' : 'missing';
+
+    if (authMode === 'oauth') {
+      return { mode: 'oauth', source: oauthSource };
+    }
+    if (authMode === 'api_key') {
+      return { mode: 'api_key', source: apiSource };
+    }
+    if (apiSource !== 'missing') {
+      return { mode: 'auto->api_key', source: apiSource };
+    }
+    return { mode: 'auto->oauth', source: oauthSource };
+  }
+
+  if (cfg.provider === 'anthropic') {
+    const authMode = getEffectiveAuthMode(cfg);
+    const apiSource = firstTruthySource([
+      [Boolean(cfg.api_keys && cfg.api_keys.length > 0), 'config.api_keys'],
+      [Boolean(cfg.api_key?.trim()), 'config.api_key'],
+      [Boolean(process.env.ANTHROPIC_API_KEY?.trim()), 'env:ANTHROPIC_API_KEY'],
+      [credentialState.anthropicApiKeyStored, 'auth.json:anthropic.api_key'],
+    ]);
+    const oauthSource = firstTruthySource([
+      [Boolean(cfg.auth_token?.trim()), 'config.auth_token'],
+      [Boolean(process.env.ANTHROPIC_AUTH_TOKEN?.trim()), 'env:ANTHROPIC_AUTH_TOKEN'],
+      [credentialState.anthropicAuthTokenStored, 'auth.json:anthropic.auth_token'],
+    ]);
+
+    if (authMode === 'oauth') {
+      return { mode: 'oauth', source: oauthSource };
+    }
+    if (authMode === 'api_key') {
+      return { mode: 'api_key', source: apiSource };
+    }
+    if (apiSource !== 'missing') {
+      return { mode: 'auto->api_key', source: apiSource };
+    }
+    return { mode: 'auto->oauth', source: oauthSource };
+  }
+
+  if (cfg.provider === 'gemini') {
+    const source = firstTruthySource([
+      [Boolean(cfg.api_key?.trim()), 'config.api_key'],
+      [Boolean(process.env.GEMINI_API_KEY?.trim()), 'env:GEMINI_API_KEY'],
+      [Boolean(process.env.GOOGLE_API_KEY?.trim()), 'env:GOOGLE_API_KEY'],
+    ]);
+    return { mode: 'api_key', source };
+  }
+
+  if (cfg.provider === 'bedrock') {
+    const source = firstTruthySource([
+      [Boolean(cfg.api_key?.trim() && cfg.auth_token?.trim()), 'config.api_key+auth_token'],
+      [Boolean(process.env.AWS_ACCESS_KEY_ID?.trim() && process.env.AWS_SECRET_ACCESS_KEY?.trim()), 'env:AWS_ACCESS_KEY_ID+AWS_SECRET_ACCESS_KEY'],
+    ]);
+    return { mode: 'aws_credentials', source };
+  }
+
+  if (cfg.provider === 'ollama' || cfg.provider === 'llamacpp' || cfg.provider === 'synthetic') {
+    return { mode: 'local', source: 'none' };
+  }
+
+  const source = firstTruthySource([
+    [Boolean(cfg.api_keys && cfg.api_keys.length > 0), 'config.api_keys'],
+    [Boolean(cfg.api_key?.trim()), 'config.api_key'],
+  ]);
+  return { mode: 'api_key', source };
+}
+
+function resolveEndpoint(cfg: ModelConfig): string {
+  if (cfg.endpoint?.trim()) {
+    return cfg.endpoint.trim();
+  }
+  return DEFAULT_ENDPOINTS[cfg.provider] ?? '(provider default)';
+}
+
+function normalizeProbeError(error: unknown): string {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+
+function isPreflightRequest(task: string): boolean {
+  return task.trim().toLowerCase() === 'preflight';
+}
+
+export function shouldRunCouncilPreflight(task: string): boolean {
+  return isPreflightRequest(task);
+}
+
+export async function buildCouncilPreflightReport(options: CouncilPreflightOptions): Promise<string> {
+  const { config, registry, delegateRunner } = options;
+  const councils = config.councils;
+  const credentialState = options.credentialState ?? getCredentialStateFromSystem();
+
+  const roles = [
+    { role: 'D.arbiter', agent: councils.groups.D.arbiter_agent, tierOverride: councils.groups.D.model_tier },
+    { role: 'D.freethinker', agent: councils.groups.D.freethinker_agent, tierOverride: councils.groups.D.model_tier },
+    ...(councils.groups.D.grounder_agent
+      ? [{ role: 'D.grounder', agent: councils.groups.D.grounder_agent, tierOverride: councils.groups.D.model_tier }]
+      : []),
+    ...(councils.groups.D.writer_agent
+      ? [{ role: 'D.writer', agent: councils.groups.D.writer_agent, tierOverride: councils.groups.D.model_tier }]
+      : []),
+    { role: 'P.arbiter', agent: councils.groups.P.arbiter_agent, tierOverride: councils.groups.P.model_tier },
+    { role: 'P.freethinker', agent: councils.groups.P.freethinker_agent, tierOverride: councils.groups.P.model_tier },
+    ...(councils.groups.P.grounder_agent
+      ? [{ role: 'P.grounder', agent: councils.groups.P.grounder_agent, tierOverride: councils.groups.P.model_tier }]
+      : []),
+    ...(councils.groups.P.writer_agent
+      ? [{ role: 'P.writer', agent: councils.groups.P.writer_agent, tierOverride: councils.groups.P.model_tier }]
+      : []),
+    { role: 'Meta.arbiter', agent: councils.meta_arbiter_agent, tierOverride: councils.meta_model_tier },
+    ...(councils.meta_writer_agent
+      ? [{ role: 'Meta.writer', agent: councils.meta_writer_agent, tierOverride: councils.meta_model_tier }]
+      : []),
+  ];
+
+  const roleLines: string[] = [];
+  const tiersToProbe = new Set<ModelTier>();
+
+  for (const role of roles) {
+    const agentConfig = registry.get(role.agent);
+    const agentTier = agentConfig?.modelTier ?? 'default';
+    const effectiveTier = role.tierOverride ?? agentTier;
+    const { modelConfig, fellBackToDefault } = resolveTierConfig(config, effectiveTier);
+    const modelLabel = `${modelConfig.provider}/${modelConfig.model}`;
+    const missingSuffix = agentConfig ? '' : ' [agent_missing]';
+    const fallbackSuffix = fellBackToDefault ? ' [tier_unconfigured->default]' : '';
+    roleLines.push(
+      `- ${role.role}: agent=${role.agent}${missingSuffix} ` +
+      `agent_tier=${agentTier} override_tier=${role.tierOverride ?? 'none'} ` +
+      `effective_tier=${effectiveTier} model=${modelLabel}${fallbackSuffix}`,
+    );
+    tiersToProbe.add(effectiveTier);
+  }
+
+  const tierLines: string[] = [];
+  for (const tier of [...tiersToProbe].sort()) {
+    const { modelConfig, fellBackToDefault } = resolveTierConfig(config, tier);
+    const auth = resolveAuth(modelConfig, credentialState);
+    const endpoint = resolveEndpoint(modelConfig);
+    tierLines.push(
+      `- ${tier}: provider=${modelConfig.provider} model=${modelConfig.model} ` +
+      `endpoint=${endpoint} auth_mode=${auth.mode} auth_source=${auth.source}` +
+      `${auth.note ? ` note=${auth.note}` : ''}` +
+      `${fellBackToDefault ? ' fallback=default' : ''}`,
+    );
+  }
+
+  const probeLines: string[] = [];
+  if (options.includeLiveProbe ?? true) {
+    const probeResults = await Promise.all(
+      [...tiersToProbe].sort().map(async (tier) => {
+        const startedAt = Date.now();
+        try {
+          await delegateRunner.delegate({
+            tier,
+            systemPrompt: 'Return exactly {"ok":true}.',
+            message: 'Return exactly {"ok":true}.',
+            maxTokens: 64,
+          });
+          return {
+            tier,
+            ok: true,
+            latencyMs: Math.max(0, Date.now() - startedAt),
+            error: '',
+          };
+        } catch (error) {
+          return {
+            tier,
+            ok: false,
+            latencyMs: Math.max(0, Date.now() - startedAt),
+            error: normalizeProbeError(error),
+          };
+        }
+      }),
+    );
+
+    for (const result of probeResults) {
+      if (result.ok) {
+        probeLines.push(`- ${result.tier}: ok (${result.latencyMs}ms)`);
+      } else {
+        probeLines.push(`- ${result.tier}: failed (${result.latencyMs}ms) ${result.error}`);
+      }
+    }
+  } else {
+    probeLines.push('- skipped');
+  }
+
+  const activeTier = options.activeTier ?? 'default';
+  const activeModel = resolveTierConfig(config, activeTier).modelConfig;
+
+  return [
+    '[Council preflight]',
+    `Councils enabled: ${councils.enabled ? 'yes' : 'no'}`,
+    `Active interactive tier: ${activeTier} (${activeModel.provider}/${activeModel.model})`,
+    `Scaffold path: ${councils.scaffold_path?.trim() ? councils.scaffold_path : '(none)'}`,
+    '',
+    'Role routing:',
+    ...roleLines,
+    '',
+    'Tier provider/auth:',
+    ...tierLines,
+    '',
+    'Live tier probes:',
+    ...probeLines,
+    '',
+    'Path comparison:',
+    '- Interactive TUI messages use the active tier above.',
+    '- Council calls use per-role effective tiers (group override, then agent tier).',
+  ].join('\n');
+}