import type { AudioTranscriptionConfig } from '../models/media.js'; import type { Attachment } from '../channels/types.js'; import { isSupportedAudio, transcribeAudio } from '../models/media.js'; import { synthesizeSpeechAttachment } from '../models/tts.js'; import { supportsAudioInput } from '../models/capabilities.js'; import { AgentOrchestrator, type DelegationConfig } from '../backends/index.js'; import { OutboundAttachmentCollector } from '../backends/native/attachments.js'; import type { ExternalBackend, ExternalBackendName } from '../backends/index.js'; import type { InboundMessage, OutboundMessage } from '../channels/index.js'; import { MemoryStore } from '../memory/index.js'; import type { Tool } from '../tools/types.js'; import { createMediaSendTool, createAgentDelegateTool, createCouncilRunTool } from '../tools/index.js'; import type { AgentDelegateDeps } from '../tools/index.js'; import { createSandboxedShellTool, createSandboxedProcessStartTool, SandboxManager } from '../sandbox/index.js'; import { MODEL_PROVIDERS, type Config, type CouncilsConfig, type ModelConfig, type ModelProvider } from '../config/index.js'; import { ModelRouter, type ModelClient, type ModelTier } from '../models/index.js'; import { ToolRegistry, ToolExecutor } from '../tools/index.js'; import { SessionManager } from '../session/index.js'; import { AgentConfigRegistry, AgentRouter } from '../agents/index.js'; import type { CommandRegistry } from '../commands/index.js'; import { executeRuntimeBackendModeCommand, formatRuntimeBackendStatusLine, type RuntimeBackendMode, } from '../commands/index.js'; import type { ComponentRegistry } from '../intents/index.js'; import type { RoutingPolicy } from '../routing/index.js'; import type { HookEngine } from '../hooks/index.js'; import { createClientFromConfig } from './models.js'; import { matchReactionPrompt } from '../automation/reactions.js'; import { loadSkillRegistryCatalog } from '../skills/index.js'; import type { SkillInstaller, SkillRegistry, SkillRegistryEntry, SkillRegistrySource } from '../skills/index.js'; import { auditLogger } from '../audit/index.js'; import { getElevationStatusMessage, setElevationFromInput } from '../security/elevation.js'; import type { MetricsCollector } from '../gateway/metrics.js'; import { dirname, resolve } from 'path'; import { loadCouncilScaffoldSafe } from '../councils/scaffold.js'; import { buildCouncilPreflightReport, shouldRunCouncilPreflight } from '../councils/preflight.js'; export type BackendRuntimeMode = RuntimeBackendMode; function buildProviderConfigMap(config: Config): Partial> { const providerConfigs: Partial> = {}; const modelConfigs: ModelConfig[] = [ config.models.default, ...(config.models.fast ? [config.models.fast] : []), ...(config.models.complex ? [config.models.complex] : []), ...(config.models.local ? [config.models.local] : []), ...Object.values(config.models.local_providers ?? {}), ]; for (const modelConfig of modelConfigs) { providerConfigs[modelConfig.provider] = modelConfig; if (modelConfig.fallback) { providerConfigs[modelConfig.fallback.provider] = modelConfig.fallback; } } return providerConfigs; } function tierFromUseCase(config: Config, useCaseRaw: unknown): ModelTier | undefined { if (typeof useCaseRaw !== 'string') { return undefined; } const normalized = useCaseRaw.trim().toLowerCase(); if (!normalized) { return undefined; } const mappings: Array<{ tier: ModelTier; tags: string[] | undefined }> = [ { tier: 'fast', tags: config.models.fast?.for }, { tier: 'default', tags: config.models.default.for }, { tier: 'complex', tags: config.models.complex?.for }, { tier: 'local', tags: config.models.local?.for }, ]; for (const { tier, tags } of mappings) { if (!tags || tags.length === 0) { continue; } if (tags.some((tag) => tag.trim().toLowerCase() === normalized)) { return tier; } } return undefined; } function buildBackgroundModelOverrides(config: Config): Partial> { const overrides: Partial> = {}; const configured = config.agents?.background_models ?? {}; const providerConfigs = buildProviderConfigMap(config); const tasks: Array = [ 'compaction', 'memory_extraction', 'classification', 'tool_summarisation', 'complex_reasoning', ]; for (const task of tasks) { const entry = configured[task]; if (!entry || entry.enabled === false) { continue; } const template = providerConfigs[entry.provider]; try { const client = createClientFromConfig( template ? { ...template, provider: entry.provider, model: entry.model } : { provider: entry.provider, model: entry.model }, ); overrides[task] = { client, label: `${entry.provider}/${entry.model}`, fallbackTier: entry.fallback_tier, }; } catch (error) { console.warn( `[Flynn:routing] Failed to initialize background model override for ${task} ` + `(${entry.provider}/${entry.model}): ${error instanceof Error ? error.message : String(error)}`, ); } } return overrides; } function parseResearchPrefix(text: string): string | undefined { const trimmed = text.trim(); const researchMatch = trimmed.match(/^research(?:\s*[:,-])?\s+(.+)$/i); if (researchMatch?.[1]) { return researchMatch[1].trim(); } const lookupMatch = trimmed.match(/^(?:look\s+up|lookup)(?:\s*[:,-])?\s+(.+)$/i); if (lookupMatch?.[1]) { return lookupMatch[1].trim(); } return undefined; } function buildReactionFilterSummary( rule: { on?: string[]; filter?: { contains?: string; regex?: string; metadata?: Record; }; } | undefined, ): string | undefined { if (!rule) { return undefined; } const parts: string[] = []; if (rule.on && rule.on.length > 0) { parts.push(`on:${rule.on.join('|')}`); } if (rule.filter?.contains) { parts.push(`contains:${rule.filter.contains}`); } if (rule.filter?.regex) { parts.push(`regex:${rule.filter.regex}`); } if (rule.filter?.metadata && Object.keys(rule.filter.metadata).length > 0) { parts.push(`metadata:${Object.keys(rule.filter.metadata).join('|')}`); } return parts.length > 0 ? parts.join(', ') : undefined; } function shouldForceNativeForCapabilityQuery(text: string): boolean { const normalized = text.trim().toLowerCase(); if (!normalized) { return false; } if ( normalized.includes('available tools') || normalized.includes('what tools') || normalized.includes('which tools') || normalized.includes('tool list') || normalized.includes('list tools') || normalized.includes('what can you do') || normalized.includes('full access') || normalized.includes('do you have access') || normalized.includes('what access') ) { return true; } return ( /\b(?:show|list|check)\s+(?:me\s+)?(?:your\s+)?(?:available\s+|new\s+)?tools?\b/.test(normalized) || /\b(?:what|which)\s+tools?\b/.test(normalized) || /\btools?\s+(?:do\s+you\s+have|are\s+available)\b/.test(normalized) || /\b(?:show|list|what\s+are)\s+(?:your\s+)?capabilities\b/.test(normalized) || /\bdo\s+you\s+have\s+(?:full\s+)?access\b/.test(normalized) ); } function shouldForceNativeForPiNoTools(text: string): boolean { const normalized = text.trim().toLowerCase(); if (!normalized) { return false; } if ( /`(?:shell\.exec|file\.(?:read|write|edit|patch|list)|web\.(?:fetch|search)|browser\.)/.test(normalized) || /\b(?:gmail|calendar|docs|drive|tasks|k8s|docker|minio)\b/.test(normalized) ) { return true; } return ( /\b(?:run|execute)\s+(?:a\s+)?(?:shell|bash|command)\b/.test(normalized) || /\b(?:run|execute)\s+(?:a\s+)?(?:quick\s+)?check\b/.test(normalized) || /\b(?:quick\s+)?check\s+(?:access|status|logs?|health|config|setup)\b/.test(normalized) || /\b(?:verify|confirm)\s+(?:access|setup|status|config)\b/.test(normalized) || /\b(?:read|open|show|edit|write|patch|delete|list)\s+(?:the\s+)?(?:file|files|directory|repo|code)\b/.test(normalized) || /\b(?:search|fetch|browse|scrape)\s+(?:the\s+)?(?:web|internet|url|site)\b/.test(normalized) || /\b(?:use|call)\s+(?:a\s+)?tool\b/.test(normalized) ); } function providerAcceptsNativeAudioContentParts(provider: string): boolean { return ( provider === 'openai' || provider === 'github' || provider === 'gemini' || provider === 'openrouter' || provider === 'zhipuai' || provider === 'xai' || provider === 'minimax' || provider === 'moonshot' || provider === 'vercel' ); } const LAST_AUDIO_ATTACHMENT_CONFIG_KEY = 'lastAudioAttachment'; function persistLatestAudioAttachment( session: { setConfig(key: string, value: string): void }, audioAttachments: Attachment[], ): void { const latest = [...audioAttachments].reverse().find((att) => ( (typeof att.data === 'string' && att.data.length > 0) || (typeof att.url === 'string' && att.url.length > 0) )); if (!latest) { return; } const payload: { data?: string; url?: string; mimeType?: string } = { mimeType: latest.mimeType, }; if (typeof latest.data === 'string' && latest.data.length > 0) { payload.data = latest.data; } else if (typeof latest.url === 'string' && latest.url.length > 0) { payload.url = latest.url; } if (!payload.data && !payload.url) { return; } try { session.setConfig(LAST_AUDIO_ATTACHMENT_CONFIG_KEY, JSON.stringify(payload)); } catch (error) { console.warn( 'Failed to persist latest audio attachment for tool hydration:', error instanceof Error ? error.message : String(error), ); } } function extractLatestAudioToolInput(audioAttachments: Attachment[]): { data?: string; url?: string; mime_type?: string } | undefined { const latest = [...audioAttachments].reverse().find((att) => ( (typeof att.data === 'string' && att.data.length > 0) || (typeof att.url === 'string' && att.url.length > 0) )); if (!latest) { return undefined; } const data = typeof latest.data === 'string' && latest.data.length > 0 ? latest.data : undefined; const url = typeof latest.url === 'string' && latest.url.length > 0 ? latest.url : undefined; if (!data && !url) { return undefined; } return { ...(data ? { data } : {}), ...(url ? { url } : {}), mime_type: latest.mimeType, }; } function isTtsEnabledForChannel(config: Config, channel: string): boolean { if (!config.tts?.enabled) { return false; } const enabledChannels = config.tts.enabled_channels ?? []; if (enabledChannels.length === 0) { return true; } return enabledChannels.includes(channel); } function resolveRegistrySource(config: Config): { source?: SkillRegistrySource; error?: string } { const raw = config.skills.registry_source?.trim() || process.env.FLYNN_SKILLS_REGISTRY_SOURCE?.trim(); if (!raw) { return { error: 'Skills registry is not configured. Set `skills.registry_source` (or FLYNN_SKILLS_REGISTRY_SOURCE).', }; } if (raw.startsWith('http://')) { return { error: `Registry URL must use https:// (${raw})` }; } if (raw.startsWith('https://')) { return { source: { type: 'url', url: raw } }; } return { source: { type: 'file', path: raw } }; } function resolveRegistryEntryLocalPath(entry: SkillRegistryEntry, registrySource: SkillRegistrySource): string | null { const source = entry.source.trim(); if (!source) { return null; } if (source.startsWith('http://') || source.startsWith('https://') || source.startsWith('git+https://')) { return null; } if (source.startsWith('file://')) { try { return decodeURIComponent(new URL(source).pathname); } catch { return null; } } if (registrySource.type === 'file' && (source.startsWith('./') || source.startsWith('../'))) { return resolve(dirname(resolve(registrySource.path)), source); } return resolve(source); } /** * Create the unified message handler for the channel registry. * Each channel+sender pair gets its own AgentOrchestrator backed by a persistent session. * The orchestrator wraps a NativeAgent and adds delegation to different model tiers. * * Returns both the message handler function and the agents map for usage tracking. */ export function createMessageRouter(deps: { sessionManager: SessionManager; modelRouter: ModelRouter; systemPrompt: string; toolRegistry: ToolRegistry; toolExecutor: ToolExecutor; metrics?: MetricsCollector; config: Config; memoryStore?: MemoryStore; agentConfigRegistry?: AgentConfigRegistry; agentRouter?: AgentRouter; sandboxManager?: SandboxManager; commandRegistry?: CommandRegistry; hookEngine?: HookEngine; intentRegistry?: ComponentRegistry; routingPolicy?: RoutingPolicy; skillRegistry?: SkillRegistry; skillInstaller?: SkillInstaller; externalBackends?: Partial>; defaultName?: ExternalBackendName; getBackendMode?: () => BackendRuntimeMode; setBackendMode?: (mode: BackendRuntimeMode) => void; }): { handler: (msg: InboundMessage, reply: (response: OutboundMessage) => Promise) => Promise; agents: Map; } { // Cache agents by session ID + agent config name to avoid recreating on every message const agents = new Map(); const talkModeUntil = new Map(); const activeRuns = new Map(); function getBackendMode(): BackendRuntimeMode { return deps.getBackendMode?.() ?? 'config_default'; } function getConfiguredOrFallbackDefaultBackend(): ExternalBackendName | 'native' { return deps.defaultName ?? 'native'; } function getEffectiveDefaultBackend(): ExternalBackendName | 'native' { const mode = getBackendMode(); if (mode === 'force_native') { return 'native'; } if (mode === 'force_pi_embedded') { return 'pi_embedded'; } return getConfiguredOrFallbackDefaultBackend(); } function resolveRoutableBackend( requestedBackend: ExternalBackendName | 'native' | undefined, ): ExternalBackendName | 'native' { if (!requestedBackend || requestedBackend === 'native') { return 'native'; } return deps.externalBackends?.[requestedBackend] ? requestedBackend : 'native'; } function applyBackendModeOverride( requestedBackend: ExternalBackendName | 'native' | undefined, ): ExternalBackendName | 'native' | undefined { if (requestedBackend !== 'pi_embedded') { return requestedBackend; } if (getBackendMode() === 'force_native') { return 'native'; } return requestedBackend; } function listAvailableExternalBackends(): string[] { return Object.keys(deps.externalBackends ?? {}); } function formatBackendStatus(activeTier: string): string { return formatRuntimeBackendStatusLine({ getActiveTier: () => activeTier, getBackendMode, getConfiguredDefaultBackend: getConfiguredOrFallbackDefaultBackend, getEffectiveDefaultBackend: () => resolveRoutableBackend(getEffectiveDefaultBackend()), getAvailableExternalBackends: listAvailableExternalBackends, }); } function requestActiveRunCancellation(input: { sessionId: string; channel: string; senderId: string; requestId: string; }): { cancelled: boolean; latencyMs: number } { const cancelStartedAt = Date.now(); const run = activeRuns.get(input.sessionId); if (!run || !run.isCancellable()) { const latencyMs = Date.now() - cancelStartedAt; deps.metrics?.recordCancelLatency(latencyMs); auditLogger?.runCancel?.({ session_id: input.sessionId, channel: input.channel, sender: input.senderId, source: 'channel', requested: true, acknowledged: false, request_id: input.requestId, latency_ms: latencyMs, }); return { cancelled: false, latencyMs }; } run.cancel(); const cancelLatencyMs = Date.now() - cancelStartedAt; deps.metrics?.recordCancelLatency(cancelLatencyMs); auditLogger?.runCancel?.({ session_id: input.sessionId, channel: input.channel, sender: input.senderId, source: 'channel', requested: true, acknowledged: true, request_id: input.requestId, latency_ms: cancelLatencyMs, }); auditLogger?.runState?.({ session_id: input.sessionId, channel: input.channel, sender: input.senderId, source: 'channel', state: 'cancel_requested', request_id: input.requestId, duration_ms: cancelLatencyMs, }); deps.metrics?.recordRunState('cancel_requested'); return { cancelled: true, latencyMs: cancelLatencyMs }; } function executeBackendCommand(inputRaw: string, activeTier: string): string { return executeRuntimeBackendModeCommand(inputRaw, { getActiveTier: () => activeTier, getBackendMode, setBackendMode: deps.setBackendMode, getConfiguredDefaultBackend: getConfiguredOrFallbackDefaultBackend, getEffectiveDefaultBackend: () => resolveRoutableBackend(getEffectiveDefaultBackend()), getAvailableExternalBackends: listAvailableExternalBackends, }); } async function maybeBuildTtsAttachment(responseText: string, channel: string) { if (!isTtsEnabledForChannel(deps.config, channel)) { return undefined; } const provider = deps.config.tts?.provider; const endpoint = provider?.endpoint ?? (provider?.type === 'openai' ? 'https://api.openai.com/v1/audio/speech' : undefined); if (!endpoint) { return undefined; } try { return await synthesizeSpeechAttachment(responseText, { endpoint, apiKey: provider?.api_key, model: provider?.model, voice: provider?.voice, format: provider?.format, }); } catch (error) { console.warn(`TTS synthesis failed for channel ${channel}:`, error instanceof Error ? error.message : 'Unknown error'); return undefined; } } function getOrCreateAgent(channel: string, senderId: string, metadata?: Record, agentOverride?: string): { orchestrator: AgentOrchestrator; collector: OutboundAttachmentCollector } { // Resolve agent config name via routing (sender → channel → default fallback) const agentConfigName = agentOverride ?? deps.agentRouter?.resolve(channel, senderId); const agentConfig = agentConfigName ? deps.agentConfigRegistry?.get(agentConfigName) : undefined; // Cron job tier wins over agent config tier const tierFromMetadata = metadata?.modelTier as ModelTier | undefined; const tierFromUseCaseMetadata = tierFromUseCase(deps.config, metadata?.modelFor); // Include agent config name in cache key so different agents aren't shared let skillOverride = metadata?.skillOverride as string | undefined; if (skillOverride && deps.skillRegistry) { const s = deps.skillRegistry.get(skillOverride); if (!s || !s.available) { skillOverride = undefined; } } const baseSid = agentConfigName || skillOverride ? `${channel}:${senderId}:${agentConfigName ?? 'default'}:${skillOverride ?? 'none'}` : `${channel}:${senderId}`; const session = deps.sessionManager.getSession(channel, senderId); // Read per-session model tier override (persisted in SQLite) const sessionTierOverride = session.getConfig('modelTier') as ModelTier | undefined; // Resolution chain: metadata (explicit tier) → metadata modelFor -> session override -> agent config -> global default const effectiveTier = tierFromMetadata ?? tierFromUseCaseMetadata ?? sessionTierOverride ?? agentConfig?.modelTier ?? deps.config.agents.primary_tier ?? 'default'; // Cache agents by tier too so switching tiers updates context-window heuristics. const sessionId = `${baseSid}:${effectiveTier}`; let entry = agents.get(sessionId); if (!entry) { // Use agent config overrides where available, falling back to global config let effectiveSystemPrompt = agentConfig?.systemPrompt ?? deps.systemPrompt; // If an active skill is specified, annotate the system prompt for clarity. const activeSkillName = skillOverride; const activeSkill = activeSkillName ? deps.skillRegistry?.get(activeSkillName) : undefined; if (activeSkillName) { effectiveSystemPrompt += `\n\n[Active skill: ${activeSkillName}. Tool access is capability-restricted and may be sandboxed.]`; } const modelsConfig = deps.config.models as Record; const tierConfig = modelsConfig[effectiveTier] ?? deps.config.models.default; const effectiveProvider = tierConfig?.provider ?? deps.config.models.default.provider; const effectiveModelName = tierConfig?.model ?? deps.config.models.default.model; const effectiveContextWindow = tierConfig?.context_window ?? deps.config.models.default.context_window; const delegationConfig: DelegationConfig = { compaction: deps.config.agents.delegation.compaction ?? 'fast', memory_extraction: deps.config.agents.delegation.memory_extraction ?? 'fast', classification: deps.config.agents.delegation.classification ?? 'fast', tool_summarisation: deps.config.agents.delegation.tool_summarisation ?? 'fast', complex_reasoning: deps.config.agents.delegation.complex_reasoning ?? 'complex', }; const backgroundModelOverrides = buildBackgroundModelOverrides(deps.config); // Clone the tool registry and replace high-risk tools with sandboxed versions if configured. let effectiveToolRegistry = deps.toolRegistry; const skillEnvPreference = activeSkill?.manifest.permissions?.execution_environment; const executionEnvironment: 'host' | 'sandbox' = skillOverride ? (skillEnvPreference === 'host' ? 'host' : (deps.sandboxManager && deps.config.sandbox.enabled ? 'sandbox' : 'host')) : 'host'; const useSandboxTools = executionEnvironment === 'sandbox' && deps.sandboxManager && deps.config.sandbox.enabled; if ((agentConfig?.sandbox || Boolean(skillOverride)) && useSandboxTools) { effectiveToolRegistry = deps.toolRegistry.clone(); // Lazy sandbox: create the sandboxed tools with a deferred sandbox reference // The sandbox is created on first use via SandboxManager.getOrCreate() const sandboxSessionId = sessionId; const sandboxManager = deps.sandboxManager; if (!sandboxManager) { throw new Error('Sandbox manager unavailable for sandboxed agent execution'); } // Create a proxy sandbox that lazily initializes const lazySandboxShell: Tool = { name: 'shell.exec', description: 'Execute a shell command inside a sandboxed container and return stdout/stderr.', inputSchema: { type: 'object', properties: { command: { type: 'string', description: 'The shell command to execute' }, cwd: { type: 'string', description: 'Working directory inside the container (optional)' }, timeout: { type: 'number', description: 'Timeout in milliseconds (default 30000)' }, }, required: ['command'], }, execute: async (rawArgs: unknown) => { const sandbox = await sandboxManager.getOrCreate(sandboxSessionId); const tool = createSandboxedShellTool(sandbox); return tool.execute(rawArgs); }, }; const lazySandboxProcess: Tool = { name: 'process.start', description: 'Start a command in the background inside a sandboxed container.', inputSchema: { type: 'object', properties: { command: { type: 'string', description: 'The shell command to run in the background' }, cwd: { type: 'string', description: 'Working directory inside the container (optional)' }, }, required: ['command'], }, execute: async (rawArgs: unknown) => { const sandbox = await sandboxManager.getOrCreate(sandboxSessionId); const tool = createSandboxedProcessStartTool(sandbox); return tool.execute(rawArgs); }, }; effectiveToolRegistry.replace(lazySandboxShell); effectiveToolRegistry.replace(lazySandboxProcess); } // Create an attachment collector for this agent session const collector = new OutboundAttachmentCollector(); // Clone the tool registry to register the media.send tool bound to this collector effectiveToolRegistry = effectiveToolRegistry.clone(); effectiveToolRegistry.register(createMediaSendTool(collector)); // Register delegation tools with lazy orchestrator reference (resolved after construction) let resolveOrchestrator: ((o: AgentOrchestrator) => void) | undefined; if (deps.agentConfigRegistry && deps.agentConfigRegistry.list().length > 0) { let lazyOrchestrator: AgentOrchestrator | null = null; resolveOrchestrator = (o: AgentOrchestrator) => { lazyOrchestrator = o; }; effectiveToolRegistry.register(createAgentDelegateTool({ registry: deps.agentConfigRegistry, get orchestrator(): AgentOrchestrator { if (!lazyOrchestrator) { throw new Error('Agent orchestrator not yet initialized'); } return lazyOrchestrator; }, } as AgentDelegateDeps)); if (deps.config.councils?.enabled) { const scaffold = loadCouncilScaffoldSafe(deps.config.councils.scaffold_path); effectiveToolRegistry.register(createCouncilRunTool({ registry: deps.agentConfigRegistry, config: deps.config.councils as CouncilsConfig, scaffold, get orchestrator(): AgentOrchestrator { if (!lazyOrchestrator) { throw new Error('Agent orchestrator not yet initialized'); } return lazyOrchestrator; }, })); } } const toolPolicyContext = { agent: effectiveTier, provider: effectiveProvider, sessionId: session.id, channel, sender: senderId, tier: effectiveTier, autonomyLevel: deps.config.agents.autonomy_level ?? 'standard', sensitiveMode: deps.config.agents.sensitive_mode, immutableDenylist: (deps.config.agents.immutable_denylist ?? []).map((rule) => ({ tool: rule.tool, argsPattern: rule.args_pattern, reason: rule.reason, })), skillName: activeSkillName, skillPermissions: activeSkill?.manifest.permissions, allowedSecretScopes: activeSkill?.manifest.permissions?.secrets, executionEnvironment, }; const orchestrator = new AgentOrchestrator({ modelRouter: deps.modelRouter, systemPrompt: effectiveSystemPrompt, session, toolRegistry: effectiveToolRegistry, toolExecutor: deps.toolExecutor, primaryTier: effectiveTier, delegation: delegationConfig, backgroundModelOverrides, maxDelegationDepth: deps.config.agents.max_delegation_depth ?? 3, maxIterations: deps.config.agents.max_iterations, compaction: deps.config.compaction.enabled ? { thresholdPct: deps.config.compaction.threshold_pct, keepTurns: deps.config.compaction.keep_turns, summaryMaxTokens: deps.config.compaction.summary_max_tokens, importanceThreshold: deps.config.compaction.importance_threshold, proactive: { enabled: deps.config.compaction.proactive.enabled, warnPct: deps.config.compaction.proactive.warn_pct, checkpointPct: deps.config.compaction.proactive.checkpoint_pct, autoCompactPct: deps.config.compaction.proactive.auto_compact_pct, checkpointCooldownMs: deps.config.compaction.proactive.checkpoint_cooldown_ms, memoryNamespace: deps.config.compaction.proactive.memory_namespace, }, } : undefined, modelName: effectiveModelName, contextWindow: effectiveContextWindow, memoryStore: deps.memoryStore, memoryAutoExtract: deps.config.memory?.auto_extract, memoryInjectionStrategy: deps.config.memory?.injection_strategy, memoryMaxInjectionTokens: deps.config.memory?.max_injection_tokens, memoryProactiveExtractEnabled: deps.config.memory?.proactive_extract?.enabled, memoryProactiveExtractMinToolCalls: deps.config.memory?.proactive_extract?.min_tool_calls, memoryProactiveExtractNamespace: deps.config.memory?.proactive_extract?.namespace, memoryDailyLogEnabled: deps.config.memory?.daily_log?.enabled, memoryDailyLogNamespacePrefix: deps.config.memory?.daily_log?.namespace_prefix, memoryDailyLogIncludeSessionMetadata: deps.config.memory?.daily_log?.include_session_metadata, memoryDailyLogMaxUserChars: deps.config.memory?.daily_log?.max_user_chars, memoryDailyLogMaxAssistantChars: deps.config.memory?.daily_log?.max_assistant_chars, autoEscalate: deps.config.agents.auto_escalate, autoEscalateTier: 'complex', toolPolicyContext, attachmentCollector: collector, }); // Resolve the lazy orchestrator reference for agent.delegate resolveOrchestrator?.(orchestrator); entry = { orchestrator, collector }; agents.set(sessionId, entry); } return entry; } const handler = async (msg: InboundMessage, reply: (response: OutboundMessage) => Promise): Promise => { const sessionIdForRun = `${msg.channel}:${msg.senderId}`; let incomingText = msg.text; let matchedReactionName: string | undefined; const talkMode = deps.config.audio?.talk_mode; if (talkMode?.enabled && incomingText.trim().length > 0) { const key = `${msg.channel}:${msg.senderId}`; const now = Date.now(); const timeoutMs = talkMode.timeout_ms; const currentUntil = talkModeUntil.get(key) ?? 0; const lower = incomingText.trim().toLowerCase(); if (talkMode.allow_manual_toggle) { if (lower === '/talk on') { talkModeUntil.set(key, now + timeoutMs); await reply({ text: `Talk mode enabled for ${Math.ceil(timeoutMs / 1000)}s.`, replyTo: msg.id }); return; } if (lower === '/talk off') { talkModeUntil.delete(key); await reply({ text: 'Talk mode disabled.', replyTo: msg.id }); return; } if (lower === '/talk status') { if (currentUntil <= now) { await reply({ text: 'Talk mode is idle (wake phrase required).', replyTo: msg.id }); } else { await reply({ text: `Talk mode active for ${Math.ceil((currentUntil - now) / 1000)}s.`, replyTo: msg.id }); } return; } } const phrase = talkMode.wake_phrase.trim(); const wakeRegex = phrase ? new RegExp(`^\\s*${escapeRegex(phrase)}(?:[\\s,:!.-]+)?`, 'i') : null; const wakeMatched = Boolean(wakeRegex && wakeRegex.test(incomingText)); if (wakeMatched && wakeRegex) { talkModeUntil.set(key, now + timeoutMs); incomingText = incomingText.replace(wakeRegex, '').trim(); if (!incomingText) { await reply({ text: `Listening. Talk mode active for ${Math.ceil(timeoutMs / 1000)}s.`, replyTo: msg.id }); return; } } else if (currentUntil > now) { talkModeUntil.set(key, now + timeoutMs); } else { return; } } const session = deps.sessionManager.getSession(msg.channel, msg.senderId); const queueMode = session.getConfig('queue.mode') ?? deps.config.server?.queue?.mode ?? 'collect'; const rawCommand = msg.metadata?.isCommand ? msg.metadata.command : incomingText.trim().startsWith('/') ? incomingText.trim().slice(1).split(/\s+/, 1)[0] : undefined; const isCancelCommand = rawCommand === 'stop' || rawCommand === 'cancel'; if (queueMode === 'interrupt' && !isCancelCommand) { requestActiveRunCancellation({ sessionId: sessionIdForRun, channel: msg.channel, senderId: msg.senderId, requestId: msg.id, }); } const automationReactions = deps.config.automation?.reactions ?? []; if (!msg.metadata?.isCommand) { if (automationReactions.length === 0) { auditLogger?.reactionSkip?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', reason: 'no_rules', candidate_count: 0, }); deps.metrics?.recordReactionDecision({ matched: false, reason: 'no_rules' }); } else { const reactionMatch = matchReactionPrompt(automationReactions, { channel: msg.channel, senderId: msg.senderId, text: incomingText, metadata: msg.metadata, }); if (reactionMatch) { matchedReactionName = reactionMatch.name; incomingText = reactionMatch.prompt; const matchedRule = automationReactions.find((rule) => rule.name === reactionMatch.name); auditLogger?.reactionMatch?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', rule_name: reactionMatch.name, candidate_count: automationReactions.length, filter_summary: buildReactionFilterSummary(matchedRule), }); deps.metrics?.recordReactionDecision({ matched: true }); } else { auditLogger?.reactionSkip?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', reason: 'no_match', candidate_count: automationReactions.length, }); deps.metrics?.recordReactionDecision({ matched: false, reason: 'no_match' }); } } } let intentAgentOverride: string | undefined; let intentSkillOverride: string | undefined; if (!deps.config.intents?.enabled && deps.agentConfigRegistry?.get('research')) { const researchTask = parseResearchPrefix(incomingText); if (researchTask) { intentAgentOverride = 'research'; incomingText = researchTask; } } if (deps.config.intents?.enabled && deps.intentRegistry) { const intentMatch = deps.intentRegistry.match(incomingText); if (intentMatch?.rule.target.type === 'agent') { let confidence = intentMatch.score; if (deps.config.history_index?.enabled) { const historySessionId = `${msg.channel}:${msg.senderId}`; const historyHits = deps.sessionManager.searchHistory(msg.text, { sessionId: historySessionId, limit: 1, }); if (historyHits.length > 0 && historyHits[0].score >= (deps.config.history_index.min_score ?? 0.15)) { confidence = Math.min(1, confidence + (deps.config.history_index.routing_boost ?? 0.05)); } } const decision = deps.routingPolicy ? deps.routingPolicy.decide({ confidence }) : { path: 'fast' as const, reason: 'high_confidence' as const }; console.log(`[routing] intent=${intentMatch.rule.name} confidence=${confidence.toFixed(3)} path=${decision.path} reason=${decision.reason}`); if (decision.path === 'fast') { intentAgentOverride = intentMatch.rule.target.name; } } if (intentMatch?.rule.target.type === 'skill') { let confidence = intentMatch.score; const decision = deps.routingPolicy ? deps.routingPolicy.decide({ confidence }) : { path: 'fast' as const, reason: 'high_confidence' as const }; console.log(`[routing] intent=${intentMatch.rule.name} confidence=${confidence.toFixed(3)} path=${decision.path} reason=${decision.reason}`); if (decision.path === 'fast') { intentSkillOverride = intentMatch.rule.target.name; } } } const effectiveMetadata = { ...(msg.metadata ?? {}), ...(intentSkillOverride ? { skillOverride: intentSkillOverride } : {}), ...(matchedReactionName ? { automationReaction: matchedReactionName } : {}), }; const agentConfigName = intentAgentOverride ?? deps.agentRouter?.resolve(msg.channel, msg.senderId); const agentConfig = agentConfigName ? deps.agentConfigRegistry?.get(agentConfigName) : undefined; const { orchestrator: agent, collector } = getOrCreateAgent(msg.channel, msg.senderId, effectiveMetadata, agentConfigName); const commandInput = msg.metadata?.isCommand && typeof msg.metadata.command === 'string' ? `/${msg.metadata.command}${msg.metadata.commandArgs ? ` ${msg.metadata.commandArgs}` : ''}` : incomingText; const metadataCommand = typeof msg.metadata?.command === 'string' ? msg.metadata.command : undefined; const parsedCommand = msg.metadata?.isCommand ? metadataCommand : commandInput.startsWith('/') ? commandInput.slice(1).split(/\s+/, 1)[0] : undefined; auditLogger?.userAction({ session_id: `${msg.channel}:${msg.senderId}`, channel: msg.channel, sender: msg.senderId, source: 'channel', action_type: parsedCommand ? 'command' : 'message', content_length: commandInput.length, attachments_count: msg.attachments?.length ?? 0, command: parsedCommand, }); if (deps.commandRegistry && deps.commandRegistry.isCommand(commandInput)) { const commandResult = await deps.commandRegistry.execute(commandInput, { channel: msg.channel, senderId: msg.senderId, sessionId: session.id, rawInput: commandInput, services: { getStatus: () => { return formatBackendStatus(agent.getModelTier()); }, getTools: () => { const names = new Set(deps.toolRegistry.list().map((tool: Tool) => tool.name)); names.add('media.send'); if (deps.agentConfigRegistry && deps.agentConfigRegistry.list().length > 0) { names.add('agent.delegate'); if (deps.config.councils?.enabled) { names.add('council.run'); } } const sorted = [...names].sort(); return [ `Available tools (${sorted.length}):`, ...sorted.map((name) => `- ${name}`), ].join('\n'); }, getUsage: () => { const usage = agent.getUsage(); const lines = [ '**Token Usage**', '', `Primary: ${usage.primary.inputTokens.toLocaleString()} in / ${usage.primary.outputTokens.toLocaleString()} out (${usage.primary.calls} calls)`, ]; const delegationEntries = Object.entries(usage.delegation); if (delegationEntries.length > 0) { lines.push(''); lines.push('Delegation:'); for (const [tier, stats] of delegationEntries) { lines.push(` ${tier}: ${stats.inputTokens.toLocaleString()} in / ${stats.outputTokens.toLocaleString()} out (${stats.calls} calls)`); } } lines.push(''); lines.push(`**Total:** ${usage.total.inputTokens.toLocaleString()} in / ${usage.total.outputTokens.toLocaleString()} out (${usage.total.calls} calls)`); if (usage.total.estimatedCost > 0) { lines.push(`**Estimated cost:** $${usage.total.estimatedCost.toFixed(4)}`); } return lines.join('\n'); }, getModel: () => { const currentTier = agent.getModelTier(); const sessionOverride = session.getConfig('modelTier'); const available = deps.modelRouter.getAvailableTiers(); const labels = deps.modelRouter.getAllLabels(); const lines = [`Active tier: ${currentTier}${sessionOverride ? ' (session override)' : ''}`]; for (const tier of available) { const label = labels[tier] ?? 'unknown'; const marker = tier === currentTier ? ' ←' : ''; lines.push(` ${tier}: ${label}${marker}`); } return lines.join('\n'); }, setModel: (tier) => { const raw = tier.trim(); if (!raw) { return 'Usage: /model OR /model OR /model reset'; } const parts = raw.split(/\s+/); const requestedTier = parts[0]; const validTiers = deps.modelRouter.getAvailableTiers(); if (!validTiers.includes(requestedTier as ModelTier)) { return `Model tier not available: ${requestedTier}`; } const modelTier = requestedTier as ModelTier; // /model if (parts.length === 1) { session.setConfig('modelTier', modelTier); agent.setModelTier(modelTier); const label = deps.modelRouter.getLabel(modelTier); return `Switched to model: ${modelTier} (${label})`; } const arg2 = parts[1]; // /model reset — restore configured provider/model and re-enable fallbacks if (arg2.toLowerCase() === 'reset') { const configured: ModelConfig | undefined = modelTier === 'default' ? deps.config.models.default : modelTier === 'fast' ? deps.config.models.fast : modelTier === 'complex' ? deps.config.models.complex : modelTier === 'local' ? deps.config.models.local : undefined; if (!configured) { return `No configured model for tier: ${modelTier}`; } const client = createClientFromConfig(configured); const label = `${configured.provider}/${configured.model}`; deps.modelRouter.setClient(modelTier, client, label); deps.modelRouter.setTierStrict(modelTier, false); session.setConfig('modelTier', modelTier); agent.setModelTier(modelTier); return `Reset ${modelTier} to: ${label}`; } // /model const providerModel = arg2; if (!providerModel.includes('/')) { return 'Invalid format. Use: /model (e.g. /model default github/gpt-5-mini)'; } const slashIdx = providerModel.indexOf('/'); const provider = providerModel.slice(0, slashIdx); const model = providerModel.slice(slashIdx + 1); if (!MODEL_PROVIDERS.includes(provider as ModelProvider)) { return `Unknown provider "${provider}". Known providers: ${MODEL_PROVIDERS.join(', ')}`; } const providerType = provider as ModelProvider; const providerConfigs = buildProviderConfigMap(deps.config); const template = providerConfigs[providerType]; try { const client = createClientFromConfig( template ? { ...template, provider: providerType, model } : { provider: providerType, model }, ); deps.modelRouter.setClient(modelTier, client, providerModel); deps.modelRouter.setTierStrict(modelTier, true); session.setConfig('modelTier', modelTier); agent.setModelTier(modelTier); const lines = [ `Set ${modelTier} to: ${providerModel}`, `Fallbacks for ${modelTier} disabled (strict tier mode).`, ]; if (parts.length > 2) { lines.push(`Note: ignored extra args: ${parts.slice(2).join(' ')}`); } return lines.join('\n'); } catch (error) { const message = error instanceof Error ? error.message : String(error); return `Failed to switch ${modelTier} to ${providerModel}: ${message}`; } }, compact: async () => { const result = await agent.compact(); if (result && result.compactedCount > 0) { return `Compacted ${result.compactedCount} messages: ${result.tokensBefore} → ${result.tokensAfter} tokens`; } return 'Nothing to compact.'; }, reset: () => { agent.reset(); session.deleteConfig('modelTier'); return ''; }, cancelRun: () => { const result = requestActiveRunCancellation({ sessionId: session.id, channel: msg.channel, senderId: msg.senderId, requestId: msg.id, }); return result.cancelled ? 'Cancellation requested. The active operation will stop at the next safe point.' : 'No active operation to cancel.'; }, delegateAgent: async (agentName: string, task: string) => { const target = agentName.trim(); const message = task.trim(); if (!target || !message) { return 'Usage: /research '; } if (!deps.agentConfigRegistry) { return 'No agent configurations are registered. Add agent_configs.research in config.'; } const agentConfig = deps.agentConfigRegistry.get(target); if (!agentConfig) { const available = deps.agentConfigRegistry.list().map((c) => c.name); return `Agent "${target}" not found. Available agents: ${available.length > 0 ? available.join(', ') : 'none'}`; } const tier: ModelTier = agentConfig.modelTier ?? 'default'; const systemPrompt = agentConfig.systemPrompt ?? `You are a sub-agent named "${target}". Complete the assigned task concisely and accurately.`; const result = await agent.delegate({ tier, systemPrompt, message, maxTokens: 4096, }); return `[Agent: ${target} | Tier: ${result.tier} | Tokens: ${result.usage.inputTokens}+${result.usage.outputTokens}]\n\n${result.content}`; }, runCouncil: async (task: string) => { const message = task.trim(); if (!message) { return 'Usage: /council | /council preflight'; } if (!deps.config.councils?.enabled) { return 'Councils are disabled. Set councils.enabled: true in config.'; } if (!deps.agentConfigRegistry || deps.agentConfigRegistry.list().length === 0) { return 'No agent configurations are registered. Add council_* agent_configs first.'; } if (shouldRunCouncilPreflight(message)) { return buildCouncilPreflightReport({ config: deps.config, registry: deps.agentConfigRegistry, delegateRunner: agent, activeTier: agent.getModelTier(), includeLiveProbe: true, }); } const tool = createCouncilRunTool({ registry: deps.agentConfigRegistry, orchestrator: agent, config: deps.config.councils as CouncilsConfig, scaffold: loadCouncilScaffoldSafe(deps.config.councils.scaffold_path), }); const result = await tool.execute({ task: message }); if (!result.success) { return `Council run failed: ${result.error ?? 'unknown error'}`; } return result.output; }, getElevation: () => { return getElevationStatusMessage({ get: (key) => session.getConfig(key), set: (key, value) => session.setConfig(key, value), delete: (key) => session.deleteConfig(key), }, { showExpiredSuffix: true, auditContext: { sessionId: session.id, channel: msg.channel, sender: msg.senderId, }, }); }, setElevation: (input: string) => { return setElevationFromInput({ get: (key) => session.getConfig(key), set: (key, value) => session.setConfig(key, value), delete: (key) => session.deleteConfig(key), }, input, { auditContext: { sessionId: session.id, channel: msg.channel, sender: msg.senderId, }, }); }, getQueue: () => { const mode = session.getConfig('queue.mode') ?? deps.config.server.queue.mode; const cap = session.getConfig('queue.cap') ?? String(deps.config.server.queue.cap); const overflow = session.getConfig('queue.overflow') ?? deps.config.server.queue.overflow; const debounceMs = session.getConfig('queue.debounce_ms') ?? String(deps.config.server.queue.debounce_ms); const summarizeOverflow = session.getConfig('queue.summarize_overflow') ?? String(deps.config.server.queue.summarize_overflow); const source = session.getConfig('queue.mode') || session.getConfig('queue.cap') || session.getConfig('queue.overflow') || session.getConfig('queue.debounce_ms') || session.getConfig('queue.summarize_overflow') ? 'session override' : 'default config'; return [ '**Queue policy**', `mode: ${mode}`, `cap: ${cap}`, `overflow: ${overflow}`, `debounce_ms: ${debounceMs}`, `summarize_overflow: ${summarizeOverflow}`, `source: ${source}`, ].join('\n'); }, setQueue: (input: string) => { const [rawKey, ...rest] = input.trim().split(/\s+/); const value = rest.join(' ').trim(); if (!rawKey || !value) { return 'Usage: /queue '; } const key = rawKey.toLowerCase(); if (key === 'mode') { if (!['collect', 'followup', 'steer', 'steer_backlog', 'interrupt'].includes(value)) { return 'Invalid mode. Use one of: collect, followup, steer, steer_backlog, interrupt'; } session.setConfig('queue.mode', value); return `Set queue.mode=${value} for this session`; } if (key === 'cap') { const cap = Number.parseInt(value, 10); if (!Number.isFinite(cap) || cap < 1 || cap > 1000) { return 'Invalid cap. Use an integer between 1 and 1000'; } session.setConfig('queue.cap', String(cap)); return `Set queue.cap=${cap} for this session`; } if (key === 'overflow') { if (value !== 'drop_old' && value !== 'drop_new') { return 'Invalid overflow. Use drop_old or drop_new'; } session.setConfig('queue.overflow', value); return `Set queue.overflow=${value} for this session`; } if (key === 'debounce_ms') { const debounceMs = Number.parseInt(value, 10); if (!Number.isFinite(debounceMs) || debounceMs < 0 || debounceMs > 60_000) { return 'Invalid debounce_ms. Use an integer between 0 and 60000'; } session.setConfig('queue.debounce_ms', String(debounceMs)); return `Set queue.debounce_ms=${debounceMs} for this session`; } if (key === 'summarize_overflow') { const normalized = value.toLowerCase(); if (normalized !== 'true' && normalized !== 'false') { return 'Invalid summarize_overflow. Use true or false'; } session.setConfig('queue.summarize_overflow', normalized); return `Set queue.summarize_overflow=${normalized} for this session`; } return 'Unknown queue key. Use one of: mode, cap, overflow, debounce_ms, summarize_overflow'; }, resetQueue: () => { session.deleteConfig('queue.mode'); session.deleteConfig('queue.cap'); session.deleteConfig('queue.overflow'); session.deleteConfig('queue.debounce_ms'); session.deleteConfig('queue.summarize_overflow'); return 'Reset session queue overrides.'; }, transferSession: (targetRaw: string) => { const target = targetRaw.trim().toLowerCase(); if (!target) { return 'Usage: /transfer '; } let toFrontend: string; let toUserId: string; let destinationLabel: string; if (target === 'tui') { toFrontend = 'tui'; toUserId = 'local'; destinationLabel = 'TUI (local)'; } else if (target === 'telegram') { if (msg.channel === 'telegram') { toFrontend = 'telegram'; toUserId = msg.senderId; } else { const chatId = deps.config.telegram?.allowed_chat_ids?.[0]; if (chatId === undefined) { return 'Telegram not configured'; } toFrontend = 'telegram'; toUserId = String(chatId); } destinationLabel = `Telegram (${toUserId})`; } else { return `Unknown transfer target: ${target}. Supported targets: tui, telegram`; } if (msg.channel === toFrontend && msg.senderId === toUserId) { return `Session is already active on ${destinationLabel}`; } deps.sessionManager.transferSession(msg.channel, msg.senderId, toFrontend, toUserId); return `Session transferred to ${destinationLabel}`; }, backendCommand: (inputRaw: string) => executeBackendCommand(inputRaw, agent.getModelTier()), getApprovals: () => { if (!deps.hookEngine) { return 'Approval gates are not enabled in this runtime.'; } const pending = deps.hookEngine.getPendingConfirmations({ sessionId: session.id }); if (pending.length === 0) { return 'No pending approvals for this session.'; } const lines = ['Pending approvals:']; for (const item of pending) { const ageSec = Math.max(0, Math.round((Date.now() - item.createdAt.getTime()) / 1000)); lines.push(`- ${item.id} | ${item.tool} | ${ageSec}s old`); } lines.push(''); lines.push('Use `/approve ` or `/deny ` (id optional: latest is used).'); return lines.join('\n'); }, approvePending: (inputRaw: string) => { if (!deps.hookEngine) { return 'Approval gates are not enabled in this runtime.'; } const pending = deps.hookEngine.getPendingConfirmations({ sessionId: session.id }); if (pending.length === 0) { return 'No pending approvals for this session.'; } const input = inputRaw.trim(); const selected = input ? pending.find((item) => item.id === input) : pending[pending.length - 1]; if (!selected) { return `Approval id not found in this session: ${input}`; } const resolved = deps.hookEngine.resolveConfirmation(selected.id, { approved: true }); return resolved ? `Approved: ${selected.tool} (${selected.id})` : `Approval request is no longer pending: ${selected.id}`; }, denyPending: (inputRaw: string) => { if (!deps.hookEngine) { return 'Approval gates are not enabled in this runtime.'; } const pending = deps.hookEngine.getPendingConfirmations({ sessionId: session.id }); if (pending.length === 0) { return 'No pending approvals for this session.'; } const input = inputRaw.trim(); let targetId: string | undefined; let reason = 'Denied by user'; if (input) { const [first, ...rest] = input.split(/\s+/); const matched = pending.find((item) => item.id === first); if (matched) { targetId = matched.id; reason = rest.join(' ').trim() || reason; } else { targetId = pending[pending.length - 1].id; reason = input; } } else { targetId = pending[pending.length - 1].id; } const selected = pending.find((item) => item.id === targetId); if (!selected) { return `Approval request is no longer pending: ${targetId}`; } const resolved = deps.hookEngine.resolveConfirmation(selected.id, { approved: false, reason }); return resolved ? `Denied: ${selected.tool} (${selected.id}) — ${reason}` : `Approval request is no longer pending: ${selected.id}`; }, skillCommand: async (inputRaw: string) => { const input = inputRaw.trim(); const [actionRaw, ...rest] = input.split(/\s+/).filter(Boolean); const action = actionRaw?.toLowerCase(); if (!action || action === 'help') { return [ 'Usage: /skill ', '/skill list', '/skill search ', '/skill install ', ].join('\n'); } if (action === 'list') { const skills = deps.skillRegistry?.listAvailable() ?? []; if (skills.length === 0) { return 'No available skills are currently loaded.'; } return [ `Available skills (${skills.length}):`, ...skills.map((skill) => `- ${skill.manifest.name} (${skill.manifest.tier}, v${skill.manifest.version})`), ].join('\n'); } if (action === 'search') { const term = rest.join(' ').trim().toLowerCase(); if (!term) { return 'Usage: /skill search '; } const sourceResolved = resolveRegistrySource(deps.config); if (!sourceResolved.source) { return sourceResolved.error ?? 'Failed to resolve registry source.'; } try { const catalog = await loadSkillRegistryCatalog(sourceResolved.source); const matches = catalog.skills.filter((entry) => { const haystack = `${entry.id} ${entry.name} ${entry.summary} ${entry.publisher ?? ''}`.toLowerCase(); return haystack.includes(term); }).slice(0, 12); if (matches.length === 0) { return `No registry skills found for "${term}".`; } return [ `Registry matches (${matches.length}):`, ...matches.map((entry) => `- ${entry.id} (${entry.version}) — ${entry.summary}`), ].join('\n'); } catch (error) { return `Registry lookup failed: ${error instanceof Error ? error.message : String(error)}`; } } if (action === 'install') { const registryId = rest.join(' ').trim().toLowerCase(); if (!registryId) { return 'Usage: /skill install '; } if (!deps.skillInstaller || !deps.skillRegistry) { return 'Skill installation is unavailable in this runtime.'; } const sourceResolved = resolveRegistrySource(deps.config); if (!sourceResolved.source) { return sourceResolved.error ?? 'Failed to resolve registry source.'; } try { const catalog = await loadSkillRegistryCatalog(sourceResolved.source); const entry = catalog.skills.find((item) => item.id.toLowerCase() === registryId); if (!entry) { return `Registry skill not found: ${registryId}`; } const localPath = resolveRegistryEntryLocalPath(entry, sourceResolved.source); if (!localPath) { return `Registry entry '${entry.id}' points to a remote source. Use CLI install for remote sources.`; } const installed = deps.skillInstaller.install(localPath); if (!installed) { return `Failed to install skill from ${entry.source}`; } deps.skillRegistry.register(installed); return `Installed skill '${installed.manifest.name}' (${installed.manifest.version}) from registry id '${entry.id}'.`; } catch (error) { return `Skill install failed: ${error instanceof Error ? error.message : String(error)}`; } } return 'Unknown skill action. Use: list, search, install'; }, }, }); if (commandResult.handled) { if (commandResult.text.trim()) { await reply({ text: commandResult.text, replyTo: msg.id }); } return; } } try { const runStartedAt = Date.now(); auditLogger?.runState?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', state: 'start', request_id: msg.id, }); deps.metrics?.recordRunState('start'); // Determine if the active model supports native audio input let effectiveTier: string = deps.config.agents.primary_tier ?? 'default'; const sessionTierOverride = session.getConfig('modelTier'); const tierFromUseCaseMetadata = tierFromUseCase(deps.config, msg.metadata?.modelFor); if (msg.metadata?.modelTier) { effectiveTier = msg.metadata.modelTier as string; } else if (tierFromUseCaseMetadata) { effectiveTier = tierFromUseCaseMetadata; } else if (sessionTierOverride) { effectiveTier = sessionTierOverride; } else if (deps.agentRouter && deps.agentConfigRegistry) { const agentName = deps.agentRouter.resolve(msg.channel, msg.senderId); if (agentName) { const agentCfg = deps.agentConfigRegistry.get(agentName); if (agentCfg?.modelTier) { effectiveTier = agentCfg.modelTier; } } } // Look up provider/model for the effective tier const modelsConfig = deps.config.models as Record; const tierConfig = modelsConfig[effectiveTier] ?? deps.config.models.default; const modelProvider = tierConfig?.provider ?? deps.config.models.default.provider; const modelName = tierConfig?.model ?? deps.config.models.default.model; const supportsAudioOverride = (tierConfig as Record | undefined)?.supports_audio as boolean | undefined; const nativeAudioSupported = supportsAudioInput(modelProvider, modelName, supportsAudioOverride); let messageText = incomingText; let attachments = msg.attachments; const audioAttachments = (msg.attachments ?? []).filter((a: Attachment) => isSupportedAudio(a)); const turnAudioToolInput = extractLatestAudioToolInput(audioAttachments); if (audioAttachments.length > 0) { persistLatestAudioAttachment(session, audioAttachments); } if (audioAttachments.length > 0 && !nativeAudioSupported) { // Model doesn't support native audio — transcribe via Whisper and strip audio attachments const audioConfig: AudioTranscriptionConfig | undefined = deps.config.audio?.enabled && deps.config.audio.provider ? { endpoint: deps.config.audio.provider.endpoint, apiKey: deps.config.audio.provider.api_key, model: deps.config.audio.provider.model, } : undefined; if (!audioConfig?.endpoint) { // Without transcription, we cannot safely send audio to a non-audio-capable model. // Fast-path a deterministic, user-friendly reply instead of invoking the agent loop. await reply({ text: [ 'I received your voice message, but I cannot transcribe it yet because audio transcription is not configured.', '', 'To enable voice messages, set `audio.enabled: true` and configure an `audio.provider` in `config.yaml` (OpenAI/Groq/custom Whisper-compatible `/v1/audio/transcriptions`).', '', 'Workarounds:', '1. Paste the transcription text.', '2. Upload the audio file somewhere and send me a direct URL.', ].join('\n'), replyTo: msg.id, }); auditLogger?.runState?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', state: 'complete', request_id: msg.id, duration_ms: Date.now() - runStartedAt, }); deps.metrics?.recordRunState('complete'); return; } for (const att of audioAttachments) { const transcript = await transcribeAudio(att, audioConfig); messageText = `[Voice message]: ${transcript}\n\n${messageText}`; } // For providers that cannot ingest native audio content parts (e.g. Anthropic), // keep the original audio attachment available in the tool loop so // audio.transcribe can still be hydrated from bytes if the model requests it. // For providers that do accept native audio parts (OpenAI-compatible/Gemini), // strip audio to avoid sending raw audio to a model tier that was marked as non-audio. if (providerAcceptsNativeAudioContentParts(modelProvider)) { attachments = (msg.attachments ?? []).filter((a: Attachment) => !isSupportedAudio(a)); if (attachments.length === 0) { attachments = undefined; } } } // If native audio IS supported, we pass attachments through unchanged — // buildUserMessage() in the agent will create native audio content parts const requestedBackend = applyBackendModeOverride(agentConfig?.backend ?? getEffectiveDefaultBackend()); const forceNativeForCapabilityQuery = shouldForceNativeForCapabilityQuery(messageText); const hasAttachmentsForExternalBackend = Boolean(attachments && attachments.length > 0); const selectedBackend = requestedBackend && requestedBackend !== 'native' ? deps.externalBackends?.[requestedBackend] : undefined; const externalBackendRequested = Boolean(selectedBackend && requestedBackend && requestedBackend !== 'native'); const forceNativeForPiNoTools = requestedBackend === 'pi_embedded' && deps.config.backends.pi_embedded.no_tools_mode && shouldForceNativeForPiNoTools(messageText); let forcedNativeGuardReason: 'capability_query' | 'pi_no_tools_mode' | 'attachments_present' | undefined; if (externalBackendRequested) { if (forceNativeForCapabilityQuery) { forcedNativeGuardReason = 'capability_query'; } else if (forceNativeForPiNoTools) { forcedNativeGuardReason = 'pi_no_tools_mode'; } else if (hasAttachmentsForExternalBackend) { forcedNativeGuardReason = 'attachments_present'; } } const selectedBackendForAudit: 'native' | ExternalBackendName = selectedBackend && requestedBackend && !forcedNativeGuardReason ? requestedBackend : 'native'; auditLogger?.backendRoute?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, selected_backend: selectedBackendForAudit, source: forcedNativeGuardReason ? 'forced_native_guard' : agentConfig?.backend ? 'agent_override' : selectedBackend ? 'default_external' : 'native', ...(forcedNativeGuardReason ? { guard_reason: forcedNativeGuardReason } : {}), }); if (selectedBackend && !hasAttachmentsForExternalBackend && !forceNativeForCapabilityQuery && !forceNativeForPiNoTools) { const backendStartedAt = Date.now(); try { const history = toExternalHistory(session.getHistory()); session.addMessage({ role: 'user', content: messageText }); const externalSystemPrompt = requestedBackend === 'pi_embedded' ? agent.getSystemPrompt(messageText) : undefined; const response = await selectedBackend.process({ prompt: messageText, history, ...(externalSystemPrompt ? { systemPrompt: externalSystemPrompt } : {}), }); auditLogger?.backendSuccess?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, backend: selectedBackend.name, duration_ms: Date.now() - backendStartedAt, response_length: response.length, }); session.addMessage({ role: 'assistant', content: response }); const ttsAttachment = await maybeBuildTtsAttachment(response, msg.channel); await reply({ text: response, replyTo: msg.id, attachments: ttsAttachment ? [ttsAttachment] : undefined, }); auditLogger?.runState?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', state: 'complete', request_id: msg.id, duration_ms: Date.now() - runStartedAt, }); deps.metrics?.recordRunState('complete'); return; } catch (error) { const detail = error instanceof Error ? error.message : String(error); console.warn(`External backend "${selectedBackend.name}" failed, falling back to native: ${detail}`); auditLogger?.backendFallback?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, from_backend: (requestedBackend && requestedBackend !== 'native') ? requestedBackend : (selectedBackend.name as ExternalBackendName), to_backend: 'native', reason: detail, duration_ms: Date.now() - backendStartedAt, }); } } let response: string; activeRuns.set(sessionIdForRun, agent); try { response = await agent.process(messageText, attachments, turnAudioToolInput); } catch (error) { const currentTier = agent.getModelTier(); const canEscalate = deps.config.agents.auto_escalate && currentTier !== 'complex'; if (!canEscalate) { throw error; } console.warn(`Auto-escalating session ${msg.channel}:${msg.senderId} from ${currentTier} to complex after processing failure.`); agent.setModelTier('complex'); response = await agent.process(messageText, attachments, turnAudioToolInput); } const outboundAttachments = collector.drain(); const ttsAttachment = await maybeBuildTtsAttachment(response, msg.channel); const mergedAttachments = ttsAttachment ? [...outboundAttachments, ttsAttachment] : outboundAttachments; await reply({ text: response, replyTo: msg.id, attachments: mergedAttachments.length > 0 ? mergedAttachments : undefined, }); const finalState = response.trim().toLowerCase() === 'operation cancelled by user.' ? 'cancelled' : 'complete'; auditLogger?.runState?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', state: finalState, request_id: msg.id, duration_ms: Date.now() - runStartedAt, }); deps.metrics?.recordRunState(finalState); } catch (error) { console.error(`Error processing message from ${msg.channel}:${msg.senderId}:`, error); await reply({ text: 'Sorry, an error occurred while processing your message.', replyTo: msg.id, }); auditLogger?.runState?.({ session_id: sessionIdForRun, channel: msg.channel, sender: msg.senderId, source: 'channel', state: 'error', request_id: msg.id, error: error instanceof Error ? error.message : String(error), }); deps.metrics?.recordRunState('error'); } finally { activeRuns.delete(sessionIdForRun); } }; return { handler, agents }; } function escapeRegex(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } function toExternalHistory(history: Array<{ role: string; content: unknown }>): Array<{ role: 'user' | 'assistant'; content: string }> { return history .filter((message): message is { role: 'user' | 'assistant'; content: unknown } => ( message.role === 'user' || message.role === 'assistant' )) .map((message) => ({ role: message.role, content: messageContentToText(message.content), })) .filter((message) => message.content.trim().length > 0); } function messageContentToText(content: unknown): string { if (typeof content === 'string') { return content; } if (!Array.isArray(content)) { return ''; } return content .map((part) => { if (!part || typeof part !== 'object') { return ''; } const partObj = part as { type?: string; text?: string }; if (partObj.type === 'text' && typeof partObj.text === 'string') { return partObj.text; } return ''; }) .filter(Boolean) .join('\n'); }