From c01de7d09750903f64bdfb9d43aaa632724b78fc Mon Sep 17 00:00:00 2001 From: William Valentin Date: Wed, 11 Feb 2026 09:33:21 -0800 Subject: [PATCH] feat: native tool calling message normalization for Ollama and llama.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ollama.ts: add normalizeMessagesForOllama() converting Anthropic-style tool_use/tool_result blocks to Ollama's native tool_calls + role:tool format - llamacpp.ts: add normalizeMessagesForLlamaCpp() with hybrid approach — assistant tool_calls in native format, but tool results as structured user messages (many GGUF templates silently drop role:tool messages) - llamacpp.ts: add configurable requestTimeout with AbortController (default 3min) - Both use fast-path when no tool blocks are present (zero overhead) - Full test coverage for both normalizers: plain text passthrough, tool_use conversion, tool_result mapping, multi-tool round trips, error results --- src/models/local/llamacpp.test.ts | 173 +++++++++++++++++++++++++++++- src/models/local/llamacpp.ts | 154 ++++++++++++++++++++++++-- src/models/local/ollama.test.ts | 135 ++++++++++++++++++++++- src/models/local/ollama.ts | 113 ++++++++++++++++++- 4 files changed, 560 insertions(+), 15 deletions(-) diff --git a/src/models/local/llamacpp.test.ts b/src/models/local/llamacpp.test.ts index 1735f14..ad446ac 100644 --- a/src/models/local/llamacpp.test.ts +++ b/src/models/local/llamacpp.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { LlamaCppClient } from './llamacpp.js'; -import type { ChatStreamEvent } from '../types.js'; +import { LlamaCppClient, normalizeMessagesForLlamaCpp } from './llamacpp.js'; +import type { ChatStreamEvent, Message } from '../types.js'; describe('LlamaCppClient', () => { const mockFetch = vi.fn(); @@ -341,3 +341,172 @@ describe('LlamaCppClient', () => { expect(requestBody.stream).toBe(true); }); }); + +describe('normalizeMessagesForLlamaCpp', () => { + it('passes plain text messages through', () => { + const messages: Message[] = [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + ]; + + const result = normalizeMessagesForLlamaCpp('System prompt', messages); + + expect(result).toEqual([ + { role: 'system', content: 'System prompt' }, + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + ]); + }); + + it('converts assistant tool_use blocks to OpenAI tool_calls format', () => { + const messages: Message[] = [ + { role: 'user', content: 'Search for news' }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Searching...' }, + { type: 'tool_use', id: 'call_1', name: 'web.search', input: { query: 'news' } }, + ] as any, + }, + ]; + + const result = normalizeMessagesForLlamaCpp(undefined, messages); + + expect(result).toHaveLength(2); + expect(result[1]).toEqual({ + role: 'assistant', + content: 'Searching...', + tool_calls: [{ + id: 'call_1', + type: 'function', + function: { + name: 'web.search', + arguments: '{"query":"news"}', + }, + }], + }); + }); + + it('converts user tool_result blocks to user messages with text formatting', () => { + const messages: Message[] = [ + { role: 'user', content: 'Search' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'call_1', name: 'web.search', input: { query: 'news' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'Results here', is_error: false }, + ] as any, + }, + ]; + + const result = normalizeMessagesForLlamaCpp(undefined, messages); + + expect(result).toHaveLength(3); + expect(result[2]).toEqual({ + role: 'user', + content: '[Tool "web.search" result]\nResults here', + }); + }); + + it('handles multiple tool results in a single user message', () => { + const messages: Message[] = [ + { role: 'user', content: 'Do two things' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'call_a', name: 'tool.a', input: {} }, + { type: 'tool_use', id: 'call_b', name: 'tool.b', input: { x: 1 } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_a', content: 'A result' }, + { type: 'tool_result', tool_use_id: 'call_b', content: 'B result' }, + ] as any, + }, + ]; + + const result = normalizeMessagesForLlamaCpp(undefined, messages); + + expect(result[1].tool_calls).toHaveLength(2); + expect(result[1].tool_calls![0].id).toBe('call_a'); + expect(result[1].tool_calls![1].function.arguments).toBe('{"x":1}'); + // Multiple results merged into one user message + expect(result).toHaveLength(3); + expect(result[2]).toEqual({ + role: 'user', + content: '[Tool "tool.a" result]\nA result\n\n[Tool "tool.b" result]\nB result', + }); + }); + + it('marks error results in text formatting', () => { + const messages: Message[] = [ + { role: 'user', content: 'Do it' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'call_1', name: 'file.read', input: { path: '/tmp/x' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'File not found', is_error: true }, + ] as any, + }, + ]; + + const result = normalizeMessagesForLlamaCpp(undefined, messages); + + expect(result[2]).toEqual({ + role: 'user', + content: '[Tool "file.read" result (error)]\nFile not found', + }); + }); + + it('handles full tool round-trip conversation', () => { + const messages: Message[] = [ + { role: 'user', content: 'What is the weather?' }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Checking...' }, + { type: 'tool_use', id: 'tc_0', name: 'weather.get', input: { city: 'NYC' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'tc_0', content: 'Sunny, 72F' }, + ] as any, + }, + { role: 'assistant', content: 'The weather in NYC is sunny, 72F.' }, + ]; + + const result = normalizeMessagesForLlamaCpp('You are helpful.', messages); + + expect(result).toHaveLength(5); + expect(result[0]).toEqual({ role: 'system', content: 'You are helpful.' }); + expect(result[1]).toEqual({ role: 'user', content: 'What is the weather?' }); + expect(result[2]).toEqual({ + role: 'assistant', + content: 'Checking...', + tool_calls: [{ + id: 'tc_0', + type: 'function', + function: { name: 'weather.get', arguments: '{"city":"NYC"}' }, + }], + }); + expect(result[3]).toEqual({ + role: 'user', + content: '[Tool "weather.get" result]\nSunny, 72F', + }); + expect(result[4]).toEqual({ role: 'assistant', content: 'The weather in NYC is sunny, 72F.' }); + }); +}); diff --git a/src/models/local/llamacpp.ts b/src/models/local/llamacpp.ts index 82be1b0..f4103c1 100644 --- a/src/models/local/llamacpp.ts +++ b/src/models/local/llamacpp.ts @@ -1,10 +1,12 @@ -import type { ChatRequest, ChatResponse, ChatStreamEvent, ModelClient, ModelToolCall } from '../types.js'; +import type { ChatRequest, ChatResponse, ChatStreamEvent, ModelClient, ModelToolCall, Message } from '../types.js'; import { normalizeMessagesForLocal } from '../media.js'; export interface LlamaCppClientConfig { endpoint: string; model: string; authToken?: string; + /** Per-request timeout in ms. Default: 180000 (3 minutes). */ + requestTimeout?: number; } interface LlamaCppToolCall { @@ -46,19 +48,145 @@ interface LlamaCppStreamChunk { usage?: { prompt_tokens: number; completion_tokens: number }; } +/** Message format for OpenAI-compatible chat completions API. */ +interface LlamaCppChatMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + tool_calls?: Array<{ + id: string; + type: 'function'; + function: { name: string; arguments: string }; + }>; + tool_call_id?: string; +} + +/** + * Normalize messages for llama.cpp's OpenAI-compatible API, converting + * Anthropic-style tool_use/tool_result blocks for local model consumption. + * + * Uses a hybrid approach for maximum template compatibility: + * - Assistant messages with tool_use blocks → assistant with tool_calls array + * (native format, so the model knows what it called) + * - User messages with tool_result blocks → single user message with structured + * text (NOT role: 'tool', because many GGUF chat templates silently drop + * unknown roles, making tool results invisible to the model) + * - All other messages → plain text via normalizeMessagesForLocal + */ +export function normalizeMessagesForLlamaCpp( + system: string | undefined, + messages: Message[], +): LlamaCppChatMessage[] { + // Check if any messages contain structured tool blocks + const hasToolBlocks = messages.some( + m => Array.isArray(m.content) && (m.content as Record[]).some( + b => b.type === 'tool_use' || b.type === 'tool_result', + ), + ); + + // Fast path: no tool blocks, use the simple normalizer + if (!hasToolBlocks) { + return normalizeMessagesForLocal(system, messages) as LlamaCppChatMessage[]; + } + + const result: LlamaCppChatMessage[] = []; + + // Track tool_use_id → tool_name for labeling results + const toolNameMap = new Map(); + + if (system) { + result.push({ role: 'system', content: system }); + } + + for (const msg of messages) { + if (typeof msg.content === 'string') { + result.push({ role: msg.role, content: msg.content }); + continue; + } + + const blocks = msg.content as Record[]; + + if (msg.role === 'assistant') { + const textParts: string[] = []; + const toolCalls: LlamaCppChatMessage['tool_calls'] = []; + + for (const block of blocks) { + if (block.type === 'text' && typeof block.text === 'string') { + textParts.push(block.text); + } else if (block.type === 'tool_use') { + const name = block.name as string; + const id = block.id as string; + if (id) toolNameMap.set(id, name); + let argsStr: string; + try { + argsStr = JSON.stringify(block.input); + } catch { + argsStr = '{}'; + } + toolCalls!.push({ + id, + type: 'function', + function: { + name, + arguments: argsStr, + }, + }); + } + } + + const chatMsg: LlamaCppChatMessage = { + role: 'assistant', + content: textParts.join('\n'), + }; + if (toolCalls!.length > 0) { + chatMsg.tool_calls = toolCalls; + } + result.push(chatMsg); + } else if (msg.role === 'user') { + const toolResults = blocks.filter(b => b.type === 'tool_result'); + + if (toolResults.length > 0) { + // Send tool results as a user message with clear text formatting. + // Many GGUF chat templates don't handle role: 'tool', silently + // dropping those messages. A user message always works. + const parts: string[] = []; + for (const tr of toolResults) { + const toolUseId = tr.tool_use_id as string; + const toolName = toolUseId ? toolNameMap.get(toolUseId) : undefined; + const content = (tr.content as string) ?? ''; + const isError = tr.is_error ? ' (error)' : ''; + const label = toolName ?? 'unknown'; + parts.push(`[Tool "${label}" result${isError}]\n${content}`); + } + result.push({ role: 'user', content: parts.join('\n\n') }); + } else { + const textParts = blocks + .filter(b => b.type === 'text' && typeof b.text === 'string') + .map(b => b.text as string); + if (textParts.length > 0) { + result.push({ role: 'user', content: textParts.join('\n') }); + } + } + } + } + + return result; +} + export class LlamaCppClient implements ModelClient { private endpoint: string; private model: string; private authToken?: string; + private requestTimeout: number; constructor(config: LlamaCppClientConfig) { this.endpoint = config.endpoint.replace(/\/$/, ''); this.model = config.model; this.authToken = config.authToken; + this.requestTimeout = config.requestTimeout ?? 180_000; // 3 minutes default } async chat(request: ChatRequest): Promise { - const messages = normalizeMessagesForLocal(request.system, request.messages); + const messages = normalizeMessagesForLlamaCpp(request.system, request.messages); const headers: Record = { 'Content-Type': 'application/json', @@ -88,12 +216,22 @@ export class LlamaCppClient implements ModelClient { })); } - response = await fetch(`${this.endpoint}/v1/chat/completions`, { - method: 'POST', - headers, - body: JSON.stringify(body), - }); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), this.requestTimeout); + try { + response = await fetch(`${this.endpoint}/v1/chat/completions`, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: controller.signal, + }); + } finally { + clearTimeout(timer); + } } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new Error(`llama-server request timed out after ${Math.round(this.requestTimeout / 1000)}s`); + } if (error instanceof TypeError && error.message.includes('fetch failed')) { throw new Error(`llama-server not running at ${this.endpoint}`); } @@ -129,7 +267,7 @@ export class LlamaCppClient implements ModelClient { } async *chatStream(request: ChatRequest): AsyncIterable { - const messages = normalizeMessagesForLocal(request.system, request.messages); + const messages = normalizeMessagesForLlamaCpp(request.system, request.messages); const headers: Record = { 'Content-Type': 'application/json', diff --git a/src/models/local/ollama.test.ts b/src/models/local/ollama.test.ts index c5bcd54..c0580fb 100644 --- a/src/models/local/ollama.test.ts +++ b/src/models/local/ollama.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { OllamaClient } from './ollama.js'; +import { OllamaClient, normalizeMessagesForOllama } from './ollama.js'; +import type { Message } from '../types.js'; const mockChat = vi.fn(); const mockShow = vi.fn(); @@ -348,3 +349,135 @@ describe('OllamaClient', () => { expect(callArgs.tools).toBeDefined(); }); }); + +describe('normalizeMessagesForOllama', () => { + it('passes plain text messages through', () => { + const messages: Message[] = [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + ]; + + const result = normalizeMessagesForOllama('System prompt', messages); + + expect(result).toEqual([ + { role: 'system', content: 'System prompt' }, + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + ]); + }); + + it('converts assistant tool_use blocks to Ollama tool_calls format', () => { + const messages: Message[] = [ + { role: 'user', content: 'Search for news' }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Let me search for that.' }, + { type: 'tool_use', id: 'call_1', name: 'web.search', input: { query: 'latest news' } }, + ] as any, + }, + ]; + + const result = normalizeMessagesForOllama(undefined, messages); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ role: 'user', content: 'Search for news' }); + expect(result[1]).toEqual({ + role: 'assistant', + content: 'Let me search for that.', + tool_calls: [{ + function: { + name: 'web.search', + arguments: { query: 'latest news' }, + }, + }], + }); + }); + + it('converts user tool_result blocks to Ollama tool role messages with tool_name', () => { + const messages: Message[] = [ + { role: 'user', content: 'Search for news' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'call_1', name: 'web.search', input: { query: 'news' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'Breaking news: ...', is_error: false }, + ] as any, + }, + ]; + + const result = normalizeMessagesForOllama(undefined, messages); + + expect(result).toHaveLength(3); + expect(result[2]).toEqual({ + role: 'tool', + content: 'Breaking news: ...', + tool_name: 'web.search', + }); + }); + + it('handles multiple tool calls in a single assistant message', () => { + const messages: Message[] = [ + { role: 'user', content: 'Search two things' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'call_1', name: 'web.search', input: { query: 'a' } }, + { type: 'tool_use', id: 'call_2', name: 'web.search', input: { query: 'b' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'Result A' }, + { type: 'tool_result', tool_use_id: 'call_2', content: 'Result B' }, + ] as any, + }, + ]; + + const result = normalizeMessagesForOllama(undefined, messages); + + expect(result[1].tool_calls).toHaveLength(2); + // Each tool_result becomes a separate tool message with tool_name + expect(result[2]).toEqual({ role: 'tool', content: 'Result A', tool_name: 'web.search' }); + expect(result[3]).toEqual({ role: 'tool', content: 'Result B', tool_name: 'web.search' }); + }); + + it('handles full tool round-trip conversation', () => { + const messages: Message[] = [ + { role: 'user', content: 'What is the weather?' }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Let me check.' }, + { type: 'tool_use', id: 'tc_0', name: 'weather.get', input: { city: 'NYC' } }, + ] as any, + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'tc_0', content: 'Sunny, 72F' }, + ] as any, + }, + { role: 'assistant', content: 'The weather in NYC is sunny, 72F.' }, + ]; + + const result = normalizeMessagesForOllama('You are helpful.', messages); + + expect(result).toHaveLength(5); + expect(result[0]).toEqual({ role: 'system', content: 'You are helpful.' }); + expect(result[1]).toEqual({ role: 'user', content: 'What is the weather?' }); + expect(result[2]).toEqual({ + role: 'assistant', + content: 'Let me check.', + tool_calls: [{ function: { name: 'weather.get', arguments: { city: 'NYC' } } }], + }); + expect(result[3]).toEqual({ role: 'tool', content: 'Sunny, 72F', tool_name: 'weather.get' }); + expect(result[4]).toEqual({ role: 'assistant', content: 'The weather in NYC is sunny, 72F.' }); + }); +}); diff --git a/src/models/local/ollama.ts b/src/models/local/ollama.ts index 181ad3a..757a2f1 100644 --- a/src/models/local/ollama.ts +++ b/src/models/local/ollama.ts @@ -1,7 +1,112 @@ -import { Ollama, type Tool } from 'ollama'; -import type { ChatRequest, ChatResponse, ChatStreamEvent, ModelClient, ToolDefinition, ModelToolCall } from '../types.js'; +import { Ollama, type Tool, type Message as OllamaMessage } from 'ollama'; +import type { ChatRequest, ChatResponse, ChatStreamEvent, ModelClient, ToolDefinition, ModelToolCall, Message } from '../types.js'; import { normalizeMessagesForLocal } from '../media.js'; +/** + * Normalize messages for Ollama, converting Anthropic-style tool_use/tool_result + * blocks to Ollama's native tool calling format. + * + * - Assistant messages with tool_use blocks → assistant with tool_calls array + * - User messages with tool_result blocks → one { role: 'tool' } message per result + * - All other messages → plain text via normalizeMessagesForLocal + */ +export function normalizeMessagesForOllama( + system: string | undefined, + messages: Message[], +): OllamaMessage[] { + // Check if any messages contain structured tool blocks + const hasToolBlocks = messages.some( + m => Array.isArray(m.content) && (m.content as Record[]).some( + b => b.type === 'tool_use' || b.type === 'tool_result', + ), + ); + + // Fast path: no tool blocks, use the simple normalizer + if (!hasToolBlocks) { + return normalizeMessagesForLocal(system, messages) as OllamaMessage[]; + } + + const result: OllamaMessage[] = []; + + if (system) { + result.push({ role: 'system', content: system }); + } + + // Track tool_use_id → tool_name so tool_result messages can set tool_name + const toolNameMap = new Map(); + + for (const msg of messages) { + if (typeof msg.content === 'string') { + // Plain text message + result.push({ role: msg.role, content: msg.content }); + continue; + } + + const blocks = msg.content as Record[]; + + if (msg.role === 'assistant') { + // Extract text and tool_use blocks + const textParts: string[] = []; + const toolCalls: Array<{ function: { name: string; arguments: Record } }> = []; + + for (const block of blocks) { + if (block.type === 'text' && typeof block.text === 'string') { + textParts.push(block.text); + } else if (block.type === 'tool_use') { + const name = block.name as string; + const id = block.id as string; + if (id) toolNameMap.set(id, name); + toolCalls.push({ + function: { + name, + arguments: (block.input ?? {}) as Record, + }, + }); + } + } + + const ollamaMsg: OllamaMessage = { + role: 'assistant', + content: textParts.join('\n'), + }; + if (toolCalls.length > 0) { + ollamaMsg.tool_calls = toolCalls; + } + result.push(ollamaMsg); + } else if (msg.role === 'user') { + // Check if this is a tool_result message + const toolResults = blocks.filter(b => b.type === 'tool_result'); + + if (toolResults.length > 0) { + // Convert each tool_result to a separate 'tool' role message + for (const tr of toolResults) { + const toolUseId = tr.tool_use_id as string; + const ollamaMsg: OllamaMessage = { + role: 'tool', + content: (tr.content as string) ?? '', + }; + // Set tool_name so Ollama associates this result with the correct tool call + const toolName = toolUseId ? toolNameMap.get(toolUseId) : undefined; + if (toolName) { + ollamaMsg.tool_name = toolName; + } + result.push(ollamaMsg); + } + } else { + // Regular user message with content parts (e.g. images) + const textParts = blocks + .filter(b => b.type === 'text' && typeof b.text === 'string') + .map(b => b.text as string); + if (textParts.length > 0) { + result.push({ role: 'user', content: textParts.join('\n') }); + } + } + } + } + + return result; +} + export interface OllamaClientConfig { host?: string; model: string; @@ -61,7 +166,7 @@ export class OllamaClient implements ModelClient { } async chat(request: ChatRequest): Promise { - const messages = normalizeMessagesForLocal(request.system, request.messages); + const messages = normalizeMessagesForOllama(request.system, request.messages); // Build the chat params, optionally including tools const chatParams: Parameters[0] = { @@ -112,7 +217,7 @@ export class OllamaClient implements ModelClient { } async *chatStream(request: ChatRequest): AsyncIterable { - const messages = normalizeMessagesForLocal(request.system, request.messages); + const messages = normalizeMessagesForOllama(request.system, request.messages); try { // Build tools array if provided and model supports them