feat: add LlamaCppClient with basic chat support

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
William Valentin
2026-02-05 13:11:56 -08:00
parent 3cf28df111
commit a20156f8db
2 changed files with 107 additions and 0 deletions
+71
View File
@@ -0,0 +1,71 @@
import type { ChatRequest, ChatResponse, ModelClient } from '../types.js';
export interface LlamaCppClientConfig {
endpoint: string;
authToken?: string;
}
interface LlamaCppMessage {
role: 'system' | 'user' | 'assistant';
content: string;
}
interface LlamaCppResponse {
choices: Array<{ message: { content: string } }>;
usage: { prompt_tokens: number; completion_tokens: number };
}
export class LlamaCppClient implements ModelClient {
private endpoint: string;
private authToken?: string;
constructor(config: LlamaCppClientConfig) {
this.endpoint = config.endpoint.replace(/\/$/, '');
this.authToken = config.authToken;
}
async chat(request: ChatRequest): Promise<ChatResponse> {
const messages: LlamaCppMessage[] = [];
if (request.system) {
messages.push({ role: 'system', content: request.system });
}
for (const msg of request.messages) {
messages.push({ role: msg.role, content: msg.content });
}
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
if (this.authToken) {
headers['Authorization'] = `Bearer ${this.authToken}`;
}
const response = await fetch(`${this.endpoint}/v1/chat/completions`, {
method: 'POST',
headers,
body: JSON.stringify({
messages,
max_tokens: request.maxTokens ?? 2048,
}),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`llama-server error (${response.status}): ${text}`);
}
const data = (await response.json()) as LlamaCppResponse;
return {
content: data.choices[0]?.message?.content ?? '',
stopReason: 'stop',
usage: {
inputTokens: data.usage?.prompt_tokens ?? 0,
outputTokens: data.usage?.completion_tokens ?? 0,
},
};
}
}