feat: add LlamaCppClient with basic chat support
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
import type { ChatRequest, ChatResponse, ModelClient } from '../types.js';
|
||||
|
||||
export interface LlamaCppClientConfig {
|
||||
endpoint: string;
|
||||
authToken?: string;
|
||||
}
|
||||
|
||||
interface LlamaCppMessage {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
interface LlamaCppResponse {
|
||||
choices: Array<{ message: { content: string } }>;
|
||||
usage: { prompt_tokens: number; completion_tokens: number };
|
||||
}
|
||||
|
||||
export class LlamaCppClient implements ModelClient {
|
||||
private endpoint: string;
|
||||
private authToken?: string;
|
||||
|
||||
constructor(config: LlamaCppClientConfig) {
|
||||
this.endpoint = config.endpoint.replace(/\/$/, '');
|
||||
this.authToken = config.authToken;
|
||||
}
|
||||
|
||||
async chat(request: ChatRequest): Promise<ChatResponse> {
|
||||
const messages: LlamaCppMessage[] = [];
|
||||
|
||||
if (request.system) {
|
||||
messages.push({ role: 'system', content: request.system });
|
||||
}
|
||||
|
||||
for (const msg of request.messages) {
|
||||
messages.push({ role: msg.role, content: msg.content });
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
|
||||
if (this.authToken) {
|
||||
headers['Authorization'] = `Bearer ${this.authToken}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.endpoint}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
messages,
|
||||
max_tokens: request.maxTokens ?? 2048,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`llama-server error (${response.status}): ${text}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as LlamaCppResponse;
|
||||
|
||||
return {
|
||||
content: data.choices[0]?.message?.content ?? '',
|
||||
stopReason: 'stop',
|
||||
usage: {
|
||||
inputTokens: data.usage?.prompt_tokens ?? 0,
|
||||
outputTokens: data.usage?.completion_tokens ?? 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user