feat(audio): add native audio support to type system and model clients

- Add AudioSource interface and 'audio' variant to MessageContentPart union
- Update buildUserMessage() to create audio content parts from attachments
- Add attachmentToAudioSource(), hasAudio(), stripAudioParts() helpers
- Gemini: native audio via inlineData (same format as images)
- OpenAI/GitHub: native audio via input_audio content parts
- Anthropic/Bedrock: graceful fallback to transcript text
- Update getMessageTextWithTools() to handle audio blocks for local models
This commit is contained in:
William Valentin
2026-02-11 18:17:33 -08:00
parent a875bcc4ae
commit 32e1a2724a
8 changed files with 169 additions and 22 deletions
+9
View File
@@ -188,6 +188,15 @@ function convertMessages(messages: Message[]): Content[] {
// so we pass as a text description. In production, you'd want to fetch + base64 encode.
return { text: `[Image: ${part.source.url ?? 'unavailable'}]` };
}
// Audio part — Gemini supports native audio via inlineData (same format as images)
if (part.type === 'audio') {
return {
inlineData: {
mimeType: part.source.media_type,
data: part.source.data,
},
};
}
return { text: JSON.stringify(part) };
});