feat: add multimodal media pipeline for image support across all providers and channels
Widen Message.content from string to string | MessageContentPart[] to support multimodal content. Add Attachment type to channel layer, media conversion utilities, and image extraction to all channel adapters (Telegram, Discord, Slack, WhatsApp). Update all model clients (Anthropic, OpenAI, Gemini, Bedrock) to convert structured content to provider-specific formats. Fix downstream consumers (tokens, compaction, TUI, local models) to handle the widened type via getMessageText() helper.
This commit is contained in:
@@ -2,6 +2,7 @@ import { Bot } from 'grammy';
|
||||
|
||||
import type { HookEngine } from '../../hooks/index.js';
|
||||
import type {
|
||||
Attachment,
|
||||
InboundMessage,
|
||||
OutboundMessage,
|
||||
ChannelAdapter,
|
||||
@@ -44,6 +45,26 @@ export class TelegramAdapter implements ChannelAdapter {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/** Download a file from Telegram API and convert to base64. */
|
||||
private async downloadFileToBase64(fileId: string): Promise<string | null> {
|
||||
try {
|
||||
const file = await this.bot?.api.getFile(fileId);
|
||||
if (!file || !file.file_path) return null;
|
||||
|
||||
const token = this.config.botToken;
|
||||
const url = `https://api.telegram.org/file/bot${token}/${file.file_path}`;
|
||||
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) return null;
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return buffer.toString('base64');
|
||||
} catch (error) {
|
||||
console.error(`Failed to download file ${fileId}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Register the inbound message handler. Called by the registry before connect(). */
|
||||
onMessage(handler: (msg: InboundMessage) => void): void {
|
||||
this.messageHandler = handler;
|
||||
@@ -164,6 +185,84 @@ export class TelegramAdapter implements ChannelAdapter {
|
||||
});
|
||||
});
|
||||
|
||||
// ── Photo message handler ──
|
||||
|
||||
this.bot.on('message:photo', async (ctx) => {
|
||||
if (!this.messageHandler) return;
|
||||
|
||||
const photo = ctx.message.photo;
|
||||
if (!photo || photo.length === 0) return;
|
||||
|
||||
const largestPhoto = photo[photo.length - 1];
|
||||
|
||||
await ctx.replyWithChatAction('typing');
|
||||
|
||||
const imageData = await this.downloadFileToBase64(largestPhoto.file_id);
|
||||
if (!imageData) {
|
||||
console.error(`Failed to download photo ${largestPhoto.file_id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const caption = ctx.message.caption ?? '';
|
||||
|
||||
this.messageHandler({
|
||||
id: String(ctx.message.message_id),
|
||||
channel: 'telegram',
|
||||
senderId: String(ctx.chat.id),
|
||||
senderName: ctx.from?.first_name,
|
||||
text: caption,
|
||||
attachments: [
|
||||
{
|
||||
mimeType: 'image/jpeg',
|
||||
data: imageData,
|
||||
filename: `photo_${largestPhoto.file_unique_id}.jpg`,
|
||||
size: largestPhoto.file_size,
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
});
|
||||
|
||||
// ── Image document handler ──
|
||||
|
||||
this.bot.on('message:document', async (ctx) => {
|
||||
if (!this.messageHandler) return;
|
||||
|
||||
const document = ctx.message.document;
|
||||
if (!document) return;
|
||||
|
||||
const mimeType = document.mime_type ?? '';
|
||||
if (!mimeType.startsWith('image/')) return;
|
||||
|
||||
await ctx.replyWithChatAction('typing');
|
||||
|
||||
const fileData = await this.downloadFileToBase64(document.file_id);
|
||||
if (!fileData) {
|
||||
console.error(`Failed to download document ${document.file_id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const caption = ctx.message.caption ?? '';
|
||||
const filename = document.file_name ?? document.file_unique_id;
|
||||
|
||||
this.messageHandler({
|
||||
id: String(ctx.message.message_id),
|
||||
channel: 'telegram',
|
||||
senderId: String(ctx.chat.id),
|
||||
senderName: ctx.from?.first_name,
|
||||
text: caption,
|
||||
attachments: [
|
||||
{
|
||||
mimeType,
|
||||
data: fileData,
|
||||
filename,
|
||||
size: document.file_size,
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
});
|
||||
|
||||
// ── Start long polling ──
|
||||
|
||||
this.bot.start({
|
||||
|
||||
Reference in New Issue
Block a user