feat: add multimodal media pipeline for image support across all providers and channels
Widen Message.content from string to string | MessageContentPart[] to support multimodal content. Add Attachment type to channel layer, media conversion utilities, and image extraction to all channel adapters (Telegram, Discord, Slack, WhatsApp). Update all model clients (Anthropic, OpenAI, Gemini, Bedrock) to convert structured content to provider-specific formats. Fix downstream consumers (tokens, compaction, TUI, local models) to handle the widened type via getMessageText() helper.
This commit is contained in:
@@ -9,6 +9,7 @@
|
||||
|
||||
import { Client, LocalAuth } from 'whatsapp-web.js';
|
||||
import type {
|
||||
Attachment,
|
||||
InboundMessage,
|
||||
OutboundMessage,
|
||||
ChannelAdapter,
|
||||
@@ -37,6 +38,12 @@ interface WhatsAppMessage {
|
||||
fromMe: boolean;
|
||||
author?: string;
|
||||
_data?: { notifyName?: string };
|
||||
/** Whether this message contains media (image, video, audio, document). */
|
||||
hasMedia?: boolean;
|
||||
/** Message type (e.g. "image", "video", "chat"). */
|
||||
type?: string;
|
||||
/** Download the media attached to this message. */
|
||||
downloadMedia?: () => Promise<{ mimetype: string; data: string; filename?: string } | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -149,7 +156,7 @@ export class WhatsAppAdapter implements ChannelAdapter {
|
||||
}
|
||||
|
||||
/** Internal: process an inbound WhatsApp message. */
|
||||
private handleMessage(message: WhatsAppMessage): void {
|
||||
private async handleMessage(message: WhatsAppMessage): Promise<void> {
|
||||
if (!this.messageHandler) return;
|
||||
|
||||
// Ignore messages from the bot itself
|
||||
@@ -204,6 +211,26 @@ export class WhatsAppAdapter implements ChannelAdapter {
|
||||
|
||||
const senderName = message._data?.notifyName;
|
||||
|
||||
// Extract image attachments if the message has media
|
||||
const attachments: Attachment[] = [];
|
||||
if (message.hasMedia) {
|
||||
try {
|
||||
const media = await (message as any).downloadMedia();
|
||||
if (media && typeof media.mimetype === 'string' && media.mimetype.startsWith('image/')) {
|
||||
attachments.push({
|
||||
mimeType: media.mimetype,
|
||||
data: media.data,
|
||||
filename: media.filename,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'Failed to download WhatsApp media:',
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Detect reset command
|
||||
if (text === '!reset' || text === 'reset') {
|
||||
this.messageHandler({
|
||||
@@ -214,6 +241,7 @@ export class WhatsAppAdapter implements ChannelAdapter {
|
||||
text: '!reset',
|
||||
timestamp: Date.now(),
|
||||
metadata: { isCommand: true, command: 'reset' },
|
||||
...(attachments.length > 0 ? { attachments } : {}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -226,6 +254,7 @@ export class WhatsAppAdapter implements ChannelAdapter {
|
||||
senderName,
|
||||
text,
|
||||
timestamp: Date.now(),
|
||||
...(attachments.length > 0 ? { attachments } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user