feat: add multimodal media pipeline for image support across all providers and channels
Widen Message.content from string to string | MessageContentPart[] to support multimodal content. Add Attachment type to channel layer, media conversion utilities, and image extraction to all channel adapters (Telegram, Discord, Slack, WhatsApp). Update all model clients (Anthropic, OpenAI, Gemini, Bedrock) to convert structured content to provider-specific formats. Fix downstream consumers (tokens, compaction, TUI, local models) to handle the widened type via getMessageText() helper.
This commit is contained in:
@@ -10,6 +10,7 @@ import { Client, GatewayIntentBits, Events } from 'discord.js';
|
||||
import type { Message as DiscordMessage } from 'discord.js';
|
||||
|
||||
import type {
|
||||
Attachment,
|
||||
InboundMessage,
|
||||
OutboundMessage,
|
||||
ChannelAdapter,
|
||||
@@ -50,6 +51,20 @@ export class DiscordAdapter implements ChannelAdapter {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/** Infer MIME type from URL if contentType is not provided. */
|
||||
private _inferMimeTypeFromUrl(url: string): string | null {
|
||||
const ext = url.split('.').pop()?.toLowerCase();
|
||||
const mimeTypes: Record<string, string> = {
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
gif: 'image/gif',
|
||||
webp: 'image/webp',
|
||||
svg: 'image/svg+xml',
|
||||
};
|
||||
return mimeTypes[ext || ''] || null;
|
||||
}
|
||||
|
||||
/** Register the inbound message handler. Called by the registry before connect(). */
|
||||
onMessage(handler: (msg: InboundMessage) => void): void {
|
||||
this.messageHandler = handler;
|
||||
@@ -159,6 +174,22 @@ export class DiscordAdapter implements ChannelAdapter {
|
||||
// Strip bot mention from the message text
|
||||
const text = message.content.replace(/<@!?\d+>/g, '').trim();
|
||||
|
||||
// ── Extract image attachments ──
|
||||
const attachments: Attachment[] = [];
|
||||
if (message.attachments && message.attachments.size > 0) {
|
||||
for (const attachment of message.attachments.values()) {
|
||||
const mimeType = attachment.contentType || this._inferMimeTypeFromUrl(attachment.url);
|
||||
if (mimeType && mimeType.startsWith('image/')) {
|
||||
attachments.push({
|
||||
mimeType,
|
||||
url: attachment.url,
|
||||
filename: attachment.name,
|
||||
size: attachment.size,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Reset command ──
|
||||
if (text === '!reset' || text === 'reset') {
|
||||
this.messageHandler({
|
||||
@@ -180,6 +211,7 @@ export class DiscordAdapter implements ChannelAdapter {
|
||||
senderId: message.channelId,
|
||||
senderName: message.author.username,
|
||||
text,
|
||||
attachments: attachments.length > 0 ? attachments : undefined,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user