feat: add multimodal media pipeline for image support across all providers and channels
Widen Message.content from string to string | MessageContentPart[] to support multimodal content. Add Attachment type to channel layer, media conversion utilities, and image extraction to all channel adapters (Telegram, Discord, Slack, WhatsApp). Update all model clients (Anthropic, OpenAI, Gemini, Bedrock) to convert structured content to provider-specific formats. Fix downstream consumers (tokens, compaction, TUI, local models) to handle the widened type via getMessageText() helper.
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
|
||||
import { App } from '@slack/bolt';
|
||||
import type {
|
||||
Attachment,
|
||||
InboundMessage,
|
||||
OutboundMessage,
|
||||
ChannelAdapter,
|
||||
@@ -35,6 +36,14 @@ interface SlackMessageEvent {
|
||||
text?: string;
|
||||
bot_id?: string;
|
||||
subtype?: string;
|
||||
files?: Array<{
|
||||
id?: string;
|
||||
mimetype?: string;
|
||||
name?: string;
|
||||
size?: number;
|
||||
url_private?: string;
|
||||
url_private_download?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -160,6 +169,56 @@ export class SlackAdapter implements ChannelAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download image files from a Slack message and convert to base64 Attachments.
|
||||
* Non-image files are skipped. Download errors are logged but don't crash the handler.
|
||||
*/
|
||||
private async extractImageAttachments(
|
||||
files?: SlackMessageEvent['files'],
|
||||
): Promise<Attachment[]> {
|
||||
if (!files || files.length === 0) return [];
|
||||
|
||||
const attachments: Attachment[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
// Only process image files
|
||||
if (!file.mimetype?.startsWith('image/')) continue;
|
||||
|
||||
const downloadUrl = file.url_private_download || file.url_private;
|
||||
if (!downloadUrl) continue;
|
||||
|
||||
try {
|
||||
const response = await fetch(downloadUrl, {
|
||||
headers: { Authorization: `Bearer ${this.config.botToken}` },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(
|
||||
`Slack: failed to download file ${file.name ?? file.id ?? 'unknown'}: HTTP ${response.status}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const base64 = Buffer.from(arrayBuffer).toString('base64');
|
||||
|
||||
attachments.push({
|
||||
mimeType: file.mimetype,
|
||||
data: base64,
|
||||
filename: file.name,
|
||||
size: file.size,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Slack: error downloading file ${file.name ?? file.id ?? 'unknown'}:`,
|
||||
error instanceof Error ? error.message : 'Unknown error',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return attachments;
|
||||
}
|
||||
|
||||
/** Internal: process an inbound Slack message event. */
|
||||
private async handleMessage(message: SlackMessageEvent): Promise<void> {
|
||||
if (!this.messageHandler) return;
|
||||
@@ -200,6 +259,9 @@ export class SlackAdapter implements ChannelAdapter {
|
||||
? await this.resolveUserName(message.user)
|
||||
: undefined;
|
||||
|
||||
// Extract image attachments from Slack file uploads
|
||||
const attachments = await this.extractImageAttachments(message.files);
|
||||
|
||||
// Detect reset command
|
||||
if (text === '!reset' || text === 'reset') {
|
||||
this.messageHandler({
|
||||
@@ -210,6 +272,7 @@ export class SlackAdapter implements ChannelAdapter {
|
||||
text: '!reset',
|
||||
timestamp: Date.now(),
|
||||
metadata: { isCommand: true, command: 'reset' },
|
||||
...(attachments.length > 0 && { attachments }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -222,6 +285,7 @@ export class SlackAdapter implements ChannelAdapter {
|
||||
senderName,
|
||||
text,
|
||||
timestamp: Date.now(),
|
||||
...(attachments.length > 0 && { attachments }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user