feat(tools): add minio.share upload and presigned link tool
This commit is contained in:
@@ -26,6 +26,7 @@ Self-hosted personal AI assistant with Telegram and Terminal interfaces.
|
||||
- **Inbound Webhooks**: HTTP endpoints that trigger agent processing with HMAC auth and template rendering
|
||||
- **Heartbeat Monitor**: Periodic health checks (gateway, model, channels, memory, disk) with failure notifications
|
||||
- **Scheduled Backups**: Interval- or cron-based snapshot backups with optional startup run
|
||||
- **MinIO File Sharing Tool**: Upload a local file and return a temporary MinIO share link via `minio.share`
|
||||
- **Gmail Pub/Sub Watcher**: Monitor Gmail inbox via Google Cloud Pub/Sub push notifications with polling fallback
|
||||
- **Vector Memory Search**: Hybrid keyword + semantic search with embeddings (OpenAI, Gemini, Ollama, llama.cpp, Voyage AI)
|
||||
- **Docker Deployment**: Multi-stage Dockerfile and docker-compose.yml for production containers
|
||||
@@ -603,6 +604,14 @@ backup:
|
||||
secure: true
|
||||
```
|
||||
|
||||
## MinIO Share Tool
|
||||
|
||||
When `backup.minio.enabled` is configured, Flynn also exposes a `minio.share` tool:
|
||||
|
||||
- Uploads a local file to the configured MinIO bucket
|
||||
- Returns a temporary download URL (`mc share download`)
|
||||
- Useful for sharing CSVs, logs, images, and generated artifacts without dumping file contents in chat
|
||||
|
||||
## Inbound Webhooks
|
||||
|
||||
HTTP endpoints that trigger agent processing. Each webhook accepts POST requests, optionally verifies an HMAC signature, renders a message template, and routes the agent's response to an output channel.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"status": "completed",
|
||||
"date": "2026-02-16",
|
||||
"updated": "2026-02-16",
|
||||
"summary": "Added first-class automation presets and scheduling upgrades: `automation.daily_briefing` now auto-registers an opinionated cron job for morning briefings, and backup scheduling now supports cron expressions via `backup.schedule` plus optional `backup.run_on_start` while preserving interval fallback. Added `BackupScheduler` with `backup.notify` channel alerts, configurable `backup.failure_threshold`, and recovery notifications (`backup.notify_recovery`) so backup failures/recoveries proactively notify operators. Extended heartbeat monitoring with `process_memory`, `backup`, and `provider_errors` checks (with thresholds) so high RSS usage, backup failure streaks, and model-provider error spikes proactively trigger health alerts. Added timezone-safe daily briefing dedupe via `automation.daily_briefing.dedupe_per_local_day` and cron-level `once_per_local_day` so morning briefings do not send twice on the same local day.",
|
||||
"summary": "Added first-class automation presets and scheduling upgrades: `automation.daily_briefing` now auto-registers an opinionated cron job for morning briefings, and backup scheduling now supports cron expressions via `backup.schedule` plus optional `backup.run_on_start` while preserving interval fallback. Added `BackupScheduler` with `backup.notify` channel alerts, configurable `backup.failure_threshold`, and recovery notifications (`backup.notify_recovery`) so backup failures/recoveries proactively notify operators. Extended heartbeat monitoring with `process_memory`, `backup`, and `provider_errors` checks (with thresholds) so high RSS usage, backup failure streaks, and model-provider error spikes proactively trigger health alerts. Added timezone-safe daily briefing dedupe via `automation.daily_briefing.dedupe_per_local_day` and cron-level `once_per_local_day` so morning briefings do not send twice on the same local day. Added `minio.share` tool to upload local artifacts and return temporary MinIO share links using existing backup MinIO credentials.",
|
||||
"files_modified": [
|
||||
"src/config/schema.ts",
|
||||
"src/config/schema.test.ts",
|
||||
@@ -23,6 +23,11 @@
|
||||
"src/backup/scheduler.test.ts",
|
||||
"src/backup/status.ts",
|
||||
"src/backup/status.test.ts",
|
||||
"src/tools/builtin/minio-share.ts",
|
||||
"src/tools/builtin/minio-share.test.ts",
|
||||
"src/tools/builtin/index.ts",
|
||||
"src/tools/index.ts",
|
||||
"src/tools/policy.ts",
|
||||
"src/daemon/channels.ts",
|
||||
"src/daemon/channels.test.ts",
|
||||
"src/daemon/index.ts",
|
||||
@@ -33,7 +38,7 @@
|
||||
"config/default.yaml",
|
||||
"README.md"
|
||||
],
|
||||
"test_status": "pnpm test:run src/automation/presets.test.ts src/automation/cron.test.ts src/automation/heartbeat.test.ts src/backup/scheduler.test.ts src/backup/status.test.ts src/config/schema.test.ts src/daemon/channels.test.ts src/gateway/handlers/services.test.ts + pnpm typecheck passing"
|
||||
"test_status": "pnpm test:run src/automation/presets.test.ts src/automation/cron.test.ts src/automation/heartbeat.test.ts src/backup/scheduler.test.ts src/backup/status.test.ts src/tools/builtin/minio-share.test.ts src/tools/policy.test.ts src/config/schema.test.ts src/daemon/channels.test.ts src/gateway/handlers/services.test.ts + pnpm typecheck passing"
|
||||
},
|
||||
"backup-session-summary-audit-trail": {
|
||||
"status": "completed",
|
||||
@@ -3319,7 +3324,7 @@
|
||||
}
|
||||
},
|
||||
"overall_progress": {
|
||||
"total_test_count": 1844,
|
||||
"total_test_count": 1848,
|
||||
"all_tests_passing": true,
|
||||
"p0_completion": "3/3 (100%)",
|
||||
"p1_completion": "4/4 (100%)",
|
||||
|
||||
+4
-1
@@ -27,7 +27,7 @@ import { RoutingPolicy } from '../routing/index.js';
|
||||
import type { ModelRouter } from '../models/index.js';
|
||||
import { SessionStore, SessionManager, parseDuration } from '../session/index.js';
|
||||
import { HookEngine } from '../hooks/index.js';
|
||||
import { createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools, createGmailTools, createGcalTools, createGdocsTools, createGdriveTools, createGtasksTools } from '../tools/index.js';
|
||||
import { createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools, createGmailTools, createGcalTools, createGdocsTools, createGdriveTools, createGtasksTools, createMinioShareTool } from '../tools/index.js';
|
||||
import { ChannelRegistry } from '../channels/index.js';
|
||||
import type { McpManager } from '../mcp/index.js';
|
||||
import type { SkillRegistry, SkillInstaller } from '../skills/index.js';
|
||||
@@ -185,6 +185,9 @@ export async function startDaemon(config: Config, options?: StartDaemonOptions):
|
||||
if (config.automation.gtasks?.enabled) {
|
||||
for (const tool of createGtasksTools(config.automation.gtasks)) { toolRegistry.register(tool); }
|
||||
}
|
||||
if (config.backup.minio.enabled) {
|
||||
toolRegistry.register(createMinioShareTool(config.backup));
|
||||
}
|
||||
|
||||
// ── Lifecycle ──
|
||||
await startServices({ config, lifecycle, channelRegistry, gateway, modelRouter, memoryDir, dataDir });
|
||||
|
||||
@@ -27,6 +27,7 @@ export { createGcalTools } from './gcal.js';
|
||||
export { createGdocsTools } from './gdocs.js';
|
||||
export { createGdriveTools } from './gdrive.js';
|
||||
export { createGtasksTools } from './gtasks.js';
|
||||
export { createMinioShareTool } from './minio-share.js';
|
||||
export { screenCaptureTool, cameraCaptureTool } from './capture.js';
|
||||
|
||||
import type { Tool } from '../types.js';
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createMinioShareTool, minioShareInternals } from './minio-share.js';
|
||||
import type { BackupConfig } from '../../config/schema.js';
|
||||
|
||||
function makeBackupConfig(overrides?: Partial<BackupConfig>): BackupConfig {
|
||||
return {
|
||||
enabled: true,
|
||||
schedule: undefined,
|
||||
interval: '24h',
|
||||
run_on_start: false,
|
||||
notify: undefined,
|
||||
failure_threshold: 1,
|
||||
notify_recovery: true,
|
||||
local_dir: '~/.local/share/flynn/backups',
|
||||
include_vectors: true,
|
||||
minio: {
|
||||
enabled: true,
|
||||
endpoint: 'localhost:9000',
|
||||
access_key: 'minio-admin',
|
||||
secret_key: 'minio-secret',
|
||||
bucket: 'flynn-shared',
|
||||
prefix: 'flynn',
|
||||
secure: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('minio share internals', () => {
|
||||
it('parses share URL from json output', () => {
|
||||
const out = '{"status":"success","share":"https://minio.local/share/some-token"}\n';
|
||||
expect(minioShareInternals.parseShareUrl(out)).toBe('https://minio.local/share/some-token');
|
||||
});
|
||||
|
||||
it('parses share URL from plain text output', () => {
|
||||
const out = 'Share: https://minio.local/share/some-token\n';
|
||||
expect(minioShareInternals.parseShareUrl(out)).toBe('https://minio.local/share/some-token');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createMinioShareTool', () => {
|
||||
let tempDir = '';
|
||||
|
||||
afterEach(() => {
|
||||
if (tempDir) {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
tempDir = '';
|
||||
}
|
||||
});
|
||||
|
||||
it('uploads file and returns share URL', async () => {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'flynn-minio-share-'));
|
||||
const filePath = join(tempDir, 'report.csv');
|
||||
writeFileSync(filePath, 'a,b\n1,2\n');
|
||||
|
||||
const execRunner = vi.fn(async (_file: string, args: string[]) => {
|
||||
if (args[0] === 'share') {
|
||||
return { stdout: '{"share":"https://minio.local/share/abc"}\n', stderr: '' };
|
||||
}
|
||||
return { stdout: '', stderr: '' };
|
||||
});
|
||||
|
||||
const tool = createMinioShareTool(makeBackupConfig(), {
|
||||
execRunner,
|
||||
now: () => new Date('2026-02-16T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
const result = await tool.execute({ path: filePath, expires: '12h', prefix: 'exports' });
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toContain('https://minio.local/share/abc');
|
||||
expect(execRunner).toHaveBeenCalledTimes(3);
|
||||
expect(execRunner.mock.calls[1]?.[1]).toContain('cp');
|
||||
expect(execRunner.mock.calls[2]?.[1]).toContain('share');
|
||||
});
|
||||
|
||||
it('returns error when minio is disabled', async () => {
|
||||
const tool = createMinioShareTool(makeBackupConfig({
|
||||
minio: {
|
||||
enabled: false,
|
||||
endpoint: undefined,
|
||||
access_key: undefined,
|
||||
secret_key: undefined,
|
||||
bucket: undefined,
|
||||
prefix: 'flynn',
|
||||
secure: true,
|
||||
},
|
||||
}));
|
||||
|
||||
const result = await tool.execute({ path: '/tmp/test.txt' });
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('backup.minio.enabled=true');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
import { promisify } from 'node:util';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { basename, resolve } from 'node:path';
|
||||
import type { BackupConfig } from '../../config/schema.js';
|
||||
import type { Tool, ToolResult } from '../types.js';
|
||||
import { backupInternals } from '../../backup/index.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
type ExecRunner = (file: string, args: string[], options?: { env?: NodeJS.ProcessEnv }) => Promise<{ stdout: string; stderr: string }>;
|
||||
|
||||
export interface MinioShareDeps {
|
||||
execRunner?: ExecRunner;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
function timestamp(now: Date): string {
|
||||
const year = now.getUTCFullYear();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
const day = String(now.getUTCDate()).padStart(2, '0');
|
||||
const hour = String(now.getUTCHours()).padStart(2, '0');
|
||||
const minute = String(now.getUTCMinutes()).padStart(2, '0');
|
||||
const second = String(now.getUTCSeconds()).padStart(2, '0');
|
||||
return `${year}${month}${day}_${hour}${minute}${second}`;
|
||||
}
|
||||
|
||||
function parseShareUrl(stdout: string): string | null {
|
||||
const lines = stdout.split('\n').map((line) => line.trim()).filter(Boolean);
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Record<string, unknown>;
|
||||
const share = typeof parsed.share === 'string' ? parsed.share : undefined;
|
||||
const url = typeof parsed.url === 'string' ? parsed.url : undefined;
|
||||
if (share) {return share;}
|
||||
if (url) {return url;}
|
||||
} catch {
|
||||
const match = line.match(/https?:\/\/\S+/);
|
||||
if (match) {
|
||||
return match[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export const minioShareInternals = {
|
||||
parseShareUrl,
|
||||
timestamp,
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a tool that uploads a local file to configured MinIO and returns a presigned download URL.
|
||||
* Uses backup.minio credentials so operators can reuse existing MinIO setup.
|
||||
*/
|
||||
export function createMinioShareTool(config: BackupConfig, deps?: MinioShareDeps): Tool {
|
||||
return {
|
||||
name: 'minio.share',
|
||||
description: 'Upload a local file to MinIO and return a temporary share link.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
description: 'Local file path to upload',
|
||||
},
|
||||
expires: {
|
||||
type: 'string',
|
||||
description: 'Share link expiry for mc (e.g. "24h", "7d"). Default: 24h',
|
||||
},
|
||||
prefix: {
|
||||
type: 'string',
|
||||
description: 'Object-key prefix under the bucket. Default: "shared"',
|
||||
},
|
||||
object_key: {
|
||||
type: 'string',
|
||||
description: 'Explicit object key (overrides generated key)',
|
||||
},
|
||||
},
|
||||
required: ['path'],
|
||||
},
|
||||
execute: async (rawArgs: unknown): Promise<ToolResult> => {
|
||||
const args = rawArgs as { path: string; expires?: string; prefix?: string; object_key?: string };
|
||||
const minio = config.minio;
|
||||
|
||||
if (!minio.enabled) {
|
||||
return { success: false, output: '', error: 'MinIO sharing requires backup.minio.enabled=true' };
|
||||
}
|
||||
if (!minio.endpoint || !minio.access_key || !minio.secret_key || !minio.bucket) {
|
||||
return {
|
||||
success: false,
|
||||
output: '',
|
||||
error: 'Missing MinIO credentials in backup.minio (endpoint/access_key/secret_key/bucket)',
|
||||
};
|
||||
}
|
||||
|
||||
const localPath = resolve(args.path);
|
||||
if (!existsSync(localPath)) {
|
||||
return { success: false, output: '', error: `File not found: ${localPath}` };
|
||||
}
|
||||
|
||||
const now = deps?.now ? deps.now() : new Date();
|
||||
const prefix = args.prefix ?? 'shared';
|
||||
const fileName = basename(localPath);
|
||||
const objectKey = args.object_key ?? backupInternals.buildObjectKey(prefix, `${timestamp(now)}_${fileName}`);
|
||||
const alias = 'flynnshare';
|
||||
const remotePath = `${alias}/${minio.bucket}/${objectKey}`;
|
||||
const expires = args.expires ?? '24h';
|
||||
|
||||
const host = backupInternals.buildMinioHost({
|
||||
endpoint: minio.endpoint,
|
||||
accessKey: minio.access_key,
|
||||
secretKey: minio.secret_key,
|
||||
secure: minio.secure,
|
||||
});
|
||||
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
|
||||
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv }) => {
|
||||
return execFileAsync(file, cmdArgs, options);
|
||||
});
|
||||
|
||||
try {
|
||||
await runner('mc', ['mb', '--ignore-existing', `${alias}/${minio.bucket}`], { env });
|
||||
await runner('mc', ['cp', localPath, remotePath], { env });
|
||||
const { stdout } = await runner('mc', ['share', 'download', '--json', '--expire', expires, remotePath], { env });
|
||||
const shareUrl = parseShareUrl(typeof stdout === 'string' ? stdout : stdout.toString('utf-8'));
|
||||
if (!shareUrl) {
|
||||
return { success: false, output: '', error: 'Failed to parse MinIO share URL from mc output' };
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: `Shared file uploaded.\nPath: ${localPath}\nObject: ${objectKey}\nURL: ${shareUrl}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
output: '',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
+1
-1
@@ -5,7 +5,7 @@ export { ToolExecutor } from './executor.js';
|
||||
export type { ToolExecutorConfig } from './executor.js';
|
||||
export { ToolPolicy } from './policy.js';
|
||||
export type { ToolPolicyContext } from './policy.js';
|
||||
export { allBuiltinTools, createWebSearchTools, createProcessTools, ProcessManager, BrowserManager, createBrowserTools, createMediaSendTool, createAudioTranscribeTool, createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools, createGmailTools, createGcalTools, createGdocsTools, createGdriveTools, createGtasksTools } from './builtin/index.js';
|
||||
export { allBuiltinTools, createWebSearchTools, createProcessTools, ProcessManager, BrowserManager, createBrowserTools, createMediaSendTool, createAudioTranscribeTool, createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools, createGmailTools, createGcalTools, createGdocsTools, createGdriveTools, createGtasksTools, createMinioShareTool } from './builtin/index.js';
|
||||
export type { WebSearchConfig } from './builtin/web-search.js';
|
||||
export type { ProcessManagerConfig } from './builtin/process/index.js';
|
||||
export type { BrowserManagerConfig } from './builtin/browser/index.js';
|
||||
|
||||
@@ -39,6 +39,7 @@ const PROFILE_TOOLS: Record<ToolProfile, Set<string>> = {
|
||||
'cron.trigger',
|
||||
'cron.create',
|
||||
'cron.delete',
|
||||
'minio.share',
|
||||
]),
|
||||
coding: new Set([
|
||||
'file.read',
|
||||
@@ -67,6 +68,7 @@ const PROFILE_TOOLS: Record<ToolProfile, Set<string>> = {
|
||||
'cron.trigger',
|
||||
'cron.create',
|
||||
'cron.delete',
|
||||
'minio.share',
|
||||
'file.write',
|
||||
'file.edit',
|
||||
'file.patch',
|
||||
@@ -102,6 +104,7 @@ export const TOOL_GROUPS: Record<string, string[]> = {
|
||||
'group:gdrive': ['drive.list', 'drive.search', 'drive.read'],
|
||||
'group:gtasks': ['tasks.lists', 'tasks.list'],
|
||||
'group:cron': ['cron.list', 'cron.trigger', 'cron.create', 'cron.delete'],
|
||||
'group:minio': ['minio.share'],
|
||||
};
|
||||
|
||||
/** Expand group references in a list of tool names/patterns. */
|
||||
|
||||
Reference in New Issue
Block a user