diff --git a/README.md b/README.md index d15b807..a860931 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Flynn provides a full CLI via the `flynn` binary (or `npx tsx src/cli/index.ts` | `flynn sessions` | List active sessions | | `flynn doctor` | Validate config and check system health | | `flynn config` | Show resolved configuration (secrets redacted) | +| `flynn backup` | Create a snapshot backup and optionally upload to MinIO | | `flynn completion ` | Generate shell completions (bash, zsh, fish) | | `flynn setup` | Interactive setup wizard | | `flynn gmail-auth` | Authenticate with Gmail via OAuth2 | @@ -89,6 +90,9 @@ flynn config # List sessions flynn sessions +# Create backup now (uses config backup + MinIO settings) +flynn backup + # Generate shell completions flynn completion bash # Print bash completions to stdout flynn completion zsh --install # Install zsh completions to ~/.zshrc diff --git a/config/default.yaml b/config/default.yaml index b9dda27..a2e63c5 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -284,6 +284,24 @@ hooks: # failure_threshold: 2 # disk_threshold_mb: 100 +# ── Backup ────────────────────────────────────────────────────────── +# Snapshot sessions.db, vectors.db (optional), and memory/ into a tarball. +# If MinIO is enabled, upload with `mc` using ephemeral credentials. +# +# backup: +# enabled: false +# interval: "24h" +# local_dir: ~/.local/share/flynn/backups +# include_vectors: true +# minio: +# enabled: false +# endpoint: localhost:9000 +# access_key: ${MINIO_ACCESS_KEY} +# secret_key: ${MINIO_SECRET_KEY} +# bucket: flynn-backups +# prefix: flynn +# secure: true + # ── Audio ──────────────────────────────────────────────────────────── # Configure a Whisper-compatible endpoint for audio transcription. # Models that support native audio input (Gemini, OpenAI, GitHub) will diff --git a/scripts/backup-to-minio.sh b/scripts/backup-to-minio.sh new file mode 100644 index 0000000..837672f --- /dev/null +++ b/scripts/backup-to-minio.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Required env vars: +# FLYNN_DATA_DIR (default: ~/.local/share/flynn) +# FLYNN_BACKUP_DIR (default: ~/.local/share/flynn/backups) +# MINIO_ENDPOINT +# MINIO_ACCESS_KEY +# MINIO_SECRET_KEY +# MINIO_BUCKET +# +# Optional: +# MINIO_PREFIX (default: flynn) +# MINIO_SECURE (default: true) + +DATA_DIR="${FLYNN_DATA_DIR:-$HOME/.local/share/flynn}" +BACKUP_DIR="${FLYNN_BACKUP_DIR:-$HOME/.local/share/flynn/backups}" +MINIO_PREFIX="${MINIO_PREFIX:-flynn}" +MINIO_SECURE="${MINIO_SECURE:-true}" + +if [[ -z "${MINIO_ENDPOINT:-}" || -z "${MINIO_ACCESS_KEY:-}" || -z "${MINIO_SECRET_KEY:-}" || -z "${MINIO_BUCKET:-}" ]]; then + echo "Missing MinIO config. Set MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_BUCKET." >&2 + exit 1 +fi + +if ! command -v mc >/dev/null 2>&1; then + echo "MinIO client (mc) not found in PATH." >&2 + exit 1 +fi + +mkdir -p "${BACKUP_DIR}" + +STAMP="$(date -u +%Y%m%d_%H%M%S)" +ARCHIVE="flynn_${STAMP}.tar.gz" +ARCHIVE_PATH="${BACKUP_DIR}/${ARCHIVE}" + +ENTRIES=() +[[ -f "${DATA_DIR}/sessions.db" ]] && ENTRIES+=("sessions.db") +[[ -f "${DATA_DIR}/vectors.db" ]] && ENTRIES+=("vectors.db") +[[ -d "${DATA_DIR}/memory" ]] && ENTRIES+=("memory") + +if [[ ${#ENTRIES[@]} -eq 0 ]]; then + echo "No backup inputs found under ${DATA_DIR}" >&2 + exit 1 +fi + +tar -czf "${ARCHIVE_PATH}" -C "${DATA_DIR}" "${ENTRIES[@]}" + +if [[ "${MINIO_SECURE}" == "true" ]]; then + PROTO="https" +else + PROTO="http" +fi + +export MC_HOST_flynnbackup="${PROTO}://${MINIO_ACCESS_KEY}:${MINIO_SECRET_KEY}@${MINIO_ENDPOINT}" +mc mb --ignore-existing "flynnbackup/${MINIO_BUCKET}" >/dev/null +mc cp "${ARCHIVE_PATH}" "flynnbackup/${MINIO_BUCKET}/${MINIO_PREFIX}/${ARCHIVE}" >/dev/null + +echo "Backup created: ${ARCHIVE_PATH}" +echo "Uploaded to: flynnbackup/${MINIO_BUCKET}/${MINIO_PREFIX}/${ARCHIVE}" diff --git a/src/backup/index.ts b/src/backup/index.ts new file mode 100644 index 0000000..e0d44a3 --- /dev/null +++ b/src/backup/index.ts @@ -0,0 +1 @@ +export { runBackupSnapshot, backupInternals, type BackupRunOptions, type BackupResult } from './run.js'; diff --git a/src/backup/run.test.ts b/src/backup/run.test.ts new file mode 100644 index 0000000..8c42fda --- /dev/null +++ b/src/backup/run.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect } from 'vitest'; +import { backupInternals } from './run.js'; + +describe('backup internals', () => { + it('builds minio host URL with protocol and encoded credentials', () => { + const host = backupInternals.buildMinioHost({ + endpoint: 'localhost:9000', + accessKey: 'minio-admin', + secretKey: 's3cr3t/with:chars', + secure: false, + }); + + expect(host).toBe('http://minio-admin:s3cr3t%2Fwith%3Achars@localhost:9000'); + }); + + it('builds object key from prefix and file name', () => { + expect(backupInternals.buildObjectKey('flynn/daily', 'a.tar.gz')).toBe('flynn/daily/a.tar.gz'); + expect(backupInternals.buildObjectKey('/flynn/daily/', 'a.tar.gz')).toBe('flynn/daily/a.tar.gz'); + expect(backupInternals.buildObjectKey('', 'a.tar.gz')).toBe('a.tar.gz'); + }); +}); diff --git a/src/backup/run.ts b/src/backup/run.ts new file mode 100644 index 0000000..b754bc4 --- /dev/null +++ b/src/backup/run.ts @@ -0,0 +1,168 @@ +import { execFile } from 'node:child_process'; +import { existsSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { homedir } from 'node:os'; +import type { BackupConfig } from '../config/index.js'; +import { auditLogger } from '../audit/index.js'; + +const execFileAsync = promisify(execFile); + +export interface BackupTargetFiles { + sessionsDb: string; + vectorsDb?: string; + memoryDir?: string; +} + +export interface BackupRunOptions { + dataDir: string; + backupConfig: BackupConfig; + now?: Date; +} + +export interface BackupResult { + archivePath: string; + fileName: string; + uploaded: boolean; + remotePath?: string; +} + +function expandHomePath(pathValue: string): string { + if (!pathValue.startsWith('~/')) { + return pathValue; + } + return resolve(homedir(), pathValue.slice(2)); +} + +function fileTimestamp(now: Date): string { + const year = now.getUTCFullYear(); + const month = String(now.getUTCMonth() + 1).padStart(2, '0'); + const day = String(now.getUTCDate()).padStart(2, '0'); + const hour = String(now.getUTCHours()).padStart(2, '0'); + const minute = String(now.getUTCMinutes()).padStart(2, '0'); + const second = String(now.getUTCSeconds()).padStart(2, '0'); + return `${year}${month}${day}_${hour}${minute}${second}`; +} + +function buildMinioHost(opts: { + endpoint: string; + accessKey: string; + secretKey: string; + secure: boolean; +}): string { + const protocol = opts.secure ? 'https' : 'http'; + const encodedAccess = encodeURIComponent(opts.accessKey); + const encodedSecret = encodeURIComponent(opts.secretKey); + return `${protocol}://${encodedAccess}:${encodedSecret}@${opts.endpoint}`; +} + +function buildObjectKey(prefix: string, fileName: string): string { + const trimmed = prefix.replace(/^\/+|\/+$/g, ''); + return trimmed.length > 0 ? `${trimmed}/${fileName}` : fileName; +} + +function collectExistingEntries(opts: { + dataDir: string; + backupConfig: BackupConfig; +}): string[] { + const entries: string[] = []; + const sessionsPath = join(opts.dataDir, 'sessions.db'); + if (existsSync(sessionsPath)) { + entries.push('sessions.db'); + } + + if (opts.backupConfig.include_vectors) { + const vectorsPath = join(opts.dataDir, 'vectors.db'); + if (existsSync(vectorsPath)) { + entries.push('vectors.db'); + } + } + + const memoryPath = join(opts.dataDir, 'memory'); + if (existsSync(memoryPath)) { + entries.push('memory'); + } + + return entries; +} + +export async function runBackupSnapshot(opts: BackupRunOptions): Promise { + const now = opts.now ?? new Date(); + const localDir = expandHomePath(opts.backupConfig.local_dir); + mkdirSync(localDir, { recursive: true }); + + const entries = collectExistingEntries({ + dataDir: opts.dataDir, + backupConfig: opts.backupConfig, + }); + + if (entries.length === 0) { + throw new Error(`No backup inputs found under ${opts.dataDir}`); + } + + const fileName = `flynn_${fileTimestamp(now)}.tar.gz`; + const archivePath = join(localDir, fileName); + await execFileAsync('tar', ['-czf', archivePath, '-C', opts.dataDir, ...entries]); + + auditLogger?.systemConfig('backup', 'snapshot', { + archive_path: archivePath, + entry_count: entries.length, + }); + + if (!opts.backupConfig.minio.enabled) { + return { + archivePath, + fileName, + uploaded: false, + }; + } + + const endpoint = opts.backupConfig.minio.endpoint; + const accessKey = opts.backupConfig.minio.access_key; + const secretKey = opts.backupConfig.minio.secret_key; + const bucket = opts.backupConfig.minio.bucket; + + if (!endpoint || !accessKey || !secretKey || !bucket) { + throw new Error('backup.minio.enabled=true requires endpoint, access_key, secret_key, and bucket'); + } + + const alias = 'flynnbackup'; + const host = buildMinioHost({ + endpoint, + accessKey, + secretKey, + secure: opts.backupConfig.minio.secure, + }); + const env = { + ...process.env, + [`MC_HOST_${alias}`]: host, + }; + + await execFileAsync('mc', ['mb', '--ignore-existing', `${alias}/${bucket}`], { env }); + + const objectKey = buildObjectKey(opts.backupConfig.minio.prefix, fileName); + const remotePath = `${alias}/${bucket}/${objectKey}`; + await execFileAsync('mc', ['cp', archivePath, remotePath], { env }); + + auditLogger?.systemConfig('backup', 'upload', { + archive_path: archivePath, + remote_path: remotePath, + endpoint, + bucket, + }); + + return { + archivePath, + fileName, + uploaded: true, + remotePath, + }; +} + +export const backupInternals = { + buildMinioHost, + buildObjectKey, + collectExistingEntries, + expandHomePath, + fileTimestamp, +}; diff --git a/src/cli/backup.ts b/src/cli/backup.ts new file mode 100644 index 0000000..acfac80 --- /dev/null +++ b/src/cli/backup.ts @@ -0,0 +1,40 @@ +import type { Command } from 'commander'; +import { getDataDir, loadConfigSafe } from './shared.js'; +import { runBackupSnapshot } from '../backup/index.js'; + +export function registerBackupCommand(program: Command): void { + program + .command('backup') + .description('Create a snapshot backup and optionally upload to MinIO') + .option('-c, --config ', 'Config file path') + .option('--data-dir ', 'Data directory (defaults to FLYNN_DATA_DIR or ~/.local/share/flynn)') + .option('--no-upload', 'Disable MinIO upload for this run') + .action(async (opts: { config?: string; dataDir?: string; upload?: boolean }) => { + const { config, error } = loadConfigSafe(opts.config); + if (!config) { + console.error(error); + process.exit(1); + } + + const backupConfig = { + ...config.backup, + minio: { + ...config.backup.minio, + enabled: opts.upload === false ? false : config.backup.minio.enabled, + }, + }; + + const dataDir = opts.dataDir ?? getDataDir(); + const result = await runBackupSnapshot({ + dataDir, + backupConfig, + }); + + console.log(`Backup archive: ${result.archivePath}`); + if (result.uploaded && result.remotePath) { + console.log(`Uploaded to MinIO: ${result.remotePath}`); + } else { + console.log('MinIO upload skipped.'); + } + }); +} diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index fdcd9e0..df3cf11 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -13,6 +13,7 @@ describe('CLI program', () => { expect(commandNames).toContain('doctor'); expect(commandNames).toContain('config'); expect(commandNames).toContain('skills'); + expect(commandNames).toContain('backup'); expect(commandNames).toContain('openai-auth'); expect(commandNames).toContain('openai-key'); diff --git a/src/cli/index.ts b/src/cli/index.ts index 8024e35..2a30f14 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -27,6 +27,7 @@ import { registerOpenaiKeyCommand } from './openai-key.js'; import { registerZaiAuthCommand } from './zai-auth.js'; import { registerAnthropicAuthCommand } from './anthropic-auth.js'; import { registerSkillsCommand } from './skills.js'; +import { registerBackupCommand } from './backup.js'; export function createProgram(): Command { const program = new Command(); @@ -54,6 +55,7 @@ export function createProgram(): Command { registerZaiAuthCommand(program); registerAnthropicAuthCommand(program); registerSkillsCommand(program); + registerBackupCommand(program); return program; } diff --git a/src/config/index.ts b/src/config/index.ts index 7200740..82ac0ce 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,3 +1,3 @@ export { loadConfig, deepMerge } from './loader.js'; export { persistConfig } from './persistence.js'; -export { configSchema, MODEL_PROVIDERS, type ModelProvider, type Config, type TelegramConfig, type ModelConfig, type CronJobConfig, type AgentsConfig, type CompactionConfig, type ToolProfile, type ToolOverrideConfig, type ToolsConfig, type SandboxConfig, type AgentConfigEntry, type RoutingConfig, type ServerConfig } from './schema.js'; +export { configSchema, MODEL_PROVIDERS, type ModelProvider, type Config, type TelegramConfig, type ModelConfig, type CronJobConfig, type AgentsConfig, type CompactionConfig, type ToolProfile, type ToolOverrideConfig, type ToolsConfig, type SandboxConfig, type AgentConfigEntry, type RoutingConfig, type ServerConfig, type BackupConfig } from './schema.js'; diff --git a/src/config/schema.test.ts b/src/config/schema.test.ts index 2783446..c354b5b 100644 --- a/src/config/schema.test.ts +++ b/src/config/schema.test.ts @@ -196,6 +196,54 @@ describe('configSchema — server', () => { }); }); +describe('configSchema — backup', () => { + const minimalConfig = { + telegram: { bot_token: 'test', allowed_chat_ids: [1] }, + models: { default: { provider: 'anthropic', model: 'claude-3' } }, + }; + + it('defaults backup settings', () => { + const result = configSchema.parse(minimalConfig); + expect(result.backup.enabled).toBe(false); + expect(result.backup.interval).toBe('24h'); + expect(result.backup.include_vectors).toBe(true); + expect(result.backup.minio.enabled).toBe(false); + expect(result.backup.minio.prefix).toBe('flynn'); + expect(result.backup.minio.secure).toBe(true); + }); + + it('accepts custom backup settings', () => { + const result = configSchema.parse({ + ...minimalConfig, + backup: { + enabled: true, + interval: '12h', + local_dir: '/tmp/flynn-backups', + include_vectors: false, + minio: { + enabled: true, + endpoint: 'localhost:9000', + access_key: 'key', + secret_key: 'secret', + bucket: 'flynn-backups', + prefix: 'daily', + secure: false, + }, + }, + }); + + expect(result.backup.enabled).toBe(true); + expect(result.backup.interval).toBe('12h'); + expect(result.backup.local_dir).toBe('/tmp/flynn-backups'); + expect(result.backup.include_vectors).toBe(false); + expect(result.backup.minio.enabled).toBe(true); + expect(result.backup.minio.endpoint).toBe('localhost:9000'); + expect(result.backup.minio.bucket).toBe('flynn-backups'); + expect(result.backup.minio.prefix).toBe('daily'); + expect(result.backup.minio.secure).toBe(false); + }); +}); + describe('configSchema — agent_configs', () => { const minimalConfig = { telegram: { bot_token: 'test', allowed_chat_ids: [1] }, diff --git a/src/config/schema.ts b/src/config/schema.ts index d222166..b7b4b8d 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -672,6 +672,22 @@ const sessionsSchema = z.object({ ttl: z.string().default('30d'), }).default({}); +const backupSchema = z.object({ + enabled: z.boolean().default(false), + interval: z.string().default('24h'), + local_dir: z.string().default('~/.local/share/flynn/backups'), + include_vectors: z.boolean().default(true), + minio: z.object({ + enabled: z.boolean().default(false), + endpoint: z.string().optional(), + access_key: z.string().optional(), + secret_key: z.string().optional(), + bucket: z.string().optional(), + prefix: z.string().default('flynn'), + secure: z.boolean().default(true), + }).default({}), +}).default({}); + const historyIndexSchema = z.object({ enabled: z.boolean().default(false), max_keywords: z.number().min(1).max(20).default(8), @@ -736,6 +752,7 @@ export const configSchema = z.object({ routing_policy: routingPolicySchema, history_index: historyIndexSchema, sessions: sessionsSchema, + backup: backupSchema, pairing: pairingSchema, }); @@ -777,6 +794,7 @@ export type RoutingPolicyConfig = z.infer; export type HistoryIndexConfig = z.infer; export type ServerConfig = z.infer; export type SessionsConfig = z.infer; +export type BackupConfig = z.infer; export type ThinkingConfig = z.infer; export type HeartbeatConfig = z.infer; export type HeartbeatCheck = z.infer; diff --git a/src/daemon/index.ts b/src/daemon/index.ts index afef8d6..855d2e0 100644 --- a/src/daemon/index.ts +++ b/src/daemon/index.ts @@ -33,6 +33,7 @@ import type { McpManager } from '../mcp/index.js'; import type { SkillRegistry, SkillInstaller } from '../skills/index.js'; import type { GatewayServer } from '../gateway/index.js'; import { AuditLogger, initAuditLogger } from '../audit/index.js'; +import { runBackupSnapshot } from '../backup/index.js'; export interface DaemonContext { config: Config; @@ -103,6 +104,39 @@ export async function startDaemon(config: Config, options?: StartDaemonOptions): lifecycle.onShutdown(async () => { clearInterval(pruneInterval); }); } + if (config.backup.enabled) { + const backupIntervalMs = parseDuration(config.backup.interval); + if (!backupIntervalMs) { + console.warn(`Backup enabled but interval is invalid: ${config.backup.interval}`); + } else { + let backupRunning = false; + const runScheduledBackup = async (): Promise => { + if (backupRunning) { + return; + } + backupRunning = true; + try { + const result = await runBackupSnapshot({ + dataDir, + backupConfig: config.backup, + }); + console.log(`Backup completed: ${result.archivePath}${result.uploaded && result.remotePath ? ` -> ${result.remotePath}` : ''}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`Backup failed: ${message}`); + } finally { + backupRunning = false; + } + }; + + const backupInterval = setInterval(() => { + void runScheduledBackup(); + }, backupIntervalMs); + lifecycle.onShutdown(async () => { clearInterval(backupInterval); }); + console.log(`Backup scheduler enabled (${config.backup.interval})`); + } + } + // ── Core Services ── const hookEngine = new HookEngine(config.hooks); const { toolRegistry, toolExecutor, browserManager } = initTools({ config, lifecycle, hookEngine });