will/flynn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(minio): support mc_path and harden sync against transient objects

Browse Source
This commit is contained in:
William Valentin
2026-02-19 13:18:20 -08:00
parent b5d691a99f
commit d4f4be068c
12 changed files with 238 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/builtin/minio-ingest.test.ts
+13 -2
View File
@@ -49,7 +49,18 @@ describe('createMinioIngestTool', () => {
stderr: '',
}));
const tool = createMinioIngestTool(makeBackupConfig(), store, {
const tool = createMinioIngestTool(makeBackupConfig({
minio: {
enabled: true,
endpoint: 'localhost:9000',
access_key: 'minio-admin',
secret_key: 'minio-secret',
bucket: 'flynn-knowledge',
prefix: 'flynn',
secure: false,
mc_path: '/opt/bin/mc',
},
}), store, {
execRunner,
now: () => new Date('2026-02-16T15:00:00.000Z'),
});
@@ -68,7 +79,7 @@ describe('createMinioIngestTool', () => {
'append',
);
expect(execRunner).toHaveBeenCalledWith(
'mc',
'/opt/bin/mc',
['cat', 'flynningest/flynn-knowledge/knowledge/runbook.md'],
expect.objectContaining({ env: expect.any(Object) }),
);
src/tools/builtin/minio-ingest.ts
+6 -4
View File
@@ -58,20 +58,21 @@ function isExtractableBinaryObject(objectKey: string): boolean {
async function readObjectText(
runner: ExecRunner,
mcPath: string,
remotePath: string,
objectKey: string,
env: NodeJS.ProcessEnv,
): Promise<string> {
const ext = extname(objectKey).toLowerCase();
if (!isExtractableBinaryObject(objectKey)) {
const { stdout } = await runner('mc', ['cat', remotePath], { env, maxBuffer: 20 * 1024 * 1024 });
const { stdout } = await runner(mcPath, ['cat', remotePath], { env, maxBuffer: 20 * 1024 * 1024 });
return toText(stdout);
}
const tempDir = mkdtempSync(join(tmpdir(), 'flynn-minio-ingest-'));
const localPath = join(tempDir, 'object.bin');
try {
await runner('mc', ['cp', remotePath, localPath], { env, maxBuffer: 20 * 1024 * 1024 });
await runner(mcPath, ['cp', remotePath, localPath], { env, maxBuffer: 20 * 1024 * 1024 });
if (ext === '.pdf') {
const { stdout } = await runner('pdftotext', ['-q', localPath, '-'], { maxBuffer: 20 * 1024 * 1024 });
return toText(stdout);
@@ -187,13 +188,14 @@ export function createMinioIngestTool(config: BackupConfig, store: MemoryStore,
secure: minio.secure,
});
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
const mcPath = backupInternals.resolveMcPath(minio.mc_path);
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv; maxBuffer?: number }) => {
return execFileAsync(file, cmdArgs, options);
});
const remotePath = `${alias}/${bucket}/${objectKey}`;
try {
const text = await readObjectText(runner, remotePath, objectKey, env);
const text = await readObjectText(runner, mcPath, remotePath, objectKey, env);
if (!force && !isExtractableBinaryObject(objectKey) && !isLikelyText(text)) {
return {
@@ -227,7 +229,7 @@ export function createMinioIngestTool(config: BackupConfig, store: MemoryStore,
return {
success: false,
output: '',
error: error instanceof Error ? error.message : String(error),
error: backupInternals.formatMinioCliError(error, mcPath),
};
}
},
src/tools/builtin/minio-share.test.ts
+13 -1
View File
@@ -63,7 +63,18 @@ describe('createMinioShareTool', () => {
return { stdout: '', stderr: '' };
});
const tool = createMinioShareTool(makeBackupConfig(), {
const tool = createMinioShareTool(makeBackupConfig({
minio: {
enabled: true,
endpoint: 'localhost:9000',
access_key: 'minio-admin',
secret_key: 'minio-secret',
bucket: 'flynn-shared',
prefix: 'flynn',
secure: false,
mc_path: '/opt/bin/mc',
},
}), {
execRunner,
now: () => new Date('2026-02-16T10:00:00.000Z'),
});
@@ -72,6 +83,7 @@ describe('createMinioShareTool', () => {
expect(result.success).toBe(true);
expect(result.output).toContain('https://minio.local/share/abc');
expect(execRunner).toHaveBeenCalledTimes(3);
expect(execRunner.mock.calls[0]?.[0]).toBe('/opt/bin/mc');
expect(execRunner.mock.calls[1]?.[1]).toContain('cp');
expect(execRunner.mock.calls[2]?.[1]).toContain('share');
});
src/tools/builtin/minio-share.ts
+5 -4
View File
@@ -116,14 +116,15 @@ export function createMinioShareTool(config: BackupConfig, deps?: MinioShareDeps
secure: minio.secure,
});
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
const mcPath = backupInternals.resolveMcPath(minio.mc_path);
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv }) => {
return execFileAsync(file, cmdArgs, options);
});
try {
await runner('mc', ['mb', '--ignore-existing', `${alias}/${minio.bucket}`], { env });
await runner('mc', ['cp', localPath, remotePath], { env });
const { stdout } = await runner('mc', ['share', 'download', '--json', '--expire', expires, remotePath], { env });
await runner(mcPath, ['mb', '--ignore-existing', `${alias}/${minio.bucket}`], { env });
await runner(mcPath, ['cp', localPath, remotePath], { env });
const { stdout } = await runner(mcPath, ['share', 'download', '--json', '--expire', expires, remotePath], { env });
const shareUrl = parseShareUrl(typeof stdout === 'string' ? stdout : stdout.toString('utf-8'));
if (!shareUrl) {
return { success: false, output: '', error: 'Failed to parse MinIO share URL from mc output' };
@@ -137,7 +138,7 @@ export function createMinioShareTool(config: BackupConfig, deps?: MinioShareDeps
return {
success: false,
output: '',
error: error instanceof Error ? error.message : String(error),
error: backupInternals.formatMinioCliError(error, mcPath),
};
}
},
src/tools/builtin/minio-sync.test.ts
+96
View File
@@ -41,6 +41,17 @@ describe('minio sync internals', () => {
]);
});
it('ignores keep-marker objects from recursive listings', () => {
const stdout = [
'{"status":"success","type":"file","key":".keep"}',
'{"status":"success","type":"file","key":"reports/.keep"}',
'{"status":"success","type":"file","key":"reports/daily.md"}',
].join('\n');
expect(minioSyncInternals.parseListedObjectKeys(stdout)).toEqual([
'reports/daily.md',
]);
});
it('normalizes object paths into namespace-safe segments', () => {
expect(minioSyncInternals.normalizeNamespaceSegment('knowledge/team runbook.v2.md')).toBe('knowledge/team_runbook_v2');
});
@@ -163,4 +174,89 @@ describe('createMinioSyncTool', () => {
expect(result.success).toBe(false);
expect(result.error).toContain('backup.minio.enabled=true');
});
it('uses configured backup.minio.mc_path and returns setup hint when missing', async () => {
const write = vi.fn();
const store = { write } as unknown as MemoryStore;
const execRunner = vi.fn(async () => {
const error = new Error('spawn /opt/minio/mc ENOENT') as Error & { code?: string };
error.code = 'ENOENT';
throw error;
});
const tool = createMinioSyncTool(makeBackupConfig({
minio: {
enabled: true,
endpoint: 'localhost:9000',
access_key: 'minio-admin',
secret_key: 'minio-secret',
bucket: 'flynn-knowledge',
prefix: 'flynn',
secure: false,
mc_path: '/opt/minio/mc',
},
}), store, { execRunner });
const result = await tool.execute({ prefix: 'knowledge/' });
expect(result.success).toBe(false);
expect(result.error).toContain('MinIO client binary not found: /opt/minio/mc');
expect(result.error).toContain('backup.minio.mc_path');
expect(execRunner).toHaveBeenCalledWith(
'/opt/minio/mc',
['ls', '--json', '--recursive', 'flynnsync/flynn-knowledge/knowledge/'],
expect.objectContaining({ env: expect.any(Object) }),
);
});
it('skips objects that disappear between ls and read', async () => {
const write = vi.fn();
const store = { write } as unknown as MemoryStore;
const execRunner = vi.fn(async (_file: string, args: string[]) => {
if (args[0] === 'ls') {
return {
stdout: [
'{"status":"success","type":"file","key":"reports/.keep"}',
'{"status":"success","type":"file","key":"reports/summary.md"}',
].join('\n'),
stderr: '',
};
}
if (args[0] === 'cat' && args[1]?.endsWith('reports/summary.md')) {
return { stdout: 'OK summary', stderr: '' };
}
throw new Error(`Unexpected call: ${args.join(' ')}`);
});
const tool = createMinioSyncTool(makeBackupConfig(), store, { execRunner });
const result = await tool.execute({ prefix: 'reports/' });
expect(result.success).toBe(true);
expect(result.output).toContain('Scanned: 1 object(s)');
expect(result.output).toContain('Imported: 1');
expect(result.output).toContain('Skipped: 0');
expect(write).toHaveBeenCalledTimes(1);
});
it('treats missing-object read errors as skip instead of task failure', async () => {
const write = vi.fn();
const store = { write } as unknown as MemoryStore;
const execRunner = vi.fn(async (_file: string, args: string[]) => {
if (args[0] === 'ls') {
return {
stdout: '{"status":"success","type":"file","key":"reports/missing.md"}',
stderr: '',
};
}
if (args[0] === 'cat') {
throw new Error('mcli: <ERROR> Unable to read from flynnsync/flynn/reports/missing.md. Object does not exist.');
}
return { stdout: '', stderr: '' };
});
const tool = createMinioSyncTool(makeBackupConfig(), store, { execRunner });
const result = await tool.execute({ prefix: 'reports/' });
expect(result.success).toBe(true);
expect(result.output).toContain('Imported: 0');
expect(result.output).toContain('Skipped: 1');
expect(write).not.toHaveBeenCalled();
});
});
src/tools/builtin/minio-sync.ts
+25 -4
View File
@@ -44,6 +44,7 @@ function parseListedObjectKeys(stdout: string): string[] {
if (!key) {continue;}
if (type && type !== 'file') {continue;}
if (key.endsWith('/')) {continue;}
if (key === '.keep' || key.endsWith('/.keep')) {continue;}
keys.push(key);
} catch {
continue;
@@ -52,6 +53,15 @@ function parseListedObjectKeys(stdout: string): string[] {
return keys;
}
function isBenignMissingObjectError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
const lowered = message.toLowerCase();
return lowered.includes('object does not exist')
|| lowered.includes('unable to read from')
|| lowered.includes('no such key')
|| lowered.includes('not found');
}
function normalizeNamespaceSegment(value: string): string {
return value
.replace(/\.[^.]+$/, '')
@@ -61,6 +71,7 @@ function normalizeNamespaceSegment(value: string): string {
}
export const minioSyncInternals = {
isBenignMissingObjectError,
parseListedObjectKeys,
normalizeNamespaceSegment,
};
@@ -137,13 +148,14 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
secure: minio.secure,
});
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
const mcPath = backupInternals.resolveMcPath(minio.mc_path);
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv; maxBuffer?: number }) => {
return execFileAsync(file, cmdArgs, options);
});
try {
const basePath = `${alias}/${bucket}/${prefix}`;
const { stdout: listed } = await runner('mc', ['ls', '--json', '--recursive', basePath], {
const { stdout: listed } = await runner(mcPath, ['ls', '--json', '--recursive', basePath], {
env,
maxBuffer: 20 * 1024 * 1024,
});
@@ -167,8 +179,17 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
continue;
}
const remotePath = `${alias}/${bucket}/${key}`;
const text = await minioIngestInternals.readObjectText(runner, remotePath, key, env);
let text = '';
try {
const remotePath = `${alias}/${bucket}/${key}`;
text = await minioIngestInternals.readObjectText(runner, mcPath, remotePath, key, env);
} catch (error) {
if (isBenignMissingObjectError(error)) {
skipped++;
continue;
}
throw error;
}
if (!force && !minioIngestInternals.isExtractableBinaryObject(key) && !minioIngestInternals.isLikelyText(text)) {
skipped++;
@@ -207,7 +228,7 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
return {
success: false,
output: '',
error: error instanceof Error ? error.message : String(error),
error: backupInternals.formatMinioCliError(error, mcPath),
};
}
},