fix(minio): support mc_path and harden sync against transient objects

This commit is contained in:
William Valentin
2026-02-19 13:18:20 -08:00
parent b5d691a99f
commit d4f4be068c
12 changed files with 238 additions and 21 deletions
+25 -4
View File
@@ -44,6 +44,7 @@ function parseListedObjectKeys(stdout: string): string[] {
if (!key) {continue;}
if (type && type !== 'file') {continue;}
if (key.endsWith('/')) {continue;}
if (key === '.keep' || key.endsWith('/.keep')) {continue;}
keys.push(key);
} catch {
continue;
@@ -52,6 +53,15 @@ function parseListedObjectKeys(stdout: string): string[] {
return keys;
}
function isBenignMissingObjectError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
const lowered = message.toLowerCase();
return lowered.includes('object does not exist')
|| lowered.includes('unable to read from')
|| lowered.includes('no such key')
|| lowered.includes('not found');
}
function normalizeNamespaceSegment(value: string): string {
return value
.replace(/\.[^.]+$/, '')
@@ -61,6 +71,7 @@ function normalizeNamespaceSegment(value: string): string {
}
export const minioSyncInternals = {
isBenignMissingObjectError,
parseListedObjectKeys,
normalizeNamespaceSegment,
};
@@ -137,13 +148,14 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
secure: minio.secure,
});
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
const mcPath = backupInternals.resolveMcPath(minio.mc_path);
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv; maxBuffer?: number }) => {
return execFileAsync(file, cmdArgs, options);
});
try {
const basePath = `${alias}/${bucket}/${prefix}`;
const { stdout: listed } = await runner('mc', ['ls', '--json', '--recursive', basePath], {
const { stdout: listed } = await runner(mcPath, ['ls', '--json', '--recursive', basePath], {
env,
maxBuffer: 20 * 1024 * 1024,
});
@@ -167,8 +179,17 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
continue;
}
const remotePath = `${alias}/${bucket}/${key}`;
const text = await minioIngestInternals.readObjectText(runner, remotePath, key, env);
let text = '';
try {
const remotePath = `${alias}/${bucket}/${key}`;
text = await minioIngestInternals.readObjectText(runner, mcPath, remotePath, key, env);
} catch (error) {
if (isBenignMissingObjectError(error)) {
skipped++;
continue;
}
throw error;
}
if (!force && !minioIngestInternals.isExtractableBinaryObject(key) && !minioIngestInternals.isLikelyText(text)) {
skipped++;
@@ -207,7 +228,7 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
return {
success: false,
output: '',
error: error instanceof Error ? error.message : String(error),
error: backupInternals.formatMinioCliError(error, mcPath),
};
}
},