fix(minio): support mc_path and harden sync against transient objects
This commit is contained in:
@@ -44,6 +44,7 @@ function parseListedObjectKeys(stdout: string): string[] {
|
||||
if (!key) {continue;}
|
||||
if (type && type !== 'file') {continue;}
|
||||
if (key.endsWith('/')) {continue;}
|
||||
if (key === '.keep' || key.endsWith('/.keep')) {continue;}
|
||||
keys.push(key);
|
||||
} catch {
|
||||
continue;
|
||||
@@ -52,6 +53,15 @@ function parseListedObjectKeys(stdout: string): string[] {
|
||||
return keys;
|
||||
}
|
||||
|
||||
function isBenignMissingObjectError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const lowered = message.toLowerCase();
|
||||
return lowered.includes('object does not exist')
|
||||
|| lowered.includes('unable to read from')
|
||||
|| lowered.includes('no such key')
|
||||
|| lowered.includes('not found');
|
||||
}
|
||||
|
||||
function normalizeNamespaceSegment(value: string): string {
|
||||
return value
|
||||
.replace(/\.[^.]+$/, '')
|
||||
@@ -61,6 +71,7 @@ function normalizeNamespaceSegment(value: string): string {
|
||||
}
|
||||
|
||||
export const minioSyncInternals = {
|
||||
isBenignMissingObjectError,
|
||||
parseListedObjectKeys,
|
||||
normalizeNamespaceSegment,
|
||||
};
|
||||
@@ -137,13 +148,14 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
|
||||
secure: minio.secure,
|
||||
});
|
||||
const env = { ...process.env, [`MC_HOST_${alias}`]: host };
|
||||
const mcPath = backupInternals.resolveMcPath(minio.mc_path);
|
||||
const runner = deps?.execRunner ?? (async (file: string, cmdArgs: string[], options?: { env?: NodeJS.ProcessEnv; maxBuffer?: number }) => {
|
||||
return execFileAsync(file, cmdArgs, options);
|
||||
});
|
||||
|
||||
try {
|
||||
const basePath = `${alias}/${bucket}/${prefix}`;
|
||||
const { stdout: listed } = await runner('mc', ['ls', '--json', '--recursive', basePath], {
|
||||
const { stdout: listed } = await runner(mcPath, ['ls', '--json', '--recursive', basePath], {
|
||||
env,
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
});
|
||||
@@ -167,8 +179,17 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
|
||||
continue;
|
||||
}
|
||||
|
||||
const remotePath = `${alias}/${bucket}/${key}`;
|
||||
const text = await minioIngestInternals.readObjectText(runner, remotePath, key, env);
|
||||
let text = '';
|
||||
try {
|
||||
const remotePath = `${alias}/${bucket}/${key}`;
|
||||
text = await minioIngestInternals.readObjectText(runner, mcPath, remotePath, key, env);
|
||||
} catch (error) {
|
||||
if (isBenignMissingObjectError(error)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!force && !minioIngestInternals.isExtractableBinaryObject(key) && !minioIngestInternals.isLikelyText(text)) {
|
||||
skipped++;
|
||||
@@ -207,7 +228,7 @@ export function createMinioSyncTool(config: BackupConfig, store: MemoryStore, de
|
||||
return {
|
||||
success: false,
|
||||
output: '',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
error: backupInternals.formatMinioCliError(error, mcPath),
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user