#!/usr/bin/env bash set -euo pipefail # ------------------------------------------------------------ # rclone → MinIO sync (HTTP/no TLS, small-file optimized, fast LAN) # ------------------------------------------------------------ # Env (required) # MINIO_ENDPOINT e.g. "http://minio.local:9000" (HTTP on purpose) # MINIO_ACCESS_KEY # MINIO_SECRET_KEY # MINIO_BUCKET # # Optional # MINIO_PREFIX # RCLONE_REMOTE_NAME (default: minio) # RCLONE_CONFIG_FILE (default: ~/.config/rclone/rclone.conf) # LOG_FILE (default: ./minio-sync.log.json) # # Usage: ./minio-sync.sh /path/to/src [--dry-run] # # Notes: # - This is tuned for many small files on a LAN: high concurrency, tiny buffers, # and minimal per-file overhead. We avoid --checksum to skip extra HEAD calls. # - 'sync' mirrors destination (deletes extras). Use 'copy' to avoid deletes. # ------------------------------------------------------------ SRC_DIR="${1:-}" DRY_RUN="${2:-}" if [[ -z "${SRC_DIR}" ]]; then echo "Usage: $0 /path/to/source [--dry-run]" >&2 exit 1 fi if [[ ! -d "${SRC_DIR}" ]]; then echo "Source directory not found: ${SRC_DIR}" >&2 exit 1 fi : "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required (e.g. http://minio.local:9000)}" : "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}" : "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}" : "${MINIO_BUCKET:?MINIO_BUCKET is required}" # Enforce plaintext (per your setup) if [[ "${MINIO_ENDPOINT}" =~ ^https:// ]]; then echo "Warning: MINIO_ENDPOINT uses HTTPS. You said no TLS; consider http:// instead." >&2 fi RCLONE_REMOTE_NAME="${RCLONE_REMOTE_NAME:-minio}" RCLONE_CONFIG_FILE="${RCLONE_CONFIG_FILE:-$HOME/.config/rclone/rclone.conf}" MINIO_PREFIX="${MINIO_PREFIX:-}" DEST_PATH="${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" [[ -n "${MINIO_PREFIX}" ]] && DEST_PATH="${DEST_PATH}/${MINIO_PREFIX}" LOG_FILE="${LOG_FILE:-./minio-sync.log.json}" command -v rclone >/dev/null 2>&1 || { echo "rclone not found. Install: https://rclone.org/install/"; exit 1; } mkdir -p "$(dirname "${RCLONE_CONFIG_FILE}")" # Create/update remote for MinIO (path-style, HTTP allowed) if ! rclone listremotes --config "${RCLONE_CONFIG_FILE}" | grep -qE "^${RCLONE_REMOTE_NAME}:"; then echo "Creating rclone remote '${RCLONE_REMOTE_NAME}' for MinIO…" rclone config create "${RCLONE_REMOTE_NAME}" s3 \ provider Minio \ access_key_id "${MINIO_ACCESS_KEY}" \ secret_access_key "${MINIO_SECRET_KEY}" \ endpoint "${MINIO_ENDPOINT}" \ acl private \ --config "${RCLONE_CONFIG_FILE}" >/dev/null else rclone config update "${RCLONE_REMOTE_NAME}" \ provider Minio \ access_key_id "${MINIO_ACCESS_KEY}" \ secret_access_key "${MINIO_SECRET_KEY}" \ endpoint "${MINIO_ENDPOINT}" \ acl private \ --config "${RCLONE_CONFIG_FILE}" >/dev/null fi # Ensure bucket exists (idempotent) rclone mkdir "${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" --config "${RCLONE_CONFIG_FILE}" >/dev/null || true # --- Performance tuning for small files on LAN --- # Rationale: # - High --transfers & --checkers to parallelize tiny objects. # - Small --buffer-size to avoid RAM blowup. # - Skip --checksum to reduce extra HEADs; rely on modtime/size. # - --use-server-modtime to avoid metadata roundtrips when possible. # - Slightly larger s3-upload concurrency helps occasional multi-part. # - --fast-list to cut listing calls. # - Built-in excludes to avoid common noise. RCLONE_FLAGS=( "--config" "${RCLONE_CONFIG_FILE}" # "--fast-list" "--transfers=32" # tune up/down based on CPU/IO "--checkers=500" "--buffer-size=512k" "--use-server-modtime" "--s3-chunk-size=8M" "--s3-upload-concurrency=16" "--s3-no-check-bucket" # speed up in some setups "--retries=8" "--retries-sleep=2s" "--low-level-retries=20" "--bwlimit=off" "--use-json-log" "--log-file" "${LOG_FILE}" "--log-level=INFO" "--delete-excluded" "-P" ) # Junk excludes baked into flags RCLONE_FLAGS+=( "--exclude" ".git/**" "--exclude" "node_modules/**" "--exclude" "*.tmp" "--exclude" "*.swp" "--exclude" ".DS_Store" "--exclude" "Thumbs.db" "--exclude" "__pycache__/**" "--exclude" "*.log" "--exclude" ".cache/**" "--exclude" ".venv/**" ) # Dry run if requested [[ "${DRY_RUN}" == "--dry-run" ]] && RCLONE_FLAGS+=("--dry-run") echo "Source: ${SRC_DIR}" echo "Destination: ${DEST_PATH}" [[ "${DRY_RUN}" == "--dry-run" ]] && echo "Mode: DRY RUN" echo "Logging to: ${LOG_FILE}" echo "Starting sync…" echo # The business end: mirror local → MinIO rclone sync "${SRC_DIR%/}/" "${DEST_PATH%/}/" "${RCLONE_FLAGS[@]}" echo "Done."