140 lines
4.5 KiB
Bash
Executable File
140 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# ------------------------------------------------------------
|
|
# rclone → MinIO sync (HTTP/no TLS, small-file optimized, fast LAN)
|
|
# ------------------------------------------------------------
|
|
# Env (required)
|
|
# MINIO_ENDPOINT e.g. "http://minio.local:9000" (HTTP on purpose)
|
|
# MINIO_ACCESS_KEY
|
|
# MINIO_SECRET_KEY
|
|
# MINIO_BUCKET
|
|
#
|
|
# Optional
|
|
# MINIO_PREFIX
|
|
# RCLONE_REMOTE_NAME (default: minio)
|
|
# RCLONE_CONFIG_FILE (default: ~/.config/rclone/rclone.conf)
|
|
# LOG_FILE (default: ./minio-sync.log.json)
|
|
#
|
|
# Usage: ./minio-sync.sh /path/to/src [--dry-run]
|
|
#
|
|
# Notes:
|
|
# - This is tuned for many small files on a LAN: high concurrency, tiny buffers,
|
|
# and minimal per-file overhead. We avoid --checksum to skip extra HEAD calls.
|
|
# - 'sync' mirrors destination (deletes extras). Use 'copy' to avoid deletes.
|
|
# ------------------------------------------------------------
|
|
|
|
SRC_DIR="${1:-}"
|
|
DRY_RUN="${2:-}"
|
|
|
|
if [[ -z "${SRC_DIR}" ]]; then
|
|
echo "Usage: $0 /path/to/source [--dry-run]" >&2
|
|
exit 1
|
|
fi
|
|
if [[ ! -d "${SRC_DIR}" ]]; then
|
|
echo "Source directory not found: ${SRC_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
: "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required (e.g. http://minio.local:9000)}"
|
|
: "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}"
|
|
: "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}"
|
|
: "${MINIO_BUCKET:?MINIO_BUCKET is required}"
|
|
|
|
# Enforce plaintext (per your setup)
|
|
if [[ "${MINIO_ENDPOINT}" =~ ^https:// ]]; then
|
|
echo "Warning: MINIO_ENDPOINT uses HTTPS. You said no TLS; consider http:// instead." >&2
|
|
fi
|
|
|
|
RCLONE_REMOTE_NAME="${RCLONE_REMOTE_NAME:-minio}"
|
|
RCLONE_CONFIG_FILE="${RCLONE_CONFIG_FILE:-$HOME/.config/rclone/rclone.conf}"
|
|
MINIO_PREFIX="${MINIO_PREFIX:-}"
|
|
DEST_PATH="${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}"
|
|
[[ -n "${MINIO_PREFIX}" ]] && DEST_PATH="${DEST_PATH}/${MINIO_PREFIX}"
|
|
LOG_FILE="${LOG_FILE:-./minio-sync.log.json}"
|
|
|
|
command -v rclone >/dev/null 2>&1 || { echo "rclone not found. Install: https://rclone.org/install/"; exit 1; }
|
|
mkdir -p "$(dirname "${RCLONE_CONFIG_FILE}")"
|
|
|
|
# Create/update remote for MinIO (path-style, HTTP allowed)
|
|
if ! rclone listremotes --config "${RCLONE_CONFIG_FILE}" | grep -qE "^${RCLONE_REMOTE_NAME}:"; then
|
|
echo "Creating rclone remote '${RCLONE_REMOTE_NAME}' for MinIO…"
|
|
rclone config create "${RCLONE_REMOTE_NAME}" s3 \
|
|
provider Minio \
|
|
access_key_id "${MINIO_ACCESS_KEY}" \
|
|
secret_access_key "${MINIO_SECRET_KEY}" \
|
|
endpoint "${MINIO_ENDPOINT}" \
|
|
acl private \
|
|
--config "${RCLONE_CONFIG_FILE}" >/dev/null
|
|
else
|
|
rclone config update "${RCLONE_REMOTE_NAME}" \
|
|
provider Minio \
|
|
access_key_id "${MINIO_ACCESS_KEY}" \
|
|
secret_access_key "${MINIO_SECRET_KEY}" \
|
|
endpoint "${MINIO_ENDPOINT}" \
|
|
acl private \
|
|
--config "${RCLONE_CONFIG_FILE}" >/dev/null
|
|
fi
|
|
|
|
# Ensure bucket exists (idempotent)
|
|
rclone mkdir "${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" --config "${RCLONE_CONFIG_FILE}" >/dev/null || true
|
|
|
|
# --- Performance tuning for small files on LAN ---
|
|
# Rationale:
|
|
# - High --transfers & --checkers to parallelize tiny objects.
|
|
# - Small --buffer-size to avoid RAM blowup.
|
|
# - Skip --checksum to reduce extra HEADs; rely on modtime/size.
|
|
# - --use-server-modtime to avoid metadata roundtrips when possible.
|
|
# - Slightly larger s3-upload concurrency helps occasional multi-part.
|
|
# - --fast-list to cut listing calls.
|
|
# - Built-in excludes to avoid common noise.
|
|
RCLONE_FLAGS=(
|
|
"--config" "${RCLONE_CONFIG_FILE}"
|
|
# "--fast-list"
|
|
"--transfers=32" # tune up/down based on CPU/IO
|
|
"--checkers=500"
|
|
"--buffer-size=512k"
|
|
"--use-server-modtime"
|
|
"--s3-chunk-size=8M"
|
|
"--s3-upload-concurrency=16"
|
|
"--s3-no-check-bucket" # speed up in some setups
|
|
"--retries=8"
|
|
"--retries-sleep=2s"
|
|
"--low-level-retries=20"
|
|
"--bwlimit=off"
|
|
"--use-json-log"
|
|
"--log-file" "${LOG_FILE}"
|
|
"--log-level=INFO"
|
|
"--delete-excluded"
|
|
"-P"
|
|
)
|
|
|
|
# Junk excludes baked into flags
|
|
RCLONE_FLAGS+=(
|
|
"--exclude" ".git/**"
|
|
"--exclude" "node_modules/**"
|
|
"--exclude" "*.tmp"
|
|
"--exclude" "*.swp"
|
|
"--exclude" ".DS_Store"
|
|
"--exclude" "Thumbs.db"
|
|
"--exclude" "__pycache__/**"
|
|
"--exclude" "*.log"
|
|
"--exclude" ".cache/**"
|
|
"--exclude" ".venv/**"
|
|
)
|
|
|
|
# Dry run if requested
|
|
[[ "${DRY_RUN}" == "--dry-run" ]] && RCLONE_FLAGS+=("--dry-run")
|
|
|
|
echo "Source: ${SRC_DIR}"
|
|
echo "Destination: ${DEST_PATH}"
|
|
[[ "${DRY_RUN}" == "--dry-run" ]] && echo "Mode: DRY RUN"
|
|
echo "Logging to: ${LOG_FILE}"
|
|
echo "Starting sync…"
|
|
echo
|
|
|
|
# The business end: mirror local → MinIO
|
|
rclone sync "${SRC_DIR%/}/" "${DEST_PATH%/}/" "${RCLONE_FLAGS[@]}"
|
|
|
|
echo "Done."
|