Add various scripts for MinIO and Restic backup management

scripts/rclone-sync-gpt5.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ------------------------------------------------------------
+# rclone → MinIO sync (HTTP/no TLS, small-file optimized, fast LAN)
+# ------------------------------------------------------------
+# Env (required)
+#   MINIO_ENDPOINT   e.g. "http://minio.local:9000"  (HTTP on purpose)
+#   MINIO_ACCESS_KEY
+#   MINIO_SECRET_KEY
+#   MINIO_BUCKET
+#
+# Optional
+#   MINIO_PREFIX
+#   RCLONE_REMOTE_NAME  (default: minio)
+#   RCLONE_CONFIG_FILE  (default: ~/.config/rclone/rclone.conf)
+#   LOG_FILE            (default: ./minio-sync.log.json)
+#
+# Usage: ./minio-sync.sh /path/to/src [--dry-run]
+#
+# Notes:
+# - This is tuned for many small files on a LAN: high concurrency, tiny buffers,
+#   and minimal per-file overhead. We avoid --checksum to skip extra HEAD calls.
+# - 'sync' mirrors destination (deletes extras). Use 'copy' to avoid deletes.
+# ------------------------------------------------------------
+
+SRC_DIR="${1:-}"
+DRY_RUN="${2:-}"
+
+if [[ -z "${SRC_DIR}" ]]; then
+  echo "Usage: $0 /path/to/source [--dry-run]" >&2
+  exit 1
+fi
+if [[ ! -d "${SRC_DIR}" ]]; then
+  echo "Source directory not found: ${SRC_DIR}" >&2
+  exit 1
+fi
+
+: "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required (e.g. http://minio.local:9000)}"
+: "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}"
+: "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}"
+: "${MINIO_BUCKET:?MINIO_BUCKET is required}"
+
+# Enforce plaintext (per your setup)
+if [[ "${MINIO_ENDPOINT}" =~ ^https:// ]]; then
+  echo "Warning: MINIO_ENDPOINT uses HTTPS. You said no TLS; consider http:// instead." >&2
+fi
+
+RCLONE_REMOTE_NAME="${RCLONE_REMOTE_NAME:-minio}"
+RCLONE_CONFIG_FILE="${RCLONE_CONFIG_FILE:-$HOME/.config/rclone/rclone.conf}"
+MINIO_PREFIX="${MINIO_PREFIX:-}"
+DEST_PATH="${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}"
+[[ -n "${MINIO_PREFIX}" ]] && DEST_PATH="${DEST_PATH}/${MINIO_PREFIX}"
+LOG_FILE="${LOG_FILE:-./minio-sync.log.json}"
+
+command -v rclone >/dev/null 2>&1 || { echo "rclone not found. Install: https://rclone.org/install/"; exit 1; }
+mkdir -p "$(dirname "${RCLONE_CONFIG_FILE}")"
+
+# Create/update remote for MinIO (path-style, HTTP allowed)
+if ! rclone listremotes --config "${RCLONE_CONFIG_FILE}" | grep -qE "^${RCLONE_REMOTE_NAME}:"; then
+  echo "Creating rclone remote '${RCLONE_REMOTE_NAME}' for MinIO…"
+  rclone config create "${RCLONE_REMOTE_NAME}" s3 \
+    provider Minio \
+    access_key_id "${MINIO_ACCESS_KEY}" \
+    secret_access_key "${MINIO_SECRET_KEY}" \
+    endpoint "${MINIO_ENDPOINT}" \
+    acl private \
+    --config "${RCLONE_CONFIG_FILE}" >/dev/null
+else
+  rclone config update "${RCLONE_REMOTE_NAME}" \
+    provider Minio \
+    access_key_id "${MINIO_ACCESS_KEY}" \
+    secret_access_key "${MINIO_SECRET_KEY}" \
+    endpoint "${MINIO_ENDPOINT}" \
+    acl private \
+    --config "${RCLONE_CONFIG_FILE}" >/dev/null
+fi
+
+# Ensure bucket exists (idempotent)
+rclone mkdir "${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" --config "${RCLONE_CONFIG_FILE}" >/dev/null || true
+
+# --- Performance tuning for small files on LAN ---
+# Rationale:
+# - High --transfers & --checkers to parallelize tiny objects.
+# - Small --buffer-size to avoid RAM blowup.
+# - Skip --checksum to reduce extra HEADs; rely on modtime/size.
+# - --use-server-modtime to avoid metadata roundtrips when possible.
+# - Slightly larger s3-upload concurrency helps occasional multi-part.
+# - --fast-list to cut listing calls.
+# - Built-in excludes to avoid common noise.
+RCLONE_FLAGS=(
+  "--config" "${RCLONE_CONFIG_FILE}"
+  # "--fast-list"
+  "--transfers=32"                 # tune up/down based on CPU/IO
+  "--checkers=500"
+  "--buffer-size=512k"
+  "--use-server-modtime"
+  "--s3-chunk-size=8M"
+  "--s3-upload-concurrency=16"
+  "--s3-no-check-bucket"            # speed up in some setups
+  "--retries=8"
+  "--retries-sleep=2s"
+  "--low-level-retries=20"
+  "--bwlimit=off"
+  "--use-json-log"
+  "--log-file" "${LOG_FILE}"
+  "--log-level=INFO"
+  "--delete-excluded"
+  "-P"
+)
+
+# Junk excludes baked into flags
+RCLONE_FLAGS+=(
+  "--exclude" ".git/**"
+  "--exclude" "node_modules/**"
+  "--exclude" "*.tmp"
+  "--exclude" "*.swp"
+  "--exclude" ".DS_Store"
+  "--exclude" "Thumbs.db"
+  "--exclude" "__pycache__/**"
+  "--exclude" "*.log"
+  "--exclude" ".cache/**"
+  "--exclude"  ".venv/**"
+)
+
+# Dry run if requested
+[[ "${DRY_RUN}" == "--dry-run" ]] && RCLONE_FLAGS+=("--dry-run")
+
+echo "Source:      ${SRC_DIR}"
+echo "Destination: ${DEST_PATH}"
+[[ "${DRY_RUN}" == "--dry-run" ]] && echo "Mode:        DRY RUN"
+echo "Logging to:  ${LOG_FILE}"
+echo "Starting sync…"
+echo
+
+# The business end: mirror local → MinIO
+rclone sync "${SRC_DIR%/}/" "${DEST_PATH%/}/" "${RCLONE_FLAGS[@]}"
+
+echo "Done."