Add various scripts for MinIO and Restic backup management

2025-10-24 11:48:52 -07:00
parent 7bd5974ac4
commit 27560832a2
10 changed files with 500 additions and 0 deletions
--- a/scripts/bridge-up.sh
+++ b/scripts/bridge-up.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+ip link add name br0 type bridge
+ip link set dev br0 up
+ip address add 192.168.153.211/24 dev br0
+ip route append default via 192.168.153.1 dev br0
+ip link set enp1s0f0 master br0
+ip address del 192.168.153.211/24 dev enp1s0f0
--- a/scripts/chezmoi-sync.sh
+++ b/scripts/chezmoi-sync.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/bash
+
+current_time=$(date +"%d-%m-%Y-%H")
+log_filename="${current_time}.log"
+
+# Redirect stdout and stderr to the log file
+exec > >(tee -a "$log_filename") 2>&1
+
+cd $HOME || { echo "Failed to change directory to $HOME"; exit 1; }
+
+# Run the 'chezmoi managed' command and save the output to a variable
+managed_files=$(chezmoi managed)
+
+
+if [ $? -ne 0 ]; then
+  echo "Error during the execution of 'chezmoi managed'"
+  exit 1
+fi
+
+
+while IFS= read -r file; do
+  # Run the 'chezmoi add' command for each file
+  chezmoi add "$file"
+done <<< "$managed_files"
--- a/scripts/curl-s3.sh
+++ b/scripts/curl-s3.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/bash
+
+MINIO_ENDPOINT="192.168.153.245:9000"
+MINIO_ACCESS_KEY="Tg4UTucrnhJ8FPaPsqra"
+MINIO_SECRET_KEY="mGEJ6CSYWfSYdaQ90yAY2hfvNtMXV6at9T34o3Kc"
+MINIO_BUCKET="tmp"
+
+DATE=$(date -R --utc)
+SIG_STRING="GET\\n\\napplication/zstd\\n${DATE}\\n/${MINIO_BUCKET}/$1"
+SIGNATURE=`echo -en "${SIG_STRING}" | openssl sha1 -hmac "${MINIO_SECRET_KEY}" -binary | base64`
+
+curl -o $1 \
+-H "Host: ${MINIO_ENDPOINT}" \
+-H "Date: ${DATE}" \
+-H "Content-Type: application/zstd" \
+-H "Authorization: AWS ${MINIO_ACCESS_KEY}:${SIGNATURE}" \
+http://${MINIO_ENDPOINT}/${MINIO_BUCKET}/$1
+
--- a/scripts/kvm-bridge-setup.sh
+++ b/scripts/kvm-bridge-setup.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+# configure-bridge.sh
+# This script configures a network bridge interface for KVM on Arch Linux
+# using systemd-networkd. It creates configuration files for:
+#   - the bridge interface (default: br0)
+#   - the physical interface to be enslaved to the bridge
+#
+# Usage:
+#   sudo ./configure-bridge.sh <physical_interface> [bridge_name]
+#
+# Example:
+#   sudo ./configure-bridge.sh enp2s0 br0
+#
+# Reference: Arch Linux Wiki, "Bridge networking" https://wiki.archlinux.org/title/Bridge_networking
+
+set -euo pipefail
+
+if [ "$EUID" -ne 0 ]; then
+    echo "Please run as root."
+    exit 1
+fi
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: $0 <physical_interface> [bridge_name]"
+    exit 1
+fi
+
+PHYS_IF="$1"
+BRIDGE_IF="${2:-br0}"
+
+NETWORK_DIR="/etc/systemd/network"
+if [ ! -d "$NETWORK_DIR" ]; then
+    echo "Directory $NETWORK_DIR does not exist. Please ensure systemd-networkd is installed."
+    exit 1
+fi
+
+echo "Configuring bridge interface '${BRIDGE_IF}' for physical interface '${PHYS_IF}'..."
+
+# Create the .netdev file for the bridge
+BRIDGE_NETDEV_FILE="${NETWORK_DIR}/25-${BRIDGE_IF}.netdev"
+cat > "$BRIDGE_NETDEV_FILE" <<EOF
+[NetDev]
+Name=${BRIDGE_IF}
+Kind=bridge
+EOF
+echo "Created ${BRIDGE_NETDEV_FILE}"
+
+# Create the .network file for the bridge (using DHCP)
+BRIDGE_NETWORK_FILE="${NETWORK_DIR}/30-${BRIDGE_IF}.network"
+cat > "$BRIDGE_NETWORK_FILE" <<EOF
+[Match]
+Name=${BRIDGE_IF}
+
+[Network]
+DHCP=yes
+EOF
+echo "Created ${BRIDGE_NETWORK_FILE}"
+
+# Create the .network file for the physical interface to attach it to the bridge
+PHYS_NETWORK_FILE="${NETWORK_DIR}/20-${PHYS_IF}.network"
+cat > "$PHYS_NETWORK_FILE" <<EOF
+[Match]
+Name=${PHYS_IF}
+
+[Network]
+Bridge=${BRIDGE_IF}
+EOF
+echo "Created ${PHYS_NETWORK_FILE}"
+
+echo "Restarting systemd-networkd to apply changes..."
+systemctl restart systemd-networkd
+
+echo "Bridge configuration complete. The bridge '${BRIDGE_IF}' should now be up."
+echo "You can check the status with: networkctl status ${BRIDGE_IF}"
--- a/scripts/rclone-sync-gpt5.sh
+++ b/scripts/rclone-sync-gpt5.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ------------------------------------------------------------
+# rclone → MinIO sync (HTTP/no TLS, small-file optimized, fast LAN)
+# ------------------------------------------------------------
+# Env (required)
+#   MINIO_ENDPOINT   e.g. "http://minio.local:9000"  (HTTP on purpose)
+#   MINIO_ACCESS_KEY
+#   MINIO_SECRET_KEY
+#   MINIO_BUCKET
+#
+# Optional
+#   MINIO_PREFIX
+#   RCLONE_REMOTE_NAME  (default: minio)
+#   RCLONE_CONFIG_FILE  (default: ~/.config/rclone/rclone.conf)
+#   LOG_FILE            (default: ./minio-sync.log.json)
+#
+# Usage: ./minio-sync.sh /path/to/src [--dry-run]
+#
+# Notes:
+# - This is tuned for many small files on a LAN: high concurrency, tiny buffers,
+#   and minimal per-file overhead. We avoid --checksum to skip extra HEAD calls.
+# - 'sync' mirrors destination (deletes extras). Use 'copy' to avoid deletes.
+# ------------------------------------------------------------
+
+SRC_DIR="${1:-}"
+DRY_RUN="${2:-}"
+
+if [[ -z "${SRC_DIR}" ]]; then
+  echo "Usage: $0 /path/to/source [--dry-run]" >&2
+  exit 1
+fi
+if [[ ! -d "${SRC_DIR}" ]]; then
+  echo "Source directory not found: ${SRC_DIR}" >&2
+  exit 1
+fi
+
+: "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required (e.g. http://minio.local:9000)}"
+: "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}"
+: "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}"
+: "${MINIO_BUCKET:?MINIO_BUCKET is required}"
+
+# Enforce plaintext (per your setup)
+if [[ "${MINIO_ENDPOINT}" =~ ^https:// ]]; then
+  echo "Warning: MINIO_ENDPOINT uses HTTPS. You said no TLS; consider http:// instead." >&2
+fi
+
+RCLONE_REMOTE_NAME="${RCLONE_REMOTE_NAME:-minio}"
+RCLONE_CONFIG_FILE="${RCLONE_CONFIG_FILE:-$HOME/.config/rclone/rclone.conf}"
+MINIO_PREFIX="${MINIO_PREFIX:-}"
+DEST_PATH="${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}"
+[[ -n "${MINIO_PREFIX}" ]] && DEST_PATH="${DEST_PATH}/${MINIO_PREFIX}"
+LOG_FILE="${LOG_FILE:-./minio-sync.log.json}"
+
+command -v rclone >/dev/null 2>&1 || { echo "rclone not found. Install: https://rclone.org/install/"; exit 1; }
+mkdir -p "$(dirname "${RCLONE_CONFIG_FILE}")"
+
+# Create/update remote for MinIO (path-style, HTTP allowed)
+if ! rclone listremotes --config "${RCLONE_CONFIG_FILE}" | grep -qE "^${RCLONE_REMOTE_NAME}:"; then
+  echo "Creating rclone remote '${RCLONE_REMOTE_NAME}' for MinIO…"
+  rclone config create "${RCLONE_REMOTE_NAME}" s3 \
+    provider Minio \
+    access_key_id "${MINIO_ACCESS_KEY}" \
+    secret_access_key "${MINIO_SECRET_KEY}" \
+    endpoint "${MINIO_ENDPOINT}" \
+    acl private \
+    --config "${RCLONE_CONFIG_FILE}" >/dev/null
+else
+  rclone config update "${RCLONE_REMOTE_NAME}" \
+    provider Minio \
+    access_key_id "${MINIO_ACCESS_KEY}" \
+    secret_access_key "${MINIO_SECRET_KEY}" \
+    endpoint "${MINIO_ENDPOINT}" \
+    acl private \
+    --config "${RCLONE_CONFIG_FILE}" >/dev/null
+fi
+
+# Ensure bucket exists (idempotent)
+rclone mkdir "${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" --config "${RCLONE_CONFIG_FILE}" >/dev/null || true
+
+# --- Performance tuning for small files on LAN ---
+# Rationale:
+# - High --transfers & --checkers to parallelize tiny objects.
+# - Small --buffer-size to avoid RAM blowup.
+# - Skip --checksum to reduce extra HEADs; rely on modtime/size.
+# - --use-server-modtime to avoid metadata roundtrips when possible.
+# - Slightly larger s3-upload concurrency helps occasional multi-part.
+# - --fast-list to cut listing calls.
+# - Built-in excludes to avoid common noise.
+RCLONE_FLAGS=(
+  "--config" "${RCLONE_CONFIG_FILE}"
+  # "--fast-list"
+  "--transfers=32"                 # tune up/down based on CPU/IO
+  "--checkers=500"
+  "--buffer-size=512k"
+  "--use-server-modtime"
+  "--s3-chunk-size=8M"
+  "--s3-upload-concurrency=16"
+  "--s3-no-check-bucket"            # speed up in some setups
+  "--retries=8"
+  "--retries-sleep=2s"
+  "--low-level-retries=20"
+  "--bwlimit=off"
+  "--use-json-log"
+  "--log-file" "${LOG_FILE}"
+  "--log-level=INFO"
+  "--delete-excluded"
+  "-P"
+)
+
+# Junk excludes baked into flags
+RCLONE_FLAGS+=(
+  "--exclude" ".git/**"
+  "--exclude" "node_modules/**"
+  "--exclude" "*.tmp"
+  "--exclude" "*.swp"
+  "--exclude" ".DS_Store"
+  "--exclude" "Thumbs.db"
+  "--exclude" "__pycache__/**"
+  "--exclude" "*.log"
+  "--exclude" ".cache/**"
+  "--exclude"  ".venv/**"
+)
+
+# Dry run if requested
+[[ "${DRY_RUN}" == "--dry-run" ]] && RCLONE_FLAGS+=("--dry-run")
+
+echo "Source:      ${SRC_DIR}"
+echo "Destination: ${DEST_PATH}"
+[[ "${DRY_RUN}" == "--dry-run" ]] && echo "Mode:        DRY RUN"
+echo "Logging to:  ${LOG_FILE}"
+echo "Starting sync…"
+echo
+
+# The business end: mirror local → MinIO
+rclone sync "${SRC_DIR%/}/" "${DEST_PATH%/}/" "${RCLONE_FLAGS[@]}"
+
+echo "Done."
--- a/scripts/rclone-sync.sh
+++ b/scripts/rclone-sync.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+# => WIP
+#
+INCLUDES=('/home/will/**')
+EXCLUDES=('/home/*/.gvfs/**' '/home/*/.cache/**' '/home/*/.local/share/Trash/**' '/home/*/.vscode-server/**' '/home/*/node_modules/**')
+LOG_FILE="/tmp/rclone.log"
+
+set -o noglob
+        filterOptions=()
+        for option in ${EXCLUDES[@]};
+        do
+            echo --filter="- $option" >> "$LOG_FILE"
+            filterOptions+=(--filter="- $option")
+        done
+
+        for option in ${INCLUDES[@]};
+        do
+            echo --filter="- $option" >> "$LOG_FILE"
+            filterOptions+=(--filter="+ $option")
+        done
+
+        echo --filter="- /**" >> "$LOG_FILE"
+        filterOptions+=(--filter="- /**")
+set +o noglob
+
+/usr/bin/rclone -P                                            \
+        --dry-run                                          \
+        -v                                                 \
+        --fast-list                                        \
+        --skip-links                                       \
+        --delete-excluded                                  \
+        "${filterOptions[@]}"                              \
+        --suffix ".$(/bin/date +\%Y\%m\%d\%H\%M\%S)"       \
+        sync                                               \
+            /home/will/Code/                               \
+            "minio-k0s:dev/"
--- a/scripts/rclone-test.sh
+++ b/scripts/rclone-test.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/bash
+
+/usr/bin/rclone \
+sync \
+-P \
+--delete-excluded \
+--exclude '/**/.cache' \
+--exclude '/**/node_modules' \
+--exclude '/**/.venv' \
+--exclude '/**/.git' \
+--exclude '/**/target' \
+--exclude '/**/build' \
+--exclude '/**/dist' \
+--skip-links \
+--fast-list \
+--checkers 1000 \
+--transfers 32 \
+--s3-upload-concurrency 16 \
+--s3-chunk-size 8M \
+--s3-upload-cutoff 32M \
+--multi-thread-streams 8 \
+--multi-thread-cutoff 50M \
+--s3-no-head \
+--s3-no-head-object \
+--s3-no-check-bucket \
+--s3-disable-checksum \
+--ignore-checksum \
+--size-only \
+--progress \
+--log-file /tmp/rclone.log \
+--log-level INFO \
+/home/will/Code/ minio-k0s:/dev/
--- a/scripts/restic-backup-new.sh
+++ b/scripts/restic-backup-new.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# --- CONFIGURATION ---
+# The bucket quotas check requires the 'mc' (MinIO client) command.
+# Download it from https://min.io/docs/minio/linux/reference/minio-client/mc.html
+# and make sure it's in your PATH.
+
+# MinIO Server Details
+MINIO_ALIAS="minio-k0s"
+MINIO_URL="http://192.168.153.245:9000"
+MINIO_ACCESS_KEY="Tg4UTucrnhJ8FPaPsqra"
+MINIO_SECRET_KEY="mGEJ6CSYWfSYdaQ90yAY2hfvNtMXV6at9T34o3Kc"
+
+# Restic Backup Details
+RESTIC_REPO="s3:http://192.168.153.245:9000/backup"
+BACKUP_SOURCE="/home/$(whoami)"
+RESTIC_EXCLUDE_FILE="~/.resticignore" # optional, create this file to list folders/files to exclude
+
+# Restic Environment Variables (for security, use these instead of passing keys directly in the command)
+export RESTIC_PASSWORD="frack666"
+export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
+export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
+
+# Quota Check Configuration
+BUCKET_NAME="backup"
+MIN_QUOTA_REQUIRED_MB=1024 # Minimum required space in megabytes (1GB)
+
+# Notification System Configuration
+# EMAIL_RECIPIENT="william.valentin.info@gmail.com"
+# EMAIL_SUBJECT="Restic Backup Report"
+
+# --- FUNCTIONS ---
+send_notification() {
+    local type="$1"
+    local message="$2"
+    local summary="Restic Backup Report"
+
+    # Send GNOME notification
+    if command -v notify-send &> /dev/null; then
+        notify-send -t 5000 -i "dialog-${type}" "${summary}" "${message}"
+    fi
+
+    # Send email notification
+    if [ -n "${EMAIL_RECIPIENT}" ]; then
+        echo -e "${message}" | mail -s "${summary} - ${type^^}" "${EMAIL_RECIPIENT}"
+    fi
+}
+
+# --- SCRIPT START ---
+echo "Starting Restic backup process for user folder..."
+echo "------------------------------------------------"
+
+# --- 1. MinIO Client Setup & Quota Check ---
+# Check if MinIO client (mc) is installed
+if ! command -v mc &> /dev/null; then
+    send_notification "error" "MinIO client 'mc' not found. Please install it to use the quota check."
+    exit 1
+fi
+
+# Set up MinIO alias if it doesn't exist
+if ! mc alias list | grep -q "${MINIO_ALIAS}"; then
+    echo "Setting up MinIO alias '${MINIO_ALIAS}'..."
+    mc alias set "${MINIO_ALIAS}" "${MINIO_URL}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
+    if [ $? -ne 0 ]; then
+        send_notification "error" "Failed to set up MinIO alias. Check your credentials and URL."
+        exit 1
+    fi
+fi
+
+# Get bucket usage and quota
+BUCKET_USAGE_MB=$(mc du "${MINIO_ALIAS}/${BUCKET_NAME}" | awk '{print $1}')
+BUCKET_QUOTA_MB=$(mc quota info "${MINIO_ALIAS}/${BUCKET_NAME}" | grep 'Hard Quota' | awk '{print $3}' | sed 's/MB//')
+
+if [ -z "${BUCKET_QUOTA_MB}" ]; then
+    echo "NOTICE: No hard quota set for bucket '${BUCKET_NAME}'. Skipping quota check."
+else
+    # Calculate remaining space
+    REMAINING_SPACE_MB=$((BUCKET_QUOTA_MB - BUCKET_USAGE_MB))
+
+    echo "Bucket '${BUCKET_NAME}' usage: ${BUCKET_USAGE_MB} MB"
+    echo "Bucket '${BUCKET_NAME}' quota: ${BUCKET_QUOTA_MB} MB"
+    echo "Remaining space: ${REMAINING_SPACE_MB} MB"
+
+    if [ "${REMAINING_SPACE_MB}" -lt "${MIN_QUOTA_REQUIRED_MB}" ]; then
+        send_notification "error" "Insufficient space in the bucket. Only ${REMAINING_SPACE_MB} MB remaining, but ${MIN_QUOTA_REQUIRED_MB} MB required."
+        exit 1
+    fi
+fi
+
+# --- 2. Restic Repository Check ---
+echo "Checking Restic repository existence..."
+if ! restic snapshots --json &> /dev/null; then
+    echo "Restic repository not found. Initializing a new one..."
+    restic init
+    if [ $? -ne 0 ]; then
+        send_notification "error" "Failed to initialize Restic repository. Check your RESTIC_REPO and credentials."
+        exit 1
+    fi
+fi
+
+# --- 3. Backup Execution ---
+echo "Starting backup of ${BACKUP_SOURCE}..."
+
+if [ -f "${RESTIC_EXCLUDE_FILE}" ]; then
+    echo "Excluding files listed in ${RESTIC_EXCLUDE_FILE}..."
+    restic backup "${BACKUP_SOURCE}" --exclude-file="${RESTIC_EXCLUDE_FILE}"
+else
+    echo "No exclude file found, backing up all files in the source directory."
+    restic backup "${BACKUP_SOURCE}"
+fi
+
+if [ $? -ne 0 ]; then
+    send_notification "error" "Restic backup failed."
+    exit 1
+else
+    send_notification "info" "Restic backup completed successfully."
+fi
+
+# --- 4. Cleanup and Maintenance ---
+echo "Running repository maintenance..."
+restic forget --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6
+
+if [ $? -ne 0 ]; then
+    send_notification "warning" "Restic maintenance failed. Check the logs for details."
+fi
+
+echo "Backup script finished. ✨"
--- a/scripts/restic-backup.sh
+++ b/scripts/restic-backup.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+export RESTIC_PASSWORD=`printf "ZnJhY2s2NjY=" | base64 -d`
+export RESTIC_REPOSITORY=s3:http://192.168.153.242:9000/backup
+
+# export AWS_ACCESS_KEY_ID=mvwfWSsT7oddu14jln0y
+# export AWS_SECRET_ACCESS_KEY=qijzP5fYRHYn7SJTZjJNtNIu1RQznQchpt2Ai7Mt
+export AWS_ACCESS_KEY_ID=pb700qhqSYJXShaybd5o
+export AWS_SECRET_ACCESS_KEY=I56D63st3HGd8h716bjOoO1xjweeSZ0XoE8FMWqH
+# export AWS_ACCESS_KEY_ID=`cat ~/.mc/config.json | jq '.aliases."minio-k0s".accessKey'`
+# export AWS_SECRET_ACCESS_KEY=`cat ~/.mc/config.json | jq '.aliases."minio-k0s".secretKey'`
+
+restic backup \
+    --exclude="node_modules" \
+        --exclude="cache/" \
+	--exclude=".cache/" \
+	--exclude="Cache/" \
+	--exclude="~/VMs" \
+	--exclude="~/ISOs" \
+	--exclude="~/Repos" \
+	--exclude-caches \
+	--insecure-tls \
+	~/
--- a/scripts/restic-clean.sh
+++ b/scripts/restic-clean.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export RESTIC_PASSWORD=frack666
+export RESTIC_REPOSITORY=s3:http://192.168.153.245:9000/backup
+export AWS_ACCESS_KEY_ID=mvwfWSsT7oddu14jln0y
+export AWS_SECRET_ACCESS_KEY=qijzP5fYRHYn7SJTZjJNtNIu1RQznQchpt2Ai7Mt
+
+restic forget \
+       	--keep-daily 24 \
+	--keep-weekly 9 \
+	--keep-monthly 12 \
+	--insecure-tls \
+	--prune
+
+restic cache --cleanup --insecure-tls
+