From 27560832a23a31885277086110f24a6bf33b2d0f Mon Sep 17 00:00:00 2001 From: William Valentin Date: Fri, 24 Oct 2025 11:48:52 -0700 Subject: [PATCH] Add various scripts for MinIO and Restic backup management --- scripts/bridge-up.sh | 8 ++ scripts/chezmoi-sync.sh | 24 ++++++ scripts/curl-s3.sh | 18 +++++ scripts/kvm-bridge-setup.sh | 74 +++++++++++++++++++ scripts/rclone-sync-gpt5.sh | 139 +++++++++++++++++++++++++++++++++++ scripts/rclone-sync.sh | 36 +++++++++ scripts/rclone-test.sh | 32 ++++++++ scripts/restic-backup-new.sh | 127 ++++++++++++++++++++++++++++++++ scripts/restic-backup.sh | 26 +++++++ scripts/restic-clean.sh | 16 ++++ 10 files changed, 500 insertions(+) create mode 100755 scripts/bridge-up.sh create mode 100755 scripts/chezmoi-sync.sh create mode 100755 scripts/curl-s3.sh create mode 100755 scripts/kvm-bridge-setup.sh create mode 100755 scripts/rclone-sync-gpt5.sh create mode 100755 scripts/rclone-sync.sh create mode 100755 scripts/rclone-test.sh create mode 100755 scripts/restic-backup-new.sh create mode 100755 scripts/restic-backup.sh create mode 100755 scripts/restic-clean.sh diff --git a/scripts/bridge-up.sh b/scripts/bridge-up.sh new file mode 100755 index 0000000..2a06aa9 --- /dev/null +++ b/scripts/bridge-up.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ip link add name br0 type bridge +ip link set dev br0 up +ip address add 192.168.153.211/24 dev br0 +ip route append default via 192.168.153.1 dev br0 +ip link set enp1s0f0 master br0 +ip address del 192.168.153.211/24 dev enp1s0f0 diff --git a/scripts/chezmoi-sync.sh b/scripts/chezmoi-sync.sh new file mode 100755 index 0000000..80ff5d4 --- /dev/null +++ b/scripts/chezmoi-sync.sh @@ -0,0 +1,24 @@ +#!/usr/bin/bash + +current_time=$(date +"%d-%m-%Y-%H") +log_filename="${current_time}.log" + +# Redirect stdout and stderr to the log file +exec > >(tee -a "$log_filename") 2>&1 + +cd $HOME || { echo "Failed to change directory to $HOME"; exit 1; } + +# Run the 'chezmoi managed' command and save the output to a variable +managed_files=$(chezmoi managed) + + +if [ $? -ne 0 ]; then + echo "Error during the execution of 'chezmoi managed'" + exit 1 +fi + + +while IFS= read -r file; do + # Run the 'chezmoi add' command for each file + chezmoi add "$file" +done <<< "$managed_files" diff --git a/scripts/curl-s3.sh b/scripts/curl-s3.sh new file mode 100755 index 0000000..43b6cf1 --- /dev/null +++ b/scripts/curl-s3.sh @@ -0,0 +1,18 @@ +#!/usr/bin/bash + +MINIO_ENDPOINT="192.168.153.245:9000" +MINIO_ACCESS_KEY="Tg4UTucrnhJ8FPaPsqra" +MINIO_SECRET_KEY="mGEJ6CSYWfSYdaQ90yAY2hfvNtMXV6at9T34o3Kc" +MINIO_BUCKET="tmp" + +DATE=$(date -R --utc) +SIG_STRING="GET\\n\\napplication/zstd\\n${DATE}\\n/${MINIO_BUCKET}/$1" +SIGNATURE=`echo -en "${SIG_STRING}" | openssl sha1 -hmac "${MINIO_SECRET_KEY}" -binary | base64` + +curl -o $1 \ +-H "Host: ${MINIO_ENDPOINT}" \ +-H "Date: ${DATE}" \ +-H "Content-Type: application/zstd" \ +-H "Authorization: AWS ${MINIO_ACCESS_KEY}:${SIGNATURE}" \ +http://${MINIO_ENDPOINT}/${MINIO_BUCKET}/$1 + diff --git a/scripts/kvm-bridge-setup.sh b/scripts/kvm-bridge-setup.sh new file mode 100755 index 0000000..795a221 --- /dev/null +++ b/scripts/kvm-bridge-setup.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# configure-bridge.sh +# This script configures a network bridge interface for KVM on Arch Linux +# using systemd-networkd. It creates configuration files for: +# - the bridge interface (default: br0) +# - the physical interface to be enslaved to the bridge +# +# Usage: +# sudo ./configure-bridge.sh [bridge_name] +# +# Example: +# sudo ./configure-bridge.sh enp2s0 br0 +# +# Reference: Arch Linux Wiki, "Bridge networking" https://wiki.archlinux.org/title/Bridge_networking + +set -euo pipefail + +if [ "$EUID" -ne 0 ]; then + echo "Please run as root." + exit 1 +fi + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [bridge_name]" + exit 1 +fi + +PHYS_IF="$1" +BRIDGE_IF="${2:-br0}" + +NETWORK_DIR="/etc/systemd/network" +if [ ! -d "$NETWORK_DIR" ]; then + echo "Directory $NETWORK_DIR does not exist. Please ensure systemd-networkd is installed." + exit 1 +fi + +echo "Configuring bridge interface '${BRIDGE_IF}' for physical interface '${PHYS_IF}'..." + +# Create the .netdev file for the bridge +BRIDGE_NETDEV_FILE="${NETWORK_DIR}/25-${BRIDGE_IF}.netdev" +cat > "$BRIDGE_NETDEV_FILE" < "$BRIDGE_NETWORK_FILE" < "$PHYS_NETWORK_FILE" <&2 + exit 1 +fi +if [[ ! -d "${SRC_DIR}" ]]; then + echo "Source directory not found: ${SRC_DIR}" >&2 + exit 1 +fi + +: "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required (e.g. http://minio.local:9000)}" +: "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}" +: "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}" +: "${MINIO_BUCKET:?MINIO_BUCKET is required}" + +# Enforce plaintext (per your setup) +if [[ "${MINIO_ENDPOINT}" =~ ^https:// ]]; then + echo "Warning: MINIO_ENDPOINT uses HTTPS. You said no TLS; consider http:// instead." >&2 +fi + +RCLONE_REMOTE_NAME="${RCLONE_REMOTE_NAME:-minio}" +RCLONE_CONFIG_FILE="${RCLONE_CONFIG_FILE:-$HOME/.config/rclone/rclone.conf}" +MINIO_PREFIX="${MINIO_PREFIX:-}" +DEST_PATH="${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" +[[ -n "${MINIO_PREFIX}" ]] && DEST_PATH="${DEST_PATH}/${MINIO_PREFIX}" +LOG_FILE="${LOG_FILE:-./minio-sync.log.json}" + +command -v rclone >/dev/null 2>&1 || { echo "rclone not found. Install: https://rclone.org/install/"; exit 1; } +mkdir -p "$(dirname "${RCLONE_CONFIG_FILE}")" + +# Create/update remote for MinIO (path-style, HTTP allowed) +if ! rclone listremotes --config "${RCLONE_CONFIG_FILE}" | grep -qE "^${RCLONE_REMOTE_NAME}:"; then + echo "Creating rclone remote '${RCLONE_REMOTE_NAME}' for MinIO…" + rclone config create "${RCLONE_REMOTE_NAME}" s3 \ + provider Minio \ + access_key_id "${MINIO_ACCESS_KEY}" \ + secret_access_key "${MINIO_SECRET_KEY}" \ + endpoint "${MINIO_ENDPOINT}" \ + acl private \ + --config "${RCLONE_CONFIG_FILE}" >/dev/null +else + rclone config update "${RCLONE_REMOTE_NAME}" \ + provider Minio \ + access_key_id "${MINIO_ACCESS_KEY}" \ + secret_access_key "${MINIO_SECRET_KEY}" \ + endpoint "${MINIO_ENDPOINT}" \ + acl private \ + --config "${RCLONE_CONFIG_FILE}" >/dev/null +fi + +# Ensure bucket exists (idempotent) +rclone mkdir "${RCLONE_REMOTE_NAME}:${MINIO_BUCKET}" --config "${RCLONE_CONFIG_FILE}" >/dev/null || true + +# --- Performance tuning for small files on LAN --- +# Rationale: +# - High --transfers & --checkers to parallelize tiny objects. +# - Small --buffer-size to avoid RAM blowup. +# - Skip --checksum to reduce extra HEADs; rely on modtime/size. +# - --use-server-modtime to avoid metadata roundtrips when possible. +# - Slightly larger s3-upload concurrency helps occasional multi-part. +# - --fast-list to cut listing calls. +# - Built-in excludes to avoid common noise. +RCLONE_FLAGS=( + "--config" "${RCLONE_CONFIG_FILE}" + # "--fast-list" + "--transfers=32" # tune up/down based on CPU/IO + "--checkers=500" + "--buffer-size=512k" + "--use-server-modtime" + "--s3-chunk-size=8M" + "--s3-upload-concurrency=16" + "--s3-no-check-bucket" # speed up in some setups + "--retries=8" + "--retries-sleep=2s" + "--low-level-retries=20" + "--bwlimit=off" + "--use-json-log" + "--log-file" "${LOG_FILE}" + "--log-level=INFO" + "--delete-excluded" + "-P" +) + +# Junk excludes baked into flags +RCLONE_FLAGS+=( + "--exclude" ".git/**" + "--exclude" "node_modules/**" + "--exclude" "*.tmp" + "--exclude" "*.swp" + "--exclude" ".DS_Store" + "--exclude" "Thumbs.db" + "--exclude" "__pycache__/**" + "--exclude" "*.log" + "--exclude" ".cache/**" + "--exclude" ".venv/**" +) + +# Dry run if requested +[[ "${DRY_RUN}" == "--dry-run" ]] && RCLONE_FLAGS+=("--dry-run") + +echo "Source: ${SRC_DIR}" +echo "Destination: ${DEST_PATH}" +[[ "${DRY_RUN}" == "--dry-run" ]] && echo "Mode: DRY RUN" +echo "Logging to: ${LOG_FILE}" +echo "Starting sync…" +echo + +# The business end: mirror local → MinIO +rclone sync "${SRC_DIR%/}/" "${DEST_PATH%/}/" "${RCLONE_FLAGS[@]}" + +echo "Done." diff --git a/scripts/rclone-sync.sh b/scripts/rclone-sync.sh new file mode 100755 index 0000000..e506796 --- /dev/null +++ b/scripts/rclone-sync.sh @@ -0,0 +1,36 @@ +#!/usr/bin/bash +# => WIP +# +INCLUDES=('/home/will/**') +EXCLUDES=('/home/*/.gvfs/**' '/home/*/.cache/**' '/home/*/.local/share/Trash/**' '/home/*/.vscode-server/**' '/home/*/node_modules/**') +LOG_FILE="/tmp/rclone.log" + +set -o noglob + filterOptions=() + for option in ${EXCLUDES[@]}; + do + echo --filter="- $option" >> "$LOG_FILE" + filterOptions+=(--filter="- $option") + done + + for option in ${INCLUDES[@]}; + do + echo --filter="- $option" >> "$LOG_FILE" + filterOptions+=(--filter="+ $option") + done + + echo --filter="- /**" >> "$LOG_FILE" + filterOptions+=(--filter="- /**") +set +o noglob + +/usr/bin/rclone -P \ + --dry-run \ + -v \ + --fast-list \ + --skip-links \ + --delete-excluded \ + "${filterOptions[@]}" \ + --suffix ".$(/bin/date +\%Y\%m\%d\%H\%M\%S)" \ + sync \ + /home/will/Code/ \ + "minio-k0s:dev/" diff --git a/scripts/rclone-test.sh b/scripts/rclone-test.sh new file mode 100755 index 0000000..1f3949c --- /dev/null +++ b/scripts/rclone-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/bash + +/usr/bin/rclone \ +sync \ +-P \ +--delete-excluded \ +--exclude '/**/.cache' \ +--exclude '/**/node_modules' \ +--exclude '/**/.venv' \ +--exclude '/**/.git' \ +--exclude '/**/target' \ +--exclude '/**/build' \ +--exclude '/**/dist' \ +--skip-links \ +--fast-list \ +--checkers 1000 \ +--transfers 32 \ +--s3-upload-concurrency 16 \ +--s3-chunk-size 8M \ +--s3-upload-cutoff 32M \ +--multi-thread-streams 8 \ +--multi-thread-cutoff 50M \ +--s3-no-head \ +--s3-no-head-object \ +--s3-no-check-bucket \ +--s3-disable-checksum \ +--ignore-checksum \ +--size-only \ +--progress \ +--log-file /tmp/rclone.log \ +--log-level INFO \ +/home/will/Code/ minio-k0s:/dev/ diff --git a/scripts/restic-backup-new.sh b/scripts/restic-backup-new.sh new file mode 100755 index 0000000..e1ee204 --- /dev/null +++ b/scripts/restic-backup-new.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# --- CONFIGURATION --- +# The bucket quotas check requires the 'mc' (MinIO client) command. +# Download it from https://min.io/docs/minio/linux/reference/minio-client/mc.html +# and make sure it's in your PATH. + +# MinIO Server Details +MINIO_ALIAS="minio-k0s" +MINIO_URL="http://192.168.153.245:9000" +MINIO_ACCESS_KEY="Tg4UTucrnhJ8FPaPsqra" +MINIO_SECRET_KEY="mGEJ6CSYWfSYdaQ90yAY2hfvNtMXV6at9T34o3Kc" + +# Restic Backup Details +RESTIC_REPO="s3:http://192.168.153.245:9000/backup" +BACKUP_SOURCE="/home/$(whoami)" +RESTIC_EXCLUDE_FILE="~/.resticignore" # optional, create this file to list folders/files to exclude + +# Restic Environment Variables (for security, use these instead of passing keys directly in the command) +export RESTIC_PASSWORD="frack666" +export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" +export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" + +# Quota Check Configuration +BUCKET_NAME="backup" +MIN_QUOTA_REQUIRED_MB=1024 # Minimum required space in megabytes (1GB) + +# Notification System Configuration +# EMAIL_RECIPIENT="william.valentin.info@gmail.com" +# EMAIL_SUBJECT="Restic Backup Report" + +# --- FUNCTIONS --- +send_notification() { + local type="$1" + local message="$2" + local summary="Restic Backup Report" + + # Send GNOME notification + if command -v notify-send &> /dev/null; then + notify-send -t 5000 -i "dialog-${type}" "${summary}" "${message}" + fi + + # Send email notification + if [ -n "${EMAIL_RECIPIENT}" ]; then + echo -e "${message}" | mail -s "${summary} - ${type^^}" "${EMAIL_RECIPIENT}" + fi +} + +# --- SCRIPT START --- +echo "Starting Restic backup process for user folder..." +echo "------------------------------------------------" + +# --- 1. MinIO Client Setup & Quota Check --- +# Check if MinIO client (mc) is installed +if ! command -v mc &> /dev/null; then + send_notification "error" "MinIO client 'mc' not found. Please install it to use the quota check." + exit 1 +fi + +# Set up MinIO alias if it doesn't exist +if ! mc alias list | grep -q "${MINIO_ALIAS}"; then + echo "Setting up MinIO alias '${MINIO_ALIAS}'..." + mc alias set "${MINIO_ALIAS}" "${MINIO_URL}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + if [ $? -ne 0 ]; then + send_notification "error" "Failed to set up MinIO alias. Check your credentials and URL." + exit 1 + fi +fi + +# Get bucket usage and quota +BUCKET_USAGE_MB=$(mc du "${MINIO_ALIAS}/${BUCKET_NAME}" | awk '{print $1}') +BUCKET_QUOTA_MB=$(mc quota info "${MINIO_ALIAS}/${BUCKET_NAME}" | grep 'Hard Quota' | awk '{print $3}' | sed 's/MB//') + +if [ -z "${BUCKET_QUOTA_MB}" ]; then + echo "NOTICE: No hard quota set for bucket '${BUCKET_NAME}'. Skipping quota check." +else + # Calculate remaining space + REMAINING_SPACE_MB=$((BUCKET_QUOTA_MB - BUCKET_USAGE_MB)) + + echo "Bucket '${BUCKET_NAME}' usage: ${BUCKET_USAGE_MB} MB" + echo "Bucket '${BUCKET_NAME}' quota: ${BUCKET_QUOTA_MB} MB" + echo "Remaining space: ${REMAINING_SPACE_MB} MB" + + if [ "${REMAINING_SPACE_MB}" -lt "${MIN_QUOTA_REQUIRED_MB}" ]; then + send_notification "error" "Insufficient space in the bucket. Only ${REMAINING_SPACE_MB} MB remaining, but ${MIN_QUOTA_REQUIRED_MB} MB required." + exit 1 + fi +fi + +# --- 2. Restic Repository Check --- +echo "Checking Restic repository existence..." +if ! restic snapshots --json &> /dev/null; then + echo "Restic repository not found. Initializing a new one..." + restic init + if [ $? -ne 0 ]; then + send_notification "error" "Failed to initialize Restic repository. Check your RESTIC_REPO and credentials." + exit 1 + fi +fi + +# --- 3. Backup Execution --- +echo "Starting backup of ${BACKUP_SOURCE}..." + +if [ -f "${RESTIC_EXCLUDE_FILE}" ]; then + echo "Excluding files listed in ${RESTIC_EXCLUDE_FILE}..." + restic backup "${BACKUP_SOURCE}" --exclude-file="${RESTIC_EXCLUDE_FILE}" +else + echo "No exclude file found, backing up all files in the source directory." + restic backup "${BACKUP_SOURCE}" +fi + +if [ $? -ne 0 ]; then + send_notification "error" "Restic backup failed." + exit 1 +else + send_notification "info" "Restic backup completed successfully." +fi + +# --- 4. Cleanup and Maintenance --- +echo "Running repository maintenance..." +restic forget --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6 + +if [ $? -ne 0 ]; then + send_notification "warning" "Restic maintenance failed. Check the logs for details." +fi + +echo "Backup script finished. ✨" diff --git a/scripts/restic-backup.sh b/scripts/restic-backup.sh new file mode 100755 index 0000000..f1a32a0 --- /dev/null +++ b/scripts/restic-backup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash + +set -x +set -e + +export RESTIC_PASSWORD=`printf "ZnJhY2s2NjY=" | base64 -d` +export RESTIC_REPOSITORY=s3:http://192.168.153.242:9000/backup + +# export AWS_ACCESS_KEY_ID=mvwfWSsT7oddu14jln0y +# export AWS_SECRET_ACCESS_KEY=qijzP5fYRHYn7SJTZjJNtNIu1RQznQchpt2Ai7Mt +export AWS_ACCESS_KEY_ID=pb700qhqSYJXShaybd5o +export AWS_SECRET_ACCESS_KEY=I56D63st3HGd8h716bjOoO1xjweeSZ0XoE8FMWqH +# export AWS_ACCESS_KEY_ID=`cat ~/.mc/config.json | jq '.aliases."minio-k0s".accessKey'` +# export AWS_SECRET_ACCESS_KEY=`cat ~/.mc/config.json | jq '.aliases."minio-k0s".secretKey'` + +restic backup \ + --exclude="node_modules" \ + --exclude="cache/" \ + --exclude=".cache/" \ + --exclude="Cache/" \ + --exclude="~/VMs" \ + --exclude="~/ISOs" \ + --exclude="~/Repos" \ + --exclude-caches \ + --insecure-tls \ + ~/ diff --git a/scripts/restic-clean.sh b/scripts/restic-clean.sh new file mode 100755 index 0000000..23b79e0 --- /dev/null +++ b/scripts/restic-clean.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +export RESTIC_PASSWORD=frack666 +export RESTIC_REPOSITORY=s3:http://192.168.153.245:9000/backup +export AWS_ACCESS_KEY_ID=mvwfWSsT7oddu14jln0y +export AWS_SECRET_ACCESS_KEY=qijzP5fYRHYn7SJTZjJNtNIu1RQznQchpt2Ai7Mt + +restic forget \ + --keep-daily 24 \ + --keep-weekly 9 \ + --keep-monthly 12 \ + --insecure-tls \ + --prune + +restic cache --cleanup --insecure-tls +