diff --git a/README.md b/README.md index 7f7bfb8..9bbf8e8 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,33 @@ A robust Debian/Ubuntu-friendly tool to update Docker Compose stacks. - Sufficient permissions to run docker commands (may require sudo) - `flock` command (usually pre-installed on Linux systems) +### backup-tool + +A simple, robust backup tool for homelabs. Dockerized with privileges, runs on a Docker Swarm manager node. + +**Features:** + +- Incremental backups via rsync (SSH hosts) and docker cp (container volumes) +- Interactive setup wizard for hosts, paths, drives, and Telegram alerts +- Time & size based retention per host/path +- Replicate important backups to multiple drives (compressed replicas, no RAID) +- Telegram bot alerts with exponential back-off (offline, disk space, errors) +- Offline detection every 30 seconds, independent of backup schedule + +**Quick start:** + +```bash +cd packages/backup-tool + +# Interactive setup +docker compose run --rm backup setup + +# Start the service +docker compose up -d +``` + +See [packages/backup-tool/README.md](packages/backup-tool/README.md) for full documentation. + ### create-package A tool to create new package directories with bash script templates. diff --git a/packages/backup-tool/Dockerfile b/packages/backup-tool/Dockerfile new file mode 100644 index 0000000..5e67296 --- /dev/null +++ b/packages/backup-tool/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ + coreutils \ + curl \ + gzip \ + openssh-client \ + rsync \ + tar \ + docker.io \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/backup-tool + +COPY . . + +RUN chmod +x backup-tool setup + +ENTRYPOINT ["./backup-tool"] +CMD ["run"] diff --git a/packages/backup-tool/README.md b/packages/backup-tool/README.md new file mode 100644 index 0000000..7689a55 --- /dev/null +++ b/packages/backup-tool/README.md @@ -0,0 +1,179 @@ +# backup-tool + +A simple, robust backup tool for homelabs. Dockerized with privileges, designed to run on a Docker Swarm manager node as a normal Compose stack on Ubuntu Server. + +## Features + +- **SSH hosts (rsync)** — incremental backups of remote directories via rsync over SSH +- **Docker volumes** — backup container volumes using docker commands +- **Interactive setup** — wizard to configure hosts, paths, drives, and alerts +- **Time & size based retention** — per host/path TTL policies +- **Important data replication** — replicate critical backups to multiple drives (no RAID) +- **Compressed replicas** — replicas are compressed after copy; main backup stays uncompressed for fast rsync +- **Telegram alerts** — error notifications with exponential back-off (no spam) +- **Offline detection** — checks every 30 seconds regardless of backup schedule +- **Timed incremental backups** — configurable interval, link-dest based incremental rsync + +## Quick Start + +```bash +# 1. Clone and navigate to the backup-tool directory +cd packages/backup-tool + +# 2. Run the interactive setup wizard +docker compose run --rm backup setup + +# 3. Start the backup service +docker compose up -d + +# 4. View logs +docker compose logs -f +``` + +## Configuration + +All configuration lives in the `config/` directory (mounted at `/etc/backup-tool` in the container). + +### Main config: `config/backup.conf` + +| Key | Description | Default | +|-----|-------------|---------| +| `backup_dest` | Where backups are stored | `/backup` | +| `backup_interval_min` | Backup interval in minutes | `60` | +| `important_drives` | Comma-separated mount points for replication | | +| `important_folders` | Comma-separated paths relative to backup\_dest | | +| `telegram_bot_token` | Telegram bot token | | +| `telegram_chat_id` | Telegram chat ID | | + +### Host configs: `config/hosts.d/.conf` + +One file per backup source. See `config/host.conf.example`. + +| Key | Description | +|-----|-------------| +| `type` | `ssh` (rsync) or `docker` (container volumes) | +| `address` | SSH address (user@host) | +| `paths` | Comma-separated paths or volume names | +| `retention_days` | Days to keep snapshots (0 = forever) | +| `retention_max_size` | Max total size (e.g. `10G`) | + +## How It Works + +### Backup Cycle + +1. For each configured host, back up each path: + - **SSH hosts**: `rsync -a --delete --link-dest=` for incremental backups + - **Docker volumes**: temporary container mount + `docker cp` +2. Apply retention policies (delete old/oversized snapshots) +3. Replicate important folders to all important drives +4. Compress replicas (main copy stays uncompressed) + +### Monitoring (runs independently) + +- **Offline detection**: pings all SSH hosts every 30 seconds +- **Disk space**: checks all important drives and backup destination every 60 seconds +- Alerts go to Telegram with exponential back-off (2min → 4min → 8min → ... → 8h max) +- Recovery messages are sent when issues clear + +### Directory Structure + +Backups are stored as timestamped snapshots: + +``` +/backup/ + myhost/ + var_data/ + 20250101-120000/ ← full snapshot + 20250102-120000/ ← incremental (hardlinked to previous) + latest -> 20250102-120000 + home_user/ + ... +``` + +Replicas on important drives: + +``` +/mnt/usb1/ + backup-replicas/ + myhost_var_data.tar.gz ← compressed copy + myhost_home_user.tar.gz +``` + +## Commands + +```bash +# Run continuous backup + monitoring (default) +docker compose run --rm backup run + +# Run a single backup cycle +docker compose run --rm backup once + +# Interactive setup wizard +docker compose run --rm backup setup + +# Run only the monitor (offline/disk checks) +docker compose run --rm backup monitor + +# Run only retention cleanup +docker compose run --rm backup retention + +# Run only replication +docker compose run --rm backup replicate +``` + +## Telegram Bot Setup + +1. Open Telegram and search for **@BotFather** +2. Send `/newbot` and follow the prompts +3. BotFather gives you a **bot token** — e.g. `123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11` +4. Open a chat with your new bot and send `/start` +5. Get your **chat ID** by visiting: + ``` + https://api.telegram.org/bot/getUpdates + ``` + Look for `"chat":{"id":NNNNNN}` in the response +6. Add both values to your config: + ``` + telegram_bot_token=123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11 + telegram_chat_id=123456789 + ``` + +### Alert Behavior + +| Alert | Check interval | Back-off | +|-------|---------------|----------| +| Host offline | 30 seconds | 2min → 4min → 8min → ... → 8h | +| Disk space > 90% | 60 seconds | 2min → 4min → 8min → ... → 8h | +| Backup/rsync error | Per backup cycle | 2min → 4min → 8min → ... → 8h | + +Recovery messages are sent when the condition clears, and the back-off timer resets. + +## Requirements + +- Docker Engine with docker compose plugin +- Ubuntu Server (host) +- SSH key-based authentication to backup targets (for SSH hosts) +- Docker socket access (for container volume backups) + +## Docker Compose Reference + +```yaml +services: + backup: + build: . + privileged: true + network_mode: host + volumes: + - ./config:/etc/backup-tool # Configuration + - backup-state:/var/lib/backup-tool # Alert state + - backup-logs:/var/log/backup-tool # Logs + - /backup:/backup # Backup destination + - /var/run/docker.sock:/var/run/docker.sock:ro + - ~/.ssh:/root/.ssh:ro # SSH keys for rsync +``` + +The container needs: +- **privileged** — access to host devices and Docker socket +- **network_mode: host** — SSH to remote hosts without port mapping +- **Docker socket** — to query Swarm nodes and copy from volumes +- **SSH keys** — for passwordless rsync to remote hosts diff --git a/packages/backup-tool/backup-tool b/packages/backup-tool/backup-tool new file mode 100755 index 0000000..4a99c07 --- /dev/null +++ b/packages/backup-tool/backup-tool @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# backup-tool — main entry point. +# Runs the monitor loop in the background and executes timed incremental backups. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Load libraries +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" +# shellcheck source=lib/telegram.sh +source "${SCRIPT_DIR}/lib/telegram.sh" +# shellcheck source=lib/monitor.sh +source "${SCRIPT_DIR}/lib/monitor.sh" +# shellcheck source=lib/backup.sh +source "${SCRIPT_DIR}/lib/backup.sh" +# shellcheck source=lib/retention.sh +source "${SCRIPT_DIR}/lib/retention.sh" +# shellcheck source=lib/replication.sh +source "${SCRIPT_DIR}/lib/replication.sh" + +ensure_dirs + +# ── Usage ──────────────────────────────────────────────────────────────────── +usage() { + cat </dev/null || true; exit 0' EXIT INT TERM + log_info "Monitor started (PID $monitor_pid)" + + while true; do + run_cycle 2>&1 | tee -a "${BACKUP_LOG_DIR}/backup.log" + log_info "Next backup in ${interval_min}m — sleeping" + sleep "$interval_sec" + done +} + +# ── Main ───────────────────────────────────────────────────────────────────── +main() { + local cmd="${1:-run}" + case "$cmd" in + run) + run_loop + ;; + once) + run_cycle + ;; + setup) + # Re-exec into the setup wizard + exec "${SCRIPT_DIR}/setup" "${@:2}" + ;; + monitor) + monitor_loop + ;; + retention) + retention_all + ;; + replicate) + replicate_important + ;; + help|--help|-h) + usage + ;; + *) + log_error "Unknown command: $cmd" + usage >&2 + exit 1 + ;; + esac +} + +main "$@" diff --git a/packages/backup-tool/config/backup.conf.example b/packages/backup-tool/config/backup.conf.example new file mode 100644 index 0000000..d9cb67a --- /dev/null +++ b/packages/backup-tool/config/backup.conf.example @@ -0,0 +1,25 @@ +# backup-tool example configuration +# +# Copy this file to config/backup.conf and edit as needed, +# or run the interactive setup: docker compose run --rm backup setup + +# Where to store backups (must match the volume mount in docker-compose.yml) +backup_dest=/backup + +# How often to run backups (minutes) +backup_interval_min=60 + +# Important drives — comma-separated mount points for replication +# After backing up important folders, latest snapshots are replicated +# to every drive listed here (no RAID — simple independent copies). +# Replicas are compressed after copy; main backup stays uncompressed for rsync. +# important_drives=/mnt/usb1,/mnt/usb2 + +# Important folders — comma-separated, relative to backup_dest +# Format: host_label/safe_path (e.g. nas/var_data, pi4/home_user) +# important_folders=nas/var_data,pi4/home_user + +# Telegram alerts (optional) +# See README.md for setup instructions. +# telegram_bot_token=123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11 +# telegram_chat_id=123456789 diff --git a/packages/backup-tool/config/host.conf.example b/packages/backup-tool/config/host.conf.example new file mode 100644 index 0000000..c64d833 --- /dev/null +++ b/packages/backup-tool/config/host.conf.example @@ -0,0 +1,25 @@ +# Host config example — SSH host backed up via rsync +# +# Place one file per host in config/hosts.d/.conf +# Or use the interactive setup wizard. + +# Host type: "ssh" for rsync over SSH, "docker" for container volumes +type=ssh + +# SSH address (must have passwordless key auth configured) +address=user@192.168.1.100 + +# Paths to back up (comma-separated) +# For SSH hosts: remote directories +# For Docker hosts: docker volume names +paths=/home/user/data,/var/lib/important + +# Retention — time-based (days, 0 = keep forever) +retention_days=30 + +# Retention — size-based (optional, e.g. 10G, 500M) +# retention_max_size=10G + +# Per-path retention overrides (use safe_path: slashes replaced with _) +# retention_days_home_user_data=7 +# retention_max_size_var_lib_important=5G diff --git a/packages/backup-tool/docker-compose.yml b/packages/backup-tool/docker-compose.yml new file mode 100644 index 0000000..2eaeba8 --- /dev/null +++ b/packages/backup-tool/docker-compose.yml @@ -0,0 +1,36 @@ +# backup-tool — Docker Compose (run on swarm manager node) +# +# Deploy as a normal compose stack on the manager node: +# docker compose up -d +# +# Interactive setup (run once before starting): +# docker compose run --rm backup setup + +services: + backup: + build: . + container_name: backup-tool + restart: unless-stopped + privileged: true + network_mode: host + volumes: + # Persist configuration and state across restarts + - ./config:/etc/backup-tool + # Persist backup state (alert timers, etc.) + - backup-state:/var/lib/backup-tool + # Logs + - backup-logs:/var/log/backup-tool + # Backup destination — change this to your preferred drive/path + - /backup:/backup + # Docker socket — needed to query swarm and copy from volumes + - /var/run/docker.sock:/var/run/docker.sock:ro + # SSH keys — mount your host SSH keys for rsync to remote hosts + - ~/.ssh:/root/.ssh:ro + environment: + - TZ=${TZ:-UTC} + # Override to run setup wizard interactively: + # docker compose run --rm backup setup + +volumes: + backup-state: + backup-logs: diff --git a/packages/backup-tool/lib/backup.sh b/packages/backup-tool/lib/backup.sh new file mode 100644 index 0000000..b463030 --- /dev/null +++ b/packages/backup-tool/lib/backup.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Backup logic — rsync for SSH/bind-mount paths, docker cp for docker volumes. + +# ── rsync-based backup (SSH hosts or local/bind-mount paths) ───────────────── +backup_rsync() { + local src="$1" dest="$2" link_dest="${3:-}" + mkdir -p "$dest" + + local rsync_opts=( -a --delete --info=progress2 --timeout=300 ) + if [[ -n "$link_dest" && -d "$link_dest" ]]; then + rsync_opts+=( --link-dest="$link_dest" ) + fi + + log_info "rsync: $src -> $dest" + if ! rsync "${rsync_opts[@]}" "$src" "$dest/" 2>&1; then + log_error "rsync failed: $src -> $dest" + alert_fire "rsync:${src}" "❌ rsync failed for \`${src}\`" + return 1 + fi + alert_resolve "rsync:${src}" "✅ rsync recovered for \`${src}\`" + return 0 +} + +# ── Docker volume backup (cp via temp container) ───────────────────────────── +backup_docker_volume() { + local volume="$1" dest="$2" host="${3:-}" + mkdir -p "$dest" + + local docker_cmd="docker" + if [[ -n "$host" ]]; then + docker_cmd="docker -H ssh://${host}" + fi + + local tmp_container + tmp_container="backup-vol-$$-$(date +%s)" + log_info "docker volume backup: $volume -> $dest" + + # Create a temporary container that mounts the volume + if ! $docker_cmd run --rm -d --name "$tmp_container" \ + -v "${volume}:/volume:ro" alpine:latest tail -f /dev/null >/dev/null 2>&1; then + log_error "Failed to start temp container for volume $volume" + alert_fire "dockervol:${volume}" "❌ Cannot backup docker volume \`${volume}\`" + return 1 + fi + + # Copy data out + if ! $docker_cmd cp "${tmp_container}:/volume/." "$dest/" 2>&1; then + log_error "docker cp failed for volume $volume" + $docker_cmd rm -f "$tmp_container" >/dev/null 2>&1 || true + alert_fire "dockervol:${volume}" "❌ docker cp failed for volume \`${volume}\`" + return 1 + fi + + $docker_cmd rm -f "$tmp_container" >/dev/null 2>&1 || true + alert_resolve "dockervol:${volume}" "✅ docker volume \`${volume}\` backup recovered" + log_ok "docker volume $volume backed up" + return 0 +} + +# ── Back up a single host (all its paths) ──────────────────────────────────── +# Reads host config file and runs appropriate backup method for each path. +# Creates timestamped snapshot directories, links to "latest" for incremental. +backup_host() { + local host_conf="$1" + local host_label + host_label=$(basename "$host_conf" .conf) + + local host_type host_address + host_type=$(cfg_get "$host_conf" type) # ssh | docker + host_address=$(cfg_get "$host_conf" address) # user@host or docker node addr + + local paths + paths=$(cfg_get "$host_conf" paths) # comma-separated + [[ -z "$paths" ]] && { log_warn "No paths for host $host_label"; return 0; } + + local backup_dest + backup_dest=$(cfg_get "$BACKUP_CONF_FILE" backup_dest "/backup") + + local ts + ts=$(date +%Y%m%d-%H%M%S) + + local IFS=',' + local path_errors=0 + for p in $paths; do + p=$(echo "$p" | xargs) # trim whitespace + [[ -z "$p" ]] && continue + + local safe_path="${p//\//_}" + safe_path="${safe_path#_}" + local host_dest="${backup_dest}/${host_label}/${safe_path}" + local snap_dir="${host_dest}/${ts}" + local latest_link="${host_dest}/latest" + + local volume_type + volume_type=$(cfg_get "$host_conf" "volume_type_${safe_path}" "") + # If not set per-path, fall back to host-level default + [[ -z "$volume_type" ]] && volume_type="$host_type" + + case "$volume_type" in + ssh) + local link_dest="" + [[ -L "$latest_link" ]] && link_dest=$(readlink -f "$latest_link") + if backup_rsync "${host_address}:${p}/" "$snap_dir" "$link_dest"; then + ln -sfn "$snap_dir" "$latest_link" + else + path_errors=$(( path_errors + 1 )) + fi + ;; + docker) + if backup_docker_volume "$p" "$snap_dir" "$host_address"; then + ln -sfn "$snap_dir" "$latest_link" + else + path_errors=$(( path_errors + 1 )) + fi + ;; + *) + log_error "Unknown volume type '$volume_type' for $host_label:$p" + path_errors=$(( path_errors + 1 )) + ;; + esac + done + + return "$path_errors" +} + +# ── Run backups for all configured hosts ───────────────────────────────────── +backup_all() { + local errors=0 + if [[ ! -d "$BACKUP_HOSTS_DIR" ]]; then + log_warn "No hosts directory at $BACKUP_HOSTS_DIR" + return 0 + fi + + for hcfg in "$BACKUP_HOSTS_DIR"/*.conf; do + [[ -f "$hcfg" ]] || continue + local label + label=$(basename "$hcfg" .conf) + log_step "Backing up host: $label" + if ! backup_host "$hcfg"; then + errors=$(( errors + 1 )) + fi + done + + if (( errors > 0 )); then + log_warn "$errors host(s) had errors" + return 1 + fi + log_ok "All hosts backed up successfully" + return 0 +} diff --git a/packages/backup-tool/lib/common.sh b/packages/backup-tool/lib/common.sh new file mode 100644 index 0000000..09edd1a --- /dev/null +++ b/packages/backup-tool/lib/common.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Common helpers for backup-tool + +# Exported via source — used by backup-tool and setup +# shellcheck disable=SC2034 +BACKUP_TOOL_VERSION="1.0.0" + +# ── Colors ─────────────────────────────────────────────────────────────────── +COL_RESET="" COL_RED="" COL_GREEN="" COL_YELLOW="" COL_CYAN="" COL_BLUE="" +if [[ -t 2 ]] && command -v tput >/dev/null 2>&1 && [[ $(tput colors 2>/dev/null || echo 0) -ge 8 ]]; then + COL_RESET="\033[0m"; COL_RED="\033[31m"; COL_GREEN="\033[32m" + COL_YELLOW="\033[33m"; COL_CYAN="\033[36m"; COL_BLUE="\033[34m" +fi + +log_info() { echo -e "${COL_BLUE}[INFO]${COL_RESET} $(date '+%H:%M:%S') $*" >&2; } +log_ok() { echo -e "${COL_GREEN}[OK]${COL_RESET} $(date '+%H:%M:%S') $*" >&2; } +log_warn() { echo -e "${COL_YELLOW}[WARN]${COL_RESET} $(date '+%H:%M:%S') $*" >&2; } +log_error() { echo -e "${COL_RED}[ERROR]${COL_RESET} $(date '+%H:%M:%S') $*" >&2; } +log_step() { echo -e "\n${COL_CYAN}[+]${COL_RESET} $*" >&2; } + +# ── Config paths ───────────────────────────────────────────────────────────── +BACKUP_CONF_DIR="${BACKUP_CONF_DIR:-/etc/backup-tool}" +# shellcheck disable=SC2034 +BACKUP_CONF_FILE="${BACKUP_CONF_DIR}/backup.conf" +BACKUP_HOSTS_DIR="${BACKUP_CONF_DIR}/hosts.d" +BACKUP_STATE_DIR="${BACKUP_STATE_DIR:-/var/lib/backup-tool}" +BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-/var/log/backup-tool}" + +ensure_dirs() { + mkdir -p "$BACKUP_CONF_DIR" "$BACKUP_HOSTS_DIR" \ + "$BACKUP_STATE_DIR" "$BACKUP_LOG_DIR" +} + +# ── Config helpers ─────────────────────────────────────────────────────────── +# Read a key from an INI-style config file (key=value) +cfg_get() { + local file="$1" key="$2" default="${3:-}" + if [[ -f "$file" ]]; then + local val + val=$(grep -m1 "^${key}=" "$file" 2>/dev/null | cut -d'=' -f2- | sed 's/^[ "]*//;s/[ "]*$//') + if [[ -n "$val" ]]; then + echo "$val" + return + fi + fi + echo "$default" +} + +# Write a key to an INI-style config file (upsert) +cfg_set() { + local file="$1" key="$2" value="$3" + mkdir -p "$(dirname "$file")" + if [[ -f "$file" ]] && grep -q "^${key}=" "$file" 2>/dev/null; then + sed -i "s|^${key}=.*|${key}=${value}|" "$file" + else + echo "${key}=${value}" >> "$file" + fi +} + +# ── Misc ───────────────────────────────────────────────────────────────────── +require_cmd() { + local cmd="$1" pkg="${2:-$1}" + if ! command -v "$cmd" >/dev/null 2>&1; then + log_error "'$cmd' is required but not found. Install it: apt-get install $pkg" + return 1 + fi +} + +ts_now() { date +%s; } + +human_size() { + local bytes="$1" + if (( bytes >= 1073741824 )); then echo "$(( bytes / 1073741824 ))G" + elif (( bytes >= 1048576 )); then echo "$(( bytes / 1048576 ))M" + elif (( bytes >= 1024 )); then echo "$(( bytes / 1024 ))K" + else echo "${bytes}B" + fi +} diff --git a/packages/backup-tool/lib/monitor.sh b/packages/backup-tool/lib/monitor.sh new file mode 100644 index 0000000..1b36da5 --- /dev/null +++ b/packages/backup-tool/lib/monitor.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Background monitor — runs independently of backup schedule. +# • Host-online checks every 30 s +# • Disk-space checks every 60 s + +MONITOR_PING_INTERVAL=30 +MONITOR_DISK_INTERVAL=60 +DISK_WARN_PERCENT=90 + +# ── Host online check ──────────────────────────────────────────────────────── +check_host_online() { + local host="$1" label="${2:-$1}" + if ssh -o ConnectTimeout=5 -o BatchMode=yes "$host" true 2>/dev/null; then + alert_resolve "offline:${label}" "✅ *${label}* is back online." + return 0 + else + alert_fire "offline:${label}" "🔴 *${label}* is OFFLINE — cannot reach via SSH." + return 1 + fi +} + +# ── Disk space check ───────────────────────────────────────────────────────── +check_disk_space() { + local mount="$1" label="${2:-$1}" + local usage + usage=$(df --output=pcent "$mount" 2>/dev/null | tail -1 | tr -d '% ') + if [[ -z "$usage" ]]; then + alert_fire "disk:${label}" "⚠️ Cannot read disk usage for *${label}* (\`${mount}\`)." + return 1 + fi + if (( usage >= DISK_WARN_PERCENT )); then + alert_fire "disk:${label}" "💾 Disk *${label}* (\`${mount}\`) is at *${usage}%* — running out of space!" + return 1 + else + alert_resolve "disk:${label}" "✅ Disk *${label}* (\`${mount}\`) back to normal (*${usage}%*)." + return 0 + fi +} + +# ── Monitor loop (meant to run as background process) ──────────────────────── +monitor_loop() { + local last_disk_check=0 + log_info "Monitor started (ping=${MONITOR_PING_INTERVAL}s, disk=${MONITOR_DISK_INTERVAL}s)" + + while true; do + local now + now=$(date +%s) + + # -- Host online checks (every 30 s) -- + if [[ -d "$BACKUP_HOSTS_DIR" ]]; then + local host_count=0 + for hcfg in "$BACKUP_HOSTS_DIR"/*.conf; do + [[ -f "$hcfg" ]] || continue + local htype haddr + htype=$(cfg_get "$hcfg" type) + haddr=$(cfg_get "$hcfg" address) + local hlabel + hlabel=$(basename "$hcfg" .conf) + if [[ "$htype" == "ssh" && -n "$haddr" ]]; then + # Limit concurrent SSH checks to avoid overwhelming the network + if (( host_count >= 10 )); then + wait + host_count=0 + fi + check_host_online "$haddr" "$hlabel" & + host_count=$(( host_count + 1 )) + fi + done + wait # reap background pings + fi + + # -- Disk space checks (every 60 s) -- + if (( now - last_disk_check >= MONITOR_DISK_INTERVAL )); then + last_disk_check=$now + local drives + drives=$(cfg_get "$BACKUP_CONF_FILE" important_drives) + if [[ -n "$drives" ]]; then + local IFS=',' + for drive in $drives; do + check_disk_space "$drive" "$drive" + done + fi + # Also check main backup destination + local dest + dest=$(cfg_get "$BACKUP_CONF_FILE" backup_dest) + if [[ -n "$dest" ]]; then + check_disk_space "$dest" "backup-dest" + fi + fi + + sleep "$MONITOR_PING_INTERVAL" + done +} diff --git a/packages/backup-tool/lib/replication.sh b/packages/backup-tool/lib/replication.sh new file mode 100644 index 0000000..f1ca927 --- /dev/null +++ b/packages/backup-tool/lib/replication.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Replication — copy important backups to every important drive. +# Main copy stays uncompressed (for fast rsync). Replicas are compressed. + +# ── Replicate a directory to a drive ───────────────────────────────────────── +replicate_to_drive() { + local src="$1" drive="$2" label="$3" + local dest="${drive}/backup-replicas/${label}" + mkdir -p "$dest" + + log_info "Replicating $label -> $dest" + if ! rsync -a --delete "$src/" "$dest/" 2>&1; then + log_error "Replication failed: $src -> $dest" + alert_fire "replicate:${label}:${drive}" \ + "❌ Replication failed for \`${label}\` to \`${drive}\`" + return 1 + fi + alert_resolve "replicate:${label}:${drive}" \ + "✅ Replication recovered for \`${label}\` to \`${drive}\`" + return 0 +} + +# ── Compress a replica directory in-place ──────────────────────────────────── +compress_replica() { + local dir="$1" + local archive="${dir}.tar.gz" + [[ -d "$dir" ]] || return 0 + + log_info "Compressing replica: $dir" + if tar -czf "$archive" -C "$(dirname "$dir")" "$(basename "$dir")" 2>&1; then + rm -rf "$dir" + log_ok "Compressed: $archive" + else + log_warn "Compression failed for $dir — keeping uncompressed copy" + fi +} + +# ── Replicate all important folders to all important drives ────────────────── +replicate_important() { + local backup_dest + backup_dest=$(cfg_get "$BACKUP_CONF_FILE" backup_dest "/backup") + local important_folders + important_folders=$(cfg_get "$BACKUP_CONF_FILE" important_folders) + local important_drives + important_drives=$(cfg_get "$BACKUP_CONF_FILE" important_drives) + + if [[ -z "$important_folders" ]]; then + log_info "No important folders configured — skipping replication" + return 0 + fi + if [[ -z "$important_drives" ]]; then + log_info "No important drives configured — skipping replication" + return 0 + fi + + local IFS=',' + local errors=0 + for folder in $important_folders; do + folder=$(echo "$folder" | xargs) + [[ -z "$folder" ]] && continue + + # folder is relative to backup_dest, e.g. "myhost/var_data" + local src="${backup_dest}/${folder}/latest" + if [[ ! -d "$src" ]]; then + log_warn "Important folder latest snapshot not found: $src" + continue + fi + + for drive in $important_drives; do + drive=$(echo "$drive" | xargs) + [[ -z "$drive" ]] && continue + if [[ ! -d "$drive" ]]; then + log_warn "Important drive not mounted: $drive" + alert_fire "drive:${drive}" "⚠️ Important drive \`${drive}\` not mounted" + errors=$(( errors + 1 )) + continue + fi + + local safe_label="${folder//\//_}" + if replicate_to_drive "$src" "$drive" "$safe_label"; then + compress_replica "${drive}/backup-replicas/${safe_label}" + else + errors=$(( errors + 1 )) + fi + done + done + + if (( errors > 0 )); then + log_warn "Replication finished with $errors error(s)" + return 1 + fi + log_ok "All important folders replicated and compressed" + return 0 +} diff --git a/packages/backup-tool/lib/retention.sh b/packages/backup-tool/lib/retention.sh new file mode 100644 index 0000000..749a068 --- /dev/null +++ b/packages/backup-tool/lib/retention.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Retention — time-based and size-based TTL per host/path. +# +# Each host config can specify: +# retention_days=30 (delete snapshots older than N days) +# retention_max_size=10G (keep total size under limit, oldest first) +# +# These can also be overridden per-path with: +# retention_days_=7 +# retention_max_size_=2G + +# Parse human size string (e.g. 10G, 500M) to bytes +parse_size() { + local s="${1^^}" + local num="${s//[^0-9.]/}" + case "$s" in + *T) awk "BEGIN{printf \"%d\", $num * 1099511627776}"; ;; + *G) awk "BEGIN{printf \"%d\", $num * 1073741824}"; ;; + *M) awk "BEGIN{printf \"%d\", $num * 1048576}"; ;; + *K) awk "BEGIN{printf \"%d\", $num * 1024}"; ;; + *) echo "${num%.*}"; ;; + esac +} + +# Get total size of a directory in bytes +dir_size_bytes() { + du -sb "$1" 2>/dev/null | awk '{print $1}' +} + +# Apply retention to a single path directory (contains timestamped snapshots). +# $1 = path directory (e.g. /backup/myhost/var_data) +# $2 = max age in days (0 = unlimited) +# $3 = max size string (empty = unlimited) +apply_retention() { + local path_dir="$1" max_days="${2:-0}" max_size_str="${3:-}" + + [[ -d "$path_dir" ]] || return 0 + + # Enumerate snapshots (directories named YYYYMMDD-HHMMSS), sorted oldest first + local snaps=() + while IFS= read -r d; do + [[ -d "$d" ]] && snaps+=("$d") + done < <(find "$path_dir" -mindepth 1 -maxdepth 1 -type d \ + -regextype posix-extended -regex '.*/[0-9]{8}-[0-9]{6}' 2>/dev/null | sort) + + if [[ ${#snaps[@]} -le 1 ]]; then + return 0 # always keep at least one snapshot + fi + + local removed=0 + + # ── Time-based retention ── + if (( max_days > 0 )); then + local cutoff + cutoff=$(date -d "${max_days} days ago" +%s 2>/dev/null || date -v-"${max_days}"d +%s 2>/dev/null || echo "") + if [[ -z "$cutoff" || "$cutoff" == "0" ]]; then + log_warn "Cannot compute cutoff date — skipping time-based retention" + return 0 + fi + local keep=() + local total_count=${#snaps[@]} + local kept_count=0 + for snap in "${snaps[@]}"; do + local snap_name + snap_name=$(basename "$snap") + # Parse YYYYMMDD-HHMMSS + local snap_ts + snap_ts=$(date -d "${snap_name:0:4}-${snap_name:4:2}-${snap_name:6:2} ${snap_name:9:2}:${snap_name:11:2}:${snap_name:13:2}" +%s 2>/dev/null || echo 0) + if (( snap_ts == 0 )); then + log_warn "Cannot parse timestamp for snapshot $snap — skipping" + keep+=("$snap") + kept_count=$(( kept_count + 1 )) + continue + fi + local remaining=$(( total_count - removed - kept_count )) + if (( snap_ts > 0 && snap_ts < cutoff && remaining > 1 )); then + log_info "Retention (age): removing $snap" + rm -rf "$snap" + removed=$(( removed + 1 )) + else + keep+=("$snap") + kept_count=$(( kept_count + 1 )) + fi + done + if [[ ${#keep[@]} -gt 0 ]]; then + snaps=("${keep[@]}") + else + snaps=() + fi + fi + + # ── Size-based retention ── + if [[ -n "$max_size_str" ]]; then + local max_bytes + max_bytes=$(parse_size "$max_size_str") + local total + total=$(dir_size_bytes "$path_dir") + + while (( total > max_bytes && ${#snaps[@]} > 1 )); do + local oldest="${snaps[0]}" + local oldest_size + oldest_size=$(dir_size_bytes "$oldest") + log_info "Retention (size): removing $oldest ($(human_size "$oldest_size"))" + rm -rf "$oldest" + snaps=("${snaps[@]:1}") + total=$(( total - oldest_size )) + removed=$(( removed + 1 )) + done + fi + + if (( removed > 0 )); then + log_info "Retention: removed $removed snapshot(s) from $path_dir" + fi +} + +# Apply retention to all hosts/paths based on their config +retention_all() { + local backup_dest + backup_dest=$(cfg_get "$BACKUP_CONF_FILE" backup_dest "/backup") + + for hcfg in "$BACKUP_HOSTS_DIR"/*.conf; do + [[ -f "$hcfg" ]] || continue + local label + label=$(basename "$hcfg" .conf) + local default_days default_size + default_days=$(cfg_get "$hcfg" retention_days 0) + default_size=$(cfg_get "$hcfg" retention_max_size "") + + local paths + paths=$(cfg_get "$hcfg" paths) + [[ -z "$paths" ]] && continue + + local IFS=',' + for p in $paths; do + p=$(echo "$p" | xargs) + [[ -z "$p" ]] && continue + local safe_path="${p//\//_}" + safe_path="${safe_path#_}" + local path_dir="${backup_dest}/${label}/${safe_path}" + + local days size + days=$(cfg_get "$hcfg" "retention_days_${safe_path}" "$default_days") + size=$(cfg_get "$hcfg" "retention_max_size_${safe_path}" "$default_size") + + apply_retention "$path_dir" "$days" "$size" + done + done +} diff --git a/packages/backup-tool/lib/telegram.sh b/packages/backup-tool/lib/telegram.sh new file mode 100644 index 0000000..7e6f714 --- /dev/null +++ b/packages/backup-tool/lib/telegram.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Telegram alert system with exponential back-off. +# +# Alert state is persisted per alert-key so the same alert is not spammed. +# Back-off: first alert at 0, then 2m, 4m, 8m, 16m … capped at 8 h. +# When the condition clears, a recovery message is sent and state is reset. + +TELEGRAM_STATE_DIR="${BACKUP_STATE_DIR:-/var/lib/backup-tool}/alerts" +ALERT_MIN_INTERVAL=120 # 2 minutes (first re-alert) +ALERT_MAX_INTERVAL=28800 # 8 hours + +# ── Send raw Telegram message ──────────────────────────────────────────────── +telegram_send() { + local token="$1" chat_id="$2" text="$3" + local url="https://api.telegram.org/bot${token}/sendMessage" + curl -sf --max-time 10 \ + -X POST "$url" \ + -d chat_id="$chat_id" \ + -d parse_mode="Markdown" \ + -d text="$text" >/dev/null 2>&1 +} + +# ── Load bot credentials from config ───────────────────────────────────────── +_tg_load_creds() { + TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-$(cfg_get "$BACKUP_CONF_FILE" telegram_bot_token)}" + TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID:-$(cfg_get "$BACKUP_CONF_FILE" telegram_chat_id)}" +} + +_tg_enabled() { + _tg_load_creds + [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$TELEGRAM_CHAT_ID" ]] +} + +# ── Stateful alerting with exponential back-off ────────────────────────────── +# alert_key : unique id for this alert (e.g. "offline:myhost") +# message : markdown text to send +alert_fire() { + local alert_key="$1" message="$2" + _tg_enabled || return 0 + + mkdir -p "$TELEGRAM_STATE_DIR" + local state_file="${TELEGRAM_STATE_DIR}/${alert_key//\//_}" + + local now last_sent interval next_due + now=$(date +%s) + + if [[ -f "$state_file" ]]; then + last_sent=$(sed -n '1p' "$state_file") + interval=$(sed -n '2p' "$state_file") + next_due=$(( last_sent + interval )) + if (( now < next_due )); then + return 0 # too soon, suppress + fi + # double the interval + interval=$(( interval * 2 )) + (( interval > ALERT_MAX_INTERVAL )) && interval=$ALERT_MAX_INTERVAL + else + interval=$ALERT_MIN_INTERVAL + fi + + if telegram_send "$TELEGRAM_BOT_TOKEN" "$TELEGRAM_CHAT_ID" "$message"; then + printf '%s\n%s\n' "$now" "$interval" > "$state_file" + fi +} + +# Called when the condition that triggered the alert has cleared. +alert_resolve() { + local alert_key="$1" message="$2" + local state_file="${TELEGRAM_STATE_DIR}/${alert_key//\//_}" + [[ -f "$state_file" ]] || return 0 # was never fired + _tg_enabled || { rm -f "$state_file"; return 0; } + + telegram_send "$TELEGRAM_BOT_TOKEN" "$TELEGRAM_CHAT_ID" "$message" || true + rm -f "$state_file" +} diff --git a/packages/backup-tool/setup b/packages/backup-tool/setup new file mode 100755 index 0000000..657f0b6 --- /dev/null +++ b/packages/backup-tool/setup @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Interactive setup wizard for backup-tool. +# Guides the user through configuring hosts, paths, drives, and Telegram alerts. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" +# shellcheck source=lib/telegram.sh +source "${SCRIPT_DIR}/lib/telegram.sh" + +ensure_dirs + +# ── TUI helpers ────────────────────────────────────────────────────────────── +prompt() { + local var="$1" msg="$2" default="${3:-}" + local input + if [[ -n "$default" ]]; then + read -rp "$msg [$default]: " input + eval "$var=\"${input:-$default}\"" + else + read -rp "$msg: " input + eval "$var=\"$input\"" + fi +} + +prompt_yesno() { + local msg="$1" default="${2:-n}" + local answer + read -rp "$msg [y/n] ($default): " answer + answer="${answer:-$default}" + [[ "$answer" =~ ^[yY] ]] +} + +# Numbered menu selector — prints chosen value(s). Allows multiple selection. +select_menu() { + local title="$1"; shift + local -a items=("$@") + local -a selected=() + + echo "" + echo -e "${COL_CYAN}${title}${COL_RESET}" + local i=1 + for item in "${items[@]}"; do + echo " $i) $item" + (( i++ )) + done + echo " 0) Done selecting" + + while true; do + local choice + read -rp "Select (number): " choice + if [[ "$choice" == "0" ]]; then + break + fi + if [[ "$choice" =~ ^[0-9]+$ ]] && (( choice >= 1 && choice <= ${#items[@]} )); then + selected+=("${items[choice-1]}") + echo -e " ${COL_GREEN}+ ${items[choice-1]}${COL_RESET}" + else + echo " Invalid choice" + fi + done + + printf '%s\n' "${selected[@]}" +} + +# ── Step 1: Backup destination ─────────────────────────────────────────────── +setup_destination() { + log_step "Backup destination" + local dest + prompt dest "Where should backups be stored?" "/backup" + mkdir -p "$dest" + cfg_set "$BACKUP_CONF_FILE" backup_dest "$dest" + log_ok "Backup destination: $dest" +} + +# ── Step 2: Backup interval ───────────────────────────────────────────────── +setup_schedule() { + log_step "Backup schedule" + local interval + prompt interval "Backup interval in minutes" "60" + cfg_set "$BACKUP_CONF_FILE" backup_interval_min "$interval" + log_ok "Backup every ${interval} minutes" +} + +# ── Step 3: Add hosts ─────────────────────────────────────────────────────── +setup_hosts() { + log_step "Configure backup hosts" + echo " You can add SSH hosts (rsync) and/or Docker Swarm nodes." + + while true; do + echo "" + echo " 1) Add SSH host (rsync)" + echo " 2) Add Docker Swarm node (container volumes)" + echo " 0) Done adding hosts" + local choice + read -rp "Choice: " choice + + case "$choice" in + 1) add_ssh_host ;; + 2) add_docker_host ;; + 0) break ;; + *) echo " Invalid choice" ;; + esac + done +} + +add_ssh_host() { + local name addr + prompt name "Short name for this host (e.g. nas, pi4)" "" + prompt addr "SSH address (user@host)" "" + + [[ -z "$name" || -z "$addr" ]] && { log_warn "Skipped — name and address required"; return; } + + log_info "Testing SSH connection to $addr ..." + if ssh -o ConnectTimeout=5 -o BatchMode=yes "$addr" true 2>/dev/null; then + log_ok "Connected to $addr" + else + log_warn "Cannot connect to $addr — saving anyway (check SSH keys)" + fi + + local hcfg="${BACKUP_HOSTS_DIR}/${name}.conf" + cfg_set "$hcfg" type "ssh" + cfg_set "$hcfg" address "$addr" + + setup_host_paths "$hcfg" "$name" "$addr" "ssh" + setup_host_retention "$hcfg" "$name" + log_ok "Host '$name' saved" +} + +add_docker_host() { + local name addr + prompt name "Short name for this Docker node (e.g. swarm1)" "" + prompt addr "SSH address of the Docker node (user@host)" "" + + [[ -z "$name" || -z "$addr" ]] && { log_warn "Skipped — name and address required"; return; } + + local hcfg="${BACKUP_HOSTS_DIR}/${name}.conf" + cfg_set "$hcfg" type "docker" + cfg_set "$hcfg" address "$addr" + + # List containers / volumes on the remote node + log_info "Fetching Docker volumes from $addr ..." + local volumes + volumes=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "$addr" \ + "docker volume ls --format '{{.Name}}'" 2>/dev/null || echo "") + + if [[ -n "$volumes" ]]; then + echo " Available volumes on $addr:" + local vol_arr=() + while IFS= read -r v; do vol_arr+=("$v"); done <<< "$volumes" + local chosen + chosen=$(select_menu "Select volumes to back up:" "${vol_arr[@]}") + if [[ -n "$chosen" ]]; then + local joined="" + while IFS= read -r v; do + joined="${joined:+${joined},}${v}" + done <<< "$chosen" + cfg_set "$hcfg" paths "$joined" + fi + else + log_warn "Could not list volumes — enter paths manually" + setup_host_paths "$hcfg" "$name" "$addr" "docker" + fi + + setup_host_retention "$hcfg" "$name" + log_ok "Docker host '$name' saved" +} + +setup_host_paths() { + local hcfg="$1" name="$2" addr="$3" + # $4 = htype (reserved for future per-type path validation) + echo "" + echo " Enter paths to back up on $name ($addr)." + echo " For SSH hosts these are remote directory paths." + echo " For Docker hosts these are volume names." + + local paths=() + while true; do + local p + read -rp " Path (empty to finish): " p + [[ -z "$p" ]] && break + paths+=("$p") + done + + if [[ ${#paths[@]} -gt 0 ]]; then + local joined + joined=$(printf '%s,' "${paths[@]}") + joined="${joined%,}" + cfg_set "$hcfg" paths "$joined" + fi +} + +setup_host_retention() { + local hcfg="$1" name="$2" + echo "" + local days size + prompt days "Retention days for $name (0 = keep forever)" "30" + prompt size "Max total size for $name (e.g. 10G, empty = unlimited)" "" + cfg_set "$hcfg" retention_days "$days" + [[ -n "$size" ]] && cfg_set "$hcfg" retention_max_size "$size" +} + +# ── Step 4: Important drives ──────────────────────────────────────────────── +setup_drives() { + log_step "Important drives (for replication)" + echo " Select mount points to replicate important backups to." + echo " Each drive gets a compressed copy. (No RAID — simple copies.)" + + # List mounted block devices (exclude tmpfs, devtmpfs, etc.) + local mounts=() + while IFS= read -r m; do + mounts+=("$m") + done < <(df -h --output=target,size,avail,fstype 2>/dev/null \ + | tail -n +2 | grep -v -E 'tmpfs|devtmpfs|squashfs|overlay' \ + | awk '{print $1 " (" $2 " total, " $3 " free, " $4 ")"}') + + if [[ ${#mounts[@]} -gt 0 ]]; then + local chosen + chosen=$(select_menu "Available mount points:" "${mounts[@]}") + if [[ -n "$chosen" ]]; then + # Extract just the mount path (first field) + local drives=() + while IFS= read -r line; do + drives+=("$(echo "$line" | awk '{print $1}')") + done <<< "$chosen" + local joined + joined=$(printf '%s,' "${drives[@]}") + joined="${joined%,}" + cfg_set "$BACKUP_CONF_FILE" important_drives "$joined" + log_ok "Important drives: $joined" + fi + else + echo " No suitable mount points detected. Enter manually:" + local drives=() + while true; do + local d + read -rp " Drive mount path (empty to finish): " d + [[ -z "$d" ]] && break + drives+=("$d") + done + if [[ ${#drives[@]} -gt 0 ]]; then + local joined + joined=$(printf '%s,' "${drives[@]}") + joined="${joined%,}" + cfg_set "$BACKUP_CONF_FILE" important_drives "$joined" + fi + fi +} + +# ── Step 5: Important folders ──────────────────────────────────────────────── +setup_important_folders() { + log_step "Important folders (replicated to all drives)" + echo " Enter folder paths relative to backup destination." + echo " Format: host/path_name (e.g. nas/var_data, pi4/home_user)" + + local folders=() + while true; do + local f + read -rp " Important folder (empty to finish): " f + [[ -z "$f" ]] && break + folders+=("$f") + done + + if [[ ${#folders[@]} -gt 0 ]]; then + local joined + joined=$(printf '%s,' "${folders[@]}") + joined="${joined%,}" + cfg_set "$BACKUP_CONF_FILE" important_folders "$joined" + log_ok "Important folders: $joined" + fi +} + +# ── Step 6: Telegram alerts ───────────────────────────────────────────────── +setup_telegram() { + log_step "Telegram alerts" + echo "" + echo " To set up Telegram alerts:" + echo " 1. Open Telegram and search for @BotFather" + echo " 2. Send /newbot and follow the prompts to create a bot" + echo " 3. Copy the bot token (e.g. 123456:ABC-DEF...)" + echo " 4. Start a chat with your new bot (send /start)" + echo " 5. To get your chat ID, visit:" + echo " https://api.telegram.org/bot/getUpdates" + echo " Look for \"chat\":{\"id\":NNNNNN}" + echo "" + + if ! prompt_yesno "Configure Telegram alerts now?"; then + log_info "Skipping Telegram setup" + return + fi + + local token chat_id + prompt token "Bot token" "" + prompt chat_id "Chat ID" "" + + if [[ -n "$token" && -n "$chat_id" ]]; then + cfg_set "$BACKUP_CONF_FILE" telegram_bot_token "$token" + cfg_set "$BACKUP_CONF_FILE" telegram_chat_id "$chat_id" + + # Send test message + if telegram_send "$token" "$chat_id" "🔧 *backup-tool* test message — alerts configured!"; then + log_ok "Test message sent — check your Telegram!" + else + log_warn "Could not send test message. Check token/chat_id." + fi + else + log_warn "Token and chat ID both required — skipping" + fi +} + +# ── Main wizard ────────────────────────────────────────────────────────────── +main() { + echo "" + echo -e "${COL_CYAN}════════════════════════════════════════════${COL_RESET}" + echo -e "${COL_CYAN} backup-tool — interactive setup ${COL_RESET}" + echo -e "${COL_CYAN}════════════════════════════════════════════${COL_RESET}" + echo "" + + setup_destination + setup_schedule + setup_hosts + setup_drives + setup_important_folders + setup_telegram + + echo "" + log_ok "Configuration complete!" + echo " Config : $BACKUP_CONF_FILE" + echo " Hosts : $BACKUP_HOSTS_DIR/" + echo "" + echo " Start the backup service:" + echo " docker compose up -d" + echo "" +} + +main "$@"