#!/usr/bin/env bash ######################################################################################### #### hetzner-snapshot-manager.sh — Create, rotate, list, audit, and restore Hetzner #### #### Cloud server snapshots via the REST API. Automated retention and cost tracking #### #### Requires: bash 4+, curl, jq #### #### #### #### Author: Phil Connor #### #### Contact: contact@mylinux.work #### #### License: MIT #### #### Version 1.01 #### #### #### #### Usage: #### #### ./hetzner-snapshot-manager.sh --snapshot --all #### #### #### #### See --help for all options. #### ######################################################################################### set -euo pipefail # ── Colors (pre-initialized) ───────────────────────────────────────── RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET="" setup_colors() { if [[ "${COLOR:-auto}" == "never" ]]; then return fi if [[ "${COLOR:-auto}" == "always" ]] || [[ -t 1 ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' fi } # ── Logging ─────────────────────────────────────────────────────────── log() { echo -e "${BLUE}[INFO]${RESET} $*"; } warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; } err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; } verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; } die() { err "$*"; exit 1; } section_header() { echo "" echo -e " ${BOLD}${CYAN}── $1 ──${RESET}" echo "" } field() { printf " ${BOLD}%-22s${RESET} %s\n" "$1" "$2" } field_color() { printf " ${BOLD}%-22s${RESET} %b\n" "$1" "$2" } elapsed() { local end_time end_time=$(date +%s) echo "$(( end_time - START_TIME ))s" } # ── Defaults ────────────────────────────────────────────────────────── RUN_MODE="" ALSO_ROTATE="false" SERVER_ID="" TARGET_ALL="false" SNAPSHOT_ID="" KEEP="${HSM_KEEP:-3}" PREFIX="${HSM_PREFIX:-auto}" MAX_AGE="${HSM_MAX_AGE:-7}" OUTPUT_FORMAT="${HSM_FORMAT:-text}" DRY_RUN="true" FORCE="false" VERBOSE="${VERBOSE:-false}" COLOR="${COLOR:-auto}" # ── Credentials ─────────────────────────────────────────────────────── HCLOUD_TOKEN="${HCLOUD_TOKEN:-}" # ── State ───────────────────────────────────────────────────────────── SCRIPT_NAME="$(basename "$0")" readonly SCRIPT_NAME START_TIME="" SNAP_CREATED=0 SNAP_DELETED=0 SNAP_ERRORS=0 # ── Cost constant ──────────────────────────────────────────────────── COST_PER_GB_MONTH="0.012" # ── API helpers ────────────────────────────────────────────────────── hetzner_api() { local method="$1" endpoint="$2" shift 2 local attempt=0 max_attempts=3 while (( attempt < max_attempts )); do local http_code http_code=$(curl -s -o /tmp/hsm_resp.json -w "%{http_code}" \ -X "$method" \ -H "Authorization: Bearer ${HCLOUD_TOKEN}" \ -H "Content-Type: application/json" \ "https://api.hetzner.cloud/v1${endpoint}" "$@") verbose "API ${method} ${endpoint} → HTTP ${http_code}" if [[ "$http_code" == "429" ]]; then ((attempt++)) || true local wait=$(( attempt * 5 )) warn "Rate limited — retrying in ${wait}s (attempt ${attempt}/${max_attempts})" sleep "$wait" continue fi cat /tmp/hsm_resp.json return 0 done err "API request failed after ${max_attempts} attempts: ${method} ${endpoint}" return 1 } check_credentials() { [[ -z "$HCLOUD_TOKEN" ]] && die "HCLOUD_TOKEN not set" } check_deps() { command -v curl &>/dev/null || die "curl is required" command -v jq &>/dev/null || die "jq is required" } # ── Server helpers ─────────────────────────────────────────────────── get_all_server_ids() { local page=1 ids="" while true; do local resp resp=$(hetzner_api GET "/servers?page=${page}&per_page=50") local page_ids page_ids=$(echo "$resp" | jq -r '.servers[].id' 2>/dev/null) [[ -z "$page_ids" ]] && break ids="${ids}${ids:+$'\n'}${page_ids}" local count count=$(echo "$page_ids" | wc -l) (( count < 50 )) && break ((page++)) || true done echo "$ids" } get_server_name() { local sid="$1" hetzner_api GET "/servers/${sid}" \ | jq -r '.server.name // "unknown"' 2>/dev/null } get_server_ids() { if [[ "$TARGET_ALL" == "true" ]]; then get_all_server_ids elif [[ -n "$SERVER_ID" ]]; then echo "$SERVER_ID" else die "Specify --server ID or --all" fi } # ── Snapshot helpers ───────────────────────────────────────────────── get_snapshots() { local page=1 result="[]" while true; do local resp resp=$(hetzner_api GET "/images?type=snapshot&page=${page}&per_page=50") local page_data page_data=$(echo "$resp" | jq '.images // []' 2>/dev/null) local count count=$(echo "$page_data" | jq 'length' 2>/dev/null || echo 0) [[ "$count" -eq 0 ]] && break result=$(echo "$result" "$page_data" | jq -s '.[0] + .[1]') (( count < 50 )) && break ((page++)) || true done echo "$result" } get_server_snapshots() { local sid="$1" get_snapshots | jq --argjson sid "$sid" \ '[.[] | select(.created_from.id == $sid)]' 2>/dev/null } # ══════════════════════════════════════════════════════════════════════ # SNAPSHOT # ══════════════════════════════════════════════════════════════════════ do_snapshot() { local ids ids=$(get_server_ids) [[ -z "$ids" ]] && die "No servers found" local count count=$(echo "$ids" | grep -c . || true) local target_label="server ${SERVER_ID}" [[ "$TARGET_ALL" == "true" ]] && target_label="all (${count} servers)" section_header "Creating Snapshots" field "Target:" "$target_label" field "Prefix:" "$PREFIX" echo "" while IFS= read -r sid; do [[ -z "$sid" ]] && continue local snap_desc snap_desc="${PREFIX}-$(date +%Y%m%d-%H%M%S)" local sname sname=$(get_server_name "$sid") verbose "Snapshotting ${sname} (${sid}) as ${snap_desc}" local resp resp=$(hetzner_api POST "/servers/${sid}/actions/create_image" \ -d "{\"description\": \"${snap_desc}\", \"type\": \"snapshot\"}" 2>/dev/null) local action_status action_status=$(echo "$resp" | jq -r '.action.status // .error.code // "error"' 2>/dev/null) if [[ "$action_status" == "running" || "$action_status" == "success" ]]; then echo -e " ${GREEN}✓${RESET} ${sname} (${sid}) ${snap_desc}" ((SNAP_CREATED++)) || true else echo -e " ${RED}✗${RESET} ${sname} (${sid}) failed" ((SNAP_ERRORS++)) || true fi sleep 1 done <<< "$ids" echo "" field_color "Created:" "${GREEN}${SNAP_CREATED}${RESET}" if [[ "$SNAP_ERRORS" -gt 0 ]]; then field_color "Errors:" "${RED}${SNAP_ERRORS}${RESET}" fi if [[ "$ALSO_ROTATE" == "true" ]]; then do_rotate fi } # ══════════════════════════════════════════════════════════════════════ # ROTATE # ══════════════════════════════════════════════════════════════════════ do_rotate() { local ids ids=$(get_server_ids) [[ -z "$ids" ]] && die "No servers found" section_header "Rotating Snapshots" field "Keep:" "$KEEP per server" field "Prefix:" "$PREFIX" if [[ "$DRY_RUN" == "true" && "$FORCE" != "true" ]]; then field_color "Mode:" "${YELLOW}DRY-RUN${RESET} (use --force to delete)" else field_color "Mode:" "${RED}LIVE${RESET}" fi echo "" local all_snaps all_snaps=$(get_snapshots) while IFS= read -r sid; do [[ -z "$sid" ]] && continue local sname sname=$(get_server_name "$sid") local managed managed=$(echo "$all_snaps" | jq -r \ --argjson sid "$sid" --arg prefix "$PREFIX" \ '[.[] | select(.created_from.id == $sid) | select(.description | startswith($prefix))] | sort_by(.created) | reverse' \ 2>/dev/null) local total total=$(echo "$managed" | jq 'length' 2>/dev/null || echo 0) if (( total <= KEEP )); then verbose "${sname}: ${total} managed snapshots, keeping all" continue fi local to_delete to_delete=$(echo "$managed" | jq -r ".[$KEEP:][] | .id" 2>/dev/null) while IFS= read -r imgid; do [[ -z "$imgid" ]] && continue if [[ "$DRY_RUN" == "true" && "$FORCE" != "true" ]]; then echo -e " ${YELLOW}⊘${RESET} Would delete: ${sname} (${sid}) → image ${imgid}" else if hetzner_api DELETE "/images/${imgid}" > /dev/null 2>&1; then echo -e " ${GREEN}✓${RESET} Deleted: ${sname} (${sid}) → image ${imgid}" ((SNAP_DELETED++)) || true else echo -e " ${RED}✗${RESET} Failed: ${sname} (${sid}) → image ${imgid}" ((SNAP_ERRORS++)) || true fi sleep 1 fi done <<< "$to_delete" done <<< "$ids" echo "" if [[ "$DRY_RUN" == "true" && "$FORCE" != "true" ]]; then log "Dry-run complete — use --force to execute" else field_color "Deleted:" "${GREEN}${SNAP_DELETED}${RESET}" if [[ "$SNAP_ERRORS" -gt 0 ]]; then field_color "Errors:" "${RED}${SNAP_ERRORS}${RESET}" fi fi } # ══════════════════════════════════════════════════════════════════════ # LIST # ══════════════════════════════════════════════════════════════════════ do_list() { section_header "Snapshots" printf " ${BOLD}%-10s %-20s %-28s %-8s %-22s${RESET}\n" \ "IMAGE_ID" "SERVER" "DESCRIPTION" "SIZE" "CREATED" printf " %s\n" "$(printf '%.0s─' {1..90})" local all_snaps all_snaps=$(get_snapshots) if [[ "$TARGET_ALL" != "true" && -n "$SERVER_ID" ]]; then all_snaps=$(echo "$all_snaps" | jq --argjson sid "$SERVER_ID" \ '[.[] | select(.created_from.id == $sid)]' 2>/dev/null) fi echo "$all_snaps" | jq -r \ '.[] | "\(.id)\t\(.created_from.name // "unknown")\t\(.description // "N/A")\t\(.disk_size)\t\(.created)"' \ 2>/dev/null | while IFS=$'\t' read -r imgid server desc size created; do printf " %-10s %-20s %-28s %5sGB %-22s\n" \ "$imgid" "${server:0:18}" "${desc:0:26}" "$size" "${created:0:20}" done } # ══════════════════════════════════════════════════════════════════════ # AUDIT # ══════════════════════════════════════════════════════════════════════ do_audit() { local ids ids=$(get_all_server_ids) [[ -z "$ids" ]] && die "No servers found" section_header "Snapshot Audit" printf " ${BOLD}%-20s %-20s %6s %6s %8s %-12s${RESET}\n" \ "SERVER" "LATEST SNAPSHOT" "AGE" "COUNT" "COST/MO" "STATUS" printf " %s\n" "$(printf '%.0s─' {1..80})" local all_snaps all_snaps=$(get_snapshots) local protected=0 stale=0 unprotected=0 total_servers=0 local total_cost=0 local now now=$(date +%s) while IFS= read -r sid; do [[ -z "$sid" ]] && continue ((total_servers++)) || true local sname sname=$(get_server_name "$sid") local snaps snaps=$(echo "$all_snaps" | jq --argjson sid "$sid" \ '[.[] | select(.created_from.id == $sid)]' 2>/dev/null) local snap_count snap_count=$(echo "$snaps" | jq 'length' 2>/dev/null || echo 0) local server_cost="0.00" if [[ "$snap_count" -gt 0 ]]; then local total_gb total_gb=$(echo "$snaps" | jq '[.[].disk_size] | add // 0' 2>/dev/null || echo 0) server_cost=$(echo "$total_gb * $COST_PER_GB_MONTH" | bc -l 2>/dev/null | xargs printf "%.2f" 2>/dev/null || echo "0.00") total_cost=$(echo "$total_cost + $server_cost" | bc -l 2>/dev/null || echo "$total_cost") fi if [[ "$snap_count" -eq 0 ]]; then printf " %-20s %-20s %6s %6s %7s€ " "${sname:0:18}" "(none)" "—" "0" "0.00" echo -e "${RED}✗ Unprotected${RESET}" ((unprotected++)) || true continue fi local latest latest=$(echo "$snaps" | jq -r \ 'sort_by(.created) | last' 2>/dev/null) local latest_desc latest_date latest_desc=$(echo "$latest" | jq -r '.description // "unknown"' 2>/dev/null) latest_date=$(echo "$latest" | jq -r '.created // ""' 2>/dev/null) local age_days="?" if [[ -n "$latest_date" ]]; then local snap_epoch snap_epoch=$(date -d "$latest_date" +%s 2>/dev/null || echo 0) if [[ "$snap_epoch" -gt 0 ]]; then age_days=$(( (now - snap_epoch) / 86400 )) fi fi local status_str status_color if [[ "$age_days" != "?" ]] && (( age_days > MAX_AGE )); then status_str="⚠ Stale" status_color="$YELLOW" ((stale++)) || true else status_str="✓ OK" status_color="$GREEN" ((protected++)) || true fi printf " %-20s %-20s %5sd %6s %7s€ " \ "${sname:0:18}" "${latest_desc:0:18}" "$age_days" "$snap_count" "$server_cost" echo -e "${status_color}${status_str}${RESET}" done <<< "$ids" local total_cost_fmt total_cost_fmt=$(printf "%.2f" "$total_cost" 2>/dev/null || echo "0.00") echo "" field "Servers:" "$total_servers" field_color "Protected:" "${GREEN}${protected}${RESET}" if [[ "$stale" -gt 0 ]]; then field_color "Stale (>${MAX_AGE}d):" "${YELLOW}${stale}${RESET}" fi if [[ "$unprotected" -gt 0 ]]; then field_color "Unprotected:" "${RED}${unprotected}${RESET}" fi field "Est. monthly cost:" "€${total_cost_fmt}" } # ══════════════════════════════════════════════════════════════════════ # RESTORE # ══════════════════════════════════════════════════════════════════════ do_restore() { [[ -z "$SERVER_ID" ]] && die "Specify --server ID" [[ -z "$SNAPSHOT_ID" ]] && die "Specify --snapshot-id IMAGE_ID" local sname sname=$(get_server_name "$SERVER_ID") section_header "Restore Snapshot" field "Server:" "${sname} (${SERVER_ID})" field "Image:" "$SNAPSHOT_ID" echo "" if [[ "$FORCE" != "true" ]]; then echo -e " ${RED}WARNING: This will rebuild the server from the snapshot image.${RESET}" echo -e " ${RED}All current data on the server will be destroyed.${RESET}" echo "" read -r -p " Type 'yes' to confirm: " confirm if [[ "$confirm" != "yes" ]]; then log "Restore cancelled" return fi fi local resp resp=$(hetzner_api POST "/servers/${SERVER_ID}/actions/rebuild" \ -d "{\"image\": ${SNAPSHOT_ID}}" 2>/dev/null) local action_status action_status=$(echo "$resp" | jq -r '.action.status // .error.code // "error"' 2>/dev/null) if [[ "$action_status" == "running" || "$action_status" == "success" ]]; then echo -e " ${GREEN}✓${RESET} Rebuild initiated — server will restore from image ${SNAPSHOT_ID}" log "Monitor server status — rebuild may take several minutes" else echo -e " ${RED}✗${RESET} Restore failed" local error_msg error_msg=$(echo "$resp" | jq -r '.error.message // ""' 2>/dev/null) [[ -n "$error_msg" ]] && err "$error_msg" fi } # ══════════════════════════════════════════════════════════════════════ # STATUS # ══════════════════════════════════════════════════════════════════════ do_status() { local ids ids=$(get_all_server_ids) [[ -z "$ids" ]] && die "No servers found" local all_snaps all_snaps=$(get_snapshots) local total_servers=0 total_snaps=0 total_gb=0 local protected=0 stale=0 unprotected=0 local now now=$(date +%s) while IFS= read -r sid; do [[ -z "$sid" ]] && continue ((total_servers++)) || true local snaps snaps=$(echo "$all_snaps" | jq --argjson sid "$sid" \ '[.[] | select(.created_from.id == $sid)]' 2>/dev/null) local snap_count snap_count=$(echo "$snaps" | jq 'length' 2>/dev/null || echo 0) total_snaps=$(( total_snaps + snap_count )) local gb gb=$(echo "$snaps" | jq '[.[].disk_size] | add // 0' 2>/dev/null || echo 0) total_gb=$(( total_gb + gb )) if [[ "$snap_count" -eq 0 ]]; then ((unprotected++)) || true continue fi local latest_date latest_date=$(echo "$snaps" | jq -r \ 'sort_by(.created) | last | .created // ""' 2>/dev/null) if [[ -n "$latest_date" ]]; then local snap_epoch snap_epoch=$(date -d "$latest_date" +%s 2>/dev/null || echo 0) if [[ "$snap_epoch" -gt 0 ]]; then local age_days=$(( (now - snap_epoch) / 86400 )) if (( age_days > MAX_AGE )); then ((stale++)) || true else ((protected++)) || true fi else ((protected++)) || true fi else ((protected++)) || true fi done <<< "$ids" local total_cost total_cost=$(echo "$total_gb * $COST_PER_GB_MONTH" | bc -l 2>/dev/null | xargs printf "%.2f" 2>/dev/null || echo "0.00") if [[ "$OUTPUT_FORMAT" == "prometheus" ]]; then cat <${MAX_AGE}d):" "${YELLOW}${stale}${RESET}" else field_color "Stale (>${MAX_AGE}d):" "${GREEN}0${RESET}" fi if [[ "$unprotected" -gt 0 ]]; then field_color "Unprotected:" "${RED}${unprotected}${RESET}" else field_color "Unprotected:" "${GREEN}0${RESET}" fi } # ══════════════════════════════════════════════════════════════════════ # HELP # ══════════════════════════════════════════════════════════════════════ show_help() { cat <