Files
linux-scripts/hetzner-backup-auditor.sh
chiefgeek a1a17e81a1 Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.
2026-05-25 03:31:08 +02:00

587 lines
23 KiB
Bash

#!/usr/bin/env bash
#########################################################################################
#### hetzner-backup-auditor.sh — Audit backup schedules, snapshot ages, and ####
#### retention policies for Hetzner Cloud servers via the REST API ####
#### Requires: bash 4+, curl, jq ####
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
#### License: MIT ####
#### Version 1.01 ####
#### ####
#### Usage: ####
#### ./hetzner-backup-auditor.sh --audit ####
#### ####
#### See --help for all options. ####
#########################################################################################
set -euo pipefail
# ── Colors (pre-initialized) ─────────────────────────────────────────
RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
setup_colors() {
if [[ "${COLOR:-auto}" == "never" ]]; then
return
fi
if [[ "${COLOR:-auto}" == "always" ]] || [[ -t 1 ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
fi
}
# ── Logging ───────────────────────────────────────────────────────────
log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
die() { err "$*"; exit 1; }
section_header() {
echo ""
echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
echo ""
}
field() {
printf " ${BOLD}%-22s${RESET} %s\n" "$1" "$2"
}
field_color() {
printf " ${BOLD}%-22s${RESET} %b\n" "$1" "$2"
}
elapsed() {
local end_time
end_time=$(date +%s)
echo "$(( end_time - START_TIME ))s"
}
# ── Defaults ──────────────────────────────────────────────────────────
RUN_MODE=""
SERVER_ID=""
LABEL_SELECTOR=""
OUTPUT_FORMAT="${HBA_FORMAT:-table}"
MAX_AGE_HOURS="${HBA_MAX_AGE:-48}"
VERBOSE="${VERBOSE:-false}"
COLOR="${COLOR:-auto}"
# ── Credentials ───────────────────────────────────────────────────────
HCLOUD_TOKEN="${HCLOUD_TOKEN:-}"
# ── State ─────────────────────────────────────────────────────────────
SCRIPT_NAME="$(basename "$0")"
readonly SCRIPT_NAME
START_TIME=""
# ── API helpers ──────────────────────────────────────────────────────
hcloud_api() {
local method="$1" endpoint="$2"
shift 2
local attempt=0 max_attempts=3
while (( attempt < max_attempts )); do
local http_code
http_code=$(curl -s -o /tmp/hba_resp.json -w "%{http_code}" \
-X "$method" \
-H "Authorization: Bearer ${HCLOUD_TOKEN}" \
-H "Content-Type: application/json" \
"https://api.hetzner.cloud/v1${endpoint}" "$@")
verbose "API ${method} ${endpoint} → HTTP ${http_code}"
if [[ "$http_code" == "429" ]]; then
((attempt++)) || true
local wait=$(( attempt * 5 ))
warn "Rate limited — retrying in ${wait}s (attempt ${attempt}/${max_attempts})"
sleep "$wait"
continue
fi
if [[ "$http_code" =~ ^[45] ]]; then
local errmsg
errmsg=$(jq -r '.error.message // empty' /tmp/hba_resp.json 2>/dev/null)
[[ -n "$errmsg" ]] && verbose "API error: ${errmsg}"
fi
cat /tmp/hba_resp.json
return 0
done
err "API request failed after ${max_attempts} attempts: ${method} ${endpoint}"
return 1
}
check_credentials() {
[[ -z "$HCLOUD_TOKEN" ]] && die "HCLOUD_TOKEN not set"
}
check_deps() {
command -v curl &>/dev/null || die "curl is required"
command -v jq &>/dev/null || die "jq is required"
}
urlencode() {
local string="$1"
python3 -c "import urllib.parse; print(urllib.parse.quote('$string', safe=''))" 2>/dev/null \
|| echo "$string"
}
# ── Pagination helper ────────────────────────────────────────────────
fetch_all() {
local endpoint="$1" key="$2"
local page=1 per_page=50 all_data="[]"
while true; do
local sep="?"
[[ "$endpoint" == *"?"* ]] && sep="&"
local resp
resp=$(hcloud_api GET "${endpoint}${sep}page=${page}&per_page=${per_page}")
local page_data
page_data=$(echo "$resp" | jq ".${key} // []" 2>/dev/null)
local page_count
page_count=$(echo "$page_data" | jq 'length' 2>/dev/null || echo 0)
[[ "$page_count" -eq 0 ]] && break
all_data=$(echo -e "${all_data}\n${page_data}" | jq -s 'add' 2>/dev/null)
(( page_count < per_page )) && break
((page++)) || true
done
echo "$all_data"
}
# ── Age helpers ──────────────────────────────────────────────────────
iso_to_epoch() {
date -d "$1" +%s 2>/dev/null || echo 0
}
age_hours() {
local created_epoch="$1"
local now
now=$(date +%s)
echo $(( (now - created_epoch) / 3600 ))
}
format_age() {
local hours="$1"
if [[ "$hours" -lt 24 ]]; then
echo "${hours}h"
else
local days=$(( hours / 24 ))
local rem=$(( hours % 24 ))
echo "${days}d ${rem}h"
fi
}
# ══════════════════════════════════════════════════════════════════════
# AUDIT
# ══════════════════════════════════════════════════════════════════════
do_audit() {
local query="/servers?"
[[ -n "$LABEL_SELECTOR" ]] && query="${query}label_selector=$(urlencode "$LABEL_SELECTOR")&"
[[ -n "$SERVER_ID" ]] && query="/servers?id=${SERVER_ID}&"
local servers
servers=$(fetch_all "${query%&}" "servers")
local server_count
server_count=$(echo "$servers" | jq 'length' 2>/dev/null || echo 0)
[[ "$server_count" -eq 0 ]] && die "No servers found"
local snapshots
snapshots=$(fetch_all "/images?type=snapshot" "images")
local now
now=$(date +%s)
local warnings=0
local no_backup=0
local stale=0
local healthy=0
local results=""
while IFS=$'\t' read -r sid sname sstatus backup_enabled; do
[[ -z "$sid" ]] && continue
# Find most recent snapshot for this server
local latest_snap
latest_snap=$(echo "$snapshots" | jq -r \
--arg sid "$sid" \
'[.[] | select(.created_from.id == ($sid | tonumber))] | sort_by(.created) | last | .created // empty' \
2>/dev/null)
# Find most recent backup (backup type images)
local latest_backup_resp
latest_backup_resp=$(hcloud_api GET "/images?type=backup&sort=created:desc&page=1&per_page=1")
# Filter backups for this specific server
local server_backups
server_backups=$(fetch_all "/images?type=backup" "images")
local latest_backup
latest_backup=$(echo "$server_backups" | jq -r \
--arg sid "$sid" \
'[.[] | select(.created_from.id == ($sid | tonumber))] | sort_by(.created) | last | .created // empty' \
2>/dev/null)
# Determine newest protection point
local newest=""
local newest_type="none"
if [[ -n "$latest_backup" && -n "$latest_snap" ]]; then
local bepoch sepoch
bepoch=$(iso_to_epoch "$latest_backup")
sepoch=$(iso_to_epoch "$latest_snap")
if [[ "$bepoch" -ge "$sepoch" ]]; then
newest="$latest_backup"
newest_type="backup"
else
newest="$latest_snap"
newest_type="snapshot"
fi
elif [[ -n "$latest_backup" ]]; then
newest="$latest_backup"
newest_type="backup"
elif [[ -n "$latest_snap" ]]; then
newest="$latest_snap"
newest_type="snapshot"
fi
local age_h="—"
local status_flag="none"
if [[ -n "$newest" ]]; then
local nepoch
nepoch=$(iso_to_epoch "$newest")
age_h=$(age_hours "$nepoch")
if [[ "$age_h" -le "$MAX_AGE_HOURS" ]]; then
status_flag="ok"
((healthy++)) || true
else
status_flag="stale"
((stale++)) || true
((warnings++)) || true
fi
else
((no_backup++)) || true
((warnings++)) || true
fi
local backup_str="disabled"
[[ "$backup_enabled" == "true" ]] && backup_str="enabled"
results="${results}${sid}\t${sname}\t${sstatus}\t${backup_str}\t${newest_type}\t${age_h}\t${status_flag}\n"
done < <(echo "$servers" | jq -r \
'.[] | "\(.id)\t\(.name // "unknown")\t\(.status)\t\(.backup_window != null)"' \
2>/dev/null)
case "$OUTPUT_FORMAT" in
json)
jq -n \
--argjson servers "$server_count" \
--argjson healthy "$healthy" \
--argjson stale "$stale" \
--argjson no_backup "$no_backup" \
--argjson warnings "$warnings" \
--argjson max_age "$MAX_AGE_HOURS" \
'{servers: $servers, healthy: $healthy, stale: $stale, no_backup: $no_backup, warnings: $warnings, max_age_hours: $max_age}'
;;
prometheus)
cat <<EOF
# HELP hetzner_backup_servers_total Total servers audited
# TYPE hetzner_backup_servers_total gauge
hetzner_backup_servers_total ${server_count}
# HELP hetzner_backup_healthy Servers with recent backup/snapshot
# TYPE hetzner_backup_healthy gauge
hetzner_backup_healthy ${healthy}
# HELP hetzner_backup_stale_total Servers with stale backup/snapshot
# TYPE hetzner_backup_stale_total gauge
hetzner_backup_stale_total ${stale}
# HELP hetzner_backup_missing_total Servers with no backup or snapshot
# TYPE hetzner_backup_missing_total gauge
hetzner_backup_missing_total ${no_backup}
# HELP hetzner_backup_warnings_total Total audit warnings
# TYPE hetzner_backup_warnings_total gauge
hetzner_backup_warnings_total ${warnings}
# HELP hetzner_backup_max_age_hours Configured max age threshold
# TYPE hetzner_backup_max_age_hours gauge
hetzner_backup_max_age_hours ${MAX_AGE_HOURS}
EOF
;;
*)
section_header "Backup Audit (max age: ${MAX_AGE_HOURS}h)"
printf " ${BOLD}%-10s %-18s %-10s %-10s %-10s %-8s %-8s${RESET}\n" \
"ID" "NAME" "STATUS" "BACKUPS" "LATEST" "AGE" "RESULT"
printf " %s\n" "$(printf '%.0s─' {1..78})"
echo -e "$results" | while IFS=$'\t' read -r sid sname sstatus backup_str newest_type age_h status_flag; do
[[ -z "$sid" ]] && continue
local result_color="$GREEN"
local result_text="✓ ok"
case "$status_flag" in
ok) result_color="$GREEN"; result_text="✓ ok" ;;
stale) result_color="$YELLOW"; result_text="⚠ stale" ;;
none) result_color="$RED"; result_text="✗ none" ;;
esac
local age_display="—"
if [[ "$age_h" != "—" ]]; then
age_display=$(format_age "$age_h")
fi
printf " %-10s %-18s %-10s %-10s %-10s %-8s " \
"$sid" "${sname:0:16}" "$sstatus" "$backup_str" "$newest_type" "$age_display"
echo -e "${result_color}${result_text}${RESET}"
done
echo ""
field "Servers:" "$server_count"
field_color "Healthy:" "${GREEN}${healthy}${RESET}"
if [[ "$stale" -gt 0 ]]; then
field_color "Stale:" "${YELLOW}${stale}${RESET}"
else
field_color "Stale:" "${GREEN}0${RESET}"
fi
if [[ "$no_backup" -gt 0 ]]; then
field_color "No backup:" "${RED}${no_backup}${RESET}"
else
field_color "No backup:" "${GREEN}0${RESET}"
fi
;;
esac
}
# ══════════════════════════════════════════════════════════════════════
# SNAPSHOTS
# ══════════════════════════════════════════════════════════════════════
do_snapshots() {
local snapshots
snapshots=$(fetch_all "/images?type=snapshot&sort=created:desc" "images")
local total
total=$(echo "$snapshots" | jq 'length' 2>/dev/null || echo 0)
[[ "$total" -eq 0 ]] && die "No snapshots found"
local now
now=$(date +%s)
case "$OUTPUT_FORMAT" in
json)
echo "$snapshots" | jq '[.[] | {
id: .id, description: .description,
size_gb: .image_size, created: .created,
server_id: .created_from.id, server_name: .created_from.name
}]'
;;
prometheus)
local stale_count=0
while IFS=$'\t' read -r iid icreated; do
[[ -z "$iid" ]] && continue
local cepoch
cepoch=$(iso_to_epoch "$icreated")
local ah
ah=$(age_hours "$cepoch")
[[ "$ah" -gt "$MAX_AGE_HOURS" ]] && ((stale_count++)) || true
done < <(echo "$snapshots" | jq -r '.[] | "\(.id)\t\(.created)"' 2>/dev/null)
cat <<EOF
# HELP hetzner_backup_snapshots_total Total snapshots
# TYPE hetzner_backup_snapshots_total gauge
hetzner_backup_snapshots_total ${total}
# HELP hetzner_backup_snapshots_stale Snapshots older than threshold
# TYPE hetzner_backup_snapshots_stale gauge
hetzner_backup_snapshots_stale ${stale_count}
EOF
;;
*)
section_header "Snapshots"
printf " ${BOLD}%-10s %-20s %-8s %-10s %-20s %-8s${RESET}\n" \
"ID" "DESCRIPTION" "SIZE" "SERVER" "CREATED" "AGE"
printf " %s\n" "$(printf '%.0s─' {1..80})"
echo "$snapshots" | jq -r \
'.[] | "\(.id)\t\(.description // "—")\t\(.image_size // 0)\t\(.created_from.name // "—")\t\(.created // "—")"' \
2>/dev/null \
| while IFS=$'\t' read -r iid idesc isize iserver icreated; do
local cepoch ah age_display age_color
cepoch=$(iso_to_epoch "$icreated")
ah=$(age_hours "$cepoch")
age_display=$(format_age "$ah")
age_color="$GREEN"
[[ "$ah" -gt "$MAX_AGE_HOURS" ]] && age_color="$YELLOW"
printf " %-10s %-20s %-8s %-10s %-20s " \
"$iid" "${idesc:0:18}" "${isize}GB" "${iserver:0:8}" "${icreated:0:19}"
echo -e "${age_color}${age_display}${RESET}"
done
echo ""
field "Snapshots:" "$total"
;;
esac
}
# ══════════════════════════════════════════════════════════════════════
# BACKUPS
# ══════════════════════════════════════════════════════════════════════
do_backups() {
local backups
backups=$(fetch_all "/images?type=backup&sort=created:desc" "images")
local total
total=$(echo "$backups" | jq 'length' 2>/dev/null || echo 0)
[[ "$total" -eq 0 ]] && die "No backups found"
case "$OUTPUT_FORMAT" in
json)
echo "$backups" | jq '[.[] | {
id: .id, description: .description,
size_gb: .image_size, created: .created,
server_id: .created_from.id, server_name: .created_from.name
}]'
;;
*)
section_header "Backups"
printf " ${BOLD}%-10s %-20s %-8s %-10s %-20s %-8s${RESET}\n" \
"ID" "DESCRIPTION" "SIZE" "SERVER" "CREATED" "AGE"
printf " %s\n" "$(printf '%.0s─' {1..80})"
echo "$backups" | jq -r \
'.[] | "\(.id)\t\(.description // "—")\t\(.image_size // 0)\t\(.created_from.name // "—")\t\(.created // "—")"' \
2>/dev/null \
| while IFS=$'\t' read -r iid idesc isize iserver icreated; do
local cepoch ah age_display age_color
cepoch=$(iso_to_epoch "$icreated")
ah=$(age_hours "$cepoch")
age_display=$(format_age "$ah")
age_color="$GREEN"
[[ "$ah" -gt "$MAX_AGE_HOURS" ]] && age_color="$YELLOW"
printf " %-10s %-20s %-8s %-10s %-20s " \
"$iid" "${idesc:0:18}" "${isize}GB" "${iserver:0:8}" "${icreated:0:19}"
echo -e "${age_color}${age_display}${RESET}"
done
echo ""
field "Backups:" "$total"
;;
esac
}
# ══════════════════════════════════════════════════════════════════════
# HELP
# ══════════════════════════════════════════════════════════════════════
show_help() {
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} — Hetzner Backup Auditor
Audit backup schedules, snapshot ages, and retention for Hetzner Cloud
servers via the REST API.
${BOLD}MODES${RESET}
--audit Audit all servers for backup coverage (default)
--snapshots List all snapshots with age
--backups List all automatic backups with age
${BOLD}TARGETING${RESET}
--server ID Audit a specific server
--label KEY=VAL Filter servers by Hetzner Cloud label selector
${BOLD}OPTIONS${RESET}
--max-age HOURS Max acceptable backup/snapshot age (default: 48)
--format FMT Output: table, json, prometheus (default: table)
--verbose Debug output
--no-color Disable colored output
--help Show this help message
${BOLD}ENVIRONMENT VARIABLES${RESET}
HCLOUD_TOKEN Hetzner Cloud API token (required)
HBA_FORMAT Default output format
HBA_MAX_AGE Default max age threshold in hours (default: 48)
VERBOSE Enable verbose output (true/false)
COLOR Color mode: auto, always, never
${BOLD}EXAMPLES${RESET}
# Audit all servers
${SCRIPT_NAME} --audit
# Audit with 24-hour threshold
${SCRIPT_NAME} --audit --max-age 24
# Audit servers by label
${SCRIPT_NAME} --audit --label env=prod
# List all snapshots
${SCRIPT_NAME} --snapshots
# List all backups
${SCRIPT_NAME} --backups
# JSON output
${SCRIPT_NAME} --audit --format json
# Prometheus metrics
${SCRIPT_NAME} --audit --format prometheus
# Cron — daily backup audit
0 6 * * * /usr/local/bin/hetzner-backup-auditor.sh --audit --format prometheus --no-color > /var/lib/node_exporter/textfile/hetzner_backup.prom 2>/dev/null
${BOLD}EXIT CODES${RESET}
0 Success
1 Runtime error
EOF
}
# ══════════════════════════════════════════════════════════════════════
# PARSE ARGS
# ══════════════════════════════════════════════════════════════════════
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--audit) RUN_MODE="audit"; shift ;;
--snapshots) RUN_MODE="snapshots"; shift ;;
--backups) RUN_MODE="backups"; shift ;;
--server) SERVER_ID="${2:?--server requires an ID}"; shift 2 ;;
--label) LABEL_SELECTOR="${2:?--label requires KEY=VALUE}"; shift 2 ;;
--max-age) MAX_AGE_HOURS="${2:?--max-age requires HOURS}"; shift 2 ;;
--format) OUTPUT_FORMAT="${2:?--format requires a value}"; shift 2 ;;
--verbose) VERBOSE="true"; shift ;;
--no-color) COLOR="never"; shift ;;
--help|-h) setup_colors; show_help; exit 0 ;;
*) die "Unknown option: $1 (see --help)" ;;
esac
done
}
# ══════════════════════════════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════════════════════════════
main() {
parse_args "$@"
setup_colors
if [[ -z "$RUN_MODE" ]]; then
RUN_MODE="audit"
fi
check_deps
check_credentials
START_TIME=$(date +%s)
case "$RUN_MODE" in
audit) do_audit ;;
snapshots) do_snapshots ;;
backups) do_backups ;;
*) die "Unknown mode: ${RUN_MODE}" ;;
esac
if [[ "$OUTPUT_FORMAT" != "prometheus" ]]; then
echo ""
field "Duration:" "$(elapsed)"
fi
}
main "$@"