Files
linux-scripts/hetzner-fleet-manager.sh
chiefgeek a1a17e81a1 Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.
2026-05-25 03:31:08 +02:00

614 lines
23 KiB
Bash
Executable File

#!/usr/bin/env bash
#########################################################################################
#### hetzner-fleet-manager.sh — Inventory, health checks, and bulk operations for ####
#### Hetzner Cloud servers via the REST API. Fleet-wide visibility and control ####
#### Requires: bash 4+, curl, jq ####
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
#### License: MIT ####
#### Version 1.01 ####
#### ####
#### Usage: ####
#### ./hetzner-fleet-manager.sh --inventory --all ####
#### ####
#### See --help for all options. ####
#########################################################################################
set -euo pipefail
# ── Colors (pre-initialized) ─────────────────────────────────────────
RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
setup_colors() {
if [[ "${COLOR:-auto}" == "never" ]]; then
return
fi
if [[ "${COLOR:-auto}" == "always" ]] || [[ -t 1 ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
fi
}
# ── Logging ───────────────────────────────────────────────────────────
log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
die() { err "$*"; exit 1; }
section_header() {
echo ""
echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
echo ""
}
field() {
printf " ${BOLD}%-22s${RESET} %s\n" "$1" "$2"
}
field_color() {
printf " ${BOLD}%-22s${RESET} %b\n" "$1" "$2"
}
elapsed() {
local end_time
end_time=$(date +%s)
echo "$(( end_time - START_TIME ))s"
}
# ── Defaults ──────────────────────────────────────────────────────────
RUN_MODE=""
SERVER_ID=""
TARGET_ALL="false"
LABEL_SELECTOR=""
LABEL_SUB_MODE=""
OUTPUT_FORMAT="${HFM_FORMAT:-table}"
PING_CHECK="false"
FORCE="false"
VERBOSE="${VERBOSE:-false}"
COLOR="${COLOR:-auto}"
# ── Credentials ───────────────────────────────────────────────────────
HCLOUD_TOKEN="${HCLOUD_TOKEN:-}"
# ── State ─────────────────────────────────────────────────────────────
SCRIPT_NAME="$(basename "$0")"
readonly SCRIPT_NAME
START_TIME=""
ACTION_OK=0
ACTION_FAIL=0
# ── API helpers ──────────────────────────────────────────────────────
hcloud_api() {
local method="$1" endpoint="$2"
shift 2
local attempt=0 max_attempts=3
while (( attempt < max_attempts )); do
local http_code
http_code=$(curl -s -o /tmp/hfm_resp.json -w "%{http_code}" \
-X "$method" \
-H "Authorization: Bearer ${HCLOUD_TOKEN}" \
-H "Content-Type: application/json" \
"https://api.hetzner.cloud/v1${endpoint}" "$@")
verbose "API ${method} ${endpoint} → HTTP ${http_code}"
if [[ "$http_code" == "429" ]]; then
((attempt++)) || true
local wait=$(( attempt * 5 ))
warn "Rate limited — retrying in ${wait}s (attempt ${attempt}/${max_attempts})"
sleep "$wait"
continue
fi
if [[ "$http_code" =~ ^[45] ]]; then
local errmsg
errmsg=$(jq -r '.error.message // empty' /tmp/hfm_resp.json 2>/dev/null)
[[ -n "$errmsg" ]] && verbose "API error: ${errmsg}"
fi
cat /tmp/hfm_resp.json
return 0
done
err "API request failed after ${max_attempts} attempts: ${method} ${endpoint}"
return 1
}
check_credentials() {
[[ -z "$HCLOUD_TOKEN" ]] && die "HCLOUD_TOKEN not set"
}
check_deps() {
command -v curl &>/dev/null || die "curl is required"
command -v jq &>/dev/null || die "jq is required"
}
# ── Server helpers ───────────────────────────────────────────────────
get_all_server_ids() {
local page=1 per_page=50 ids=""
while true; do
local resp
resp=$(hcloud_api GET "/servers?page=${page}&per_page=${per_page}")
local page_ids
page_ids=$(echo "$resp" | jq -r '.servers[].id' 2>/dev/null)
[[ -z "$page_ids" ]] && break
ids="${ids}${ids:+$'\n'}${page_ids}"
local count
count=$(echo "$page_ids" | wc -l)
(( count < per_page )) && break
((page++)) || true
done
echo "$ids"
}
get_server_ids_by_label() {
local selector="$1"
local page=1 per_page=50 ids=""
while true; do
local resp
resp=$(hcloud_api GET "/servers?page=${page}&per_page=${per_page}&label_selector=$(urlencode "$selector")")
local page_ids
page_ids=$(echo "$resp" | jq -r '.servers[].id' 2>/dev/null)
[[ -z "$page_ids" ]] && break
ids="${ids}${ids:+$'\n'}${page_ids}"
local count
count=$(echo "$page_ids" | wc -l)
(( count < per_page )) && break
((page++)) || true
done
echo "$ids"
}
urlencode() {
local string="$1"
python3 -c "import urllib.parse; print(urllib.parse.quote('$string', safe=''))" 2>/dev/null \
|| echo "$string"
}
get_server_name() {
local sid="$1"
hcloud_api GET "/servers/${sid}" \
| jq -r '.server.name // "unknown"' 2>/dev/null
}
get_server_ids() {
if [[ "$TARGET_ALL" == "true" ]]; then
get_all_server_ids
elif [[ -n "$SERVER_ID" ]]; then
echo "$SERVER_ID"
elif [[ -n "$LABEL_SELECTOR" ]]; then
get_server_ids_by_label "$LABEL_SELECTOR"
else
die "Specify --server ID, --all, or --label-selector KEY=VALUE"
fi
}
# ══════════════════════════════════════════════════════════════════════
# INVENTORY
# ══════════════════════════════════════════════════════════════════════
do_inventory() {
local page=1 per_page=50 all_data="[]"
local query="/servers?page=${page}&per_page=${per_page}"
[[ -n "$LABEL_SELECTOR" ]] && query="${query}&label_selector=$(urlencode "$LABEL_SELECTOR")"
while true; do
local resp
resp=$(hcloud_api GET "$query")
local page_data
page_data=$(echo "$resp" | jq '.servers // []' 2>/dev/null)
local page_count
page_count=$(echo "$page_data" | jq 'length' 2>/dev/null || echo 0)
[[ "$page_count" -eq 0 ]] && break
all_data=$(echo -e "${all_data}\n${page_data}" | jq -s 'add' 2>/dev/null)
(( page_count < per_page )) && break
((page++)) || true
query="/servers?page=${page}&per_page=${per_page}"
[[ -n "$LABEL_SELECTOR" ]] && query="${query}&label_selector=$(urlencode "$LABEL_SELECTOR")"
done
local total
total=$(echo "$all_data" | jq 'length' 2>/dev/null || echo 0)
[[ "$total" -eq 0 ]] && die "No servers found"
case "$OUTPUT_FORMAT" in
json)
echo "$all_data" | jq '.'
;;
ansible)
echo "[hetzner]"
echo "$all_data" | jq -r \
'.[] | (.public_net.ipv4.ip // "unknown") + " # " + (.name // "unknown") + " id=" + (.id | tostring)' \
2>/dev/null
;;
*)
section_header "Fleet Inventory"
printf " ${BOLD}%-10s %-20s %-13s %-16s %-10s %-10s${RESET}\n" \
"ID" "NAME" "STATUS" "IP" "LOCATION" "TYPE"
printf " %s\n" "$(printf '%.0s─' {1..81})"
echo "$all_data" | jq -r \
'.[] | "\(.id)\t\(.name // "unknown")\t\(.status // "unknown")\t\(.public_net.ipv4.ip // "—")\t\(.datacenter.location.name // "—")\t\(.server_type.name // "—")"' \
2>/dev/null \
| while IFS=$'\t' read -r sid name status ip location stype; do
printf " %-10s %-20s %-13s %-16s %-10s %-10s\n" \
"$sid" "${name:0:18}" "$status" "$ip" "${location:0:8}" "$stype"
done
echo ""
field "Total:" "$total"
;;
esac
}
# ══════════════════════════════════════════════════════════════════════
# HEALTH
# ══════════════════════════════════════════════════════════════════════
do_health() {
local ids
ids=$(get_server_ids)
[[ -z "$ids" ]] && die "No servers found"
local running=0 stopped=0 errored=0 total_servers=0
local results=""
while IFS= read -r sid; do
[[ -z "$sid" ]] && continue
((total_servers++)) || true
local resp
resp=$(hcloud_api GET "/servers/${sid}")
local name status ip
name=$(echo "$resp" | jq -r '.server.name // "unknown"' 2>/dev/null)
status=$(echo "$resp" | jq -r '.server.status // "unknown"' 2>/dev/null)
ip=$(echo "$resp" | jq -r '.server.public_net.ipv4.ip // ""' 2>/dev/null)
local ping_result="—"
if [[ "$PING_CHECK" == "true" && -n "$ip" ]]; then
if ping -c 1 -W 3 "$ip" &>/dev/null; then
ping_result="reachable"
else
ping_result="unreachable"
fi
fi
case "$status" in
running) ((running++)) || true ;;
off) ((stopped++)) || true ;;
*) ((errored++)) || true ;;
esac
results="${results}${sid}\t${name}\t${status}\t${ip}\t${ping_result}\n"
done <<< "$ids"
if [[ "$OUTPUT_FORMAT" == "prometheus" ]]; then
cat <<EOF
# HELP hetzner_fleet_instances_total Total Hetzner Cloud servers
# TYPE hetzner_fleet_instances_total gauge
hetzner_fleet_instances_total ${total_servers}
# HELP hetzner_fleet_running Running servers
# TYPE hetzner_fleet_running gauge
hetzner_fleet_running ${running}
# HELP hetzner_fleet_stopped Stopped servers
# TYPE hetzner_fleet_stopped gauge
hetzner_fleet_stopped ${stopped}
# HELP hetzner_fleet_error Servers in error/transitional state
# TYPE hetzner_fleet_error gauge
hetzner_fleet_error ${errored}
EOF
return
fi
section_header "Fleet Health Check"
if [[ "$PING_CHECK" == "true" ]]; then
printf " ${BOLD}%-10s %-20s %-13s %-16s %-12s${RESET}\n" \
"ID" "NAME" "STATUS" "IP" "PING"
printf " %s\n" "$(printf '%.0s─' {1..73})"
else
printf " ${BOLD}%-10s %-20s %-13s %-16s${RESET}\n" \
"ID" "NAME" "STATUS" "IP"
printf " %s\n" "$(printf '%.0s─' {1..61})"
fi
echo -e "$results" | while IFS=$'\t' read -r sid name status ip ping_res; do
[[ -z "$sid" ]] && continue
local status_color="$GREEN"
case "$status" in
running) status_color="$GREEN" ;;
off) status_color="$YELLOW" ;;
*) status_color="$RED" ;;
esac
if [[ "$PING_CHECK" == "true" ]]; then
local ping_color="$DIM"
case "$ping_res" in
reachable) ping_color="$GREEN" ;;
unreachable) ping_color="$RED" ;;
esac
printf " %-10s %-20s " "$sid" "${name:0:18}"
echo -ne "${status_color}"
printf "%-13s" "$status"
echo -ne "${RESET}"
printf " %-16s " "$ip"
echo -e "${ping_color}${ping_res}${RESET}"
else
printf " %-10s %-20s " "$sid" "${name:0:18}"
echo -ne "${status_color}"
printf "%-13s" "$status"
echo -e "${RESET} ${ip}"
fi
done
echo ""
field "Servers:" "$total_servers"
field_color "Running:" "${GREEN}${running}${RESET}"
if [[ "$stopped" -gt 0 ]]; then
field_color "Stopped:" "${YELLOW}${stopped}${RESET}"
else
field_color "Stopped:" "${GREEN}0${RESET}"
fi
if [[ "$errored" -gt 0 ]]; then
field_color "Error:" "${RED}${errored}${RESET}"
else
field_color "Error:" "${GREEN}0${RESET}"
fi
}
# ══════════════════════════════════════════════════════════════════════
# START / STOP / RESTART
# ══════════════════════════════════════════════════════════════════════
do_action() {
local action="$1"
local ids
ids=$(get_server_ids)
[[ -z "$ids" ]] && die "No servers found"
if [[ "$action" != "start" && "$FORCE" != "true" ]]; then
die "${action} is destructive — use --force to confirm"
fi
local count
count=$(echo "$ids" | grep -c . || true)
local target_label="server ${SERVER_ID}"
[[ "$TARGET_ALL" == "true" ]] && target_label="all (${count} servers)"
[[ -n "$LABEL_SELECTOR" ]] && target_label="label ${LABEL_SELECTOR} (${count} servers)"
# Map action names to Hetzner API endpoints
local api_action
case "$action" in
start) api_action="poweron" ;;
stop) api_action="poweroff" ;;
restart) api_action="reboot" ;;
*) die "Unknown action: ${action}" ;;
esac
section_header "Bulk ${action^}"
field "Target:" "$target_label"
field "Action:" "$action"
echo ""
while IFS= read -r sid; do
[[ -z "$sid" ]] && continue
local sname
sname=$(get_server_name "$sid")
verbose "Sending ${action} (${api_action}) to ${sname} (${sid})"
if hcloud_api POST "/servers/${sid}/actions/${api_action}" \
-d '{}' > /dev/null 2>&1; then
echo -e " ${GREEN}${RESET} ${sname} (${sid}) ${action} sent"
((ACTION_OK++)) || true
else
echo -e " ${RED}${RESET} ${sname} (${sid}) ${action} failed"
((ACTION_FAIL++)) || true
fi
sleep 1
done <<< "$ids"
echo ""
field_color "Succeeded:" "${GREEN}${ACTION_OK}${RESET}"
if [[ "$ACTION_FAIL" -gt 0 ]]; then
field_color "Failed:" "${RED}${ACTION_FAIL}${RESET}"
fi
}
# ══════════════════════════════════════════════════════════════════════
# LABELS
# ══════════════════════════════════════════════════════════════════════
do_labels() {
if [[ "$LABEL_SUB_MODE" == "list" ]]; then
local page=1 per_page=50 all_data="[]"
while true; do
local resp
resp=$(hcloud_api GET "/servers?page=${page}&per_page=${per_page}")
local page_data
page_data=$(echo "$resp" | jq '.servers // []' 2>/dev/null)
local page_count
page_count=$(echo "$page_data" | jq 'length' 2>/dev/null || echo 0)
[[ "$page_count" -eq 0 ]] && break
all_data=$(echo -e "${all_data}\n${page_data}" | jq -s 'add' 2>/dev/null)
(( page_count < per_page )) && break
((page++)) || true
done
local labels_json
labels_json=$(echo "$all_data" | jq '[.[].labels // {} | to_entries[]] | group_by(.key) | map({key: .[0].key, values: (map(.value) | unique)})' 2>/dev/null)
if [[ "$OUTPUT_FORMAT" == "json" ]]; then
echo "$labels_json" | jq '.'
return
fi
section_header "Labels"
printf " ${BOLD}%-25s %-50s${RESET}\n" "KEY" "VALUES"
printf " %s\n" "$(printf '%.0s─' {1..77})"
echo "$labels_json" | jq -r '.[] | "\(.key)\t\(.values | join(", "))"' 2>/dev/null \
| while IFS=$'\t' read -r lkey lvals; do
printf " %-25s %-50s\n" "${lkey:0:23}" "${lvals:0:48}"
done
elif [[ "$LABEL_SUB_MODE" == "filter" ]]; then
[[ -z "$LABEL_SELECTOR" ]] && die "Specify --label-selector KEY=VALUE with --filter"
do_inventory
else
die "Specify --list or --filter with --labels"
fi
}
# ══════════════════════════════════════════════════════════════════════
# HELP
# ══════════════════════════════════════════════════════════════════════
show_help() {
cat <<EOF
${BOLD}${SCRIPT_NAME}${RESET} — Hetzner Fleet Manager
Inventory, health checks, and bulk operations for Hetzner Cloud
servers via the REST API.
${BOLD}MODES${RESET}
--inventory Dump server inventory (table, ansible, json)
--health Health check all targeted servers
--start Start targeted servers
--stop Stop targeted servers (requires --force)
--restart Restart targeted servers (requires --force)
--labels List labels or filter servers by label
${BOLD}TARGETING${RESET}
--server ID Target a specific server
--all Target all servers
--label-selector KEY=VALUE
Target servers matching a Hetzner label selector
${BOLD}OPTIONS${RESET}
--format FMT Output: table, json, ansible, prometheus (default: table)
--ping Include ICMP ping check in health mode
--force Required for stop/restart operations
--list List all unique labels (with --labels)
--filter Filter servers by label (with --labels and --label-selector)
--verbose Debug output
--no-color Disable colored output
--help Show this help message
${BOLD}ENVIRONMENT VARIABLES${RESET}
HCLOUD_TOKEN Hetzner Cloud API token (required)
HFM_FORMAT Default output format
VERBOSE Enable verbose output (true/false)
COLOR Color mode: auto, always, never
${BOLD}EXAMPLES${RESET}
# Full fleet inventory
${SCRIPT_NAME} --inventory --all
# Inventory as Ansible inventory file
${SCRIPT_NAME} --inventory --all --format ansible
# Health check all servers
${SCRIPT_NAME} --health --all
# Health check with ping
${SCRIPT_NAME} --health --all --ping
# Prometheus metrics output
${SCRIPT_NAME} --health --all --format prometheus
# Start a single server
${SCRIPT_NAME} --start --server 12345
# Stop all servers (requires --force)
${SCRIPT_NAME} --stop --all --force
# Restart servers by label
${SCRIPT_NAME} --restart --label-selector env=production --force
# List all labels across fleet
${SCRIPT_NAME} --labels --list
# Show servers with a specific label
${SCRIPT_NAME} --labels --filter --label-selector env=staging
${BOLD}EXIT CODES${RESET}
0 Success
1 Runtime error
EOF
}
# ══════════════════════════════════════════════════════════════════════
# PARSE ARGS
# ══════════════════════════════════════════════════════════════════════
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--inventory) RUN_MODE="inventory"; shift ;;
--health) RUN_MODE="health"; shift ;;
--start) RUN_MODE="start"; shift ;;
--stop) RUN_MODE="stop"; shift ;;
--restart) RUN_MODE="restart"; shift ;;
--labels) RUN_MODE="labels"; shift ;;
--server) SERVER_ID="${2:?--server requires an ID}"; shift 2 ;;
--all) TARGET_ALL="true"; shift ;;
--label-selector) LABEL_SELECTOR="${2:?--label-selector requires KEY=VALUE}"; shift 2 ;;
--format) OUTPUT_FORMAT="${2:?--format requires a value}"; shift 2 ;;
--ping) PING_CHECK="true"; shift ;;
--force) FORCE="true"; shift ;;
--list) LABEL_SUB_MODE="list"; shift ;;
--filter) LABEL_SUB_MODE="filter"; shift ;;
--verbose) VERBOSE="true"; shift ;;
--no-color) COLOR="never"; shift ;;
--help|-h) setup_colors; show_help; exit 0 ;;
*) die "Unknown option: $1 (see --help)" ;;
esac
done
}
# ══════════════════════════════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════════════════════════════
main() {
parse_args "$@"
setup_colors
if [[ -z "$RUN_MODE" ]]; then
err "No mode specified"
echo ""
show_help
exit 1
fi
check_deps
check_credentials
START_TIME=$(date +%s)
case "$RUN_MODE" in
inventory) do_inventory ;;
health) do_health ;;
start) do_action "start" ;;
stop) do_action "stop" ;;
restart) do_action "restart" ;;
labels) do_labels ;;
*) die "Unknown mode: ${RUN_MODE}" ;;
esac
if [[ "$OUTPUT_FORMAT" != "prometheus" ]]; then
echo ""
field "Duration:" "$(elapsed)"
fi
}
main "$@"