+
+CrowdSec Exporter v1.0
+
+CrowdSec Prometheus Exporter v1.0
+Metrics
+Supplementary operational metrics from cscli commands.
+For internal CrowdSec metrics (buckets, parsers), see port 6060.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+# Main entry point - routes to appropriate output mode
+main() {
+ parse_args "$@"
+
+ if [ "$HTTP_MODE" = true ]; then
+ # Run HTTP server (blocks until killed)
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ # Textfile collector mode: write atomically using temp file
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ # Create temp file in SAME directory for atomic rename (same filesystem)
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.crowdsec_metrics.XXXXXX")
+
+ # Generate metrics to temp file
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ # Validate: file must exist, have content
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ # Set permissions before move
+ chmod 644 "$temp_file"
+
+ # Atomic rename - no gap where file is missing
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ # Default: output to stdout
+ generate_metrics
+ fi
+}
+
+# Execute main function with all script arguments
+main "$@"
diff --git a/crowdsec-install.sh b/crowdsec-install.sh
new file mode 100644
index 0000000..502ee7e
--- /dev/null
+++ b/crowdsec-install.sh
@@ -0,0 +1,444 @@
+#!/bin/bash
+################################################################################
+# Script Name: crowdsec-install.sh
+# Version: 1.0
+# Description: Automated CrowdSec installation with firewall bouncer,
+# collection selection, allowlists, and Prometheus integration
+# on Debian/Ubuntu and RHEL/Rocky/AlmaLinux
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Usage:
+# sudo ./crowdsec-install.sh
+# sudo ./crowdsec-install.sh --collections "sshd,nginx"
+# sudo ./crowdsec-install.sh --allowlist "10.0.0.0/8" --prometheus
+# sudo ./crowdsec-install.sh --dry-run
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# DEFAULTS
+# ============================================================================
+
+COLLECTIONS=""
+ALLOWLIST=""
+BOUNCER_TYPE="iptables"
+PROMETHEUS=false
+ENROLL_KEY=""
+NO_BOUNCER=false
+DRY_RUN=false
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
+log_step() { echo -e "${CYAN}[STEP]${NC} $*"; }
+
+show_usage() {
+ cat </dev/null || systemctl is-active --quiet ssh 2>/dev/null; then
+ detected+=("crowdsecurity/sshd")
+ log_info " Detected: SSH"
+ fi
+
+ # Nginx
+ if systemctl is-active --quiet nginx 2>/dev/null; then
+ detected+=("crowdsecurity/nginx")
+ log_info " Detected: Nginx"
+ fi
+
+ # Apache
+ if systemctl is-active --quiet apache2 2>/dev/null || systemctl is-active --quiet httpd 2>/dev/null; then
+ detected+=("crowdsecurity/apache2")
+ log_info " Detected: Apache"
+ fi
+
+ # Postfix
+ if systemctl is-active --quiet postfix 2>/dev/null; then
+ detected+=("crowdsecurity/postfix")
+ log_info " Detected: Postfix"
+ fi
+
+ # Dovecot
+ if systemctl is-active --quiet dovecot 2>/dev/null; then
+ detected+=("crowdsecurity/dovecot")
+ log_info " Detected: Dovecot"
+ fi
+
+ # MySQL/MariaDB
+ if systemctl is-active --quiet mysql 2>/dev/null || systemctl is-active --quiet mariadb 2>/dev/null; then
+ detected+=("crowdsecurity/mysql")
+ log_info " Detected: MySQL/MariaDB"
+ fi
+
+ # PostgreSQL
+ if systemctl is-active --quiet postgresql 2>/dev/null; then
+ detected+=("crowdsecurity/pgsql")
+ log_info " Detected: PostgreSQL"
+ fi
+
+ DETECTED_COLLECTIONS="${detected[*]}"
+}
+
+# ============================================================================
+# INSTALLATION
+# ============================================================================
+
+add_repo_debian() {
+ log_step "Adding CrowdSec repository (Debian/Ubuntu)..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would add CrowdSec apt repository"
+ return
+ fi
+
+ apt-get update -qq
+ apt-get install -y -qq curl gnupg apt-transport-https >/dev/null 2>&1
+
+ curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.deb.sh | bash >/dev/null 2>&1
+}
+
+add_repo_rhel() {
+ log_step "Adding CrowdSec repository (RHEL/Rocky)..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would add CrowdSec yum repository"
+ return
+ fi
+
+ curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.rpm.sh | bash >/dev/null 2>&1
+}
+
+install_agent() {
+ log_step "Installing CrowdSec agent..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would install crowdsec package"
+ return
+ fi
+
+ case "$OS_FAMILY" in
+ debian) apt-get install -y -qq crowdsec >/dev/null 2>&1 ;;
+ rhel) dnf install -y -q crowdsec >/dev/null 2>&1 ;;
+ esac
+
+ systemctl enable --now crowdsec >/dev/null 2>&1
+ log_info "CrowdSec agent installed and running"
+}
+
+install_bouncer() {
+ if [ "$NO_BOUNCER" = true ]; then
+ log_info "Skipping bouncer installation (--no-bouncer)"
+ return
+ fi
+
+ local pkg="crowdsec-firewall-bouncer-${BOUNCER_TYPE}"
+ log_step "Installing bouncer: $pkg..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would install $pkg"
+ return
+ fi
+
+ case "$OS_FAMILY" in
+ debian) apt-get install -y -qq "$pkg" >/dev/null 2>&1 ;;
+ rhel) dnf install -y -q "$pkg" >/dev/null 2>&1 ;;
+ esac
+
+ systemctl enable --now "$pkg" >/dev/null 2>&1
+ log_info "Bouncer installed: $pkg"
+}
+
+# ============================================================================
+# CONFIGURATION
+# ============================================================================
+
+install_collections() {
+ local collections_to_install=""
+
+ if [ -n "$COLLECTIONS" ]; then
+ # User-specified collections
+ IFS=',' read -ra cols <<< "$COLLECTIONS"
+ for col in "${cols[@]}"; do
+ col=$(echo "$col" | xargs)
+ # Add crowdsecurity/ prefix if not present
+ if [[ "$col" != */* ]]; then
+ col="crowdsecurity/$col"
+ fi
+ collections_to_install="$collections_to_install $col"
+ done
+ else
+ # Auto-detected collections
+ collections_to_install="$DETECTED_COLLECTIONS"
+ fi
+
+ log_step "Installing collections: $collections_to_install"
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would install: $collections_to_install"
+ return
+ fi
+
+ for col in $collections_to_install; do
+ if cscli collections install "$col" >/dev/null 2>&1; then
+ log_info " Installed: $col"
+ else
+ log_warn " Failed to install: $col"
+ fi
+ done
+}
+
+configure_allowlist() {
+ if [ -z "$ALLOWLIST" ]; then
+ return
+ fi
+
+ log_step "Configuring allowlist..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would whitelist: $ALLOWLIST"
+ return
+ fi
+
+ IFS=',' read -ra ips <<< "$ALLOWLIST"
+ for ip in "${ips[@]}"; do
+ ip=$(echo "$ip" | xargs)
+ if cscli decisions add --ip "$ip" --type whitelist --duration 87600h >/dev/null 2>&1; then
+ log_info " Whitelisted: $ip"
+ else
+ log_warn " Failed to whitelist: $ip"
+ fi
+ done
+}
+
+configure_prometheus() {
+ if [ "$PROMETHEUS" != true ]; then
+ return
+ fi
+
+ log_step "Enabling Prometheus metrics on :6060..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would enable Prometheus metrics"
+ return
+ fi
+
+ local config="/etc/crowdsec/config.yaml"
+ if [ -f "$config" ]; then
+ # Prometheus is enabled by default in CrowdSec, verify
+ if grep -q "prometheus:" "$config"; then
+ log_info "Prometheus metrics already configured"
+ fi
+ fi
+
+ log_info "Prometheus metrics available at http://localhost:6060/metrics"
+}
+
+enroll_console() {
+ if [ -z "$ENROLL_KEY" ]; then
+ return
+ fi
+
+ log_step "Enrolling with CrowdSec console..."
+
+ if [ "$DRY_RUN" = true ]; then
+ log_info "[DRY RUN] Would enroll with key: $ENROLL_KEY"
+ return
+ fi
+
+ if cscli console enroll "$ENROLL_KEY" >/dev/null 2>&1; then
+ log_info "Enrolled with CrowdSec console"
+ else
+ log_warn "Console enrollment failed — verify enrollment key"
+ fi
+}
+
+# ============================================================================
+# VERIFICATION
+# ============================================================================
+
+verify_installation() {
+ log_step "Verifying installation..."
+
+ echo ""
+
+ # CrowdSec agent
+ if systemctl is-active --quiet crowdsec 2>/dev/null; then
+ log_info "✓ CrowdSec agent: running"
+ else
+ log_error "✗ CrowdSec agent: not running"
+ fi
+
+ # Bouncer
+ if [ "$NO_BOUNCER" != true ]; then
+ local bouncer_svc="crowdsec-firewall-bouncer-${BOUNCER_TYPE}"
+ if systemctl is-active --quiet "$bouncer_svc" 2>/dev/null; then
+ log_info "✓ Firewall bouncer: running"
+ else
+ log_error "✗ Firewall bouncer: not running"
+ fi
+ fi
+
+ # Collections
+ log_info "Installed collections:"
+ cscli collections list 2>/dev/null | grep -E "enabled|installed" || true
+
+ # Bouncers
+ log_info "Registered bouncers:"
+ cscli bouncers list 2>/dev/null || true
+
+ echo ""
+ log_info "Installation complete"
+ echo ""
+ log_info "Useful commands:"
+ echo " cscli decisions list — view active decisions"
+ echo " cscli alerts list — view recent alerts"
+ echo " cscli metrics — view metrics summary"
+ echo " cscli hub list — view installed hub items"
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+main() {
+ parse_args "$@"
+ check_root
+ detect_os
+
+ echo ""
+ log_info "=== CrowdSec Installation Script v1.0 ==="
+ echo ""
+
+ if [ "$DRY_RUN" = true ]; then
+ log_warn "DRY RUN MODE — no changes will be made"
+ echo ""
+ fi
+
+ detect_services
+
+ case "$OS_FAMILY" in
+ debian) add_repo_debian ;;
+ rhel) add_repo_rhel ;;
+ esac
+
+ install_agent
+ install_collections
+ configure_allowlist
+ install_bouncer
+ configure_prometheus
+ enroll_console
+
+ if [ "$DRY_RUN" != true ]; then
+ verify_installation
+ fi
+}
+
+main "$@"
diff --git a/database-backup-exporter.sh b/database-backup-exporter.sh
new file mode 100644
index 0000000..686b6ef
--- /dev/null
+++ b/database-backup-exporter.sh
@@ -0,0 +1,313 @@
+#!/bin/bash
+#############################################################
+#### Database Backup Exporter for Prometheus ####
+#### Monitor MySQL and PostgreSQL backup freshness, ####
+#### size, and status via node_exporter textfile collector ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version: 1.0 ####
+#### ####
+#### Usage: ./database-backup-exporter.sh [OPTIONS] ####
+#############################################################
+
+set -euo pipefail
+
+# -----------------------------
+# Defaults
+# -----------------------------
+BACKUP_DIR="/opt/backups"
+MAX_AGE=86400
+PROM_FILE="/var/lib/node_exporter/database_backups.prom"
+INTERVAL=300
+RUN_ONCE=false
+
+# -----------------------------
+# Color codes
+# -----------------------------
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+# -----------------------------
+# Logging
+# -----------------------------
+log_info() {
+ echo -e "${GREEN}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*"
+}
+
+log_warn() {
+ echo -e "${YELLOW}[WARN]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*" >&2
+}
+
+log_error() {
+ echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*" >&2
+}
+
+# -----------------------------
+# Usage
+# -----------------------------
+usage() {
+ cat <_YYYYMMDD[HHMMSS].
+ Examples: myapp_20260309.sql.gz orders_20260308120000.pgdump
+
+EOF
+ exit 0
+}
+
+# -----------------------------
+# Parse arguments
+# -----------------------------
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --backup-dir)
+ BACKUP_DIR="$2"
+ shift 2
+ ;;
+ --max-age)
+ MAX_AGE="$2"
+ shift 2
+ ;;
+ --prom-file)
+ PROM_FILE="$2"
+ shift 2
+ ;;
+ --interval)
+ INTERVAL="$2"
+ shift 2
+ ;;
+ --once)
+ RUN_ONCE=true
+ shift
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ log_error "Unknown option: $1"
+ usage
+ ;;
+ esac
+ done
+}
+
+# -----------------------------
+# Detect backup type from ext
+# -----------------------------
+detect_type() {
+ local filename="$1"
+ case "$filename" in
+ *.pgdump|*.dump)
+ echo "postgres"
+ ;;
+ *.sql|*.sql.gz)
+ echo "mysql"
+ ;;
+ *)
+ echo "unknown"
+ ;;
+ esac
+}
+
+# -----------------------------
+# Extract database name
+# -----------------------------
+extract_dbname() {
+ local filename
+ filename="$(basename "$1")"
+ # Strip all known extensions
+ filename="${filename%.gz}"
+ filename="${filename%.sql}"
+ filename="${filename%.dump}"
+ filename="${filename%.pgdump}"
+ # Expect pattern: dbname_YYYYMMDD... — grab everything before the date segment
+ echo "$filename" | sed -E 's/_[0-9]{8,14}$//'
+}
+
+# -----------------------------
+# Collect and write metrics
+# -----------------------------
+collect_metrics() {
+ local backup_dir="$1"
+ local max_age="$2"
+ local now
+ now="$(date +%s)"
+
+ if [[ ! -d "$backup_dir" ]]; then
+ log_error "Backup directory does not exist: $backup_dir"
+ return 1
+ fi
+
+ # Associative arrays keyed by "dbname|type"
+ declare -A latest_ts
+ declare -A latest_size
+ declare -A file_count
+
+ # Scan for backup files
+ local found=0
+ while IFS= read -r -d '' file; do
+ local base
+ base="$(basename "$file")"
+ local btype
+ btype="$(detect_type "$base")"
+ [[ "$btype" == "unknown" ]] && continue
+
+ local dbname
+ dbname="$(extract_dbname "$file")"
+ [[ -z "$dbname" ]] && continue
+
+ local key="${dbname}|${btype}"
+ local mtime
+ mtime="$(stat -c '%Y' "$file" 2>/dev/null)" || continue
+ local fsize
+ fsize="$(stat -c '%s' "$file" 2>/dev/null)" || continue
+
+ # Track count
+ file_count[$key]=$(( ${file_count[$key]:-0} + 1 ))
+
+ # Track most recent
+ if [[ -z "${latest_ts[$key]:-}" ]] || (( mtime > latest_ts[$key] )); then
+ latest_ts[$key]="$mtime"
+ latest_size[$key]="$fsize"
+ fi
+
+ found=$((found + 1))
+ done < <(find "$backup_dir" -type f \( -name '*.sql' -o -name '*.sql.gz' -o -name '*.dump' -o -name '*.pgdump' \) -print0 2>/dev/null)
+
+ log_info "Found $found backup file(s) in $backup_dir"
+
+ # Build output
+ local output=""
+
+ output+="# HELP db_backup_last_timestamp Unix timestamp of most recent backup.\n"
+ output+="# TYPE db_backup_last_timestamp gauge\n"
+ for key in "${!latest_ts[@]}"; do
+ local dbname="${key%%|*}"
+ local btype="${key##*|}"
+ output+="db_backup_last_timestamp{database=\"${dbname}\",type=\"${btype}\"} ${latest_ts[$key]}\n"
+ done
+
+ output+="# HELP db_backup_age_seconds Seconds since most recent backup.\n"
+ output+="# TYPE db_backup_age_seconds gauge\n"
+ for key in "${!latest_ts[@]}"; do
+ local dbname="${key%%|*}"
+ local btype="${key##*|}"
+ local age=$(( now - latest_ts[$key] ))
+ output+="db_backup_age_seconds{database=\"${dbname}\",type=\"${btype}\"} ${age}\n"
+ done
+
+ output+="# HELP db_backup_size_bytes Size of most recent backup file in bytes.\n"
+ output+="# TYPE db_backup_size_bytes gauge\n"
+ for key in "${!latest_size[@]}"; do
+ local dbname="${key%%|*}"
+ local btype="${key##*|}"
+ output+="db_backup_size_bytes{database=\"${dbname}\",type=\"${btype}\"} ${latest_size[$key]}\n"
+ done
+
+ output+="# HELP db_backup_count Number of backup files found.\n"
+ output+="# TYPE db_backup_count gauge\n"
+ for key in "${!file_count[@]}"; do
+ local dbname="${key%%|*}"
+ local btype="${key##*|}"
+ output+="db_backup_count{database=\"${dbname}\",type=\"${btype}\"} ${file_count[$key]}\n"
+ done
+
+ output+="# HELP db_backup_fresh 1 if backup is within max_age, 0 if stale.\n"
+ output+="# TYPE db_backup_fresh gauge\n"
+ for key in "${!latest_ts[@]}"; do
+ local dbname="${key%%|*}"
+ local btype="${key##*|}"
+ local age=$(( now - latest_ts[$key] ))
+ local fresh=1
+ if (( age > max_age )); then
+ fresh=0
+ log_warn "Stale backup: database=${dbname} type=${btype} age=${age}s exceeds max_age=${max_age}s"
+ fi
+ output+="db_backup_fresh{database=\"${dbname}\",type=\"${btype}\"} ${fresh}\n"
+ done
+
+ output+="# HELP db_backup_exporter_last_run Timestamp of last exporter run.\n"
+ output+="# TYPE db_backup_exporter_last_run gauge\n"
+ output+="db_backup_exporter_last_run ${now}\n"
+
+ echo "$output"
+}
+
+# -----------------------------
+# Write metrics atomically
+# -----------------------------
+write_metrics() {
+ local content="$1"
+ local prom_file="$2"
+
+ local prom_dir
+ prom_dir="$(dirname "$prom_file")"
+
+ if [[ ! -d "$prom_dir" ]]; then
+ log_error "Prom directory does not exist: $prom_dir"
+ return 1
+ fi
+
+ local tmp_file
+ tmp_file="$(mktemp "${prom_dir}/.database_backups.prom.XXXXXX")"
+
+ echo -e "$content" > "$tmp_file"
+ mv "$tmp_file" "$prom_file"
+
+ log_info "Metrics written to $prom_file"
+}
+
+# -----------------------------
+# Main
+# -----------------------------
+main() {
+ parse_args "$@"
+
+ log_info "Database Backup Exporter starting"
+ log_info "Backup directory: $BACKUP_DIR"
+ log_info "Max backup age: ${MAX_AGE}s"
+ log_info "Prom file: $PROM_FILE"
+
+ while true; do
+ local metrics
+ metrics="$(collect_metrics "$BACKUP_DIR" "$MAX_AGE")" || true
+
+ if [[ -n "$metrics" ]]; then
+ write_metrics "$metrics" "$PROM_FILE"
+ fi
+
+ if [[ "$RUN_ONCE" == true ]]; then
+ log_info "Single run complete, exiting"
+ break
+ fi
+
+ log_info "Sleeping ${INTERVAL}s until next collection"
+ sleep "$INTERVAL"
+ done
+}
+
+main "$@"
diff --git a/database-smoke-tests.sh b/database-smoke-tests.sh
new file mode 100644
index 0000000..080c980
--- /dev/null
+++ b/database-smoke-tests.sh
@@ -0,0 +1,573 @@
+#!/usr/bin/env bash
+
+#####################################################################################
+#### database-smoke-tests.sh — Verify database health ####
+#### Checks connectivity, auth, replication, backup age, bloat, connections. ####
+#### Supports: PostgreSQL, MySQL/MariaDB, Redis ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version: 1.0 ####
+#### ####
+#### Usage: DB_TYPE=postgresql DB_HOST=localhost ./database-smoke-tests.sh ####
+#### DB_TYPE=redis REDIS_HOST=localhost ./database-smoke-tests.sh ####
+#### ####
+#### See --help for all options. ####
+#####################################################################################
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------------
+show_help() {
+ cat <<'EOF'
+database-smoke-tests.sh — Database health smoke testing
+
+ENVIRONMENT VARIABLES:
+ DB_TYPE (required) postgresql | mysql | redis
+ DB_HOST Database host (default: localhost)
+ DB_PORT Database port (default: auto — 5432/3306/6379)
+ DB_USER Database user (default: postgres | root | "")
+ DB_PASS Database password (default: "")
+ DB_NAME Database name (default: postgres | mysql)
+
+ REDIS_HOST Redis host (falls back to DB_HOST)
+ REDIS_PORT Redis port (falls back to DB_PORT)
+ REDIS_AUTH Redis auth (falls back to DB_PASS)
+
+ MAX_REPLICATION_LAG_S Max replication lag in seconds (default: 30)
+ MAX_BACKUP_AGE_H Max backup / last-save age in hours (default: 26)
+ MAX_CONNECTIONS_PCT Connection usage threshold % (default: 80)
+ SKIP_REPLICATION Skip replication checks (default: false)
+ SKIP_BACKUP_AGE Skip backup-age checks (default: false)
+
+ OUTPUT_FORMAT text | tap (default: text)
+ COLOR auto | always | never (default: auto)
+ VERBOSE true | false (default: false)
+
+EXAMPLES:
+ DB_TYPE=postgresql DB_HOST=db1 DB_PASS=secret ./database-smoke-tests.sh
+ DB_TYPE=mysql DB_HOST=db2 DB_USER=app DB_NAME=mydb ./database-smoke-tests.sh
+ DB_TYPE=redis REDIS_HOST=cache1 REDIS_AUTH=pass ./database-smoke-tests.sh
+EOF
+ exit 0
+}
+
+[[ "${1:-}" == "--help" || "${1:-}" == "-h" ]] && show_help
+
+# ---------------------------------------------------------------------------
+# Environment defaults
+# ---------------------------------------------------------------------------
+DB_TYPE="${DB_TYPE:-}"
+DB_HOST="${DB_HOST:-localhost}"
+DB_PORT="${DB_PORT:-}"
+DB_USER="${DB_USER:-}"
+DB_PASS="${DB_PASS:-}"
+DB_NAME="${DB_NAME:-}"
+
+REDIS_HOST="${REDIS_HOST:-$DB_HOST}"
+REDIS_PORT="${REDIS_PORT:-${DB_PORT:-6379}}"
+REDIS_AUTH="${REDIS_AUTH:-$DB_PASS}"
+
+MAX_REPLICATION_LAG_S="${MAX_REPLICATION_LAG_S:-30}"
+MAX_BACKUP_AGE_H="${MAX_BACKUP_AGE_H:-26}"
+MAX_CONNECTIONS_PCT="${MAX_CONNECTIONS_PCT:-80}"
+SKIP_REPLICATION="${SKIP_REPLICATION:-false}"
+SKIP_BACKUP_AGE="${SKIP_BACKUP_AGE:-false}"
+
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+COLOR="${COLOR:-auto}"
+VERBOSE="${VERBOSE:-false}"
+
+# ---------------------------------------------------------------------------
+# Apply per-engine defaults after DB_TYPE is known
+# ---------------------------------------------------------------------------
+apply_defaults() {
+ case "$DB_TYPE" in
+ postgresql)
+ DB_PORT="${DB_PORT:-5432}"
+ DB_USER="${DB_USER:-postgres}"
+ DB_NAME="${DB_NAME:-postgres}"
+ ;;
+ mysql)
+ DB_PORT="${DB_PORT:-3306}"
+ DB_USER="${DB_USER:-root}"
+ DB_NAME="${DB_NAME:-mysql}"
+ ;;
+ redis)
+ REDIS_PORT="${REDIS_PORT:-6379}"
+ ;;
+ *)
+ echo "ERROR: DB_TYPE must be one of: postgresql, mysql, redis" >&2
+ exit 1
+ ;;
+ esac
+}
+
+# ---------------------------------------------------------------------------
+# Colour setup
+# ---------------------------------------------------------------------------
+setup_colors() {
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ local use_color=false
+ case "$COLOR" in
+ always) use_color=true ;;
+ never) use_color=false ;;
+ auto) [[ -t 1 ]] && use_color=true ;;
+ esac
+ if $use_color; then
+ RED=$'\033[0;31m'
+ GREEN=$'\033[0;32m'
+ YELLOW=$'\033[1;33m'
+ BLUE=$'\033[0;34m'
+ BOLD=$'\033[1m'
+ RESET=$'\033[0m'
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# Counters
+# ---------------------------------------------------------------------------
+PASS_COUNT=0
+FAIL_COUNT=0
+SKIP_COUNT=0
+TEST_NUM=0
+
+# ---------------------------------------------------------------------------
+# run_test "description" command...
+# ---------------------------------------------------------------------------
+run_test() {
+ local desc="$1"; shift
+ TEST_NUM=$((TEST_NUM + 1))
+ local output rc
+ output=$("$@" 2>&1) && rc=0 || rc=$?
+
+ if [[ $rc -eq 0 ]]; then
+ PASS_COUNT=$((PASS_COUNT + 1))
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok $TEST_NUM - $desc"
+ else
+ echo " ${GREEN}PASS${RESET} $desc"
+ fi
+ elif [[ $rc -eq 2 ]]; then
+ SKIP_COUNT=$((SKIP_COUNT + 1))
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok $TEST_NUM - $desc # SKIP ${output:-skipped}"
+ else
+ echo " ${YELLOW}SKIP${RESET} $desc — ${output:-skipped}"
+ fi
+ else
+ FAIL_COUNT=$((FAIL_COUNT + 1))
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "not ok $TEST_NUM - $desc"
+ [[ -n "$output" ]] && echo "# $output"
+ else
+ echo " ${RED}FAIL${RESET} $desc"
+ [[ -n "$output" ]] && echo " $output"
+ fi
+ fi
+
+ if [[ "$VERBOSE" == "true" && -n "$output" && $rc -eq 0 ]]; then
+ echo " ${BLUE}→${RESET} $output"
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# skip_test "description" "reason"
+# ---------------------------------------------------------------------------
+skip_test() {
+ local desc="$1" reason="${2:-skipped}"
+ TEST_NUM=$((TEST_NUM + 1))
+ SKIP_COUNT=$((SKIP_COUNT + 1))
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok $TEST_NUM - $desc # SKIP $reason"
+ else
+ echo " ${YELLOW}SKIP${RESET} $desc — $reason"
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# check_port host port [timeout]
+# ---------------------------------------------------------------------------
+check_port() {
+ local host="$1" port="$2" timeout="${3:-5}"
+ if command -v nc &>/dev/null; then
+ nc -z -w "$timeout" "$host" "$port" 2>/dev/null
+ elif [[ -e /dev/tcp ]]; then
+ timeout "$timeout" bash -c "echo >/dev/tcp/$host/$port" 2>/dev/null
+ else
+ (echo >/dev/tcp/"$host"/"$port") 2>/dev/null
+ fi
+}
+
+# ---------------------------------------------------------------------------
+# Helper: build psql / mysql invocations
+# ---------------------------------------------------------------------------
+run_psql() {
+ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" \
+ -d "${1:-$DB_NAME}" -t -A -c "$2" 2>&1
+}
+
+run_mysql() {
+ MYSQL_PWD="$DB_PASS" mysql -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" \
+ -D "${1:-$DB_NAME}" -N -s -e "$2" 2>&1
+}
+
+run_redis() {
+ local auth_args=()
+ [[ -n "$REDIS_AUTH" ]] && auth_args=(-a "$REDIS_AUTH" --no-auth-warning)
+ redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" "${auth_args[@]}" "$@" 2>&1
+}
+
+# ===========================================================================
+# PostgreSQL tests
+# ===========================================================================
+run_postgresql_tests() {
+ echo "${BOLD}PostgreSQL smoke tests — ${DB_HOST}:${DB_PORT}${RESET}"
+ echo ""
+
+ # 1. TCP connectivity
+ run_test "TCP connectivity to ${DB_HOST}:${DB_PORT}" \
+ check_port "$DB_HOST" "$DB_PORT"
+
+ # 2. Authentication
+ run_test "Authentication as ${DB_USER}" \
+ bash -c 'PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c "SELECT 1" >/dev/null'
+
+ # 3. Version
+ run_test "Server version" \
+ bash -c '
+ ver=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c "SHOW server_version" 2>&1)
+ echo "PostgreSQL $ver"
+ '
+
+ # 4. Database accessible
+ run_test "Database '${DB_NAME}' accessible" \
+ bash -c 'PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c "SELECT current_database()" >/dev/null'
+
+ # 5. Replication lag
+ if [[ "$SKIP_REPLICATION" == "true" ]]; then
+ skip_test "Replication lag" "SKIP_REPLICATION=true"
+ else
+ run_test "Replication lag < ${MAX_REPLICATION_LAG_S}s" \
+ bash -c '
+ is_replica=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c "SELECT pg_is_in_recovery()" 2>&1)
+ if [[ "$is_replica" == "t" ]]; then
+ lag=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c \
+ "SELECT COALESCE(EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::int, 0)" 2>&1)
+ if [[ "$lag" -gt '"$MAX_REPLICATION_LAG_S"' ]]; then
+ echo "lag=${lag}s exceeds ${'"$MAX_REPLICATION_LAG_S"'}s"; exit 1
+ fi
+ echo "replica lag=${lag}s"
+ else
+ echo "not a replica"; exit 2
+ fi
+ '
+ fi
+
+ # 6. Connection count
+ run_test "Connection usage < ${MAX_CONNECTIONS_PCT}%" \
+ bash -c '
+ read -r used max_c <<< $(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c \
+ "SELECT sum(numbackends), (SELECT setting::int FROM pg_settings WHERE name='"'"'max_connections'"'"') FROM pg_stat_database" 2>&1 | tr "|" " ")
+ pct=$((used * 100 / max_c))
+ if [[ $pct -ge '"$MAX_CONNECTIONS_PCT"' ]]; then
+ echo "${used}/${max_c} (${pct}%)"; exit 1
+ fi
+ echo "${used}/${max_c} (${pct}%)"
+ '
+
+ # 7. Long-running queries
+ run_test "No queries running > 300s" \
+ bash -c '
+ count=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c \
+ "SELECT count(*) FROM pg_stat_activity WHERE state='"'"'active'"'"' AND now()-query_start > interval '"'"'300 seconds'"'"' AND pid <> pg_backend_pid()" 2>&1)
+ if [[ "$count" -gt 0 ]]; then
+ echo "${count} long-running queries found"; exit 1
+ fi
+ echo "none"
+ '
+
+ # 8. Table bloat
+ run_test "Table bloat (dead tuple ratio < 20%)" \
+ bash -c '
+ worst=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c \
+ "SELECT schemaname||'"'"'.'"'"'||relname||'"'"' '"'"'||round(100.0*n_dead_tup/(n_live_tup+n_dead_tup+1),1)||'"'"'%'"'"' FROM pg_stat_user_tables WHERE n_live_tup+n_dead_tup>1000 AND 100.0*n_dead_tup/(n_live_tup+n_dead_tup+1)>20 ORDER BY n_dead_tup DESC LIMIT 3" 2>&1)
+ if [[ -n "$worst" ]]; then
+ echo "bloated: $worst"; exit 1
+ fi
+ echo "ok"
+ '
+
+ # 9. Disk usage
+ run_test "Disk usage for '${DB_NAME}'" \
+ bash -c '
+ size=$(PGPASSWORD="'"$DB_PASS"'" psql -h "'"$DB_HOST"'" -p "'"$DB_PORT"'" -U "'"$DB_USER"'" -d "'"$DB_NAME"'" -t -A -c \
+ "SELECT pg_size_pretty(pg_database_size(current_database()))" 2>&1)
+ echo "$size"
+ '
+}
+
+# ===========================================================================
+# MySQL / MariaDB tests
+# ===========================================================================
+run_mysql_tests() {
+ echo "${BOLD}MySQL smoke tests — ${DB_HOST}:${DB_PORT}${RESET}"
+ echo ""
+
+ # 1. TCP connectivity
+ run_test "TCP connectivity to ${DB_HOST}:${DB_PORT}" \
+ check_port "$DB_HOST" "$DB_PORT"
+
+ # 2. Authentication
+ run_test "Authentication as ${DB_USER}" \
+ bash -c 'MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -e "SELECT 1" >/dev/null'
+
+ # 3. Version
+ run_test "Server version" \
+ bash -c '
+ ver=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SELECT version()" 2>&1)
+ echo "MySQL $ver"
+ '
+
+ # 4. Database accessible
+ run_test "Database '${DB_NAME}' accessible" \
+ bash -c 'MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -D "'"$DB_NAME"'" -e "SELECT 1" >/dev/null'
+
+ # 5. Replication lag
+ if [[ "$SKIP_REPLICATION" == "true" ]]; then
+ skip_test "Replication lag" "SKIP_REPLICATION=true"
+ else
+ run_test "Replication lag < ${MAX_REPLICATION_LAG_S}s" \
+ bash -c '
+ status=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SHOW REPLICA STATUS\G" 2>&1)
+ if [[ -z "$status" ]]; then
+ status=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SHOW SLAVE STATUS\G" 2>&1)
+ fi
+ if [[ -z "$status" ]]; then
+ echo "not a replica"; exit 2
+ fi
+ lag=$(echo "$status" | grep -i "Seconds_Behind" | awk "{print \$NF}")
+ if [[ "$lag" == "NULL" || -z "$lag" ]]; then
+ echo "replication not running (lag=NULL)"; exit 1
+ fi
+ if [[ "$lag" -gt '"$MAX_REPLICATION_LAG_S"' ]]; then
+ echo "lag=${lag}s exceeds '"$MAX_REPLICATION_LAG_S"'s"; exit 1
+ fi
+ echo "replica lag=${lag}s"
+ '
+ fi
+
+ # 6. Connection count
+ run_test "Connection usage < ${MAX_CONNECTIONS_PCT}%" \
+ bash -c '
+ used=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SELECT count(*) FROM information_schema.processlist" 2>&1)
+ max_c=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SHOW VARIABLES LIKE '"'"'max_connections'"'"'" 2>&1 | awk "{print \$2}")
+ pct=$((used * 100 / max_c))
+ if [[ $pct -ge '"$MAX_CONNECTIONS_PCT"' ]]; then
+ echo "${used}/${max_c} (${pct}%)"; exit 1
+ fi
+ echo "${used}/${max_c} (${pct}%)"
+ '
+
+ # 7. Slow query log
+ run_test "Slow query log enabled" \
+ bash -c '
+ val=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SHOW VARIABLES LIKE '"'"'slow_query_log'"'"'" 2>&1 | awk "{print \$2}")
+ if [[ "$val" != "ON" ]]; then
+ echo "slow_query_log=$val"; exit 1
+ fi
+ echo "enabled"
+ '
+
+ # 8. Binary log space
+ run_test "Binary log disk usage" \
+ bash -c '
+ logs=$(MYSQL_PWD="'"$DB_PASS"'" mysql -h "'"$DB_HOST"'" -P "'"$DB_PORT"'" -u "'"$DB_USER"'" -N -s -e "SHOW BINARY LOGS" 2>&1)
+ if [[ "$logs" == *"not enabled"* || -z "$logs" ]]; then
+ echo "binary logging disabled"; exit 2
+ fi
+ total=$(echo "$logs" | awk "{s+=\$2} END {printf \"%.1f MB\", s/1048576}")
+ echo "$total"
+ '
+}
+
+# ===========================================================================
+# Redis tests
+# ===========================================================================
+run_redis_tests() {
+ echo "${BOLD}Redis smoke tests — ${REDIS_HOST}:${REDIS_PORT}${RESET}"
+ echo ""
+
+ local auth_args=()
+ [[ -n "$REDIS_AUTH" ]] && auth_args=(-a "$REDIS_AUTH" --no-auth-warning)
+
+ # 1. TCP connectivity
+ run_test "TCP connectivity to ${REDIS_HOST}:${REDIS_PORT}" \
+ check_port "$REDIS_HOST" "$REDIS_PORT"
+
+ # 2. PING/PONG
+ run_test "PING/PONG" \
+ bash -c '
+ reply=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' PING 2>&1)
+ if [[ "$reply" != "PONG" ]]; then
+ echo "got: $reply"; exit 1
+ fi
+ echo "PONG"
+ '
+
+ # 3. Server info
+ run_test "Server info" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO server 2>&1)
+ ver=$(echo "$info" | grep "^redis_version:" | cut -d: -f2 | tr -d "\r")
+ up=$(echo "$info" | grep "^uptime_in_days:" | cut -d: -f2 | tr -d "\r")
+ echo "v${ver}, uptime ${up}d"
+ '
+
+ # 4. Memory usage
+ run_test "Memory usage vs maxmemory" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO memory 2>&1)
+ used=$(echo "$info" | grep "^used_memory_human:" | cut -d: -f2 | tr -d "\r")
+ max_raw=$(echo "$info" | grep "^maxmemory:" | cut -d: -f2 | tr -d "\r")
+ max_h=$(echo "$info" | grep "^maxmemory_human:" | cut -d: -f2 | tr -d "\r")
+ if [[ "$max_raw" == "0" ]]; then
+ echo "used=${used}, maxmemory=unlimited"; exit 0
+ fi
+ echo "used=${used}, max=${max_h}"
+ '
+
+ # 5. Connected clients
+ run_test "Connected clients" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO clients 2>&1)
+ count=$(echo "$info" | grep "^connected_clients:" | cut -d: -f2 | tr -d "\r")
+ echo "${count} clients"
+ '
+
+ # 6. Replication status
+ if [[ "$SKIP_REPLICATION" == "true" ]]; then
+ skip_test "Replication status" "SKIP_REPLICATION=true"
+ else
+ run_test "Replication status" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO replication 2>&1)
+ role=$(echo "$info" | grep "^role:" | cut -d: -f2 | tr -d "\r")
+ if [[ "$role" == "master" ]]; then
+ slaves=$(echo "$info" | grep "^connected_slaves:" | cut -d: -f2 | tr -d "\r")
+ echo "role=master, replicas=${slaves}"
+ elif [[ "$role" == "slave" ]]; then
+ link=$(echo "$info" | grep "^master_link_status:" | cut -d: -f2 | tr -d "\r")
+ if [[ "$link" != "up" ]]; then
+ echo "replica link $link"; exit 1
+ fi
+ echo "role=replica, link=up"
+ else
+ echo "role=$role"
+ fi
+ '
+ fi
+
+ # 7. Last save time
+ if [[ "$SKIP_BACKUP_AGE" == "true" ]]; then
+ skip_test "Last RDB/AOF save" "SKIP_BACKUP_AGE=true"
+ else
+ run_test "Last RDB save < ${MAX_BACKUP_AGE_H}h" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO persistence 2>&1)
+ last_save=$(echo "$info" | grep "^rdb_last_save_time:" | cut -d: -f2 | tr -d "\r")
+ if [[ -z "$last_save" || "$last_save" == "0" ]]; then
+ echo "no RDB save recorded"; exit 2
+ fi
+ now=$(date +%s)
+ age_h=$(( (now - last_save) / 3600 ))
+ if [[ $age_h -gt '"$MAX_BACKUP_AGE_H"' ]]; then
+ echo "last save ${age_h}h ago (max '"$MAX_BACKUP_AGE_H"'h)"; exit 1
+ fi
+ echo "last save ${age_h}h ago"
+ '
+ fi
+
+ # 8. Keyspace
+ run_test "Keyspace info" \
+ bash -c '
+ info=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' INFO keyspace 2>&1)
+ dbs=$(echo "$info" | grep "^db[0-9]" || true)
+ if [[ -z "$dbs" ]]; then
+ echo "no databases with keys"; exit 2
+ fi
+ total=0
+ while IFS= read -r line; do
+ keys=$(echo "$line" | grep -oP "keys=\K[0-9]+")
+ total=$((total + keys))
+ done <<< "$dbs"
+ echo "${total} keys across $(echo "$dbs" | wc -l) database(s)"
+ '
+
+ # 9. Eviction policy
+ run_test "Eviction policy" \
+ bash -c '
+ policy=$(redis-cli -h "'"$REDIS_HOST"'" -p "'"$REDIS_PORT"'" '"$(printf '%q ' "${auth_args[@]}")"' CONFIG GET maxmemory-policy 2>&1 | tail -1)
+ echo "policy=$policy"
+ '
+}
+
+# ===========================================================================
+# Summary
+# ===========================================================================
+print_summary() {
+ local total=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))
+ echo ""
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "1..$total"
+ echo "# pass $PASS_COUNT"
+ echo "# fail $FAIL_COUNT"
+ echo "# skip $SKIP_COUNT"
+ else
+ echo "${BOLD}───────────────────────────────────────${RESET}"
+ echo " ${GREEN}PASS${RESET} $PASS_COUNT"
+ echo " ${RED}FAIL${RESET} $FAIL_COUNT"
+ echo " ${YELLOW}SKIP${RESET} $SKIP_COUNT"
+ echo " Total $total"
+ echo "${BOLD}───────────────────────────────────────${RESET}"
+ if [[ $FAIL_COUNT -gt 0 ]]; then
+ echo " ${RED}${BOLD}RESULT: FAIL${RESET}"
+ else
+ echo " ${GREEN}${BOLD}RESULT: PASS${RESET}"
+ fi
+ fi
+}
+
+# ===========================================================================
+# Main
+# ===========================================================================
+main() {
+ if [[ -z "$DB_TYPE" ]]; then
+ echo "ERROR: DB_TYPE is required (postgresql, mysql, redis)" >&2
+ echo "Run with --help for usage information." >&2
+ exit 1
+ fi
+
+ apply_defaults
+ setup_colors
+
+ case "$DB_TYPE" in
+ postgresql) run_postgresql_tests ;;
+ mysql) run_mysql_tests ;;
+ redis) run_redis_tests ;;
+ *)
+ echo "ERROR: Unsupported DB_TYPE '${DB_TYPE}'" >&2
+ exit 1
+ ;;
+ esac
+
+ print_summary
+
+ [[ $FAIL_COUNT -gt 0 ]] && exit 1
+ exit 0
+}
+
+main "$@"
diff --git a/deploy-exporter.sh b/deploy-exporter.sh
new file mode 100755
index 0000000..cbe721b
--- /dev/null
+++ b/deploy-exporter.sh
@@ -0,0 +1,588 @@
+#!/bin/bash
+################################################################################
+# Script Name: deploy-exporter.sh
+# Version: 1.0
+# Description: Deployment tool for Prometheus exporters from mylinux.work.
+# Downloads, installs, configures cron jobs, validates output,
+# and manages lifecycle (install, update, remove, status) for
+# any exporter script hosted at mylinux.work/downloads/.
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - wget or curl
+# - root access (for /usr/local/bin/ and /etc/cron.d/)
+#
+# Usage:
+# deploy-exporter.sh list # list available
+# deploy-exporter.sh install process-metrics-exporter # install one
+# deploy-exporter.sh install process-metrics-exporter --cron "*/3 * * * *"
+# deploy-exporter.sh install process-metrics-exporter journal-error-exporter
+# deploy-exporter.sh status # check installed
+# deploy-exporter.sh remove process-metrics-exporter # remove one
+# deploy-exporter.sh update # update all
+#
+################################################################################
+
+set -uo pipefail
+
+# ============================================================================
+# CONFIGURATION
+# ============================================================================
+
+BASE_URL="https://mylinux.work/downloads"
+INSTALL_DIR="/usr/local/bin"
+CRON_DIR="/etc/cron.d"
+TEXTFILE_DIR="/var/lib/node_exporter"
+
+# ============================================================================
+# AVAILABLE EXPORTERS
+# ============================================================================
+
+declare -A EXPORTER_DESC=(
+ [alertmanager-exporter]="Alertmanager notification and silence metrics"
+ [apache-metrics-exporter]="Apache HTTP server performance metrics"
+ [apt-updates-exporter]="Pending apt package updates"
+ [artifactory-exporter]="JFrog Artifactory repository metrics"
+ [backup-status-exporter]="Backup job status and age metrics"
+ [borg-backup-exporter]="Borg backup repository and archive metrics"
+ [caprover-exporter]="CapRover app deployment metrics"
+ [clickhouse-exporter]="ClickHouse query, memory, merge, and replication metrics"
+ [consul-exporter]="HashiCorp Consul service health metrics"
+ [container-health-exporter]="Docker container health and resource metrics"
+ [coolify-exporter]="Coolify deployment platform metrics"
+ [dokku-exporter]="Dokku deployment platform metrics"
+ [dokploy-exporter]="Dokploy deployment platform metrics"
+ [cron-job-exporter]="Cron job execution status and timing"
+ [crowdsec-decisions-exporter]="CrowdSec active decisions and ban metrics"
+ [crowdsec-exporter]="CrowdSec intrusion detection metrics"
+ [database-backup-exporter]="Database backup status and size metrics"
+ [dhcp-lease-exporter]="DHCP lease allocation metrics"
+ [directory-size-exporter]="Directory size and file count metrics"
+ [disk-io-exporter]="Disk I/O throughput and latency metrics"
+ [docker-swarm-exporter]="Docker Swarm node and service metrics"
+ [dovecot-metrics-exporter]="Dovecot mail server metrics"
+ [duplicati-exporter]="Duplicati backup job metrics"
+ [elasticsearch-exporter]="Elasticsearch cluster health and index metrics"
+ [fail2ban-exporter]="Fail2ban jail and ban metrics"
+ [freeradius-exporter]="FreeRADIUS authentication metrics"
+ [game-server-exporter]="Game server player count and status metrics"
+ [gitea-exporter]="Gitea repository and user metrics"
+ [glpi-exporter]="GLPI ITSM ticket and asset metrics"
+ [gitlab-metrics-exporter]="GitLab instance performance metrics"
+ [gitlab-migration-exporter]="GitLab migration progress metrics"
+ [gpu-exporter]="GPU utilization and temperature metrics"
+ [graylog-exporter]="Graylog log management metrics"
+ [headscale-metrics-exporter]="Headscale coordination server metrics"
+ [ip-intel-exporter]="IP intelligence from nginx access logs"
+ [jenkins-exporter]="Jenkins build and queue metrics"
+ [incus-metrics-exporter]="Incus storage pool, snapshot, and instance inventory metrics"
+ [journal-error-exporter]="Journalctl error and warning metrics"
+ [keepalived-exporter]="Keepalived VRRP failover metrics"
+ [login-attempt-exporter]="SSH and system login attempt metrics"
+ [ufw-blocklist-metrics]="UFW blocklist feed, ipset, and block count metrics"
+ [users-logged-in]="User login sessions, terminals, sudo, and failed login metrics"
+ [logrotate-check-exporter]="Logrotate configuration health metrics"
+ [lynis-metrics-exporter]="Lynis security audit score metrics"
+ [mailcow-exporter]="Mailcow mail server metrics"
+ [memory-pressure-exporter]="Memory pressure and swap usage metrics"
+ [mysql-exporter]="MySQL/MariaDB performance metrics"
+ [n8n-exporter]="n8n workflow automation metrics"
+ [network-info-exporter]="Network interface and routing metrics"
+ [nexus-exporter]="Sonatype Nexus Repository metrics"
+ [nextcloud-exporter]="Nextcloud instance health metrics"
+ [nfs-exporter]="NFS client mount and performance metrics"
+ [nfs-server-exporter]="NFS server export and connection metrics"
+ [nginx-metrics-exporter]="Nginx connection and request metrics"
+ [ntp-drift-exporter]="NTP clock drift and sync metrics"
+ [ollama-exporter]="Ollama LLM model and inference metrics"
+ [openvpn-exporter]="OpenVPN tunnel and client metrics"
+ [password-expiry-exporter]="System user password expiry metrics"
+ [pihole-exporter]="Pi-hole DNS filtering metrics"
+ [plex-exporter]="Plex media server activity metrics"
+ [podman-container-exporter]="Podman container health and resource metrics"
+ [postgresql-exporter]="PostgreSQL database performance metrics"
+ [postgresql-ha-exporter]="PostgreSQL HA replication metrics"
+ [process-metrics-exporter]="Process CPU/memory/state metrics"
+ [textfile-health-exporter]="Textfile collector health monitoring"
+ [rabbitmq-exporter]="RabbitMQ queue and connection metrics"
+ [redis-metrics-exporter]="Redis server performance metrics"
+ [redis-sentinel-exporter]="Redis Sentinel failover metrics"
+ [restic-backup-exporter]="Restic backup snapshot and size metrics"
+ [rsyslog-metrics-exporter]="Rsyslog message processing metrics"
+ [samba-exporter]="Samba file share and session metrics"
+ [seo-exporter]="SEO health and crawl metrics"
+ [smart-drive-exporter]="SMART disk health and temperature metrics"
+ [snipeit-exporter]="Snipe-IT asset management metrics"
+ [sonarqube-exporter]="SonarQube code quality metrics"
+ [squid-exporter]="Squid proxy cache and request metrics"
+ [storage-health-exporter]="Storage pool and volume health metrics"
+ [suricata-exporter]="Suricata IDS/IPS alert metrics"
+ [syncthing-exporter]="Syncthing folder sync and device metrics"
+ [systemd-boot-time-exporter]="Systemd boot and service startup timing"
+ [systemd-service-exporter]="Systemd service state and restart metrics"
+ [systemd-timer-exporter]="Systemd timer schedule and execution metrics"
+ [tailscale-exporter]="Tailscale node and network metrics"
+ [trivy-cve-auditor]="Trivy container image vulnerability metrics"
+ [vault-exporter]="HashiCorp Vault seal and token metrics"
+ [vaultwarden-exporter]="Vaultwarden password manager metrics"
+ [wazuh-exporter]="Wazuh SIEM alert and agent metrics"
+ [web-traffic-exporter]="Web traffic request and response metrics"
+ [webtop-selkies-exporter]="Webtop and Selkies container desktop metrics"
+ [wickr-io-exporter]="Wickr.io bot and message metrics"
+ [wickr-metrics-exporter]="Wickr messaging platform metrics"
+ [wireguard-exporter]="WireGuard tunnel and peer metrics"
+ [yum-updates-exporter]="Pending yum/dnf package updates"
+)
+
+declare -A EXPORTER_CRON=(
+ [alertmanager-exporter]="*/5 * * * *"
+ [apache-metrics-exporter]="*/3 * * * *"
+ [apt-updates-exporter]="0 0 * * *"
+ [artifactory-exporter]="*/5 * * * *"
+ [backup-status-exporter]="*/15 * * * *"
+ [borg-backup-exporter]="*/15 * * * *"
+ [caprover-exporter]="*/5 * * * *"
+ [clickhouse-exporter]="*/3 * * * *"
+ [consul-exporter]="*/3 * * * *"
+ [container-health-exporter]="*/3 * * * *"
+ [coolify-exporter]="*/5 * * * *"
+ [dokku-exporter]="*/5 * * * *"
+ [dokploy-exporter]="*/5 * * * *"
+ [cron-job-exporter]="*/5 * * * *"
+ [crowdsec-decisions-exporter]="*/5 * * * *"
+ [crowdsec-exporter]="*/5 * * * *"
+ [database-backup-exporter]="*/15 * * * *"
+ [dhcp-lease-exporter]="*/5 * * * *"
+ [directory-size-exporter]="*/15 * * * *"
+ [disk-io-exporter]="*/3 * * * *"
+ [docker-swarm-exporter]="*/3 * * * *"
+ [dovecot-metrics-exporter]="*/5 * * * *"
+ [duplicati-exporter]="*/15 * * * *"
+ [elasticsearch-exporter]="*/3 * * * *"
+ [fail2ban-exporter]="*/5 * * * *"
+ [freeradius-exporter]="*/5 * * * *"
+ [game-server-exporter]="*/3 * * * *"
+ [gitea-exporter]="*/5 * * * *"
+ [glpi-exporter]="*/5 * * * *"
+ [gitlab-metrics-exporter]="*/5 * * * *"
+ [gitlab-migration-exporter]="*/5 * * * *"
+ [gpu-exporter]="*/3 * * * *"
+ [graylog-exporter]="*/5 * * * *"
+ [headscale-metrics-exporter]="*/5 * * * *"
+ [ip-intel-exporter]="*/5 * * * *"
+ [jenkins-exporter]="*/5 * * * *"
+ [incus-metrics-exporter]="*/5 * * * *"
+ [journal-error-exporter]="*/5 * * * *"
+ [keepalived-exporter]="*/5 * * * *"
+ [login-attempt-exporter]="*/5 * * * *"
+ [ufw-blocklist-metrics]="*/5 * * * *"
+ [users-logged-in]="*/3 * * * *"
+ [logrotate-check-exporter]="0 */6 * * *"
+ [lynis-metrics-exporter]="0 0 * * *"
+ [mailcow-exporter]="*/5 * * * *"
+ [memory-pressure-exporter]="*/3 * * * *"
+ [mysql-exporter]="*/3 * * * *"
+ [n8n-exporter]="*/5 * * * *"
+ [network-info-exporter]="*/5 * * * *"
+ [nexus-exporter]="*/5 * * * *"
+ [nextcloud-exporter]="*/5 * * * *"
+ [nfs-exporter]="*/5 * * * *"
+ [nfs-server-exporter]="*/5 * * * *"
+ [nginx-metrics-exporter]="*/3 * * * *"
+ [ntp-drift-exporter]="*/5 * * * *"
+ [ollama-exporter]="*/5 * * * *"
+ [openvpn-exporter]="*/5 * * * *"
+ [password-expiry-exporter]="0 0 * * *"
+ [pihole-exporter]="*/5 * * * *"
+ [plex-exporter]="*/5 * * * *"
+ [podman-container-exporter]="*/3 * * * *"
+ [postgresql-exporter]="*/3 * * * *"
+ [postgresql-ha-exporter]="*/3 * * * *"
+ [process-metrics-exporter]="*/3 * * * *"
+ [textfile-health-exporter]="*/5 * * * *"
+ [rabbitmq-exporter]="*/5 * * * *"
+ [redis-metrics-exporter]="*/3 * * * *"
+ [redis-sentinel-exporter]="*/5 * * * *"
+ [restic-backup-exporter]="*/15 * * * *"
+ [rsyslog-metrics-exporter]="*/5 * * * *"
+ [samba-exporter]="*/5 * * * *"
+ [seo-exporter]="0 */6 * * *"
+ [smart-drive-exporter]="*/15 * * * *"
+ [snipeit-exporter]="*/5 * * * *"
+ [sonarqube-exporter]="*/5 * * * *"
+ [squid-exporter]="*/5 * * * *"
+ [storage-health-exporter]="*/15 * * * *"
+ [suricata-exporter]="*/5 * * * *"
+ [syncthing-exporter]="*/5 * * * *"
+ [systemd-boot-time-exporter]="*/15 * * * *"
+ [systemd-service-exporter]="*/5 * * * *"
+ [systemd-timer-exporter]="*/5 * * * *"
+ [tailscale-exporter]="*/5 * * * *"
+ [trivy-cve-auditor]="*/30 * * * *"
+ [vault-exporter]="*/5 * * * *"
+ [vaultwarden-exporter]="*/5 * * * *"
+ [wazuh-exporter]="*/5 * * * *"
+ [web-traffic-exporter]="*/5 * * * *"
+ [webtop-selkies-exporter]="*/3 * * * *"
+ [wickr-io-exporter]="*/5 * * * *"
+ [wickr-metrics-exporter]="*/5 * * * *"
+ [wireguard-exporter]="*/5 * * * *"
+ [yum-updates-exporter]="0 0 * * *"
+)
+
+# ============================================================================
+# HELPERS
+# ============================================================================
+
+log() { echo "# $*"; }
+warn() { echo "# WARN: $*" >&2; }
+err() { echo "# ERROR: $*" >&2; }
+die() { err "$@"; exit 1; }
+
+check_root() {
+ [[ $EUID -eq 0 ]] || die "Must run as root (need write access to ${INSTALL_DIR}/ and ${CRON_DIR}/)"
+}
+
+download() {
+ local url="$1" dest="$2"
+ if command -v wget &>/dev/null; then
+ wget -q -O "$dest" "$url"
+ elif command -v curl &>/dev/null; then
+ curl -fsSL -o "$dest" "$url"
+ else
+ die "Neither wget nor curl found"
+ fi
+}
+
+get_script_version() {
+ local file="$1"
+ grep -m1 '^# Version:' "$file" 2>/dev/null | awk '{print $3}' || echo "unknown"
+}
+
+# ============================================================================
+# LIST
+# ============================================================================
+
+cmd_list() {
+ log "Available exporters from mylinux.work (${#EXPORTER_DESC[@]} total)"
+ log ""
+ printf "# %-40s %-15s %s\n" "EXPORTER" "DEFAULT CRON" "DESCRIPTION"
+ printf "# %-40s %-15s %s\n" "--------" "------------" "-----------"
+
+ for name in $(echo "${!EXPORTER_DESC[@]}" | tr ' ' '\n' | sort); do
+ local cron="${EXPORTER_CRON[$name]:-*/5 * * * *}"
+ local desc="${EXPORTER_DESC[$name]}"
+ local installed=""
+ [[ -f "${INSTALL_DIR}/${name}.sh" ]] && installed=" [installed]"
+ printf "# %-40s %-15s %s%s\n" "$name" "$cron" "$desc" "$installed"
+ done
+}
+
+# ============================================================================
+# INSTALL
+# ============================================================================
+
+cmd_install() {
+ check_root
+
+ local names=()
+ local cron_schedule=""
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --cron)
+ cron_schedule="$2"
+ shift 2
+ ;;
+ -*)
+ die "Unknown option: $1"
+ ;;
+ *)
+ names+=("$1")
+ shift
+ ;;
+ esac
+ done
+
+ [[ ${#names[@]} -gt 0 ]] || die "No exporter name(s) specified"
+
+ for name in "${names[@]}"; do
+ install_one "$name" "$cron_schedule"
+ done
+}
+
+install_one() {
+ local name="$1"
+ local cron_schedule="$2"
+ local url="${BASE_URL}/${name}.sh"
+ local dest="${INSTALL_DIR}/${name}.sh"
+ local temp_file
+
+ if [[ -z "${EXPORTER_DESC[$name]+x}" ]]; then
+ warn "Unknown exporter '${name}' — not in the built-in list, attempting download anyway"
+ fi
+
+ log "Installing ${name}..."
+
+ temp_file=$(mktemp "/tmp/${name}.XXXXXX")
+ if ! download "$url" "$temp_file"; then
+ rm -f "$temp_file"
+ err "Failed to download ${url}"
+ return 1
+ fi
+
+ if [[ ! -s "$temp_file" ]]; then
+ rm -f "$temp_file"
+ err "Downloaded file is empty: ${url}"
+ return 1
+ fi
+
+ chmod +x "$temp_file"
+
+ log "Validating ${name}..."
+ local test_output
+ test_output=$("$temp_file" 2>/dev/null || true)
+ local line_count
+ line_count=$(echo "$test_output" | wc -l)
+
+ if [[ "$line_count" -lt 3 ]]; then
+ rm -f "$temp_file"
+ err "Validation failed — ${name} produced only ${line_count} lines of output"
+ return 1
+ fi
+
+ mv -f "$temp_file" "$dest"
+ chmod +x "$dest"
+ log "Installed ${dest}"
+
+ if [[ -n "$cron_schedule" ]]; then
+ local default_cron="$cron_schedule"
+ elif [[ -n "${EXPORTER_CRON[$name]+x}" ]]; then
+ local default_cron="${EXPORTER_CRON[$name]}"
+ log "No --cron specified, using default: ${default_cron}"
+ else
+ local default_cron=""
+ fi
+
+ if [[ -n "$default_cron" ]]; then
+ mkdir -p "$TEXTFILE_DIR"
+ local cron_file="${CRON_DIR}/${name}"
+ cat > "$cron_file" <&1
+EOF
+ log "Created cron job: ${cron_file}"
+ log " Schedule: ${default_cron}"
+ log " Command: ${INSTALL_DIR}/${name}.sh --textfile"
+ fi
+
+ log "${name} installed successfully"
+ log ""
+}
+
+# ============================================================================
+# STATUS
+# ============================================================================
+
+cmd_status() {
+ local found=0
+
+ log "Installed exporters in ${INSTALL_DIR}/:"
+ log ""
+ printf "# %-40s %-12s %-10s %s\n" "EXPORTER" "VERSION" "CRON" "LAST .prom UPDATE"
+ printf "# %-40s %-12s %-10s %s\n" "--------" "-------" "----" "-----------------"
+
+ for script in "${INSTALL_DIR}"/*-exporter.sh; do
+ [[ -f "$script" ]] || continue
+ found=1
+
+ local name
+ name=$(basename "$script" .sh)
+ local version
+ version=$(get_script_version "$script")
+
+ local cron_status="none"
+ [[ -f "${CRON_DIR}/${name}" ]] && cron_status="active"
+
+ local prom_name
+ prom_name=$(echo "$name" | tr '-' '_')
+ local prom_file="${TEXTFILE_DIR}/${prom_name}.prom"
+ local prom_age="no .prom file"
+
+ if [[ -f "$prom_file" ]]; then
+ local mod_time now age_sec
+ mod_time=$(stat -c %Y "$prom_file" 2>/dev/null || echo 0)
+ now=$(date +%s)
+ age_sec=$(( now - mod_time ))
+
+ if [[ $age_sec -lt 60 ]]; then
+ prom_age="${age_sec}s ago"
+ elif [[ $age_sec -lt 3600 ]]; then
+ prom_age="$(( age_sec / 60 ))m ago"
+ elif [[ $age_sec -lt 86400 ]]; then
+ prom_age="$(( age_sec / 3600 ))h ago"
+ else
+ prom_age="$(( age_sec / 86400 ))d ago (STALE)"
+ fi
+ fi
+
+ printf "# %-40s %-12s %-10s %s\n" "$name" "$version" "$cron_status" "$prom_age"
+ done
+
+ if [[ $found -eq 0 ]]; then
+ log "No exporters installed in ${INSTALL_DIR}/"
+ fi
+}
+
+# ============================================================================
+# REMOVE
+# ============================================================================
+
+cmd_remove() {
+ check_root
+ [[ $# -gt 0 ]] || die "No exporter name specified"
+
+ for name in "$@"; do
+ remove_one "$name"
+ done
+}
+
+remove_one() {
+ local name="$1"
+ local script="${INSTALL_DIR}/${name}.sh"
+ local cron_file="${CRON_DIR}/${name}"
+ local prom_name
+ prom_name=$(echo "$name" | tr '-' '_')
+ local prom_file="${TEXTFILE_DIR}/${prom_name}.prom"
+
+ if [[ ! -f "$script" ]]; then
+ warn "${name} is not installed in ${INSTALL_DIR}/"
+ return 1
+ fi
+
+ rm -f "$script"
+ log "Removed ${script}"
+
+ if [[ -f "$cron_file" ]]; then
+ rm -f "$cron_file"
+ log "Removed cron job: ${cron_file}"
+ fi
+
+ if [[ -f "$prom_file" ]]; then
+ rm -f "$prom_file"
+ log "Removed .prom file: ${prom_file}"
+ fi
+
+ log "${name} removed"
+ log ""
+}
+
+# ============================================================================
+# UPDATE
+# ============================================================================
+
+cmd_update() {
+ check_root
+
+ local found=0
+
+ for script in "${INSTALL_DIR}"/*-exporter.sh; do
+ [[ -f "$script" ]] || continue
+ found=1
+
+ local name
+ name=$(basename "$script" .sh)
+ local old_version
+ old_version=$(get_script_version "$script")
+
+ log "Updating ${name} (current: v${old_version})..."
+
+ local url="${BASE_URL}/${name}.sh"
+ local temp_file
+ temp_file=$(mktemp "/tmp/${name}.XXXXXX")
+
+ if ! download "$url" "$temp_file"; then
+ rm -f "$temp_file"
+ err "Failed to download ${name}, skipping"
+ continue
+ fi
+
+ if [[ ! -s "$temp_file" ]]; then
+ rm -f "$temp_file"
+ err "Downloaded file is empty for ${name}, skipping"
+ continue
+ fi
+
+ local new_version
+ new_version=$(get_script_version "$temp_file")
+
+ chmod +x "$temp_file"
+ mv -f "$temp_file" "$script"
+ chmod +x "$script"
+
+ if [[ "$old_version" == "$new_version" ]]; then
+ log "${name}: v${new_version} (unchanged)"
+ else
+ log "${name}: v${old_version} → v${new_version}"
+ fi
+ done
+
+ if [[ $found -eq 0 ]]; then
+ log "No exporters installed in ${INSTALL_DIR}/"
+ fi
+}
+
+# ============================================================================
+# USAGE
+# ============================================================================
+
+show_usage() {
+ cat <&2; }
+info() { echo -e "${BOLD}[INFO]${RESET} $*"; }
+
+usage() {
+ cat </dev/null || true
+ log "Stopped FreshRSS containers"
+ fi
+ fi
+
+ # Remove nginx config
+ for f in /etc/nginx/conf.d/freshrss.conf /etc/nginx/sites-enabled/freshrss.conf /etc/nginx/sites-available/freshrss.conf; do
+ if [[ -f "$f" ]]; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would remove: $f"
+ else
+ rm -f "$f"
+ log "Removed $f"
+ fi
+ fi
+ done
+
+ if [[ "$DRY_RUN" != "true" ]] && command -v nginx &>/dev/null; then
+ nginx -t 2>/dev/null && systemctl reload nginx 2>/dev/null && log "Reloaded Nginx"
+ fi
+
+ echo ""
+ if [[ "$DRY_RUN" != "true" ]]; then
+ log "Containers stopped and Nginx config removed."
+ info "Data preserved at ${INSTALL_DIR}/ - remove manually if desired:"
+ echo " rm -rf ${INSTALL_DIR}"
+ fi
+ exit 0
+fi
+
+# -- Validation --
+
+if [[ -z "$DOMAIN" ]]; then
+ err "Domain is required: --domain rss.example.com"
+ exit 1
+fi
+
+if ! command -v docker &>/dev/null; then
+ err "Docker is not installed. Install Docker first."
+ exit 1
+fi
+
+if ! docker compose version &>/dev/null 2>&1; then
+ err "Docker Compose v2 is not available. Install docker-compose-plugin."
+ exit 1
+fi
+
+# Generate DB password if not provided
+if [[ -z "$DB_PASSWORD" ]]; then
+ DB_PASSWORD=$(openssl rand -base64 24 | tr -d '/+=' | head -c 24)
+fi
+
+# -- Install mode --
+
+info "Deploying FreshRSS..."
+echo ""
+info "Domain: ${DOMAIN}"
+info "Port: ${PORT}"
+info "Install dir: ${INSTALL_DIR}"
+info "Timezone: ${TZ}"
+info "Feed cron: ${CRON_MIN}"
+echo ""
+
+# 1. Create directory
+if [[ -d "$INSTALL_DIR" ]]; then
+ info "Directory ${INSTALL_DIR} already exists"
+else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: ${INSTALL_DIR}"
+ else
+ mkdir -p "$INSTALL_DIR"
+ log "Created ${INSTALL_DIR}"
+ fi
+fi
+
+# 2. Docker Compose file
+COMPOSE_FILE="${INSTALL_DIR}/docker-compose.yml"
+
+if [[ -f "$COMPOSE_FILE" ]]; then
+ info "docker-compose.yml already exists - skipping (delete to recreate)"
+else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: ${COMPOSE_FILE}"
+ else
+ cat > "$COMPOSE_FILE" </dev/null | grep -q '^freshrss$'; then
+ info "FreshRSS container is already running"
+else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would run: docker compose up -d"
+ else
+ cd "$INSTALL_DIR"
+ docker compose up -d
+ log "Started FreshRSS containers"
+ fi
+fi
+
+# 4. Nginx reverse proxy
+if [[ "$SKIP_NGINX" == "true" ]]; then
+ info "Skipping Nginx config (--skip-nginx)"
+elif ! command -v nginx &>/dev/null; then
+ warn "Nginx not installed - skipping reverse proxy config"
+ SKIP_NGINX=true
+else
+ NGINX_CONF=""
+ # Detect config directory style
+ if [[ -d /etc/nginx/conf.d ]]; then
+ NGINX_CONF="/etc/nginx/conf.d/freshrss.conf"
+ elif [[ -d /etc/nginx/sites-available ]]; then
+ NGINX_CONF="/etc/nginx/sites-available/freshrss.conf"
+ else
+ warn "Could not detect Nginx config directory - skipping"
+ SKIP_NGINX=true
+ fi
+
+ if [[ "$SKIP_NGINX" != "true" && -n "$NGINX_CONF" ]]; then
+ if [[ -f "$NGINX_CONF" ]]; then
+ info "Nginx config already exists at ${NGINX_CONF} - skipping"
+ else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: ${NGINX_CONF}"
+ else
+ if [[ "$SKIP_SSL" == "true" ]]; then
+ # HTTP only
+ cat > "$NGINX_CONF" < "$NGINX_CONF" </dev/null; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would run: certbot certonly --nginx -d ${DOMAIN}"
+ else
+ certbot certonly --nginx -d "$DOMAIN" --non-interactive --agree-tos --register-unsafely-without-email || {
+ warn "Certbot failed - configure SSL manually"
+ warn "Run: certbot certonly --nginx -d ${DOMAIN}"
+ }
+ fi
+ else
+ warn "certbot not installed - configure SSL manually"
+ fi
+fi
+
+# 6. Reload Nginx
+if [[ "$SKIP_NGINX" != "true" && "$DRY_RUN" != "true" ]]; then
+ if nginx -t 2>/dev/null; then
+ systemctl reload nginx
+ log "Reloaded Nginx"
+ else
+ warn "Nginx config test failed - check config manually"
+ fi
+fi
+
+# -- Summary --
+
+echo ""
+echo -e "${BOLD}Deployment summary:${RESET}"
+echo " Docker Compose: ${INSTALL_DIR}/docker-compose.yml"
+echo " FreshRSS: http://127.0.0.1:${PORT}"
+if [[ "$SKIP_NGINX" != "true" ]]; then
+ if [[ "$SKIP_SSL" == "true" ]]; then
+ echo " Public URL: http://${DOMAIN}"
+ else
+ echo " Public URL: https://${DOMAIN}"
+ fi
+ echo " Nginx config: ${NGINX_CONF:-/etc/nginx/conf.d/freshrss.conf}"
+fi
+echo " Database: PostgreSQL (freshrss-db container)"
+echo " Feed updates: Every ${CRON_MIN} minutes"
+echo " Data directory: ${INSTALL_DIR}/data/"
+echo ""
+echo -e "${BOLD}Next steps:${RESET}"
+if [[ "$SKIP_SSL" == "true" ]]; then
+ echo " 1. Open http://${DOMAIN} and complete the setup wizard"
+else
+ echo " 1. Open https://${DOMAIN} and complete the setup wizard"
+fi
+echo " 2. Database config in wizard:"
+echo " Type: PostgreSQL"
+echo " Host: freshrss-db"
+echo " Database: freshrss"
+echo " User: freshrss"
+echo " Password: (saved in ${INSTALL_DIR}/docker-compose.yml)"
+echo " 3. Create your admin account"
+echo " 4. Add your first feed: https://mylinux.work/index.xml"
+echo ""
+info "Remove with: $(basename "$0") --remove"
diff --git a/deploy-password-expiry-checker.ps1 b/deploy-password-expiry-checker.ps1
new file mode 100644
index 0000000..5d8e65e
--- /dev/null
+++ b/deploy-password-expiry-checker.ps1
@@ -0,0 +1,389 @@
+<#
+.SYNOPSIS
+ Deploy the password expiry checker to Windows machines.
+.DESCRIPTION
+ Downloads password-expiry-check.ps1, installs it to a configurable
+ directory, creates a scheduled task for recurring checks, and
+ optionally copies the script to NETLOGON for GPO deployment.
+.NOTES
+ Author: Phil Connor
+ License: MIT (https://opensource.org/licenses/MIT)
+ Version: 1.01
+#>
+
+param(
+ [string]$InstallDir = "C:\Scripts",
+ [int]$WarningDays = 14,
+ [int]$IntervalHours = 4,
+ [switch]$NetlogonCopy,
+ [switch]$CmdPrompt,
+ [switch]$NoProfile,
+ [switch]$Remove,
+ [switch]$DryRun,
+ [Alias("h")]
+ [switch]$Help
+)
+
+$ScriptUrl = "https://mylinux.work/downloads/password-expiry-check.ps1.zip"
+$ScriptName = "password-expiry-check.ps1"
+$TaskName = "PasswordExpiryCheck"
+
+# ── Colors ────────────────────────────────────────────────────────────
+
+function Write-OK { param([string]$Msg) Write-Host "[OK] $Msg" -ForegroundColor Green }
+function Write-Warn { param([string]$Msg) Write-Host "[WARN] $Msg" -ForegroundColor Yellow }
+function Write-Err { param([string]$Msg) Write-Host "[ERROR] $Msg" -ForegroundColor Red }
+function Write-Info { param([string]$Msg) Write-Host "[INFO] $Msg" -ForegroundColor Cyan }
+
+# ── Help ──────────────────────────────────────────────────────────────
+
+if ($Help) {
+ Write-Host @"
+Usage: .\deploy-password-expiry-checker.ps1 [OPTIONS]
+
+Deploy password expiry notifications on Windows machines.
+
+Installs:
+ 1. password-expiry-check.ps1 to C:\Scripts\ (configurable)
+ 2. Scheduled task - runs every 4 hours (configurable) under logged-on user
+ 3. Logon-triggered task - fires on every user logon
+ 4. PowerShell profile hook - warning banner in every new PowerShell window
+ 5. Optional cmd.exe AutoRun hook - warning banner in every new cmd window
+ 6. Optional NETLOGON copy for GPO deployment
+
+Options:
+ -InstallDir PATH Installation directory (default: C:\Scripts)
+ -WarningDays N Warning threshold in days (default: 14)
+ -IntervalHours N Scheduled task interval in hours (default: 4)
+ -CmdPrompt Also add warning to cmd.exe via AutoRun registry key
+ -NoProfile Skip PowerShell profile hook (scheduled tasks only)
+ -NetlogonCopy Copy script to NETLOGON share for GPO deployment
+ -Remove Remove deployed components
+ -DryRun Show what would be done without making changes
+ -Help Show this help
+
+Examples:
+ .\deploy-password-expiry-checker.ps1 # install with defaults
+ .\deploy-password-expiry-checker.ps1 -CmdPrompt # also hook into cmd.exe
+ .\deploy-password-expiry-checker.ps1 -NoProfile # skip profile hook
+ .\deploy-password-expiry-checker.ps1 -WarningDays 30 # 30-day warning threshold
+ .\deploy-password-expiry-checker.ps1 -IntervalHours 8 # check every 8 hours
+ .\deploy-password-expiry-checker.ps1 -NetlogonCopy # also copy to NETLOGON
+ .\deploy-password-expiry-checker.ps1 -DryRun # preview changes
+ .\deploy-password-expiry-checker.ps1 -Remove # uninstall
+"@
+ exit 0
+}
+
+# ── Admin check ───────────────────────────────────────────────────────
+
+$currentUser = [Security.Principal.WindowsIdentity]::GetCurrent()
+$principal = New-Object Security.Principal.WindowsPrincipal($currentUser)
+if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
+ Write-Err "Must run as Administrator"
+ exit 1
+}
+
+$ScriptPath = Join-Path $InstallDir $ScriptName
+
+# ── Remove mode ───────────────────────────────────────────────────────
+
+if ($Remove) {
+ Write-Info "Removing password expiry checker deployment..."
+ Write-Host ""
+
+ # Remove scheduled tasks
+ foreach ($name in @($TaskName, "${TaskName}Logon")) {
+ $task = Get-ScheduledTask -TaskName $name -ErrorAction SilentlyContinue
+ if ($task) {
+ if ($DryRun) {
+ Write-Info "Would remove scheduled task: $name"
+ } else {
+ Unregister-ScheduledTask -TaskName $name -Confirm:$false
+ Write-OK "Removed scheduled task: $name"
+ }
+ } else {
+ Write-Info "Scheduled task '$name' not found, skipping"
+ }
+ }
+
+ # Remove script
+ if (Test-Path $ScriptPath) {
+ if ($DryRun) {
+ Write-Info "Would remove: $ScriptPath"
+ } else {
+ Remove-Item -Path $ScriptPath -Force
+ Write-OK "Removed $ScriptPath"
+ }
+ }
+
+ # Remove PowerShell profile hook
+ $profileMarker = "# PasswordExpiryCheck"
+ $allUsersProfile = $PROFILE.AllUsersAllHosts
+ if ((Test-Path $allUsersProfile) -and (Select-String -Path $allUsersProfile -Pattern $profileMarker -Quiet)) {
+ if ($DryRun) {
+ Write-Info "Would remove profile hook from $allUsersProfile"
+ } else {
+ $content = Get-Content $allUsersProfile | Where-Object { $_ -notmatch $profileMarker }
+ if ($content) {
+ Set-Content -Path $allUsersProfile -Value $content
+ } else {
+ Remove-Item -Path $allUsersProfile -Force
+ }
+ Write-OK "Removed PowerShell profile hook"
+ }
+ }
+
+ # Remove cmd.exe AutoRun
+ $cmdAutoRun = Get-ItemProperty -Path "HKLM:\Software\Microsoft\Command Processor" -Name "AutoRun" -ErrorAction SilentlyContinue
+ if ($cmdAutoRun -and $cmdAutoRun.AutoRun -match "password-expiry-check") {
+ if ($DryRun) {
+ Write-Info "Would remove cmd.exe AutoRun registry key"
+ } else {
+ $existing = $cmdAutoRun.AutoRun
+ # Remove our command, handle single command or chained with ampersand
+ $cleaned = ($existing -split '\s*&\s*' | Where-Object { $_ -notmatch 'password-expiry-check' }) -join ' & '
+ if ($cleaned.Trim()) {
+ Set-ItemProperty -Path "HKLM:\Software\Microsoft\Command Processor" -Name "AutoRun" -Value $cleaned.Trim()
+ } else {
+ Remove-ItemProperty -Path "HKLM:\Software\Microsoft\Command Processor" -Name "AutoRun" -ErrorAction SilentlyContinue
+ }
+ Write-OK "Removed cmd.exe AutoRun hook"
+ }
+ }
+
+ # Remove install dir if empty
+ if ((Test-Path $InstallDir) -and @(Get-ChildItem $InstallDir -Force).Count -eq 0) {
+ if ($DryRun) {
+ Write-Info "Would remove empty directory: $InstallDir"
+ } else {
+ Remove-Item -Path $InstallDir -Force
+ Write-OK "Removed empty directory: $InstallDir"
+ }
+ }
+
+ Write-Host ""
+ if (-not $DryRun) {
+ Write-OK "Removal complete."
+ }
+ exit 0
+}
+
+# ── Install mode ──────────────────────────────────────────────────────
+
+Write-Info "Deploying password expiry checker..."
+Write-Host ""
+
+# 1. Create install directory
+if (-not (Test-Path $InstallDir)) {
+ if ($DryRun) {
+ Write-Info "Would create directory: $InstallDir"
+ } else {
+ New-Item -Path $InstallDir -ItemType Directory -Force | Out-Null
+ Write-OK "Created directory: $InstallDir"
+ }
+}
+
+# 2. Download script
+if (Test-Path $ScriptPath) {
+ Write-Info "Script already exists at $ScriptPath - downloading latest version"
+}
+
+if ($DryRun) {
+ Write-Info "Would download $ScriptUrl and extract to $ScriptPath"
+} else {
+ $zipPath = Join-Path $env:TEMP "password-expiry-check.ps1.zip"
+ try {
+ Invoke-WebRequest -Uri $ScriptUrl -OutFile $zipPath -UseBasicParsing -ErrorAction Stop
+ Expand-Archive -Path $zipPath -DestinationPath $InstallDir -Force
+ Remove-Item $zipPath -Force -ErrorAction SilentlyContinue
+ if (Test-Path $ScriptPath) {
+ Write-OK "Downloaded and extracted $ScriptPath"
+ } else {
+ Write-Err "Zip extracted but $ScriptName not found in $InstallDir"
+ exit 1
+ }
+ } catch {
+ Write-Err "Failed to download: $($_.Exception.Message)"
+ exit 1
+ }
+}
+
+# 3. Scheduled task - recurring interval
+$taskArgs = "-NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File `"$ScriptPath`" -Quiet -WarningDays $WarningDays"
+
+$existingTask = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue
+if ($existingTask) {
+ Write-Info "Scheduled task '$TaskName' already exists - recreating"
+ if (-not $DryRun) {
+ Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false
+ }
+}
+
+if ($DryRun) {
+ Write-Info "Would create scheduled task: $TaskName (every ${IntervalHours}h)"
+} else {
+ $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $taskArgs
+ $trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).Date.AddHours(9) `
+ -RepetitionInterval (New-TimeSpan -Hours $IntervalHours) `
+ -RepetitionDuration (New-TimeSpan -Days 365)
+ $settings = New-ScheduledTaskSettingsSet `
+ -AllowStartIfOnBatteries `
+ -DontStopIfGoingOnBatteries `
+ -StartWhenAvailable `
+ -RunOnlyIfNetworkAvailable:$false
+ $principal = New-ScheduledTaskPrincipal -GroupId "S-1-5-32-545" -RunLevel Limited
+
+ Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger `
+ -Settings $settings -Principal $principal `
+ -Description "Check password expiry every $IntervalHours hours (mylinux.work)" | Out-Null
+ Write-OK "Created scheduled task: $TaskName (every ${IntervalHours}h)"
+}
+
+# 4. Logon trigger task
+$logonTaskName = "${TaskName}Logon"
+$existingLogon = Get-ScheduledTask -TaskName $logonTaskName -ErrorAction SilentlyContinue
+if ($existingLogon) {
+ Write-Info "Logon task '$logonTaskName' already exists - recreating"
+ if (-not $DryRun) {
+ Unregister-ScheduledTask -TaskName $logonTaskName -Confirm:$false
+ }
+}
+
+if ($DryRun) {
+ Write-Info "Would create logon trigger task: $logonTaskName"
+} else {
+ $logonAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $taskArgs
+ $logonTrigger = New-ScheduledTaskTrigger -AtLogOn
+ $logonSettings = New-ScheduledTaskSettingsSet `
+ -AllowStartIfOnBatteries `
+ -DontStopIfGoingOnBatteries `
+ -StartWhenAvailable `
+ -ExecutionTimeLimit (New-TimeSpan -Minutes 5)
+ # Delay 30 seconds after logon to let the desktop load
+ $logonTrigger.Delay = "PT30S"
+ $logonPrincipal = New-ScheduledTaskPrincipal -GroupId "S-1-5-32-545" -RunLevel Limited
+
+ Register-ScheduledTask -TaskName $logonTaskName -Action $logonAction -Trigger $logonTrigger `
+ -Settings $logonSettings -Principal $logonPrincipal `
+ -Description "Check password expiry at logon (mylinux.work)" | Out-Null
+ Write-OK "Created logon trigger task: $logonTaskName"
+}
+
+# 5. NETLOGON copy (optional)
+if ($NetlogonCopy) {
+ $logonServer = $env:LOGONSERVER
+ if ($logonServer) {
+ $netlogonPath = Join-Path "$logonServer\NETLOGON" $ScriptName
+ if ($DryRun) {
+ Write-Info "Would copy $ScriptPath to $netlogonPath"
+ } else {
+ try {
+ Copy-Item -Path $ScriptPath -Destination $netlogonPath -Force -ErrorAction Stop
+ Write-OK "Copied to $netlogonPath"
+ } catch {
+ Write-Warn "Could not copy to NETLOGON: $($_.Exception.Message)"
+ Write-Warn "Copy manually: Copy-Item '$ScriptPath' '$netlogonPath'"
+ }
+ }
+ } else {
+ Write-Warn "LOGONSERVER not set - machine may not be domain-joined"
+ Write-Warn "Copy manually to \\DC\NETLOGON\$ScriptName"
+ }
+}
+
+# 6. PowerShell profile hook (default, skip with -NoProfile)
+$profileMarker = "# PasswordExpiryCheck"
+$profileLine = "& `"$ScriptPath`" -Quiet -WarningDays $WarningDays $profileMarker"
+
+if (-not $NoProfile) {
+ $allUsersProfile = $PROFILE.AllUsersAllHosts
+ $profileDir = Split-Path $allUsersProfile -Parent
+
+ # Check if hook already exists
+ $hookExists = (Test-Path $allUsersProfile) -and (Select-String -Path $allUsersProfile -Pattern $profileMarker -Quiet)
+
+ if ($hookExists) {
+ Write-Info "PowerShell profile hook already present - updating"
+ if (-not $DryRun) {
+ $content = Get-Content $allUsersProfile | Where-Object { $_ -notmatch $profileMarker }
+ $content += $profileLine
+ Set-Content -Path $allUsersProfile -Value $content
+ }
+ } else {
+ if ($DryRun) {
+ Write-Info "Would add profile hook to $allUsersProfile"
+ } else {
+ if (-not (Test-Path $profileDir)) {
+ New-Item -Path $profileDir -ItemType Directory -Force | Out-Null
+ }
+ Add-Content -Path $allUsersProfile -Value $profileLine
+ }
+ }
+ Write-OK "PowerShell profile hook: $allUsersProfile"
+} else {
+ Write-Info "Skipping PowerShell profile hook (-NoProfile)"
+}
+
+# 7. cmd.exe AutoRun hook (optional, enable with -CmdPrompt)
+if ($CmdPrompt) {
+ $cmdCommand = '@powershell.exe -NoProfile -ExecutionPolicy Bypass -File "' + $ScriptPath + '" -Quiet -WarningDays ' + $WarningDays
+ $regPath = "HKLM:\Software\Microsoft\Command Processor"
+
+ $existing = Get-ItemProperty -Path $regPath -Name "AutoRun" -ErrorAction SilentlyContinue
+ if ($existing -and $existing.AutoRun -match "password-expiry-check") {
+ Write-Info "cmd.exe AutoRun hook already present - updating"
+ if (-not $DryRun) {
+ $cleaned = ($existing.AutoRun -split '\s*&\s*' | Where-Object { $_ -notmatch 'password-expiry-check' }) -join ' & '
+ if ($cleaned.Trim()) {
+ $newValue = $cleaned.Trim() + " & " + $cmdCommand
+ } else {
+ $newValue = $cmdCommand
+ }
+ Set-ItemProperty -Path $regPath -Name "AutoRun" -Value $newValue
+ }
+ } elseif ($existing -and $existing.AutoRun.Trim()) {
+ if ($DryRun) {
+ Write-Info "Would append to existing cmd.exe AutoRun"
+ } else {
+ $newValue = $existing.AutoRun.Trim() + " & " + $cmdCommand
+ Set-ItemProperty -Path $regPath -Name "AutoRun" -Value $newValue
+ }
+ } else {
+ if ($DryRun) {
+ Write-Info "Would create cmd.exe AutoRun registry key"
+ } else {
+ Set-ItemProperty -Path $regPath -Name "AutoRun" -Value $cmdCommand
+ }
+ }
+ Write-OK "cmd.exe AutoRun hook: $regPath"
+}
+
+# ── Summary ───────────────────────────────────────────────────────────
+
+Write-Host ""
+Write-Host "Deployment summary:" -ForegroundColor White
+Write-Host " Script: $ScriptPath"
+Write-Host " Warning: $WarningDays days"
+Write-Host " Interval task: $TaskName (every ${IntervalHours}h)"
+Write-Host " Logon task: $logonTaskName (at user logon, 30s delay)"
+if (-not $NoProfile) {
+ Write-Host " PS profile: $($PROFILE.AllUsersAllHosts) (all users)"
+}
+if ($CmdPrompt) {
+ Write-Host " cmd.exe: AutoRun registry hook (HKLM)"
+}
+if ($NetlogonCopy) {
+ Write-Host " NETLOGON: $env:LOGONSERVER\NETLOGON\$ScriptName"
+}
+Write-Host ""
+Write-Host "Users will see warnings via:" -ForegroundColor White
+Write-Host " MessageBox popup every $IntervalHours hours (scheduled task)"
+Write-Host " MessageBox popup at logon (logon trigger task)"
+Write-Host " Terminal banner in new PowerShell windows (profile hook)"
+if ($CmdPrompt) {
+ Write-Host " Terminal banner in new cmd.exe windows (AutoRun hook)"
+}
+Write-Host ""
+Write-Info "Test with: & '$ScriptPath' -Test"
+Write-Info "Remove with: .\deploy-password-expiry-checker.ps1 -Remove"
diff --git a/deploy-password-expiry-timer.sh b/deploy-password-expiry-timer.sh
new file mode 100644
index 0000000..7757cae
--- /dev/null
+++ b/deploy-password-expiry-timer.sh
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### deploy-password-expiry-timer.sh — Deploy password expiry desktop notifications ####
+#### Sets up systemd user timer + /etc/bashrc integration for all users. ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### sudo ./deploy-password-expiry-timer.sh ####
+#### sudo ./deploy-password-expiry-timer.sh --dry-run ####
+#### sudo ./deploy-password-expiry-timer.sh --remove ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+DRY_RUN=false
+REMOVE=false
+SCRIPT_PATH="/usr/local/bin/password-expiry-check.sh"
+SCRIPT_URL="https://mylinux.work/downloads/password-expiry-check.sh"
+
+# ── Colors ────────────────────────────────────────────────────────────
+if [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+else
+ RED="" GREEN="" YELLOW="" BOLD="" RESET=""
+fi
+
+log() { echo -e "${GREEN}[OK]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+info() { echo -e "${BOLD}[INFO]${RESET} $*"; }
+
+# ── Usage ─────────────────────────────────────────────────────────────
+usage() {
+ cat </dev/null && \
+ log "Disabled global user timer" || info "Timer was not enabled"
+ fi
+
+ # Remove systemd files
+ for f in /etc/systemd/user/password-expiry-check.service /etc/systemd/user/password-expiry-check.timer; do
+ if [[ -f "$f" ]]; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would remove: $f"
+ else
+ rm -f "$f"
+ log "Removed $f"
+ fi
+ fi
+ done
+
+ # Remove XDG autostart
+ if [[ -f /etc/xdg/autostart/password-expiry-check.desktop ]]; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would remove: /etc/xdg/autostart/password-expiry-check.desktop"
+ else
+ rm -f /etc/xdg/autostart/password-expiry-check.desktop
+ log "Removed XDG autostart"
+ fi
+ fi
+
+ # Remove bashrc entry
+ if grep -q "$BASHRC_MARKER" /etc/bashrc 2>/dev/null; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would remove password-expiry lines from /etc/bashrc"
+ else
+ sed -i "/${BASHRC_MARKER}/d" /etc/bashrc
+ sed -i "/password-expiry-check/d" /etc/bashrc
+ log "Removed /etc/bashrc entry"
+ fi
+ fi
+
+ echo ""
+ if [[ "$DRY_RUN" != "true" ]]; then
+ log "Removal complete. Script left at ${SCRIPT_PATH} (remove manually if desired)"
+ fi
+ exit 0
+fi
+
+# ── Install mode ──────────────────────────────────────────────────────
+info "Deploying password expiry notifications..."
+echo ""
+
+# 1. Install script
+if [[ -f "$SCRIPT_PATH" ]]; then
+ info "Script already exists at ${SCRIPT_PATH}"
+else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would download ${SCRIPT_URL} to ${SCRIPT_PATH}"
+ else
+ if command -v curl &>/dev/null; then
+ curl -sSL -o "$SCRIPT_PATH" "$SCRIPT_URL"
+ elif command -v wget &>/dev/null; then
+ wget -q -O "$SCRIPT_PATH" "$SCRIPT_URL"
+ else
+ err "Neither curl nor wget found — copy password-expiry-check.sh to ${SCRIPT_PATH} manually"
+ exit 1
+ fi
+ chmod +x "$SCRIPT_PATH"
+ log "Installed ${SCRIPT_PATH}"
+ fi
+fi
+
+# 2. Systemd user service
+SERVICE_CONTENT="[Unit]
+Description=Password Expiry Checker
+After=graphical-session.target
+
+[Service]
+Type=oneshot
+ExecStart=${SCRIPT_PATH} -q
+Environment=DISPLAY=:0"
+
+if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: /etc/systemd/user/password-expiry-check.service"
+else
+ mkdir -p /etc/systemd/user
+ echo "$SERVICE_CONTENT" > /etc/systemd/user/password-expiry-check.service
+ log "Created /etc/systemd/user/password-expiry-check.service"
+fi
+
+# 3. Systemd user timer — every 4 hours
+TIMER_CONTENT="[Unit]
+Description=Check password expiry every 4 hours
+
+[Timer]
+OnStartupSec=60
+OnUnitActiveSec=4h
+Persistent=true
+
+[Install]
+WantedBy=timers.target"
+
+if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: /etc/systemd/user/password-expiry-check.timer"
+ info "Would run: systemctl --global enable password-expiry-check.timer"
+else
+ echo "$TIMER_CONTENT" > /etc/systemd/user/password-expiry-check.timer
+ log "Created /etc/systemd/user/password-expiry-check.timer"
+
+ systemctl --global enable password-expiry-check.timer 2>/dev/null
+ log "Enabled timer globally for all users"
+fi
+
+# 4. XDG autostart (graphical login trigger with delay)
+DESKTOP_CONTENT="[Desktop Entry]
+Type=Application
+Name=Password Expiry Checker
+Comment=Check password expiry on login
+Exec=bash -c 'sleep 10 && ${SCRIPT_PATH} -q'
+Terminal=false
+NoDisplay=true
+X-GNOME-Autostart-enabled=true"
+
+if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would create: /etc/xdg/autostart/password-expiry-check.desktop"
+else
+ mkdir -p /etc/xdg/autostart
+ echo "$DESKTOP_CONTENT" > /etc/xdg/autostart/password-expiry-check.desktop
+ log "Created /etc/xdg/autostart/password-expiry-check.desktop"
+fi
+
+# 5. /etc/bashrc entry (terminal warning)
+if grep -q "$BASHRC_MARKER" /etc/bashrc 2>/dev/null; then
+ info "/etc/bashrc entry already exists"
+else
+ if [[ "$DRY_RUN" == "true" ]]; then
+ info "Would add to /etc/bashrc:"
+ echo " ${BASHRC_LINE}"
+ echo " ${BASHRC_EXEC}"
+ else
+ {
+ echo ""
+ echo "$BASHRC_LINE"
+ echo "$BASHRC_EXEC ${BASHRC_MARKER}"
+ } >> /etc/bashrc
+ log "Added /etc/bashrc entry"
+ fi
+fi
+
+echo ""
+echo -e "${BOLD}Deployment summary:${RESET}"
+echo " • Script: ${SCRIPT_PATH}"
+echo " • Timer: /etc/systemd/user/password-expiry-check.timer (every 4h)"
+echo " • XDG autostart: /etc/xdg/autostart/password-expiry-check.desktop (login + 10s delay)"
+echo " • Terminal: /etc/bashrc (quiet mode — warns only when near expiry)"
+echo ""
+echo -e "${BOLD}Users will see warnings via:${RESET}"
+echo " • Desktop popup every 4 hours (systemd timer)"
+echo " • Desktop popup on graphical login (XDG autostart)"
+echo " • Terminal banner on every new shell (bashrc)"
+echo ""
+info "Test with: ${SCRIPT_PATH} --test"
+info "Remove with: $(basename "$0") --remove"
diff --git a/dhcp-lease-exporter.sh b/dhcp-lease-exporter.sh
new file mode 100644
index 0000000..4072830
--- /dev/null
+++ b/dhcp-lease-exporter.sh
@@ -0,0 +1,668 @@
+#!/bin/bash
+################################################################################
+# Script Name: dhcp-lease-exporter.sh
+# Version: 1.01
+# Description: Prometheus exporter for DHCP lease metrics — pool utilization,
+# active leases per subnet, lease expirations, reservation status,
+# DORA packet counts, and lease duration tracking for ISC DHCP
+# (dhcpd) and ISC Kea.
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Usage:
+# # Output to stdout
+# sudo ./dhcp-lease-exporter.sh
+#
+# # Textfile collector mode
+# sudo ./dhcp-lease-exporter.sh --textfile
+#
+# # HTTP server mode
+# sudo ./dhcp-lease-exporter.sh --http
+#
+# # Custom port
+# sudo ./dhcp-lease-exporter.sh --http --port 9533
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# CONFIGURATION
+# ============================================================================
+
+readonly VERSION="1.0"
+readonly SCRIPT_NAME="${0##*/}"
+
+# DHCP backend — auto, dhcpd, or kea
+DHCP_BACKEND="auto"
+
+# dhcpd paths
+DHCPD_LEASES="/var/lib/dhcp/dhcpd.leases"
+DHCPD_CONF="/etc/dhcp/dhcpd.conf"
+
+# Kea paths and API
+KEA_LEASES="/var/lib/kea/kea-leases4.csv"
+KEA_API="http://127.0.0.1:8000"
+KEA_USE_API="true"
+
+# Output settings
+TEXTFILE_DIR="/var/lib/node_exporter/textfile_collector"
+HTTP_PORT=9533
+LOCK_FILE="/tmp/dhcp-lease-exporter.lock"
+
+# Runtime
+MODE="stdout"
+ONCE=false
+DETECTED_BACKEND=""
+
+# ============================================================================
+# COLORS
+# ============================================================================
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+NC='\033[0m'
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; }
+log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
+
+show_usage() {
+ cat </dev/null || true)
+ if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+ log_error "Another instance is running (PID $pid)"
+ exit 1
+ fi
+ rm -f "$LOCK_FILE"
+ fi
+ echo $$ > "$LOCK_FILE"
+ trap 'rm -f "$LOCK_FILE"' EXIT
+}
+
+# ============================================================================
+# BACKEND DETECTION
+# ============================================================================
+
+detect_backend() {
+ if [ "$DHCP_BACKEND" != "auto" ]; then
+ DETECTED_BACKEND="$DHCP_BACKEND"
+ return
+ fi
+
+ if systemctl is-active --quiet isc-kea-dhcp4-server 2>/dev/null || \
+ systemctl is-active --quiet kea-dhcp4 2>/dev/null; then
+ DETECTED_BACKEND="kea"
+ elif systemctl is-active --quiet isc-dhcp-server 2>/dev/null || \
+ systemctl is-active --quiet dhcpd 2>/dev/null; then
+ DETECTED_BACKEND="dhcpd"
+ elif [ -f "$KEA_LEASES" ]; then
+ DETECTED_BACKEND="kea"
+ elif [ -f "$DHCPD_LEASES" ]; then
+ DETECTED_BACKEND="dhcpd"
+ else
+ DETECTED_BACKEND="unknown"
+ fi
+}
+
+# ============================================================================
+# DHCPD FUNCTIONS
+# ============================================================================
+
+# Parse dhcpd.conf for subnet definitions and pool ranges
+parse_dhcpd_subnets() {
+ local conf="$DHCPD_CONF"
+ [ -f "$conf" ] || return
+
+ local current_subnet="" current_name="" range_start="" range_end=""
+ local in_subnet=false
+
+ while IFS= read -r line; do
+ # Match subnet declaration
+ if [[ "$line" =~ ^[[:space:]]*subnet[[:space:]]+([0-9.]+)[[:space:]]+netmask[[:space:]]+([0-9.]+) ]]; then
+ current_subnet="${BASH_REMATCH[1]}"
+ local netmask="${BASH_REMATCH[2]}"
+ current_name="$current_subnet"
+ in_subnet=true
+ range_start=""
+ range_end=""
+ # Calculate CIDR from netmask
+ local cidr
+ cidr=$(netmask_to_cidr "$netmask")
+ current_subnet="${current_subnet}/${cidr}"
+ fi
+
+ # Check for comment-based name
+ if $in_subnet && [[ "$line" =~ ^[[:space:]]*#[[:space:]]*(.+) ]]; then
+ if [ "$current_name" = "${current_subnet%%/*}" ]; then
+ current_name="${BASH_REMATCH[1]}"
+ fi
+ fi
+
+ # Match range declaration
+ if $in_subnet && [[ "$line" =~ ^[[:space:]]*range[[:space:]]+([0-9.]+)[[:space:]]+([0-9.]+) ]]; then
+ range_start="${BASH_REMATCH[1]}"
+ range_end="${BASH_REMATCH[2]}"
+ fi
+
+ # End of subnet block
+ if $in_subnet && [[ "$line" =~ ^[[:space:]]*\} ]]; then
+ if [ -n "$range_start" ] && [ -n "$range_end" ]; then
+ local total
+ total=$(ip_range_count "$range_start" "$range_end")
+ echo "${current_subnet}|${current_name}|${total}|${range_start}|${range_end}"
+ fi
+ in_subnet=false
+ fi
+ done < "$conf"
+}
+
+netmask_to_cidr() {
+ local netmask="$1"
+ local cidr=0
+ for octet in $(echo "$netmask" | tr '.' ' '); do
+ case $octet in
+ 255) cidr=$((cidr + 8)) ;;
+ 254) cidr=$((cidr + 7)) ;;
+ 252) cidr=$((cidr + 6)) ;;
+ 248) cidr=$((cidr + 5)) ;;
+ 240) cidr=$((cidr + 4)) ;;
+ 224) cidr=$((cidr + 3)) ;;
+ 192) cidr=$((cidr + 2)) ;;
+ 128) cidr=$((cidr + 1)) ;;
+ 0) ;;
+ esac
+ done
+ echo "$cidr"
+}
+
+ip_to_int() {
+ local a b c d
+ IFS='.' read -r a b c d <<< "$1"
+ echo $(( (a << 24) + (b << 16) + (c << 8) + d ))
+}
+
+ip_range_count() {
+ local start_int end_int
+ start_int=$(ip_to_int "$1")
+ end_int=$(ip_to_int "$2")
+ echo $(( end_int - start_int + 1 ))
+}
+
+# Count active leases per subnet from dhcpd.leases
+count_dhcpd_leases() {
+ local lease_file="$DHCPD_LEASES"
+ [ -f "$lease_file" ] || return
+
+ local now
+ now=$(date +%s)
+
+ awk -v now="$now" '
+ /^lease / { ip = $2 }
+ /ends / {
+ gsub(/[;\/:]/, " ", $0)
+ if ($2 != "never") {
+ t = mktime($3 " " $4 " " $5 " " $6 " " $7 " " $8)
+ if (t > now) active[ip] = t - now
+ }
+ }
+ /binding state active/ { state[ip] = "active" }
+ END {
+ for (ip in active) {
+ if (state[ip] == "active") {
+ print ip, active[ip]
+ }
+ }
+ }' "$lease_file"
+}
+
+# Count reservations from dhcpd.conf
+count_dhcpd_reservations() {
+ local conf="$DHCPD_CONF"
+ [ -f "$conf" ] || return
+ grep -c "fixed-address" "$conf" 2>/dev/null || true
+}
+
+# Parse DORA stats from syslog
+parse_dhcpd_dora() {
+ local logfile="/var/log/syslog"
+ [ -f "$logfile" ] || logfile="/var/log/messages"
+ [ -f "$logfile" ] || return
+
+ local discovers offers requests acks naks declines releases
+ discovers=$(grep -c "DHCPDISCOVER" "$logfile" 2>/dev/null || true)
+ offers=$(grep -c "DHCPOFFER" "$logfile" 2>/dev/null || true)
+ requests=$(grep -c "DHCPREQUEST" "$logfile" 2>/dev/null || true)
+ acks=$(grep -c "DHCPACK" "$logfile" 2>/dev/null || true)
+ naks=$(grep -c "DHCPNAK" "$logfile" 2>/dev/null || true)
+ declines=$(grep -c "DHCPDECLINE" "$logfile" 2>/dev/null || true)
+ releases=$(grep -c "DHCPRELEASE" "$logfile" 2>/dev/null || true)
+
+ echo "${discovers}|${offers}|${requests}|${acks}|${naks}|${declines}|${releases}"
+}
+
+# ============================================================================
+# KEA FUNCTIONS
+# ============================================================================
+
+kea_api_call() {
+ local command="$1"
+ curl -s --max-time 5 -X POST "${KEA_API}" \
+ -H "Content-Type: application/json" \
+ -d "{\"command\": \"${command}\", \"service\": [\"dhcp4\"]}" 2>/dev/null
+}
+
+parse_kea_leases_file() {
+ local lease_file="$KEA_LEASES"
+ [ -f "$lease_file" ] || return
+
+ local now
+ now=$(date +%s)
+
+ awk -F',' -v now="$now" '
+ NR > 1 && NF >= 9 {
+ ip = $1
+ expire = $7
+ state = $9
+ if (state == 0 && expire > now) {
+ remaining = expire - now
+ print ip, remaining
+ }
+ }' "$lease_file"
+}
+
+parse_kea_api_subnets() {
+ local response
+ response=$(kea_api_call "subnet4-list")
+ if [ -z "$response" ]; then
+ return 1
+ fi
+
+ echo "$response" | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+if data[0]['result'] == 0:
+ for s in data[0].get('arguments', {}).get('subnets', []):
+ sid = s.get('id', 0)
+ subnet = s.get('subnet', '')
+ print(f'{sid}|{subnet}')
+" 2>/dev/null
+}
+
+parse_kea_api_stats() {
+ local response
+ response=$(kea_api_call "statistic-get-all")
+ if [ -z "$response" ]; then
+ return 1
+ fi
+ echo "$response"
+}
+
+# ============================================================================
+# METRIC COLLECTION
+# ============================================================================
+
+collect_metrics() {
+ local start_time
+ start_time=$(date +%s%N)
+ local metrics=""
+
+ # Exporter status
+ metrics+="$(write_metric_header "dhcp_up" "gauge" "Exporter status (1=up, 0=down)")"$'\n'
+ metrics+="$(write_metric_header "dhcp_exporter_info" "gauge" "Exporter version and backend")"$'\n'
+
+ if [ "$DETECTED_BACKEND" = "unknown" ]; then
+ metrics+="dhcp_up 0"$'\n'
+ echo "$metrics"
+ return
+ fi
+
+ metrics+="dhcp_up 1"$'\n'
+ metrics+="dhcp_exporter_info{version=\"${VERSION}\",backend=\"${DETECTED_BACKEND}\"} 1"$'\n'
+
+ local subnet_count=0
+ local total_active=0
+
+ if [ "$DETECTED_BACKEND" = "dhcpd" ]; then
+ collect_dhcpd_metrics
+ elif [ "$DETECTED_BACKEND" = "kea" ]; then
+ collect_kea_metrics
+ fi
+
+ # Subnet count
+ metrics+="$(write_metric_header "dhcp_subnets_total" "gauge" "Total number of configured subnets")"$'\n'
+ metrics+="dhcp_subnets_total ${subnet_count}"$'\n'
+
+ # Total active leases
+ metrics+="$(write_metric_header "dhcp_leases_active_total" "gauge" "Total active leases across all subnets")"$'\n'
+ metrics+="dhcp_leases_active_total ${total_active}"$'\n'
+
+ # Lease file info
+ if [ "$DETECTED_BACKEND" = "dhcpd" ] && [ -f "$DHCPD_LEASES" ]; then
+ local file_age file_size
+ file_age=$(( $(date +%s) - $(stat -c %Y "$DHCPD_LEASES") ))
+ file_size=$(stat -c %s "$DHCPD_LEASES")
+ metrics+="$(write_metric_header "dhcp_lease_file_age_seconds" "gauge" "Seconds since the lease file was last modified")"$'\n'
+ metrics+="dhcp_lease_file_age_seconds ${file_age}"$'\n'
+ metrics+="$(write_metric_header "dhcp_lease_file_size_bytes" "gauge" "Size of the lease file")"$'\n'
+ metrics+="dhcp_lease_file_size_bytes ${file_size}"$'\n'
+ elif [ "$DETECTED_BACKEND" = "kea" ] && [ -f "$KEA_LEASES" ]; then
+ local file_age file_size
+ file_age=$(( $(date +%s) - $(stat -c %Y "$KEA_LEASES") ))
+ file_size=$(stat -c %s "$KEA_LEASES")
+ metrics+="$(write_metric_header "dhcp_lease_file_age_seconds" "gauge" "Seconds since the lease file was last modified")"$'\n'
+ metrics+="dhcp_lease_file_age_seconds ${file_age}"$'\n'
+ metrics+="$(write_metric_header "dhcp_lease_file_size_bytes" "gauge" "Size of the lease file")"$'\n'
+ metrics+="dhcp_lease_file_size_bytes ${file_size}"$'\n'
+ fi
+
+ # Execution time
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $start_time) / 1000000000" | bc 2>/dev/null || echo "0")
+ metrics+="$(write_metric_header "dhcp_exporter_duration_seconds" "gauge" "Script execution time")"$'\n'
+ metrics+="dhcp_exporter_duration_seconds ${duration}"$'\n'
+ metrics+="$(write_metric_header "dhcp_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run")"$'\n'
+ metrics+="dhcp_exporter_last_run_timestamp $(date +%s)"$'\n'
+
+ echo "$metrics"
+}
+
+collect_dhcpd_metrics() {
+ # Parse subnets from config
+ local subnet_data
+ subnet_data=$(parse_dhcpd_subnets)
+
+ # Get active leases
+ local lease_data
+ lease_data=$(count_dhcpd_leases)
+
+ metrics+="$(write_metric_header "dhcp_subnet_pool_total" "gauge" "Total addresses in the pool")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_active" "gauge" "Currently leased addresses")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_free" "gauge" "Available addresses in the pool")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_utilization" "gauge" "Pool utilization percentage (0-100)")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_reserved" "gauge" "Number of static reservations")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_leases_expiring" "gauge" "Leases expiring within threshold")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_lease_longest_seconds" "gauge" "Remaining time on the longest lease")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_lease_shortest_seconds" "gauge" "Remaining time on the shortest lease")"$'\n'
+
+ while IFS='|' read -r subnet name pool_total range_start range_end; do
+ [ -z "$subnet" ] && continue
+ subnet_count=$((subnet_count + 1))
+
+ # Count active leases in this subnet range
+ local active=0 longest=0 shortest=999999999
+ local expiring_1h=0 expiring_4h=0 expiring_24h=0
+ local start_int end_int
+ start_int=$(ip_to_int "$range_start")
+ end_int=$(ip_to_int "$range_end")
+
+ while read -r lease_ip remaining; do
+ [ -z "$lease_ip" ] && continue
+ local lip
+ lip=$(ip_to_int "$lease_ip")
+ if [ "$lip" -ge "$start_int" ] && [ "$lip" -le "$end_int" ]; then
+ active=$((active + 1))
+ total_active=$((total_active + 1))
+ [ "$remaining" -gt "$longest" ] && longest=$remaining
+ [ "$remaining" -lt "$shortest" ] && shortest=$remaining
+ [ "$remaining" -le 3600 ] && expiring_1h=$((expiring_1h + 1))
+ [ "$remaining" -le 14400 ] && expiring_4h=$((expiring_4h + 1))
+ [ "$remaining" -le 86400 ] && expiring_24h=$((expiring_24h + 1))
+ fi
+ done <<< "$lease_data"
+
+ local free=$((pool_total - active))
+ [ $free -lt 0 ] && free=0
+ local util=0
+ if [ "$pool_total" -gt 0 ]; then
+ util=$(echo "scale=2; $active * 100 / $pool_total" | bc 2>/dev/null || echo "0")
+ fi
+ [ $active -eq 0 ] && shortest=0
+
+ local reserved
+ reserved=$(count_dhcpd_reservations)
+
+ metrics+="dhcp_subnet_pool_total{subnet=\"${subnet}\",name=\"${name}\"} ${pool_total}"$'\n'
+ metrics+="dhcp_subnet_pool_active{subnet=\"${subnet}\",name=\"${name}\"} ${active}"$'\n'
+ metrics+="dhcp_subnet_pool_free{subnet=\"${subnet}\",name=\"${name}\"} ${free}"$'\n'
+ metrics+="dhcp_subnet_pool_utilization{subnet=\"${subnet}\",name=\"${name}\"} ${util}"$'\n'
+ metrics+="dhcp_subnet_pool_reserved{subnet=\"${subnet}\",name=\"${name}\"} ${reserved}"$'\n'
+ metrics+="dhcp_subnet_leases_expiring{subnet=\"${subnet}\",name=\"${name}\",within=\"1h\"} ${expiring_1h}"$'\n'
+ metrics+="dhcp_subnet_leases_expiring{subnet=\"${subnet}\",name=\"${name}\",within=\"4h\"} ${expiring_4h}"$'\n'
+ metrics+="dhcp_subnet_leases_expiring{subnet=\"${subnet}\",name=\"${name}\",within=\"24h\"} ${expiring_24h}"$'\n'
+ metrics+="dhcp_subnet_lease_longest_seconds{subnet=\"${subnet}\",name=\"${name}\"} ${longest}"$'\n'
+ metrics+="dhcp_subnet_lease_shortest_seconds{subnet=\"${subnet}\",name=\"${name}\"} ${shortest}"$'\n'
+ done <<< "$subnet_data"
+
+ # DORA stats
+ local dora
+ dora=$(parse_dhcpd_dora)
+ if [ -n "$dora" ]; then
+ IFS='|' read -r discovers offers requests acks naks declines releases <<< "$dora"
+ metrics+="$(write_metric_header "dhcp_discovers_total" "counter" "Total DHCPDISCOVER packets received")"$'\n'
+ metrics+="dhcp_discovers_total ${discovers}"$'\n'
+ metrics+="$(write_metric_header "dhcp_offers_total" "counter" "Total DHCPOFFER packets sent")"$'\n'
+ metrics+="dhcp_offers_total ${offers}"$'\n'
+ metrics+="$(write_metric_header "dhcp_requests_total" "counter" "Total DHCPREQUEST packets received")"$'\n'
+ metrics+="dhcp_requests_total ${requests}"$'\n'
+ metrics+="$(write_metric_header "dhcp_acks_total" "counter" "Total DHCPACK packets sent")"$'\n'
+ metrics+="dhcp_acks_total ${acks}"$'\n'
+ metrics+="$(write_metric_header "dhcp_naks_total" "counter" "Total DHCPNAK packets sent")"$'\n'
+ metrics+="dhcp_naks_total ${naks}"$'\n'
+ metrics+="$(write_metric_header "dhcp_declines_total" "counter" "Total DHCPDECLINE packets received")"$'\n'
+ metrics+="dhcp_declines_total ${declines}"$'\n'
+ metrics+="$(write_metric_header "dhcp_releases_total" "counter" "Total DHCPRELEASE packets received")"$'\n'
+ metrics+="dhcp_releases_total ${releases}"$'\n'
+ fi
+}
+
+collect_kea_metrics() {
+ metrics+="$(write_metric_header "dhcp_subnet_pool_total" "gauge" "Total addresses in the pool")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_active" "gauge" "Currently leased addresses")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_free" "gauge" "Available addresses in the pool")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_utilization" "gauge" "Pool utilization percentage (0-100)")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_pool_reserved" "gauge" "Number of static reservations")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_leases_expiring" "gauge" "Leases expiring within threshold")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_lease_longest_seconds" "gauge" "Remaining time on the longest lease")"$'\n'
+ metrics+="$(write_metric_header "dhcp_subnet_lease_shortest_seconds" "gauge" "Remaining time on the shortest lease")"$'\n'
+
+ if [ "$KEA_USE_API" = "true" ]; then
+ collect_kea_api_metrics
+ else
+ collect_kea_file_metrics
+ fi
+}
+
+collect_kea_api_metrics() {
+ local stats_json
+ stats_json=$(kea_api_call "statistic-get-all")
+
+ if [ -z "$stats_json" ]; then
+ log_warn "Kea API not responding, falling back to file mode"
+ collect_kea_file_metrics
+ return
+ fi
+
+ # Parse stats via python3
+ echo "$stats_json" | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+if data[0]['result'] == 0:
+ args = data[0].get('arguments', {})
+ for key, val in args.items():
+ if val and isinstance(val, list):
+ v = val[0][0] if isinstance(val[0], list) else val[0]
+ print(f'{key}={v}')
+" 2>/dev/null | while IFS='=' read -r key value; do
+ case "$key" in
+ subnet*total-addresses*)
+ local sid="${key#subnet[}"
+ sid="${sid%%]*}"
+ metrics+="dhcp_subnet_pool_total{subnet=\"${sid}\"} ${value}"$'\n'
+ ;;
+ subnet*assigned-addresses*)
+ local sid="${key#subnet[}"
+ sid="${sid%%]*}"
+ metrics+="dhcp_subnet_pool_active{subnet=\"${sid}\"} ${value}"$'\n'
+ ;;
+ pkt4-discover-received)
+ metrics+="$(write_metric_header "dhcp_discovers_total" "counter" "Total DHCPDISCOVER packets received")"$'\n'
+ metrics+="dhcp_discovers_total ${value}"$'\n'
+ ;;
+ pkt4-offer-sent)
+ metrics+="$(write_metric_header "dhcp_offers_total" "counter" "Total DHCPOFFER packets sent")"$'\n'
+ metrics+="dhcp_offers_total ${value}"$'\n'
+ ;;
+ pkt4-request-received)
+ metrics+="$(write_metric_header "dhcp_requests_total" "counter" "Total DHCPREQUEST packets received")"$'\n'
+ metrics+="dhcp_requests_total ${value}"$'\n'
+ ;;
+ pkt4-ack-sent)
+ metrics+="$(write_metric_header "dhcp_acks_total" "counter" "Total DHCPACK packets sent")"$'\n'
+ metrics+="dhcp_acks_total ${value}"$'\n'
+ ;;
+ pkt4-nak-sent)
+ metrics+="$(write_metric_header "dhcp_naks_total" "counter" "Total DHCPNAK packets sent")"$'\n'
+ metrics+="dhcp_naks_total ${value}"$'\n'
+ ;;
+ pkt4-decline-received)
+ metrics+="$(write_metric_header "dhcp_declines_total" "counter" "Total DHCPDECLINE packets received")"$'\n'
+ metrics+="dhcp_declines_total ${value}"$'\n'
+ ;;
+ pkt4-release-received)
+ metrics+="$(write_metric_header "dhcp_releases_total" "counter" "Total DHCPRELEASE packets received")"$'\n'
+ metrics+="dhcp_releases_total ${value}"$'\n'
+ ;;
+ esac
+ done
+}
+
+collect_kea_file_metrics() {
+ local lease_data
+ lease_data=$(parse_kea_leases_file)
+ local now
+ now=$(date +%s)
+
+ # Simple lease counting from CSV
+ while read -r lease_ip remaining; do
+ [ -z "$lease_ip" ] && continue
+ total_active=$((total_active + 1))
+ done <<< "$lease_data"
+}
+
+# ============================================================================
+# OUTPUT
+# ============================================================================
+
+output_metrics() {
+ local all_metrics
+ all_metrics=$(collect_metrics)
+
+ case "$MODE" in
+ stdout)
+ echo "$all_metrics"
+ ;;
+ textfile)
+ mkdir -p "$TEXTFILE_DIR"
+ local tmp_file
+ tmp_file=$(mktemp "${TEXTFILE_DIR}/.dhcp-metrics.XXXXXX")
+ echo "$all_metrics" > "$tmp_file"
+ mv "$tmp_file" "${TEXTFILE_DIR}/dhcp-metrics.prom"
+ log_info "Wrote metrics to ${TEXTFILE_DIR}/dhcp-metrics.prom"
+ ;;
+ http)
+ run_http_server "$all_metrics"
+ ;;
+ esac
+}
+
+run_http_server() {
+ log_info "Starting HTTP server on port ${HTTP_PORT}"
+ while true; do
+ local all_metrics
+ all_metrics=$(collect_metrics)
+
+ {
+ echo -e "HTTP/1.1 200 OK\r"
+ echo -e "Content-Type: text/plain; version=0.0.4; charset=utf-8\r"
+ echo -e "Content-Length: ${#all_metrics}\r"
+ echo -e "\r"
+ echo "$all_metrics"
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null || \
+ {
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n${all_metrics}"
+ } | nc -l "$HTTP_PORT" 2>/dev/null
+
+ if $ONCE; then
+ break
+ fi
+ done
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+main() {
+ parse_args "$@"
+ acquire_lock
+ detect_backend
+ log_info "Detected DHCP backend: ${DETECTED_BACKEND}"
+ output_metrics
+}
+
+main "$@"
diff --git a/directory-size-exporter.sh b/directory-size-exporter.sh
index ff25c40..0b67038 100644
--- a/directory-size-exporter.sh
+++ b/directory-size-exporter.sh
@@ -9,7 +9,7 @@
# Author: Phil Connor
# Contact: contact@mylinux.work
# License: MIT
-# Version: 1.0.0
+# Version: 1.0.1
set -euo pipefail
@@ -27,28 +27,23 @@ TARGET_DIRECTORIES=()
# ── Metrics Collection ──────────────────────────────────────────────
log_verbose() {
- [[ "$VERBOSE" == true ]] && echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
+ [[ "$VERBOSE" == true ]] && echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2 || true
}
log_info() {
- [[ "$QUIET" == false ]] && echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
+ [[ "$QUIET" == false ]] && echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2 || true
}
collect_metrics() {
local start_time
start_time=$(date +%s%N)
- echo "# HELP node_directory_size_bytes Disk space used by directory"
- echo "# TYPE node_directory_size_bytes gauge"
- echo "# HELP node_directory_filesystem_usage_percent Filesystem usage percentage for the directory mount point"
- echo "# TYPE node_directory_filesystem_usage_percent gauge"
-
local success=1
+ local size_lines="" pct_lines=""
for directory in "${TARGET_DIRECTORIES[@]}"; do
log_verbose "Running du for: $directory"
- # Get directory size in bytes
local du_output
du_output=$(timeout "$TIMEOUT" du --block-size=1 --summarize "$directory" 2>/dev/null) || {
log_info "WARNING: du failed for $directory"
@@ -58,16 +53,24 @@ collect_metrics() {
local size_bytes
size_bytes=$(echo "$du_output" | awk '{print $1}')
- echo "node_directory_size_bytes{directory=\"${directory}\"} ${size_bytes}"
+ size_lines+="node_directory_size_bytes{directory=\"${directory}\"} ${size_bytes}"$'\n'
- # Get filesystem usage percentage for the mount point
local pct
pct=$(df --output=pcent "$directory" 2>/dev/null | tail -n 1 | tr -d ' %')
if [[ "$pct" =~ ^[0-9]+$ ]]; then
- echo "node_directory_filesystem_usage_percent{directory=\"${directory}\"} ${pct}"
+ pct_lines+="node_directory_filesystem_usage_percent{directory=\"${directory}\"} ${pct}"$'\n'
fi
done
+ echo "# HELP node_directory_size_bytes Disk space used by directory"
+ echo "# TYPE node_directory_size_bytes gauge"
+ printf "%s" "$size_lines"
+
+ echo ""
+ echo "# HELP node_directory_filesystem_usage_percent Filesystem usage percentage for the directory mount point"
+ echo "# TYPE node_directory_filesystem_usage_percent gauge"
+ printf "%s" "$pct_lines"
+
# ── Script runtime ──
local end_time runtime
end_time=$(date +%s%N)
@@ -78,10 +81,12 @@ collect_metrics() {
echo "# TYPE ${EXPORTER_NAME}_duration_seconds gauge"
echo "${EXPORTER_NAME}_duration_seconds ${runtime}"
+ echo ""
echo "# HELP ${EXPORTER_NAME}_last_run_timestamp Last successful run"
echo "# TYPE ${EXPORTER_NAME}_last_run_timestamp gauge"
echo "${EXPORTER_NAME}_last_run_timestamp $(date +%s)"
+ echo ""
echo "# HELP ${EXPORTER_NAME}_success Whether the exporter ran successfully"
echo "# TYPE ${EXPORTER_NAME}_success gauge"
echo "${EXPORTER_NAME}_success ${success}"
@@ -191,8 +196,8 @@ while [[ $# -gt 0 ]]; do
shift
;;
--handle-request)
- handle_request
- exit 0
+ OUTPUT_MODE="handle-request"
+ shift
;;
-h|--help)
show_help
@@ -236,6 +241,10 @@ if [[ "$DRY_RUN" == true ]]; then
fi
case "$OUTPUT_MODE" in
+ handle-request)
+ handle_request
+ exit 0
+ ;;
stdout)
collect_metrics
;;
@@ -262,6 +271,6 @@ case "$OUTPUT_MODE" in
fi
echo "${EXPORTER_NAME} listening on port ${PORT}..."
echo "Monitoring directories: ${TARGET_DIRECTORIES[*]}"
- socat TCP-LISTEN:"$PORT",reuseaddr,fork EXEC:"$0 --handle-request"
+ socat TCP-LISTEN:"$PORT",reuseaddr,fork EXEC:"$0 --handle-request ${TARGET_DIRECTORIES[*]}"
;;
esac
diff --git a/disk-cleanup.sh b/disk-cleanup.sh
new file mode 100644
index 0000000..b80387b
--- /dev/null
+++ b/disk-cleanup.sh
@@ -0,0 +1,584 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### disk-cleanup.sh — Find and clean disk space hogs on Linux servers ####
+#### Scans logs, temp files, package caches, old kernels, journal, and Docker cruft ####
+#### Dry-run by default — nothing is deleted without --force ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.01 ####
+#### ####
+#### Usage: ####
+#### ./disk-cleanup.sh --scan ####
+#### ./disk-cleanup.sh --clean --force ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+LOG_AGE_DAYS="${LOG_AGE_DAYS:-30}"
+TMP_AGE_DAYS="${TMP_AGE_DAYS:-7}"
+JOURNAL_MAX="${JOURNAL_MAX:-500M}"
+LARGE_FILE_MIN="${LARGE_FILE_MIN:-100M}"
+LARGE_FILE_DIRS="${LARGE_FILE_DIRS:-/var /home /opt /tmp /srv}"
+DRY_RUN="${DRY_RUN:-true}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+RUN_MODE=""
+TOTAL_RECLAIMABLE=0
+TOTAL_CLEANED=0
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ else
+ RED="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+
+section_header() {
+ echo ""
+ echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
+ echo ""
+}
+
+human_bytes() {
+ local bytes="$1"
+ if [[ "$bytes" -ge 1073741824 ]]; then
+ awk "BEGIN { printf \"%.1f GiB\", $bytes / 1073741824 }"
+ elif [[ "$bytes" -ge 1048576 ]]; then
+ awk "BEGIN { printf \"%.1f MiB\", $bytes / 1048576 }"
+ elif [[ "$bytes" -ge 1024 ]]; then
+ awk "BEGIN { printf \"%.1f KiB\", $bytes / 1024 }"
+ else
+ echo "${bytes} B"
+ fi
+}
+
+add_reclaimable() {
+ TOTAL_RECLAIMABLE=$((TOTAL_RECLAIMABLE + $1))
+}
+
+add_cleaned() {
+ TOTAL_CLEANED=$((TOTAL_CLEANED + $1))
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OLD LOGS
+# ══════════════════════════════════════════════════════════════════════
+
+scan_old_logs() {
+ section_header "Old Log Files (> ${LOG_AGE_DAYS} days)"
+
+ local total_size=0
+ local count=0
+
+ while IFS= read -r -d '' file; do
+ local size
+ size=$(stat -c%s "$file" 2>/dev/null || echo 0)
+ if [[ "$size" -gt 0 ]]; then
+ total_size=$((total_size + size))
+ ((count++)) || true
+ if [[ "$VERBOSE" == "true" ]]; then
+ printf " %10s %s\n" "$(human_bytes "$size")" "$file"
+ fi
+ fi
+ done < <(find /var/log -type f \( -name "*.gz" -o -name "*.xz" -o -name "*.bz2" -o -name "*.[0-9]" -o -name "*.old" \) -mtime +"$LOG_AGE_DAYS" -print0 2>/dev/null)
+
+ # Rotated logs without compression
+ while IFS= read -r -d '' file; do
+ local size
+ size=$(stat -c%s "$file" 2>/dev/null || echo 0)
+ if [[ "$size" -gt 0 ]]; then
+ total_size=$((total_size + size))
+ ((count++)) || true
+ fi
+ done < <(find /var/log -type f -name "*.log.*" -mtime +"$LOG_AGE_DAYS" -print0 2>/dev/null)
+
+ printf " %-30s %s (%d files)\n" "Rotated/old logs:" "$(human_bytes "$total_size")" "$count"
+ add_reclaimable "$total_size"
+}
+
+clean_old_logs() {
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "[DRY-RUN] Would delete old log files in /var/log"
+ return
+ fi
+
+ local cleaned=0
+ find /var/log -type f \( -name "*.gz" -o -name "*.xz" -o -name "*.bz2" -o -name "*.[0-9]" -o -name "*.old" \) -mtime +"$LOG_AGE_DAYS" -print0 2>/dev/null | while IFS= read -r -d '' file; do
+ local size
+ size=$(stat -c%s "$file" 2>/dev/null || echo 0)
+ rm -f "$file" && cleaned=$((cleaned + size))
+ done
+
+ find /var/log -type f -name "*.log.*" -mtime +"$LOG_AGE_DAYS" -delete 2>/dev/null || true
+ log "Cleaned old log files"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# JOURNAL
+# ══════════════════════════════════════════════════════════════════════
+
+scan_journal() {
+ section_header "Systemd Journal"
+
+ if ! command -v journalctl &>/dev/null; then
+ printf " %-30s %s\n" "Journal:" "N/A (no systemd)"
+ return
+ fi
+
+ local journal_size
+ journal_size=$(journalctl --disk-usage 2>/dev/null | grep -oP '[\d.]+[GMKT]' | head -1 || echo "0")
+
+ # Get bytes for tracking
+ local journal_bytes
+ journal_bytes=$(du -sb /var/log/journal/ 2>/dev/null | awk '{print $1}' || echo "0")
+ if [[ "$journal_bytes" -eq 0 ]]; then
+ journal_bytes=$(du -sb /run/log/journal/ 2>/dev/null | awk '{print $1}' || echo "0")
+ fi
+
+ printf " %-30s %s\n" "Journal size:" "${journal_size:-Unknown}"
+ printf " %-30s %s\n" "Would vacuum to:" "$JOURNAL_MAX"
+
+ # Estimate savings
+ local max_bytes=0
+ local max_num="${JOURNAL_MAX%[GMKT]*}"
+ local max_unit="${JOURNAL_MAX: -1}"
+ case "$max_unit" in
+ G) max_bytes=$((max_num * 1073741824)) ;;
+ M) max_bytes=$((max_num * 1048576)) ;;
+ K) max_bytes=$((max_num * 1024)) ;;
+ *) max_bytes=$((max_num)) ;;
+ esac
+
+ if [[ "$journal_bytes" -gt "$max_bytes" ]]; then
+ local savings=$((journal_bytes - max_bytes))
+ add_reclaimable "$savings"
+ printf " %-30s %s\n" "Reclaimable:" "$(human_bytes "$savings")"
+ fi
+}
+
+clean_journal() {
+ if ! command -v journalctl &>/dev/null; then
+ return
+ fi
+
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "[DRY-RUN] Would vacuum journal to ${JOURNAL_MAX}"
+ return
+ fi
+
+ journalctl --vacuum-size="$JOURNAL_MAX" 2>/dev/null || warn "Journal vacuum failed"
+ log "Vacuumed journal to ${JOURNAL_MAX}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# TEMP FILES
+# ══════════════════════════════════════════════════════════════════════
+
+scan_tmp() {
+ section_header "Temp Files (> ${TMP_AGE_DAYS} days)"
+
+ local total_size=0
+ local count=0
+
+ for dir in /tmp /var/tmp; do
+ if [[ -d "$dir" ]]; then
+ while IFS= read -r -d '' file; do
+ local size
+ size=$(stat -c%s "$file" 2>/dev/null || echo 0)
+ total_size=$((total_size + size))
+ ((count++)) || true
+ done < <(find "$dir" -maxdepth 2 -type f -mtime +"$TMP_AGE_DAYS" -print0 2>/dev/null)
+ fi
+ done
+
+ printf " %-30s %s (%d files)\n" "Old temp files:" "$(human_bytes "$total_size")" "$count"
+ add_reclaimable "$total_size"
+}
+
+clean_tmp() {
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "[DRY-RUN] Would delete temp files older than ${TMP_AGE_DAYS} days"
+ return
+ fi
+
+ for dir in /tmp /var/tmp; do
+ if [[ -d "$dir" ]]; then
+ find "$dir" -maxdepth 2 -type f -mtime +"$TMP_AGE_DAYS" -delete 2>/dev/null || true
+ fi
+ done
+ log "Cleaned temp files"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# PACKAGE CACHE
+# ══════════════════════════════════════════════════════════════════════
+
+scan_package_cache() {
+ section_header "Package Cache"
+
+ if command -v apt-get &>/dev/null; then
+ local apt_size
+ apt_size=$(du -sb /var/cache/apt/archives/ 2>/dev/null | awk '{print $1}' || echo "0")
+ printf " %-30s %s\n" "APT cache:" "$(human_bytes "$apt_size")"
+ add_reclaimable "$apt_size"
+ fi
+
+ if command -v yum &>/dev/null || command -v dnf &>/dev/null; then
+ local yum_size
+ yum_size=$(du -sb /var/cache/yum/ /var/cache/dnf/ 2>/dev/null | awk '{total+=$1} END {print total+0}')
+ printf " %-30s %s\n" "YUM/DNF cache:" "$(human_bytes "$yum_size")"
+ add_reclaimable "$yum_size"
+ fi
+}
+
+clean_package_cache() {
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "[DRY-RUN] Would clean package cache"
+ return
+ fi
+
+ if command -v apt-get &>/dev/null; then
+ apt-get clean -y 2>/dev/null || warn "apt-get clean failed"
+ log "Cleaned APT cache"
+ fi
+
+ if command -v dnf &>/dev/null; then
+ dnf clean all 2>/dev/null || warn "dnf clean failed"
+ log "Cleaned DNF cache"
+ elif command -v yum &>/dev/null; then
+ yum clean all 2>/dev/null || warn "yum clean failed"
+ log "Cleaned YUM cache"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OLD KERNELS
+# ══════════════════════════════════════════════════════════════════════
+
+scan_old_kernels() {
+ section_header "Old Kernels"
+
+ local current_kernel
+ current_kernel=$(uname -r)
+
+ local old_count=0
+ local total_size=0
+
+ if command -v dpkg &>/dev/null; then
+ while IFS= read -r pkg; do
+ [[ -z "$pkg" ]] && continue
+ local pkg_version
+ pkg_version=$(echo "$pkg" | sed 's/linux-image-//' | sed 's/-generic//' | sed 's/-unsigned//')
+ if [[ "$current_kernel" != *"$pkg_version"* ]]; then
+ local size
+ size=$(dpkg-query -W --showformat='${Installed-Size}' "$pkg" 2>/dev/null || echo "0")
+ total_size=$((total_size + size * 1024))
+ ((old_count++)) || true
+ verbose "Old kernel: ${pkg} ($(human_bytes $((size * 1024))))"
+ fi
+ done < <(dpkg --list 'linux-image-*' 2>/dev/null | grep '^ii' | awk '{print $2}' | grep -v "$current_kernel")
+ elif command -v rpm &>/dev/null; then
+ while IFS= read -r pkg; do
+ [[ -z "$pkg" ]] && continue
+ if [[ "$pkg" != *"$current_kernel"* ]]; then
+ local size
+ size=$(rpm -q --queryformat '%{SIZE}' "$pkg" 2>/dev/null || echo "0")
+ total_size=$((total_size + size))
+ ((old_count++)) || true
+ verbose "Old kernel: ${pkg}"
+ fi
+ done < <(rpm -qa kernel 2>/dev/null)
+ fi
+
+ printf " %-30s %s\n" "Current kernel:" "$current_kernel"
+ printf " %-30s %d ($(human_bytes "$total_size"))\n" "Old kernels:" "$old_count"
+ add_reclaimable "$total_size"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# DOCKER CLEANUP
+# ══════════════════════════════════════════════════════════════════════
+
+scan_docker() {
+ if ! command -v docker &>/dev/null; then
+ return
+ fi
+
+ if ! docker info &>/dev/null 2>&1; then
+ return
+ fi
+
+ section_header "Docker"
+
+ # Dangling images
+ local dangling_count
+ dangling_count=$(docker images -f "dangling=true" -q 2>/dev/null | wc -l)
+
+ printf " %-30s %d\n" "Dangling images:" "$dangling_count"
+
+ # Stopped containers
+ local stopped_count
+ stopped_count=$(docker ps -f "status=exited" -q 2>/dev/null | wc -l)
+ printf " %-30s %d\n" "Stopped containers:" "$stopped_count"
+
+ # Unused volumes
+ local vol_count
+ vol_count=$(docker volume ls -f "dangling=true" -q 2>/dev/null | wc -l)
+ printf " %-30s %d\n" "Unused volumes:" "$vol_count"
+
+ # Build cache
+ if docker builder prune --dry-run 2>/dev/null | grep -q "Total:"; then
+ local build_cache
+ build_cache=$(docker builder prune --dry-run 2>/dev/null | grep "Total:" | awk '{print $2}')
+ printf " %-30s %s\n" "Build cache:" "${build_cache:-0}"
+ fi
+
+ # Docker system df
+ echo ""
+ docker system df 2>/dev/null | while IFS= read -r line; do
+ printf " %s\n" "$line"
+ done
+}
+
+clean_docker() {
+ if ! command -v docker &>/dev/null || ! docker info &>/dev/null 2>&1; then
+ return
+ fi
+
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "[DRY-RUN] Would prune Docker system (stopped containers, dangling images, unused networks, build cache)"
+ return
+ fi
+
+ docker system prune -f 2>/dev/null || warn "Docker prune failed"
+ log "Pruned Docker system"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# LARGE FILES
+# ══════════════════════════════════════════════════════════════════════
+
+scan_large_files() {
+ section_header "Large Files (> ${LARGE_FILE_MIN})"
+
+ printf " ${BOLD}%-12s %s${RESET}\n" "SIZE" "FILE"
+ printf " %s\n" "$(printf '%.0s─' {1..70})"
+
+ local count=0
+ for dir in $LARGE_FILE_DIRS; do
+ [[ -d "$dir" ]] || continue
+ find "$dir" -xdev -type f -size +"$LARGE_FILE_MIN" -printf '%s %p\n' 2>/dev/null
+ done | sort -rn | head -20 | while IFS=' ' read -r size path; do
+ printf " %10s %s\n" "$(human_bytes "$size")" "$path"
+ ((count++)) || true
+ done
+
+ if [[ "$count" -eq 0 ]]; then
+ echo " No files larger than ${LARGE_FILE_MIN} found"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SUMMARY
+# ══════════════════════════════════════════════════════════════════════
+
+print_summary() {
+ echo ""
+ echo -e " ${BOLD}══════════════════════════════════════════${RESET}"
+ echo -e " ${BOLD}Disk Cleanup Summary${RESET}"
+ echo -e " ${BOLD}══════════════════════════════════════════${RESET}"
+
+ # Current disk usage
+ local root_pct
+ root_pct=$(df / 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
+ printf " %-20s %s%%\n" "Root disk usage:" "${root_pct:-?}"
+ printf " %-20s %s\n" "Reclaimable:" "$(human_bytes "$TOTAL_RECLAIMABLE")"
+
+ if [[ "$TOTAL_CLEANED" -gt 0 ]]; then
+ printf " %-20s %s\n" "Cleaned:" "$(human_bytes "$TOTAL_CLEANED")"
+ fi
+
+ if [[ "$DRY_RUN" == "true" && "$RUN_MODE" == *"clean"* ]]; then
+ echo ""
+ echo -e " ${YELLOW}Dry-run mode — nothing was deleted${RESET}"
+ echo -e " Run with --force to actually clean"
+ fi
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# USAGE
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2
+ exit 1 ;;
+ esac
+ done
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+main() {
+ parse_args "$@"
+ setup_colors
+
+ echo ""
+ echo -e "${BOLD}Disk Cleanup — $(hostname -f 2>/dev/null || hostname)${RESET}"
+ echo -e "Mode: ${RUN_MODE}"
+ if [[ "$RUN_MODE" == "clean" ]]; then
+ if [[ "$DRY_RUN" == "true" ]]; then
+ echo -e "Safety: ${YELLOW}dry-run (use --force to delete)${RESET}"
+ else
+ echo -e "Safety: ${RED}LIVE — files will be deleted${RESET}"
+ fi
+ fi
+ echo -e "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+ case "$RUN_MODE" in
+ scan)
+ scan_old_logs
+ scan_journal
+ scan_tmp
+ scan_package_cache
+ scan_old_kernels
+ scan_docker
+ scan_large_files
+ print_summary
+ ;;
+ clean)
+ scan_old_logs
+ clean_old_logs
+ scan_journal
+ clean_journal
+ scan_tmp
+ clean_tmp
+ scan_package_cache
+ clean_package_cache
+ scan_docker
+ clean_docker
+ scan_large_files
+ print_summary
+ ;;
+ large-files)
+ scan_large_files
+ ;;
+ esac
+}
+
+main "$@"
diff --git a/disk-usage-reporter.sh b/disk-usage-reporter.sh
new file mode 100755
index 0000000..669bc88
--- /dev/null
+++ b/disk-usage-reporter.sh
@@ -0,0 +1,451 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### disk-usage-reporter.sh — Find what's consuming disk space ####
+#### Scans filesystems, ranks largest directories and files, flags old data ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./disk-usage-reporter.sh ####
+#### ./disk-usage-reporter.sh --path /var ####
+#### ./disk-usage-reporter.sh --top 50 --min-size 100M ####
+#### ./disk-usage-reporter.sh --json ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# DEFAULTS
+# ============================================================================
+
+SCAN_PATH="/"
+TOP_N=20
+MIN_SIZE="1M"
+MAX_DEPTH=3
+AGE_WARN=90
+JSON_MODE=false
+NO_COLOR=false
+VERSION="1.00"
+
+# Colors
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+SCRIPT_NAME="$(basename "$0")"
+
+# ============================================================================
+# USAGE & ARGUMENT PARSING
+# ============================================================================
+
+show_usage() {
+ cat <= 100M
+ ${SCRIPT_NAME} --path /home --age-warn 365 # Flag files older than 1 year
+ ${SCRIPT_NAME} --json # JSON output for scripting
+
+EOF
+ exit 0
+}
+
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case $1 in
+ -h|--help) show_usage ;;
+ --path) SCAN_PATH="$2"; shift 2 ;;
+ --top) TOP_N="$2"; shift 2 ;;
+ --min-size) MIN_SIZE="$2"; shift 2 ;;
+ --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+ --age-warn) AGE_WARN="$2"; shift 2 ;;
+ --json) JSON_MODE=true; shift ;;
+ --no-color) NO_COLOR=true; shift ;;
+ *) echo "Unknown option: $1" >&2; exit 1 ;;
+ esac
+ done
+ if [[ "$NO_COLOR" == true ]]; then
+ RED="" YELLOW="" GREEN="" CYAN="" BOLD="" NC=""
+ fi
+}
+
+# ============================================================================
+# HELPERS
+# ============================================================================
+
+header() {
+ echo ""
+ echo -e "${CYAN}====================================================${NC}"
+ echo -e "${CYAN} ${BOLD}${1}${NC}"
+ echo -e "${CYAN}====================================================${NC}"
+ echo ""
+}
+
+format_bytes() {
+ local b="$1"
+ if [[ "$b" -ge 1073741824 ]]; then
+ awk "BEGIN {printf \"%.2f GB\", $b/1073741824}"
+ elif [[ "$b" -ge 1048576 ]]; then
+ awk "BEGIN {printf \"%.1f MB\", $b/1048576}"
+ elif [[ "$b" -ge 1024 ]]; then
+ awk "BEGIN {printf \"%.1f KB\", $b/1024}"
+ else
+ echo "${b} B"
+ fi
+}
+
+fmt_num() {
+ printf "%'d" "$1" 2>/dev/null || echo "$1"
+}
+
+# Convert human-readable size (1M, 500K, 2G) to find -size argument
+parse_min_size() {
+ echo "${MIN_SIZE}"
+}
+
+# Convert human-readable size to bytes for comparison
+size_to_bytes() {
+ local size="$1"
+ local num unit
+ num="$(echo "$size" | sed 's/[^0-9.]//g')"
+ unit="$(echo "$size" | sed 's/[0-9.]//g' | tr '[:lower:]' '[:upper:]')"
+ case "$unit" in
+ K) awk "BEGIN {printf \"%d\", $num * 1024}" ;;
+ M) awk "BEGIN {printf \"%d\", $num * 1048576}" ;;
+ G) awk "BEGIN {printf \"%d\", $num * 1073741824}" ;;
+ T) awk "BEGIN {printf \"%d\", $num * 1099511627776}" ;;
+ *) echo "$num" ;;
+ esac
+}
+
+# ============================================================================
+# FILESYSTEM OVERVIEW
+# ============================================================================
+
+filesystem_overview() {
+ header "Filesystem Overview"
+
+ printf " ${BOLD}%-30s %6s %6s %6s %5s %-20s${NC}\n" \
+ "Filesystem" "Size" "Used" "Avail" "Use%" "Mounted on"
+ echo " ────────────────────────────────────────────────────────────────────────────────────"
+
+ df -hP -x tmpfs -x devtmpfs -x squashfs 2>/dev/null | tail -n +2 | while IFS= read -r line; do
+ local fs size used avail pct mount
+ fs="$(echo "$line" | awk '{print $1}')"
+ size="$(echo "$line" | awk '{print $2}')"
+ used="$(echo "$line" | awk '{print $3}')"
+ avail="$(echo "$line" | awk '{print $4}')"
+ pct="$(echo "$line" | awk '{print $5}')"
+ mount="$(echo "$line" | awk '{print $6}')"
+
+ local pct_num="${pct%\%}"
+ local color=""
+ if [[ "$pct_num" -ge 90 ]]; then
+ color="${RED}"
+ elif [[ "$pct_num" -ge 80 ]]; then
+ color="${YELLOW}"
+ else
+ color="${GREEN}"
+ fi
+
+ printf " ${color}%-30s %6s %6s %6s %5s %-20s${NC}\n" \
+ "$fs" "$size" "$used" "$avail" "$pct" "$mount"
+ done
+}
+
+# ============================================================================
+# TOP DIRECTORIES BY SIZE
+# ============================================================================
+
+top_directories() {
+ header "Top ${TOP_N} Directories by Size"
+
+ printf " ${BOLD}%4s %-60s %10s${NC}\n" "#" "Directory" "Size"
+ echo " ────────────────────────────────────────────────────────────────────────────────────"
+
+ du -x --max-depth="${MAX_DEPTH}" "${SCAN_PATH}" 2>/dev/null \
+ | sort -rn \
+ | head -n "${TOP_N}" \
+ | while IFS=$'\t' read -r size_kb dir; do
+ local num
+ num=$((COUNTER + 1))
+ COUNTER=$num
+ local size_bytes=$((size_kb * 1024))
+ local hsize
+ hsize="$(format_bytes "$size_bytes")"
+
+ local color="${NC}"
+ if [[ "$size_bytes" -ge 10737418240 ]]; then
+ color="${RED}"
+ elif [[ "$size_bytes" -ge 1073741824 ]]; then
+ color="${YELLOW}"
+ fi
+
+ printf " ${color}%4d %-60s %10s${NC}\n" "$num" "$dir" "$hsize"
+ done
+}
+
+# ============================================================================
+# TOP FILES BY SIZE
+# ============================================================================
+
+top_files() {
+ header "Top ${TOP_N} Files by Size"
+
+ printf " ${BOLD}%4s %-60s %10s${NC}\n" "#" "File" "Size"
+ echo " ────────────────────────────────────────────────────────────────────────────────────"
+
+ find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -printf '%s\t%p\n' 2>/dev/null \
+ | sort -rn \
+ | head -n "${TOP_N}" \
+ | awk -v idx=0 '{idx++; print idx"\t"$1"\t"$2}' \
+ | while IFS=$'\t' read -r num size_bytes filepath; do
+ local hsize
+ hsize="$(format_bytes "$size_bytes")"
+
+ local color="${NC}"
+ if [[ "$size_bytes" -ge 1073741824 ]]; then
+ color="${RED}"
+ elif [[ "$size_bytes" -ge 104857600 ]]; then
+ color="${YELLOW}"
+ fi
+
+ printf " ${color}%4d %-60s %10s${NC}\n" "$num" "$filepath" "$hsize"
+ done
+}
+
+# ============================================================================
+# OLD LARGE FILES
+# ============================================================================
+
+old_large_files() {
+ header "Old Large Files (> ${MIN_SIZE}, older than ${AGE_WARN} days)"
+
+ printf " ${BOLD}%4s %-50s %10s %12s${NC}\n" "#" "File" "Size" "Last Modified"
+ echo " ────────────────────────────────────────────────────────────────────────────────────"
+
+ OLD_FILES_DATA="$(find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -mtime +"${AGE_WARN}" \
+ -printf '%s\t%T+\t%p\n' 2>/dev/null \
+ | sort -rn \
+ | head -n "${TOP_N}")"
+
+ OLD_FILE_COUNT=0
+ OLD_FILE_BYTES=0
+
+ if [[ -z "$OLD_FILES_DATA" ]]; then
+ echo " No files found matching criteria."
+ return
+ fi
+
+ echo "$OLD_FILES_DATA" | awk -v idx=0 '{idx++; print idx"\t"$0}' \
+ | while IFS=$'\t' read -r num size_bytes mtime filepath; do
+ OLD_FILE_COUNT=$((OLD_FILE_COUNT + 1))
+ OLD_FILE_BYTES=$((OLD_FILE_BYTES + size_bytes))
+
+ local hsize mdate
+ hsize="$(format_bytes "$size_bytes")"
+ mdate="$(echo "$mtime" | cut -d'+' -f1)"
+
+ printf " ${YELLOW}%4d %-50s %10s %12s${NC}\n" "$num" "$filepath" "$hsize" "$mdate"
+ done
+}
+
+# ============================================================================
+# SUMMARY
+# ============================================================================
+
+compute_summary() {
+ local total_scanned old_count old_bytes
+
+ total_scanned="$(du -sx "${SCAN_PATH}" 2>/dev/null | awk '{print $1}')"
+ total_scanned=$((total_scanned * 1024))
+
+ old_bytes="$(find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -mtime +"${AGE_WARN}" \
+ -printf '%s\n' 2>/dev/null | awk '{s+=$1} END {print s+0}')"
+ old_count="$(find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -mtime +"${AGE_WARN}" \
+ 2>/dev/null | wc -l)"
+
+ echo "$total_scanned" "$old_count" "$old_bytes"
+}
+
+print_summary() {
+ header "Summary"
+
+ local data total_scanned old_count old_bytes
+ data="$(compute_summary)"
+ total_scanned="$(echo "$data" | awk '{print $1}')"
+ old_count="$(echo "$data" | awk '{print $2}')"
+ old_bytes="$(echo "$data" | awk '{print $3}')"
+
+ echo -e " ${BOLD}Scan path:${NC} ${SCAN_PATH}"
+ echo -e " ${BOLD}Total scanned:${NC} $(format_bytes "$total_scanned")"
+ echo -e " ${BOLD}Min file size:${NC} ${MIN_SIZE}"
+ echo -e " ${BOLD}Age threshold:${NC} ${AGE_WARN} days"
+ echo ""
+ echo -e " ${BOLD}Old large files:${NC} $(fmt_num "$old_count") files"
+ echo -e " ${BOLD}Reclaimable space:${NC} ${YELLOW}$(format_bytes "$old_bytes")${NC}"
+ echo ""
+
+ if [[ "$old_bytes" -gt 0 ]]; then
+ echo -e " ${YELLOW}→ Review old files above — candidates for cleanup or archival${NC}"
+ else
+ echo -e " ${GREEN}✓ No old large files found${NC}"
+ fi
+ echo ""
+}
+
+# ============================================================================
+# JSON OUTPUT
+# ============================================================================
+
+json_output() {
+ local total_scanned old_count old_bytes
+ local data
+ data="$(compute_summary)"
+ total_scanned="$(echo "$data" | awk '{print $1}')"
+ old_count="$(echo "$data" | awk '{print $2}')"
+ old_bytes="$(echo "$data" | awk '{print $3}')"
+
+ echo "{"
+ echo " \"scan_path\": \"${SCAN_PATH}\","
+ echo " \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","
+ echo " \"min_size\": \"${MIN_SIZE}\","
+ echo " \"age_warn_days\": ${AGE_WARN},"
+ echo " \"max_depth\": ${MAX_DEPTH},"
+
+ # Filesystems
+ echo " \"filesystems\": ["
+ local fs_first=true
+ df -hP -x tmpfs -x devtmpfs -x squashfs 2>/dev/null | tail -n +2 | while IFS= read -r line; do
+ local fs size used avail pct mount
+ fs="$(echo "$line" | awk '{print $1}')"
+ size="$(echo "$line" | awk '{print $2}')"
+ used="$(echo "$line" | awk '{print $3}')"
+ avail="$(echo "$line" | awk '{print $4}')"
+ pct="$(echo "$line" | awk '{print $5}')"
+ mount="$(echo "$line" | awk '{print $6}')"
+ if [[ "$fs_first" == true ]]; then
+ fs_first=false
+ else
+ echo ","
+ fi
+ printf ' {"filesystem":"%s","size":"%s","used":"%s","avail":"%s","use_pct":"%s","mount":"%s"}' \
+ "$fs" "$size" "$used" "$avail" "$pct" "$mount"
+ done
+ echo ""
+ echo " ],"
+
+ # Top directories
+ echo " \"top_directories\": ["
+ local dir_first=true
+ du -x --max-depth="${MAX_DEPTH}" "${SCAN_PATH}" 2>/dev/null \
+ | sort -rn | head -n "${TOP_N}" \
+ | while IFS=$'\t' read -r size_kb dir; do
+ if [[ "$dir_first" == true ]]; then
+ dir_first=false
+ else
+ echo ","
+ fi
+ printf ' {"path":"%s","size_bytes":%d}' "$dir" "$((size_kb * 1024))"
+ done
+ echo ""
+ echo " ],"
+
+ # Top files
+ echo " \"top_files\": ["
+ local file_first=true
+ find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -printf '%s\t%p\n' 2>/dev/null \
+ | sort -rn | head -n "${TOP_N}" \
+ | while IFS=$'\t' read -r size_bytes filepath; do
+ if [[ "$file_first" == true ]]; then
+ file_first=false
+ else
+ echo ","
+ fi
+ printf ' {"path":"%s","size_bytes":%d}' "$filepath" "$size_bytes"
+ done
+ echo ""
+ echo " ],"
+
+ # Old files
+ echo " \"old_large_files\": ["
+ local old_first=true
+ find "${SCAN_PATH}" -xdev -type f -size +"$(parse_min_size)" -mtime +"${AGE_WARN}" \
+ -printf '%s\t%T+\t%p\n' 2>/dev/null \
+ | sort -rn | head -n "${TOP_N}" \
+ | while IFS=$'\t' read -r size_bytes mtime filepath; do
+ local mdate
+ mdate="$(echo "$mtime" | cut -d'+' -f1)"
+ if [[ "$old_first" == true ]]; then
+ old_first=false
+ else
+ echo ","
+ fi
+ printf ' {"path":"%s","size_bytes":%d,"last_modified":"%s"}' "$filepath" "$size_bytes" "$mdate"
+ done
+ echo ""
+ echo " ],"
+
+ # Summary
+ echo " \"summary\": {"
+ echo " \"total_scanned_bytes\": ${total_scanned},"
+ echo " \"old_file_count\": ${old_count},"
+ echo " \"reclaimable_bytes\": ${old_bytes}"
+ echo " }"
+ echo "}"
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+main() {
+ parse_args "$@"
+
+ if [[ ! -d "$SCAN_PATH" ]]; then
+ echo -e "${RED}[ERROR]${NC} Path does not exist: ${SCAN_PATH}" >&2
+ exit 1
+ fi
+
+ if [[ "$JSON_MODE" == true ]]; then
+ json_output
+ exit 0
+ fi
+
+ echo ""
+ echo -e "${BOLD}Disk Usage Report${NC}"
+ echo -e "$(date '+%Y-%m-%d %H:%M:%S %Z') — Scanning: ${SCAN_PATH}"
+
+ COUNTER=0
+ filesystem_overview
+ top_directories
+ top_files
+ old_large_files
+ print_summary
+}
+
+main "$@"
diff --git a/dns-lookup.sh b/dns-lookup.sh
new file mode 100644
index 0000000..fa7188a
--- /dev/null
+++ b/dns-lookup.sh
@@ -0,0 +1,429 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### dns-lookup.sh — Batch DNS lookups with record comparison across servers ####
+#### Query multiple record types and compare results across DNS servers ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./dns-lookup.sh example.com google.com ####
+#### ./dns-lookup.sh --type MX --servers 8.8.8.8,1.1.1.1 example.com ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+DNS_TIMEOUT="${DNS_TIMEOUT:-5}"
+RECORD_TYPE="${RECORD_TYPE:-A}"
+DNS_SERVERS=""
+COMPARE="${COMPARE:-false}"
+DOMAIN_FILE=""
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+DOMAINS=()
+COUNT_TOTAL=0
+COUNT_SUCCESS=0
+COUNT_FAILED=0
+COUNT_MISMATCH=0
+DIG_CMD=""
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${CYAN}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+
+# ── Helpers ───────────────────────────────────────────────────────────
+section_header() {
+ echo ""
+ echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
+ echo ""
+}
+
+field() {
+ printf " ${BOLD}%-22s${RESET} %s\n" "$1" "$2"
+}
+
+field_color() {
+ printf " ${BOLD}%-22s${RESET} %b\n" "$1" "$2"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# DNS QUERY
+# ══════════════════════════════════════════════════════════════════════
+
+detect_dns_tool() {
+ if command -v dig &>/dev/null; then
+ DIG_CMD="dig"
+ elif command -v nslookup &>/dev/null; then
+ DIG_CMD="nslookup"
+ else
+ err "Neither dig nor nslookup found. Install dnsutils or bind-utils."
+ exit 1
+ fi
+ verbose "Using DNS tool: ${DIG_CMD}"
+}
+
+query_dig() {
+ local domain="$1"
+ local rtype="$2"
+ local server="${3:-}"
+
+ local cmd_args=()
+ if [[ -n "$server" ]]; then
+ cmd_args+=("@${server}")
+ fi
+ cmd_args+=("$domain" "$rtype" "+short" "+time=${DNS_TIMEOUT}" "+tries=1")
+
+ verbose "dig ${cmd_args[*]}"
+ dig "${cmd_args[@]}" 2>/dev/null || echo ""
+}
+
+query_nslookup() {
+ local domain="$1"
+ local rtype="$2"
+ local server="${3:-}"
+
+ local result
+ if [[ -n "$server" ]]; then
+ result=$(nslookup -type="$rtype" -timeout="$DNS_TIMEOUT" "$domain" "$server" 2>/dev/null) || result=""
+ else
+ result=$(nslookup -type="$rtype" -timeout="$DNS_TIMEOUT" "$domain" 2>/dev/null) || result=""
+ fi
+
+ # Parse nslookup output — extract answer lines
+ echo "$result" | awk '/^Name:|^Address:|answer:/{found=1} found && /^[^ \t]/' | grep -v "^Server:" | grep -v "^Name:" | awk '{print $NF}'
+}
+
+do_query() {
+ local domain="$1"
+ local rtype="$2"
+ local server="${3:-}"
+
+ if [[ "$DIG_CMD" == "dig" ]]; then
+ query_dig "$domain" "$rtype" "$server"
+ else
+ query_nslookup "$domain" "$rtype" "$server"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# LOOKUP LOGIC
+# ══════════════════════════════════════════════════════════════════════
+
+lookup_single() {
+ local domain="$1"
+ local rtype="$2"
+ local server="${3:-system resolver}"
+ local server_arg="${3:-}"
+
+ COUNT_TOTAL=$((COUNT_TOTAL + 1))
+
+ local result
+ result=$(do_query "$domain" "$rtype" "$server_arg")
+
+ if [[ -z "$result" ]]; then
+ COUNT_FAILED=$((COUNT_FAILED + 1))
+ printf " %b%-30s %-6s %-18s %s%b\n" "$RED" "$domain" "$rtype" "$server" "NO RECORDS" "$RESET"
+ return
+ fi
+
+ COUNT_SUCCESS=$((COUNT_SUCCESS + 1))
+
+ # Get TTL if using dig
+ local ttl="--"
+ if [[ "$DIG_CMD" == "dig" && -n "$server_arg" ]]; then
+ ttl=$(dig "@${server_arg}" "$domain" "$rtype" +noall +answer +time="${DNS_TIMEOUT}" +tries=1 2>/dev/null \
+ | awk '{print $2}' | head -1 || echo "--")
+ elif [[ "$DIG_CMD" == "dig" ]]; then
+ ttl=$(dig "$domain" "$rtype" +noall +answer +time="${DNS_TIMEOUT}" +tries=1 2>/dev/null \
+ | awk '{print $2}' | head -1 || echo "--")
+ fi
+
+ while IFS= read -r value; do
+ [[ -z "$value" ]] && continue
+ printf " %-30s %-6s %-8s %-18s %s\n" "$domain" "$rtype" "$ttl" "$server" "$value"
+ # Only print domain on first line
+ domain=""
+ ttl=""
+ done <<< "$result"
+}
+
+lookup_compare() {
+ local domain="$1"
+ local rtype="$2"
+ local -a servers_arr
+
+ IFS=',' read -ra servers_arr <<< "$DNS_SERVERS"
+
+ if [[ ${#servers_arr[@]} -lt 2 ]]; then
+ warn "Compare mode requires at least 2 DNS servers (use --servers)"
+ return
+ fi
+
+ local -a all_results=()
+ local first_result=""
+
+ for server in "${servers_arr[@]}"; do
+ COUNT_TOTAL=$((COUNT_TOTAL + 1))
+
+ local result
+ result=$(do_query "$domain" "$rtype" "$server" | sort)
+
+ if [[ -z "$result" ]]; then
+ COUNT_FAILED=$((COUNT_FAILED + 1))
+ printf " %b%-30s %-6s %-18s %s%b\n" "$RED" "$domain" "$rtype" "$server" "NO RECORDS" "$RESET"
+ all_results+=("FAILED")
+ continue
+ fi
+
+ COUNT_SUCCESS=$((COUNT_SUCCESS + 1))
+ all_results+=("$result")
+
+ if [[ -z "$first_result" ]]; then
+ first_result="$result"
+ fi
+
+ while IFS= read -r value; do
+ [[ -z "$value" ]] && continue
+ printf " %-30s %-6s %-18s %s\n" "$domain" "$rtype" "$server" "$value"
+ domain=""
+ done <<< "$result"
+ done
+
+ # Check for mismatches
+ local mismatch=false
+ for r in "${all_results[@]}"; do
+ if [[ "$r" != "$first_result" && "$r" != "FAILED" && "$first_result" != "FAILED" ]]; then
+ mismatch=true
+ break
+ fi
+ done
+
+ if [[ "$mismatch" == "true" ]]; then
+ COUNT_MISMATCH=$((COUNT_MISMATCH + 1))
+ printf " %b ⚠ MISMATCH across servers for %s%b\n" "$RED" "$1" "$RESET"
+ else
+ printf " %b ✓ Consistent across servers for %s%b\n" "$GREEN" "$1" "$RESET"
+ fi
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# INPUT PARSING
+# ══════════════════════════════════════════════════════════════════════
+
+parse_domain() {
+ local entry="$1"
+ entry=$(echo "$entry" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ [[ -z "$entry" || "$entry" == \#* ]] && return
+ DOMAINS+=("$entry")
+}
+
+load_domains_from_file() {
+ local file="$1"
+ if [[ ! -f "$file" ]]; then
+ err "File not found: $file"
+ exit 1
+ fi
+ while IFS= read -r line; do
+ parse_domain "$line"
+ done < "$file"
+}
+
+load_domains_from_stdin() {
+ while IFS= read -r line; do
+ parse_domain "$line"
+ done
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# USAGE
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2
+ exit 1 ;;
+ *)
+ parse_domain "$1"; shift ;;
+ esac
+ done
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+main() {
+ parse_args "$@"
+ setup_colors
+ detect_dns_tool
+
+ # Load domains from file if specified
+ if [[ -n "$DOMAIN_FILE" ]]; then
+ load_domains_from_file "$DOMAIN_FILE"
+ fi
+
+ # Load from stdin if no domains yet and stdin is not a terminal
+ if [[ ${#DOMAINS[@]} -eq 0 ]] && ! [[ -t 0 ]]; then
+ load_domains_from_stdin
+ fi
+
+ if [[ ${#DOMAINS[@]} -eq 0 ]]; then
+ err "No domains specified"
+ echo "Run ${SCRIPT_NAME} --help for usage" >&2
+ exit 1
+ fi
+
+ # Validate record type
+ case "$RECORD_TYPE" in
+ A|AAAA|MX|NS|TXT|CNAME|SOA|PTR) ;;
+ *)
+ err "Unsupported record type: ${RECORD_TYPE}"
+ exit 1 ;;
+ esac
+
+ echo ""
+ echo -e "${BOLD}DNS Lookup — ${RECORD_TYPE} Records${RESET}"
+ echo -e "${DIM}$(date '+%Y-%m-%d %H:%M:%S %Z')${RESET}"
+ echo -e "${DIM}Tool: ${DIG_CMD} | Timeout: ${DNS_TIMEOUT}s${RESET}"
+
+ section_header "Results"
+
+ if [[ "$COMPARE" == "true" ]]; then
+ printf " ${BOLD}%-30s %-6s %-18s %s${RESET}\n" "DOMAIN" "TYPE" "SERVER" "VALUE"
+ printf " %s\n" "$(printf '%.0s─' {1..85})"
+
+ for domain in "${DOMAINS[@]}"; do
+ lookup_compare "$domain" "$RECORD_TYPE"
+ done
+ else
+ # Determine servers to query
+ local -a servers_list
+ if [[ -n "$DNS_SERVERS" ]]; then
+ IFS=',' read -ra servers_list <<< "$DNS_SERVERS"
+ else
+ servers_list=("")
+ fi
+
+ printf " ${BOLD}%-30s %-6s %-8s %-18s %s${RESET}\n" "DOMAIN" "TYPE" "TTL" "SERVER" "VALUE"
+ printf " %s\n" "$(printf '%.0s─' {1..90})"
+
+ for domain in "${DOMAINS[@]}"; do
+ for server in "${servers_list[@]}"; do
+ lookup_single "$domain" "$RECORD_TYPE" "$server"
+ done
+ done
+ fi
+
+ section_header "Summary"
+ field "Total lookups:" "$COUNT_TOTAL"
+ field_color "Successful:" "${GREEN}${COUNT_SUCCESS}${RESET}"
+ if [[ "$COUNT_FAILED" -gt 0 ]]; then
+ field_color "Failed:" "${RED}${COUNT_FAILED}${RESET}"
+ else
+ field "Failed:" "$COUNT_FAILED"
+ fi
+ if [[ "$COMPARE" == "true" ]]; then
+ if [[ "$COUNT_MISMATCH" -gt 0 ]]; then
+ field_color "Mismatches:" "${RED}${COUNT_MISMATCH}${RESET}"
+ else
+ field_color "Mismatches:" "${GREEN}0${RESET}"
+ fi
+ fi
+
+ echo ""
+}
+
+main "$@"
diff --git a/dns-propagation-checker.sh b/dns-propagation-checker.sh
new file mode 100755
index 0000000..0cc4ba1
--- /dev/null
+++ b/dns-propagation-checker.sh
@@ -0,0 +1,350 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### dns-propagation-checker.sh — Check DNS propagation across public resolvers ####
+#### Queries Cloudflare, Google, Quad9, OpenDNS, compares results ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./dns-propagation-checker.sh example.com ####
+#### ./dns-propagation-checker.sh example.com --type MX ####
+#### ./dns-propagation-checker.sh example.com --watch 30 ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+DOMAIN=""
+RECORD_TYPE="A"
+TIMEOUT=5
+COLOR="auto"
+JSON_OUTPUT="false"
+WATCH_INTERVAL=0
+EXPECTED=""
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+
+# ── Built-in Resolvers ───────────────────────────────────────────────
+RESOLVER_NAMES=("Cloudflare" "Google" "Quad9" "OpenDNS" "Cloudflare-2" "Google-2")
+RESOLVER_IPS=("1.1.1.1" "8.8.8.8" "9.9.9.9" "208.67.222.222" "1.0.0.1" "8.8.4.4")
+CUSTOM_RESOLVERS=()
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+
+# ══════════════════════════════════════════════════════════════════════
+# USAGE
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2
+ exit 1 ;;
+ *)
+ if [[ -z "$DOMAIN" ]]; then
+ DOMAIN="$1"
+ else
+ err "Unexpected argument: $1"
+ exit 1
+ fi
+ shift ;;
+ esac
+ done
+
+ if [[ -z "$DOMAIN" ]]; then
+ err "Domain name is required"
+ echo "Run ${SCRIPT_NAME} --help for usage" >&2
+ exit 1
+ fi
+
+ local valid_types="A AAAA MX CNAME TXT NS SOA PTR"
+ if [[ ! " $valid_types " =~ " $RECORD_TYPE " ]]; then
+ err "Invalid record type: $RECORD_TYPE"
+ exit 1
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# DNS QUERY
+# ══════════════════════════════════════════════════════════════════════
+
+query_resolver() {
+ local resolver_ip="$1" domain="$2" rtype="$3" timeout="$4"
+ local output ttl_output answer ttl
+
+ output=$(dig +time="$timeout" +tries=1 +short "@${resolver_ip}" "$domain" "$rtype" 2>/dev/null) || true
+ ttl_output=$(dig +time="$timeout" +tries=1 +noall +answer "@${resolver_ip}" "$domain" "$rtype" 2>/dev/null) || true
+ answer=$(echo "$output" | tr '\n' ' ' | sed 's/ *$//')
+ ttl=$(echo "$ttl_output" | awk 'NR==1{print $2}')
+
+ if [[ -z "$answer" ]]; then
+ echo "FAIL||"; return
+ fi
+ echo "${answer}|${ttl:-?}|"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAJORITY ANSWER
+# ══════════════════════════════════════════════════════════════════════
+
+find_majority() {
+ local -n answers_ref=$1
+ local -A counts
+ local max_count=0 majority=""
+ for answer in "${answers_ref[@]}"; do
+ [[ "$answer" == "FAIL" ]] && continue
+ counts["$answer"]=$(( ${counts["$answer"]:-0} + 1 ))
+ if [[ ${counts["$answer"]} -gt $max_count ]]; then
+ max_count=${counts["$answer"]}; majority="$answer"
+ fi
+ done
+ echo "$majority"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# RUN CHECK
+# ══════════════════════════════════════════════════════════════════════
+
+run_check() {
+ local all_names=("${RESOLVER_NAMES[@]}")
+ local all_ips=("${RESOLVER_IPS[@]}")
+
+ for custom in "${CUSTOM_RESOLVERS[@]}"; do
+ all_names+=("Custom-${custom}")
+ all_ips+=("$custom")
+ done
+
+ local total=${#all_names[@]}
+ local answers=() ttls=()
+
+ for i in $(seq 0 $(( total - 1 ))); do
+ local result
+ result=$(query_resolver "${all_ips[$i]}" "$DOMAIN" "$RECORD_TYPE" "$TIMEOUT")
+ answers+=("$(echo "$result" | cut -d'|' -f1)")
+ ttls+=("$(echo "$result" | cut -d'|' -f2)")
+ done
+
+ local majority
+ majority=$(find_majority answers)
+ local compare_to="${EXPECTED:-$majority}"
+ local agree_count=0 statuses=()
+
+ for i in $(seq 0 $(( total - 1 ))); do
+ if [[ "${answers[$i]}" == "FAIL" ]]; then
+ statuses+=("FAIL")
+ elif [[ "${answers[$i]}" == "$compare_to" ]]; then
+ statuses+=("MATCH"); agree_count=$((agree_count + 1))
+ else
+ statuses+=("MISMATCH")
+ fi
+ done
+
+ if [[ "$JSON_OUTPUT" == "true" ]]; then
+ print_json all_names all_ips answers ttls statuses "$agree_count" "$total" "$majority"
+ else
+ print_table all_names all_ips answers ttls statuses "$agree_count" "$total" "$majority" "$compare_to"
+ fi
+ [[ "$agree_count" -eq "$total" ]] && return 0 || return 1
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OUTPUT: TABLE
+# ══════════════════════════════════════════════════════════════════════
+
+print_table() {
+ local -n names_ref=$1 ips_ref=$2 ans_ref=$3 ttl_ref=$4 stat_ref=$5
+ local agree="$6" total="$7" majority="$8" compare="$9"
+
+ echo ""
+ echo -e "${BOLD}DNS Propagation Check — ${DOMAIN} (${RECORD_TYPE})${RESET}"
+ echo -e "${DIM}$(date -u '+%Y-%m-%d %H:%M:%S UTC')${RESET}"
+ echo ""
+ printf " ${BOLD}%-20s %-17s %-22s %-6s %s${RESET}\n" "RESOLVER" "IP" "RESULT" "TTL" "STATUS"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=${#names_ref[@]}
+ for i in $(seq 0 $(( count - 1 ))); do
+ local color status_str
+ case "${stat_ref[$i]}" in
+ MATCH) color="$GREEN"; status_str="MATCH" ;;
+ MISMATCH) color="$YELLOW"; status_str="MISMATCH" ;;
+ FAIL) color="$RED"; status_str="FAIL" ;;
+ esac
+
+ local display_answer="${ans_ref[$i]}"
+ if [[ ${#display_answer} -gt 20 ]]; then
+ display_answer="${display_answer:0:17}..."
+ fi
+
+ printf " %-20s %-17s %b%-22s%b %-6s %b%s%b\n" \
+ "${names_ref[$i]}" \
+ "${ips_ref[$i]}" \
+ "$color" "$display_answer" "$RESET" \
+ "${ttl_ref[$i]}" \
+ "$color" "$status_str" "$RESET"
+ done
+
+ echo ""
+ echo -e " ${BOLD}Summary${RESET}"
+ if [[ -n "$EXPECTED" ]]; then
+ printf " %-20s %s\n" "Expected answer:" "$EXPECTED"
+ fi
+ printf " %-20s %s\n" "Majority answer:" "${majority:-N/A}"
+ printf " %-20s %s\n" "Agree:" "${agree}/${total} resolvers"
+
+ if [[ "$agree" -eq "$total" ]]; then
+ printf " %-20s " "Status:"; echo -e "${GREEN}PROPAGATION COMPLETE${RESET}"
+ else
+ printf " %-20s " "Status:"; echo -e "${YELLOW}PROPAGATION PENDING${RESET}"
+ fi
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OUTPUT: JSON
+# ══════════════════════════════════════════════════════════════════════
+
+print_json() {
+ local -n jnames=$1 jips=$2 jans=$3 jttls=$4 jstats=$5
+ local agree="$6" total="$7" majority="$8"
+ local count=${#jnames[@]} propagated="false"
+ [[ "$agree" -eq "$total" ]] && propagated="true"
+
+ printf '{"domain":"%s","type":"%s","timestamp":"%s","results":[' \
+ "$DOMAIN" "$RECORD_TYPE" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+ for i in $(seq 0 $(( count - 1 ))); do
+ [[ $i -gt 0 ]] && printf ','
+ local escaped_answer
+ escaped_answer=$(echo "${jans[$i]}" | sed 's/"/\\"/g')
+ printf '{"resolver":"%s","ip":"%s","answer":"%s","ttl":"%s","status":"%s"}' \
+ "${jnames[$i]}" "${jips[$i]}" "$escaped_answer" "${jttls[$i]}" "${jstats[$i]}"
+ done
+ local escaped_majority
+ escaped_majority=$(echo "$majority" | sed 's/"/\\"/g')
+ printf '],"summary":{"majority":"%s","agree":%d,"total":%d,"propagated":%s}}\n' \
+ "$escaped_majority" "$agree" "$total" "$propagated"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+main() {
+ parse_args "$@"
+ setup_colors
+
+ if ! command -v dig &>/dev/null; then
+ err "dig is required but not found. Install dnsutils (Debian/Ubuntu) or bind-utils (RHEL/CentOS)."
+ exit 1
+ fi
+
+ if [[ "$WATCH_INTERVAL" -gt 0 ]]; then
+ local cycle=1
+ while true; do
+ if [[ "$JSON_OUTPUT" != "true" ]]; then
+ [[ $cycle -gt 1 ]] && echo -e "${DIM}────────────────────────────────────────────────${RESET}"
+ echo -e "${DIM}Watch cycle ${cycle} — checking every ${WATCH_INTERVAL}s (Ctrl+C to stop)${RESET}"
+ fi
+ if run_check; then
+ [[ "$JSON_OUTPUT" != "true" ]] && echo -e " ${GREEN}All resolvers agree. Propagation complete.${RESET}\n"
+ exit 0
+ fi
+ cycle=$((cycle + 1))
+ sleep "$WATCH_INTERVAL"
+ done
+ else
+ run_check && exit 0 || exit 1
+ fi
+}
+
+main "$@"
diff --git a/dns-smoke-tests.sh b/dns-smoke-tests.sh
new file mode 100644
index 0000000..7959749
--- /dev/null
+++ b/dns-smoke-tests.sh
@@ -0,0 +1,500 @@
+#!/usr/bin/env bash
+
+#####################################################################################
+#### dns-smoke-tests.sh — Verify DNS infrastructure is healthy ####
+#### Checks resolution, zone transfers, SOA, DNSSEC, response time, DoT. ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version: 1.0 ####
+#### ####
+#### Usage: ./dns-smoke-tests.sh ####
+#### DNS_SERVER=192.168.1.1 DOMAIN=example.com ./dns-smoke-tests.sh ####
+#### ####
+#### See --help for all options. ####
+#####################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+DNS_SERVER="${DNS_SERVER:-}"
+DOMAIN="${DOMAIN:-example.com}"
+REVERSE_IP="${REVERSE_IP:-}"
+ZONE="${ZONE:-}"
+ZONE_MASTER="${ZONE_MASTER:-}"
+DNSSEC_DOMAIN="${DNSSEC_DOMAIN:-}"
+DOT_SERVER="${DOT_SERVER:-}"
+MAX_RESPONSE_MS="${MAX_RESPONSE_MS:-500}"
+TEST_RECORDS="${TEST_RECORDS:-}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+PASS=0; FAIL=0; SKIP=0; TOTAL=0
+RESULTS=()
+START_TIME=""
+
+# ── Dig tool detection ───────────────────────────────────────────────
+DIG_CMD=""
+detect_dig() {
+ if command -v dig >/dev/null 2>&1; then
+ DIG_CMD="dig"
+ elif command -v drill >/dev/null 2>&1; then
+ DIG_CMD="drill"
+ else
+ err "Neither dig nor drill found. Install dnsutils or ldns."
+ exit 1
+ fi
+ verbose "Using ${DIG_CMD} for DNS queries"
+}
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
+
+# ── Test Result Recording ─────────────────────────────────────────────
+record_pass() {
+ local name="$1" detail="${2:-}"
+ ((PASS++)) || true; ((TOTAL++)) || true
+ RESULTS+=("PASS|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name}${detail:+ (${detail})}"
+ else echo -e " ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}"; fi
+}
+
+record_fail() {
+ local name="$1" detail="${2:-}"
+ ((FAIL++)) || true; ((TOTAL++)) || true
+ RESULTS+=("FAIL|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "not ok ${TOTAL} - ${name}"
+ [[ -n "$detail" ]] && echo " # ${detail}"
+ else echo -e " ${RED}✗${RESET} ${name}${detail:+ — ${detail}}"; fi
+}
+
+record_skip() {
+ local name="$1" reason="${2:-}"
+ ((SKIP++)) || true; ((TOTAL++)) || true
+ RESULTS+=("SKIP|${name}|${reason}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name} # SKIP ${reason}"
+ else echo -e " ${YELLOW}⊘${RESET} ${name}${reason:+ — ${reason}}"; fi
+}
+
+# ── Helpers ───────────────────────────────────────────────────────────
+has_cmd() { command -v "$1" >/dev/null 2>&1; }
+
+# Build dig command with optional @server
+dig_cmd() {
+ if [[ -n "$DNS_SERVER" ]]; then
+ "$DIG_CMD" "@${DNS_SERVER}" "$@"
+ else
+ "$DIG_CMD" "$@"
+ fi
+}
+
+# ── Output Functions ──────────────────────────────────────────────────
+section_header() {
+ local name="$1"
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo ""
+ echo -e "${BOLD}${name}${RESET}"
+ fi
+}
+
+print_header() {
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo ""
+ echo -e "${BOLD}DNS Smoke Tests${RESET}"
+ echo "Domain: ${DOMAIN}"
+ [[ -n "$DNS_SERVER" ]] && echo "Server: ${DNS_SERVER}" || echo "Server: (system resolver)"
+ echo "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ fi
+}
+
+print_tap_header() {
+ echo "TAP version 13"
+}
+
+print_summary() {
+ local end_time; end_time=$(date +%s)
+ local duration=$(( end_time - START_TIME ))
+ echo ""
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+ echo -e "${BOLD}Summary${RESET} ${DOMAIN} ${DNS_SERVER:-(system resolver)}"
+ echo -e " ${GREEN}${PASS} passed${RESET} ${RED}${FAIL} failed${RESET} ${YELLOW}${SKIP} skipped${RESET} (${duration}s)"
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+ if [[ $FAIL -eq 0 ]]; then echo -e "${GREEN}${BOLD}All tests passed.${RESET}"
+ else echo -e "${RED}${BOLD}${FAIL} test(s) failed.${RESET}"; fi
+}
+
+print_tap_footer() {
+ echo "1..${TOTAL}"
+ echo "# pass ${PASS}"
+ echo "# fail ${FAIL}"
+ echo "# skip ${SKIP}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# TESTS
+# ══════════════════════════════════════════════════════════════════════
+
+# ── 1. Resolver Reachable ─────────────────────────────────────────────
+test_resolver_reachable() {
+ section_header "Connectivity"
+ local output
+ output=$(dig_cmd +short +time=5 +tries=1 "${DOMAIN}" A 2>&1) || true
+ if [[ -n "$output" ]] && ! echo "$output" | grep -qi "timed out\|connection refused\|no servers\|SERVFAIL"; then
+ record_pass "Resolver reachable" "${DNS_SERVER:-(system resolver)}"
+ else
+ record_fail "Resolver reachable" "${DNS_SERVER:-(system resolver)} — ${output:-no response}"
+ fi
+}
+
+# ── 2. Forward Resolution (A) ────────────────────────────────────────
+test_forward_resolution() {
+ section_header "Resolution"
+ local output
+ output=$(dig_cmd +short "${DOMAIN}" A 2>/dev/null) || true
+ if [[ -n "$output" ]]; then
+ local first_ip
+ first_ip=$(echo "$output" | head -1)
+ record_pass "Forward resolution (${DOMAIN} A)" "${first_ip}"
+ else
+ record_fail "Forward resolution (${DOMAIN} A)" "no A record returned"
+ fi
+}
+
+# ── 3. AAAA Resolution ───────────────────────────────────────────────
+test_aaaa_resolution() {
+ local output
+ output=$(dig_cmd +short "${DOMAIN}" AAAA 2>/dev/null) || true
+ if [[ -n "$output" ]]; then
+ local first_ip
+ first_ip=$(echo "$output" | head -1)
+ record_pass "AAAA resolution (${DOMAIN})" "${first_ip}"
+ else
+ record_skip "AAAA resolution (${DOMAIN})" "no AAAA record"
+ fi
+}
+
+# ── 4. MX Resolution ─────────────────────────────────────────────────
+test_mx_resolution() {
+ local output
+ output=$(dig_cmd +short "${DOMAIN}" MX 2>/dev/null) || true
+ if [[ -n "$output" ]]; then
+ local first_mx
+ first_mx=$(echo "$output" | head -1)
+ record_pass "MX resolution (${DOMAIN})" "${first_mx}"
+ else
+ record_skip "MX resolution (${DOMAIN})" "no MX record"
+ fi
+}
+
+# ── 5. Reverse Lookup ────────────────────────────────────────────────
+test_reverse_lookup() {
+ if [[ -z "$REVERSE_IP" ]]; then
+ record_skip "Reverse lookup" "REVERSE_IP not set"
+ return
+ fi
+ local output
+ output=$(dig_cmd +short -x "${REVERSE_IP}" 2>/dev/null) || true
+ if [[ -n "$output" ]]; then
+ record_pass "Reverse lookup (${REVERSE_IP})" "${output}"
+ else
+ record_fail "Reverse lookup (${REVERSE_IP})" "no PTR record returned"
+ fi
+}
+
+# ── 6. Response Time ─────────────────────────────────────────────────
+test_response_time() {
+ section_header "Performance"
+ local output query_time
+ output=$(dig_cmd "${DOMAIN}" A 2>/dev/null) || true
+ # dig outputs "Query time: 12 msec" or ";; Query time: 12 msec"
+ query_time=$(echo "$output" | grep -i "query time" | grep -oP '[0-9]+' | head -1) || true
+ if [[ -z "$query_time" ]]; then
+ # drill outputs ";; Query time: 0 msec"
+ query_time=$(echo "$output" | grep -i "query time" | awk '{print $4}') || true
+ fi
+ if [[ -n "$query_time" ]]; then
+ if [[ "$query_time" -le "$MAX_RESPONSE_MS" ]]; then
+ record_pass "Response time" "${query_time}ms (<= ${MAX_RESPONSE_MS}ms)"
+ else
+ record_fail "Response time" "${query_time}ms (> ${MAX_RESPONSE_MS}ms)"
+ fi
+ else
+ record_fail "Response time" "could not parse query time"
+ fi
+}
+
+# ── 7. Authoritative Answer ──────────────────────────────────────────
+test_authoritative_answer() {
+ section_header "Authority"
+ local output
+ output=$(dig_cmd "${DOMAIN}" A 2>/dev/null) || true
+ if echo "$output" | grep -q "flags:.*aa"; then
+ record_pass "Authoritative answer (${DOMAIN})" "AA flag set"
+ else
+ record_fail "Authoritative answer (${DOMAIN})" "AA flag not set — server is not authoritative"
+ fi
+}
+
+# ── 8. SOA Serial ────────────────────────────────────────────────────
+test_soa_serial() {
+ local output serial
+ output=$(dig_cmd +short "${DOMAIN}" SOA 2>/dev/null) || true
+ if [[ -z "$output" ]]; then
+ record_fail "SOA serial (${DOMAIN})" "no SOA record returned"
+ return
+ fi
+ # SOA format: ns1.example.com. admin.example.com. 2026051201 3600 900 604800 86400
+ serial=$(echo "$output" | awk '{print $3}') || true
+ if [[ -z "$serial" ]]; then
+ record_fail "SOA serial (${DOMAIN})" "could not parse serial"
+ elif [[ "$serial" == "0" ]]; then
+ record_fail "SOA serial (${DOMAIN})" "serial is 0"
+ else
+ record_pass "SOA serial (${DOMAIN})" "${serial}"
+ fi
+}
+
+# ── 9. SOA Consistency ───────────────────────────────────────────────
+test_soa_consistency() {
+ if [[ -z "$ZONE_MASTER" ]]; then
+ record_skip "SOA consistency" "ZONE_MASTER not set"
+ return
+ fi
+ local serial_local serial_master
+ # Get serial from configured server
+ serial_local=$(dig_cmd +short "${DOMAIN}" SOA 2>/dev/null | awk '{print $3}') || true
+ # Get serial from master
+ serial_master=$("$DIG_CMD" "@${ZONE_MASTER}" +short "${DOMAIN}" SOA 2>/dev/null | awk '{print $3}') || true
+ if [[ -z "$serial_local" || -z "$serial_master" ]]; then
+ record_fail "SOA consistency" "could not retrieve serials (local=${serial_local:-?}, master=${serial_master:-?})"
+ return
+ fi
+ if [[ "$serial_local" == "$serial_master" ]]; then
+ record_pass "SOA consistency" "serial ${serial_local} matches across servers"
+ else
+ record_fail "SOA consistency" "serial mismatch — local=${serial_local}, master=${serial_master}"
+ fi
+}
+
+# ── 10. Zone Transfer ────────────────────────────────────────────────
+test_zone_transfer() {
+ section_header "Zone Transfer"
+ if [[ -z "$ZONE" ]]; then
+ record_skip "Zone transfer (AXFR)" "ZONE not set"
+ return
+ fi
+ local output exit_code=0
+ output=$(dig_cmd AXFR "${ZONE}" 2>&1) || exit_code=$?
+ # Check if transfer returned records
+ local record_count
+ record_count=$(echo "$output" | grep -c "^${ZONE}" 2>/dev/null) || record_count=0
+ if [[ $record_count -gt 0 ]]; then
+ record_pass "Zone transfer (${ZONE})" "${record_count} records transferred"
+ elif echo "$output" | grep -qi "transfer failed\|refused\|REFUSED"; then
+ record_pass "Zone transfer (${ZONE})" "AXFR refused (expected on production)"
+ else
+ record_fail "Zone transfer (${ZONE})" "transfer failed — ${output:0:100}"
+ fi
+}
+
+# ── 11. DNSSEC Validation ────────────────────────────────────────────
+test_dnssec_validation() {
+ section_header "DNSSEC"
+ if [[ -z "$DNSSEC_DOMAIN" ]]; then
+ record_skip "DNSSEC validation" "DNSSEC_DOMAIN not set"
+ return
+ fi
+ local output
+ output=$(dig_cmd +dnssec +short "${DNSSEC_DOMAIN}" A 2>/dev/null) || true
+ # Check for AD flag in full output
+ local full_output
+ full_output=$(dig_cmd +dnssec "${DNSSEC_DOMAIN}" A 2>/dev/null) || true
+ if echo "$full_output" | grep -q "flags:.*ad"; then
+ record_pass "DNSSEC validation (${DNSSEC_DOMAIN})" "AD flag set"
+ elif [[ -n "$output" ]]; then
+ record_fail "DNSSEC validation (${DNSSEC_DOMAIN})" "response received but AD flag not set"
+ else
+ record_fail "DNSSEC validation (${DNSSEC_DOMAIN})" "no response"
+ fi
+}
+
+# ── 12. DNS-over-TLS ─────────────────────────────────────────────────
+test_dot() {
+ section_header "DNS-over-TLS"
+ if [[ -z "$DOT_SERVER" ]]; then
+ record_skip "DNS-over-TLS" "DOT_SERVER not set"
+ return
+ fi
+ if ! has_cmd openssl; then
+ record_skip "DNS-over-TLS" "openssl not installed"
+ return
+ fi
+ local output exit_code=0
+ output=$(echo "" | openssl s_client -connect "${DOT_SERVER}:853" -servername "${DOT_SERVER}" 2>&1) || exit_code=$?
+ if echo "$output" | grep -qi "connected\|verify return"; then
+ # Extract certificate info if available
+ local cn
+ cn=$(echo "$output" | grep -oP 'CN\s*=\s*\K[^,/]+' | head -1) || true
+ record_pass "DNS-over-TLS (${DOT_SERVER}:853)" "TLS handshake OK${cn:+ — CN=${cn}}"
+ else
+ record_fail "DNS-over-TLS (${DOT_SERVER}:853)" "TLS handshake failed"
+ fi
+}
+
+# ── 13. Custom Record Checks ─────────────────────────────────────────
+test_custom_records() {
+ if [[ -z "$TEST_RECORDS" ]]; then return; fi
+ section_header "Custom Records"
+ local IFS=','
+ for entry in $TEST_RECORDS; do
+ local name type expected
+ name=$(echo "$entry" | cut -d: -f1)
+ type=$(echo "$entry" | cut -d: -f2)
+ expected=$(echo "$entry" | cut -d: -f3-)
+ if [[ -z "$name" || -z "$type" ]]; then
+ record_fail "Custom record" "invalid entry: ${entry}"
+ continue
+ fi
+ local output
+ output=$(dig_cmd +short "${name}" "${type}" 2>/dev/null) || true
+ if [[ -z "$output" ]]; then
+ record_fail "Custom record (${name} ${type})" "no record returned"
+ elif [[ -n "$expected" ]]; then
+ if echo "$output" | grep -q "$expected"; then
+ record_pass "Custom record (${name} ${type})" "${output}"
+ else
+ record_fail "Custom record (${name} ${type})" "expected '${expected}', got '${output}'"
+ fi
+ else
+ record_pass "Custom record (${name} ${type})" "${output}"
+ fi
+ done
+}
+
+# ── 14. Recursive Resolution ─────────────────────────────────────────
+test_recursive_resolution() {
+ section_header "Recursion"
+ local output
+ output=$(dig_cmd +short "google.com" A 2>/dev/null) || true
+ if [[ -n "$output" ]]; then
+ local first_ip
+ first_ip=$(echo "$output" | head -1)
+ record_pass "Recursive resolution (google.com)" "${first_ip}"
+ else
+ record_fail "Recursive resolution (google.com)" "could not resolve external domain"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
+
+# ── Test Result Recording ─────────────────────────────────────────────
+record_pass() {
+ local name="$1" detail="${2:-}"
+ ((PASS++)) || true; ((TOTAL++)) || true
+ RESULTS+=("PASS|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name}${detail:+ (${detail})}"
+ else echo -e " ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}"; fi
+}
+
+record_fail() {
+ local name="$1" detail="${2:-}"
+ ((FAIL++)) || true; ((TOTAL++)) || true
+ RESULTS+=("FAIL|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "not ok ${TOTAL} - ${name}"
+ [[ -n "$detail" ]] && echo " # ${detail}"
+ else echo -e " ${RED}✗${RESET} ${name}${detail:+ — ${detail}}"; fi
+}
+
+record_skip() {
+ local name="$1" reason="${2:-}"
+ ((SKIP++)) || true; ((TOTAL++)) || true
+ RESULTS+=("SKIP|${name}|${reason}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name} # SKIP ${reason}"
+ else echo -e " ${YELLOW}⊘${RESET} ${name}${reason:+ — ${reason}}"; fi
+}
+
+# ── Helpers ───────────────────────────────────────────────────────────
+has_cmd() { command -v "$1" >/dev/null 2>&1; }
+
+remove_container() {
+ local name="$1"
+ docker rm -f "$name" >/dev/null 2>&1 || true
+}
+
+section() {
+ if [[ "$OUTPUT_FORMAT" != "tap" ]]; then echo ""; echo -e "${BOLD}$1${RESET}"; fi
+}
+
+# ── Cleanup ───────────────────────────────────────────────────────────
+# shellcheck disable=SC2317
+cleanup() {
+ verbose "Cleaning up test artifacts..."
+ remove_container "$SMOKE_CONTAINER"
+ remove_container "$SMOKE_PORT_CONTAINER"
+ remove_container "$SMOKE_DNS_CONTAINER"
+ remove_container "$SMOKE_VOL_CONTAINER"
+ remove_container "$SMOKE_NET_CONTAINER"
+ remove_container "$SMOKE_MEM_CONTAINER"
+ docker volume rm -f "$SMOKE_VOLUME" >/dev/null 2>&1 || true
+ docker network rm "$SMOKE_NETWORK" >/dev/null 2>&1 || true
+ docker rmi -f "$SMOKE_BUILD_TAG" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+# ══════════════════════════════════════════════════════════════════════
+# TEST FUNCTIONS
+# ══════════════════════════════════════════════════════════════════════
+
+# ── 1. Docker daemon running ─────────────────────────────────────────
+test_daemon_running() {
+ if has_cmd systemctl; then
+ if systemctl is-active --quiet docker 2>/dev/null; then record_pass "Docker daemon running" "systemctl active"
+ else record_fail "Docker daemon running" "systemctl inactive"; fi
+ elif has_cmd service; then
+ if service docker status >/dev/null 2>&1; then record_pass "Docker daemon running" "service running"
+ else record_fail "Docker daemon running" "service stopped"; fi
+ elif docker info >/dev/null 2>&1; then record_pass "Docker daemon running" "docker info ok"
+ else record_fail "Docker daemon running" "cannot determine status"; fi
+}
+
+# ── 2. Docker API responsive ─────────────────────────────────────────
+test_api_responsive() {
+ local output
+ if output=$(timeout 10 docker info 2>&1); then
+ local ver; ver=$(echo "$output" | grep -i "Server Version" | head -1 | awk '{print $NF}') || true
+ record_pass "Docker API responsive" "server ${ver:-unknown}"
+ else record_fail "Docker API responsive" "docker info timed out or failed"; fi
+}
+
+# ── 3. Docker socket accessible ──────────────────────────────────────
+test_socket_accessible() {
+ local socket="/var/run/docker.sock"
+ if [[ -S "$socket" ]]; then
+ if [[ -r "$socket" && -w "$socket" ]]; then record_pass "Docker socket accessible" "$socket"
+ else record_fail "Docker socket accessible" "$socket not readable/writable"; fi
+ elif docker info >/dev/null 2>&1; then record_pass "Docker socket accessible" "non-default socket"
+ else record_fail "Docker socket accessible" "$socket not found"; fi
+}
+
+# ── 4. Container lifecycle ───────────────────────────────────────────
+test_container_lifecycle() {
+ if [[ "$SKIP_LIFECYCLE" == "true" ]]; then record_skip "Container lifecycle" "SKIP_LIFECYCLE=true"; return; fi
+ remove_container "$SMOKE_CONTAINER"
+
+ if ! docker create --name "$SMOKE_CONTAINER" "$TEST_IMAGE" sleep 30 >/dev/null 2>&1; then
+ record_fail "Container lifecycle" "docker create failed"
+ return
+ fi
+
+ if ! docker start "$SMOKE_CONTAINER" >/dev/null 2>&1; then
+ record_fail "Container lifecycle" "docker start failed"
+ return
+ fi
+
+ local exec_output
+ exec_output=$(docker exec "$SMOKE_CONTAINER" echo "smoke-ok" 2>&1) || true
+ if [[ "$exec_output" != "smoke-ok" ]]; then
+ record_fail "Container lifecycle" "docker exec failed"
+ return
+ fi
+
+ if ! docker stop -t 5 "$SMOKE_CONTAINER" >/dev/null 2>&1; then
+ record_fail "Container lifecycle" "docker stop failed"
+ return
+ fi
+
+ if ! docker rm "$SMOKE_CONTAINER" >/dev/null 2>&1; then
+ record_fail "Container lifecycle" "docker rm failed"
+ return
+ fi
+
+ record_pass "Container lifecycle" "create/start/exec/stop/rm"
+}
+
+# ── 5. Port binding ──────────────────────────────────────────────────
+test_port_binding() {
+ if [[ "$SKIP_LIFECYCLE" == "true" ]]; then record_skip "Port binding" "SKIP_LIFECYCLE=true"; return; fi
+ if ! has_cmd curl; then record_skip "Port binding" "curl not installed"; return; fi
+ remove_container "$SMOKE_PORT_CONTAINER"
+
+ if ! docker run -d --name "$SMOKE_PORT_CONTAINER" \
+ -p "${SMOKE_PORT}:80" \
+ "$TEST_IMAGE" sh -c 'mkdir -p /var/www && echo "smoke-ok" > /var/www/index.html && httpd -f -p 80 -h /var/www 2>/dev/null || { while true; do echo -e "HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\nsmoke-ok\n" | nc -l -p 80 2>/dev/null || break; done; }' >/dev/null 2>&1; then
+ record_fail "Port binding" "failed to start container with port mapping"
+ return
+ fi
+
+ sleep 2
+ local response
+ response=$(curl -sf --max-time 5 "http://localhost:${SMOKE_PORT}/" 2>/dev/null) || true
+ remove_container "$SMOKE_PORT_CONTAINER"
+
+ if [[ "$response" == *"smoke-ok"* ]]; then
+ record_pass "Port binding" "curl localhost:${SMOKE_PORT}"
+ else
+ record_fail "Port binding" "no response on localhost:${SMOKE_PORT}"
+ fi
+}
+
+# ── 6. Container DNS ─────────────────────────────────────────────────
+test_container_dns() {
+ if [[ "$SKIP_LIFECYCLE" == "true" ]]; then record_skip "Container DNS" "SKIP_LIFECYCLE=true"; return; fi
+ remove_container "$SMOKE_DNS_CONTAINER"
+
+ local dns_output
+ dns_output=$(docker run --rm --name "$SMOKE_DNS_CONTAINER" "$TEST_IMAGE" \
+ sh -c "nslookup ${DNS_TEST_DOMAIN} 2>/dev/null || getent hosts ${DNS_TEST_DOMAIN} 2>/dev/null || ping -c1 -W3 ${DNS_TEST_DOMAIN} 2>/dev/null" 2>&1) || true
+
+ if [[ -n "$dns_output" ]] && ! echo "$dns_output" | grep -qi "can't resolve\|not found\|failure\|NXDOMAIN"; then
+ record_pass "Container DNS" "${DNS_TEST_DOMAIN}"
+ else
+ record_fail "Container DNS" "failed to resolve ${DNS_TEST_DOMAIN}"
+ fi
+}
+
+# ── 7. Volume mount ──────────────────────────────────────────────────
+test_volume_mount() {
+ if [[ "$SKIP_VOLUME" == "true" ]]; then record_skip "Volume mount" "SKIP_VOLUME=true"; return; fi
+ docker volume rm -f "$SMOKE_VOLUME" >/dev/null 2>&1 || true
+ remove_container "$SMOKE_VOL_CONTAINER"
+
+ if ! docker volume create "$SMOKE_VOLUME" >/dev/null 2>&1; then
+ record_fail "Volume mount" "docker volume create failed"
+ return
+ fi
+
+ local write_result
+ write_result=$(docker run --rm --name "$SMOKE_VOL_CONTAINER" \
+ -v "${SMOKE_VOLUME}:/data" "$TEST_IMAGE" \
+ sh -c 'echo "smoke-vol-ok" > /data/test.txt && cat /data/test.txt' 2>&1) || true
+
+ docker volume rm -f "$SMOKE_VOLUME" >/dev/null 2>&1 || true
+
+ if [[ "$write_result" == "smoke-vol-ok" ]]; then
+ record_pass "Volume mount" "write/read verified"
+ else
+ record_fail "Volume mount" "write/read mismatch"
+ fi
+}
+
+# ── 8. Network create/connect ────────────────────────────────────────
+test_network_create() {
+ if [[ "$SKIP_NETWORK" == "true" ]]; then record_skip "Network create/connect" "SKIP_NETWORK=true"; return; fi
+ docker network rm "$SMOKE_NETWORK" >/dev/null 2>&1 || true
+ remove_container "$SMOKE_NET_CONTAINER"
+
+ if ! docker network create --driver bridge "$SMOKE_NETWORK" >/dev/null 2>&1; then
+ record_fail "Network create/connect" "docker network create failed"
+ return
+ fi
+
+ local net_output
+ net_output=$(docker run --rm --name "$SMOKE_NET_CONTAINER" \
+ --network "$SMOKE_NETWORK" "$TEST_IMAGE" \
+ sh -c 'ip addr show 2>/dev/null || ifconfig 2>/dev/null' 2>&1) || true
+
+ docker network rm "$SMOKE_NETWORK" >/dev/null 2>&1 || true
+
+ if [[ -n "$net_output" ]]; then
+ record_pass "Network create/connect" "bridge network"
+ else
+ record_fail "Network create/connect" "container failed to attach to network"
+ fi
+}
+
+# ── 9. Image pull ────────────────────────────────────────────────────
+test_image_pull() {
+ if docker pull "$TEST_IMAGE" >/dev/null 2>&1; then
+ record_pass "Image pull" "$TEST_IMAGE"
+ else
+ record_fail "Image pull" "failed to pull $TEST_IMAGE"
+ fi
+}
+
+# ── 10. Image build ──────────────────────────────────────────────────
+test_image_build() {
+ if [[ "$SKIP_BUILD" == "true" ]]; then record_skip "Image build" "SKIP_BUILD=true"; return; fi
+ docker rmi -f "$SMOKE_BUILD_TAG" >/dev/null 2>&1 || true
+
+ if echo "FROM alpine:latest" | docker build -t "$SMOKE_BUILD_TAG" - >/dev/null 2>&1; then
+ docker rmi -f "$SMOKE_BUILD_TAG" >/dev/null 2>&1 || true
+ record_pass "Image build" "inline Dockerfile"
+ else
+ record_fail "Image build" "docker build failed"
+ fi
+}
+
+# ── 11. Docker Compose ───────────────────────────────────────────────
+test_compose_stack() {
+ if [[ -z "$COMPOSE_FILE" ]]; then
+ record_skip "Compose stack" "COMPOSE_FILE not set"
+ return
+ fi
+ if [[ ! -f "$COMPOSE_FILE" ]]; then
+ record_fail "Compose stack" "${COMPOSE_FILE} not found"
+ return
+ fi
+ local compose_cmd=""
+ if docker compose version >/dev/null 2>&1; then
+ compose_cmd="docker compose"
+ elif has_cmd docker-compose; then
+ compose_cmd="docker-compose"
+ else
+ record_skip "Compose stack" "neither 'docker compose' nor 'docker-compose' available"
+ return
+ fi
+
+ local ps_output expected_count running_count
+ ps_output=$($compose_cmd -f "$COMPOSE_FILE" ps --format json 2>/dev/null) || true
+
+ if [[ -z "$ps_output" ]]; then
+ ps_output=$($compose_cmd -f "$COMPOSE_FILE" ps 2>/dev/null) || true
+ if [[ -z "$ps_output" ]]; then
+ record_fail "Compose stack" "could not read compose project status"
+ return
+ fi
+ expected_count=$(echo "$ps_output" | tail -n +2 | wc -l)
+ running_count=$(echo "$ps_output" | tail -n +2 | grep -ciE "up|running" || true)
+ else
+ expected_count=$(echo "$ps_output" | grep -c '"Service"' 2>/dev/null || echo "$ps_output" | wc -l)
+ running_count=$(echo "$ps_output" | grep -ciE '"running"' 2>/dev/null || true)
+ fi
+
+ if [[ "$expected_count" -eq 0 ]]; then
+ record_fail "Compose stack" "no services found"
+ elif [[ "$running_count" -ge "$expected_count" ]]; then
+ record_pass "Compose stack" "${running_count}/${expected_count} services running"
+ else
+ record_fail "Compose stack" "${running_count}/${expected_count} services running"
+ fi
+}
+
+# ── 12. Resource limits ──────────────────────────────────────────────
+test_resource_limits() {
+ if [[ "$SKIP_LIFECYCLE" == "true" ]]; then record_skip "Resource limits" "SKIP_LIFECYCLE=true"; return; fi
+ remove_container "$SMOKE_MEM_CONTAINER"
+
+ local mem_limit
+ mem_limit=$(docker run --rm --name "$SMOKE_MEM_CONTAINER" \
+ --memory=64m "$TEST_IMAGE" \
+ sh -c 'cat /sys/fs/cgroup/memory.max 2>/dev/null || cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null' 2>&1) || true
+
+ if [[ -z "$mem_limit" ]]; then
+ record_skip "Resource limits" "cgroup memory info not available"
+ return
+ fi
+
+ local limit_bytes=67108864 # 64 MiB
+ if [[ "$mem_limit" =~ ^[0-9]+$ ]]; then
+ if [[ "$mem_limit" -le $((limit_bytes + 1048576)) ]]; then
+ local limit_mb=$((mem_limit / 1048576))
+ record_pass "Resource limits" "memory cgroup enforced (${limit_mb}M)"
+ else
+ record_fail "Resource limits" "memory limit not enforced (got ${mem_limit})"
+ fi
+ else
+ record_skip "Resource limits" "unexpected cgroup value: ${mem_limit}"
+ fi
+}
+
+# ── 13. Disk space ───────────────────────────────────────────────────
+test_disk_space() {
+ local df_output
+ df_output=$(docker system df 2>/dev/null) || true
+
+ if [[ -z "$df_output" ]]; then
+ record_fail "Disk space" "docker system df failed"
+ return
+ fi
+
+ local docker_root used_pct
+ docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null) || docker_root="/var/lib/docker"
+ used_pct=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%') || used_pct=0
+
+ if [[ "$used_pct" -gt 80 ]]; then
+ record_fail "Disk space" "${used_pct}% used (threshold 80%)"
+ else
+ record_pass "Disk space" "${used_pct}% used"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OUTPUT
+# ══════════════════════════════════════════════════════════════════════
+
+print_tap_header() {
+ echo "TAP version 13"
+}
+
+print_tap_footer() {
+ echo "1..${TOTAL}"
+ echo "# pass ${PASS}"
+ echo "# fail ${FAIL}"
+ echo "# skip ${SKIP}"
+}
+
+print_summary() {
+ local end_time; end_time=$(date +%s)
+ local duration=$(( end_time - START_TIME ))
+ echo ""
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+ echo -e "${BOLD}Summary${RESET} Docker Smoke Tests"
+ echo -e " ${GREEN}${PASS} passed${RESET} ${RED}${FAIL} failed${RESET} ${YELLOW}${SKIP} skipped${RESET} (${duration}s)"
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+ if [[ $FAIL -eq 0 ]]; then echo -e "${GREEN}${BOLD}All tests passed.${RESET}"
+ else echo -e "${RED}${BOLD}${FAIL} test(s) failed.${RESET}"; fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat </dev/null; then
+ missing+=("$cmd")
+ fi
+ done
+ if [[ ${#missing[@]} -gt 0 ]]; then
+ echo "ERROR: Missing required commands: ${missing[*]}" >&2
+ echo "Install with: apt install ${missing[*]} OR dnf install ${missing[*]}" >&2
+ exit 1
+ fi
+}
+
+validate_config() {
+ # Export DOCKER_HOST if set so docker CLI picks it up
+ if [[ -n "$DOCKER_HOST" ]]; then
+ export DOCKER_HOST
+ fi
+}
+
+docker_cmd() {
+ # Run a docker command and return its output
+ # Returns empty string on failure
+ docker "$@" 2>/dev/null || echo ""
+}
+
+add_metric() {
+ local name="$1"
+ local type="$2"
+ local help="$3"
+ local value="$4"
+ local labels="${5:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name} ${value}
+"
+ fi
+}
+
+add_metric_value() {
+ local name="$1"
+ local value="$2"
+ local labels="${3:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="${name} ${value}
+"
+ fi
+}
+
+collect_nodes() {
+ local nodes_json
+ nodes_json=$(docker_cmd node ls --format '{{json .}}')
+
+ if [[ -z "$nodes_json" ]]; then
+ add_metric "swarm_up" "gauge" "Docker Swarm reachability (1=up, 0=down)" "0"
+ return 1
+ fi
+
+ add_metric "swarm_up" "gauge" "Docker Swarm reachability (1=up, 0=down)" "1"
+
+ # Total node count
+ local node_count
+ node_count=$(echo "$nodes_json" | wc -l)
+ add_metric "swarm_node_count" "gauge" "Total number of nodes in the swarm" "${node_count}"
+
+ # Nodes by status
+ local nodes_ready nodes_down
+ nodes_ready=$(echo "$nodes_json" | jq -r 'select(.Status == "Ready")' | jq -s 'length')
+ nodes_down=$(echo "$nodes_json" | jq -r 'select(.Status == "Down")' | jq -s 'length')
+ add_metric "swarm_nodes_ready" "gauge" "Number of nodes in ready state" "${nodes_ready}"
+ add_metric "swarm_nodes_down" "gauge" "Number of nodes in down state" "${nodes_down}"
+
+ # Manager and worker counts
+ local managers_total workers_total
+ managers_total=$(echo "$nodes_json" | jq -r 'select(.ManagerStatus != "")' | jq -s 'length')
+ workers_total=$(echo "$nodes_json" | jq -r 'select(.ManagerStatus == "")' | jq -s 'length')
+ add_metric "swarm_managers_total" "gauge" "Total number of manager nodes" "${managers_total}"
+ add_metric "swarm_workers_total" "gauge" "Total number of worker nodes" "${workers_total}"
+
+ # Leader detection — check if the current node is the leader
+ local is_leader
+ is_leader=$(echo "$nodes_json" | jq -r 'select(.Self == "true" or .Self == true) | select(.ManagerStatus == "Leader")' | jq -s 'length')
+ if [[ "$is_leader" -gt 0 ]]; then
+ add_metric "swarm_manager_leader" "gauge" "Whether this node is the leader (1=leader, 0=not leader)" "1"
+ else
+ add_metric "swarm_manager_leader" "gauge" "Whether this node is the leader (1=leader, 0=not leader)" "0"
+ fi
+
+ return 0
+}
+
+collect_services() {
+ local services_json
+ services_json=$(docker_cmd service ls --format '{{json .}}')
+
+ if [[ -z "$services_json" ]]; then
+ add_metric "swarm_services_total" "gauge" "Total number of services" "0"
+ return
+ fi
+
+ # Total service count
+ local service_count
+ service_count=$(echo "$services_json" | wc -l)
+ add_metric "swarm_services_total" "gauge" "Total number of services" "${service_count}"
+
+ # Per-service replica metrics
+ # docker service ls --format '{{json .}}' gives us Name and Replicas ("3/3" format)
+ local first_replicas=true
+ local first_running=true
+
+ while IFS= read -r line; do
+ local service_name replicas_str desired running
+
+ service_name=$(echo "$line" | jq -r '.Name')
+ replicas_str=$(echo "$line" | jq -r '.Replicas')
+
+ # Replicas format is "RUNNING/DESIRED" (e.g. "3/3") or "RUNNING/DESIRED (max N per node)"
+ # Strip any parenthetical suffix
+ replicas_str="${replicas_str%% (*}"
+
+ running=$(echo "$replicas_str" | cut -d'/' -f1)
+ desired=$(echo "$replicas_str" | cut -d'/' -f2)
+
+ # Validate numeric
+ if ! [[ "$desired" =~ ^[0-9]+$ ]]; then
+ desired=0
+ fi
+ if ! [[ "$running" =~ ^[0-9]+$ ]]; then
+ running=0
+ fi
+
+ if [[ "$first_replicas" == true ]]; then
+ OUTPUT+="# HELP swarm_service_replicas Desired replica count per service
+# TYPE swarm_service_replicas gauge
+"
+ first_replicas=false
+ fi
+ OUTPUT+="swarm_service_replicas{service=\"${service_name}\"} ${desired}
+"
+
+ if [[ "$first_running" == true ]]; then
+ first_running=false
+ fi
+ done <<< "$services_json"
+
+ # Running replicas — separate HELP/TYPE block
+ OUTPUT+="# HELP swarm_service_replicas_running Running replica count per service
+# TYPE swarm_service_replicas_running gauge
+"
+ while IFS= read -r line; do
+ local service_name replicas_str running
+
+ service_name=$(echo "$line" | jq -r '.Name')
+ replicas_str=$(echo "$line" | jq -r '.Replicas')
+ replicas_str="${replicas_str%% (*}"
+ running=$(echo "$replicas_str" | cut -d'/' -f1)
+
+ if ! [[ "$running" =~ ^[0-9]+$ ]]; then
+ running=0
+ fi
+
+ OUTPUT+="swarm_service_replicas_running{service=\"${service_name}\"} ${running}
+"
+ done <<< "$services_json"
+}
+
+collect_tasks() {
+ # Count running tasks
+ local tasks_running
+ tasks_running=$(docker_cmd node ps --format '{{json .}}' --filter 'desired-state=running' 2>/dev/null | jq -s 'length' 2>/dev/null)
+ if [[ -z "$tasks_running" || "$tasks_running" == "null" ]]; then
+ tasks_running=0
+ fi
+ add_metric "swarm_tasks_running" "gauge" "Total number of running tasks" "${tasks_running}"
+
+ # Count failed tasks across all services
+ local tasks_failed
+ tasks_failed=$(docker_cmd service ls -q 2>/dev/null | while read -r svc_id; do
+ docker service ps "$svc_id" --format '{{json .}}' --filter 'desired-state=shutdown' 2>/dev/null
+ done | jq -r 'select(.CurrentState | test("^Failed|^Rejected"; "i"))' 2>/dev/null | jq -s 'length' 2>/dev/null)
+ if [[ -z "$tasks_failed" || "$tasks_failed" == "null" ]]; then
+ tasks_failed=0
+ fi
+ add_metric "swarm_tasks_failed" "gauge" "Total number of failed tasks" "${tasks_failed}"
+}
+
+collect_networks() {
+ local networks_json
+ networks_json=$(docker_cmd network ls --filter driver=overlay --format '{{json .}}')
+
+ local network_count=0
+ if [[ -n "$networks_json" ]]; then
+ network_count=$(echo "$networks_json" | wc -l)
+ fi
+
+ add_metric "swarm_networks_total" "gauge" "Total number of overlay networks" "${network_count}"
+}
+
+collect_raft() {
+ # Get Raft index from docker info
+ local info_json
+ info_json=$(docker_cmd info --format '{{json .}}')
+
+ if [[ -z "$info_json" ]]; then
+ return
+ fi
+
+ local raft_index
+ raft_index=$(echo "$info_json" | jq -r '.Swarm.Cluster.RaftIndex // .Swarm.RaftIndex // empty' 2>/dev/null)
+
+ # Fallback — try extracting from Swarm.Cluster directly
+ if [[ -z "$raft_index" ]]; then
+ raft_index=$(echo "$info_json" | jq -r '.Swarm.Cluster.Version.Index // empty' 2>/dev/null)
+ fi
+
+ if [[ -n "$raft_index" && "$raft_index" != "null" ]]; then
+ add_metric "swarm_raft_index" "gauge" "Raft applied index" "${raft_index}"
+ else
+ add_metric "swarm_raft_index" "gauge" "Raft applied index" "0"
+ fi
+}
+
+write_output() {
+ if [[ "$TEXTFILE_MODE" == true ]]; then
+ local output_file="${TEXTFILE_DIR}/docker_swarm.prom"
+ local temp_file="${output_file}.$$"
+
+ mkdir -p "$TEXTFILE_DIR"
+ echo "$OUTPUT" > "$temp_file"
+ mv "$temp_file" "$output_file"
+ else
+ echo "$OUTPUT"
+ fi
+}
+
+install_cron() {
+ if [[ $EUID -ne 0 ]]; then
+ echo "ERROR: --install requires root" >&2
+ exit 1
+ fi
+
+ local script_path
+ script_path=$(readlink -f "$0")
+
+ local env_lines=""
+ if [[ -n "$DOCKER_HOST" ]]; then
+ env_lines="DOCKER_HOST=${DOCKER_HOST}
+"
+ fi
+
+ cat > /etc/cron.d/docker-swarm-exporter </dev/null
+EOF
+
+ chmod 644 /etc/cron.d/docker-swarm-exporter
+ echo "Installed cron job: /etc/cron.d/docker-swarm-exporter"
+ echo "Metrics will be written to: ${TEXTFILE_DIR}/docker_swarm.prom"
+}
+
+# --- Main ---
+
+main() {
+ # Parse arguments
+ for arg in "$@"; do
+ case "$arg" in
+ --textfile) TEXTFILE_MODE=true ;;
+ --install)
+ check_dependencies
+ validate_config
+ install_cron
+ exit 0
+ ;;
+ --help|-h) usage ;;
+ *) echo "Unknown option: $arg" >&2; usage ;;
+ esac
+ done
+
+ check_dependencies
+ validate_config
+
+ START_TIME=$(date +%s%N)
+
+ # Exporter info
+ add_metric "swarm_exporter_info" "gauge" "Exporter version information" "1" "version=\"${VERSION}\""
+
+ # Collect metrics
+ if collect_nodes; then
+ collect_services
+ collect_tasks
+ collect_networks
+ collect_raft
+ fi
+
+ # Exporter performance
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
+ add_metric "swarm_exporter_duration_seconds" "gauge" "Script execution time" "$duration"
+ add_metric "swarm_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run" "$(date +%s)"
+
+ write_output
+}
+
+main "$@"
diff --git a/docker-volume-backup.sh b/docker-volume-backup.sh
new file mode 100644
index 0000000..9f26908
--- /dev/null
+++ b/docker-volume-backup.sh
@@ -0,0 +1,269 @@
+#!/bin/bash
+#############################################################
+#### Docker Volume Backup Script ####
+#### Backup and restore Docker named volumes using ####
+#### tar archives with optional compression ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version: 1.0 ####
+#### ####
+#### Usage: ./docker-volume-backup.sh [OPTIONS] ####
+#############################################################
+
+set -euo pipefail
+
+SCRIPT_NAME=$(basename "$0")
+readonly SCRIPT_NAME
+readonly DEFAULT_BACKUP_DIR="/opt/docker-backups"
+readonly DEFAULT_RETAIN=7
+readonly ALPINE_IMAGE="alpine:latest"
+
+BACKUP_DIR="$DEFAULT_BACKUP_DIR"
+RETAIN="$DEFAULT_RETAIN"
+MODE=""
+TARGET_VOLUME=""
+RESTORE_ARCHIVE=""
+
+# Colors
+readonly RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m'
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*" >&2; }
+log_step() { echo -e "${BLUE}[STEP]${NC} $(date '+%Y-%m-%d %H:%M:%S') $*"; }
+
+show_help() {
+ cat << EOF
+Usage: $SCRIPT_NAME [OPTIONS]
+
+Backup and restore Docker named volumes using tar archives with compression.
+
+OPTIONS:
+ --backup [VOLUME] Backup all named volumes, or a specific volume if given
+ --restore ARCHIVE Restore a volume from the specified tar.gz archive
+ --list List available backups
+ --backup-dir PATH Backup directory (default: $DEFAULT_BACKUP_DIR)
+ --retain N Number of backups to keep per volume (default: $DEFAULT_RETAIN)
+ --help, -h Show this help message
+
+EXAMPLES:
+ $SCRIPT_NAME --backup
+ $SCRIPT_NAME --backup my_volume
+ $SCRIPT_NAME --restore $DEFAULT_BACKUP_DIR/my_volume_20260309_143022.tar.gz
+ $SCRIPT_NAME --list
+ $SCRIPT_NAME --backup --backup-dir /mnt/backups --retain 14
+EOF
+ exit 0
+}
+
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --backup)
+ MODE="backup"; shift
+ [[ $# -gt 0 && ! "$1" =~ ^-- ]] && { TARGET_VOLUME="$1"; shift; }
+ ;;
+ --restore)
+ MODE="restore"
+ [[ $# -lt 2 ]] && { log_error "--restore requires an archive path"; exit 1; }
+ RESTORE_ARCHIVE="$2"; shift 2
+ ;;
+ --list) MODE="list"; shift ;;
+ --backup-dir)
+ [[ $# -lt 2 ]] && { log_error "--backup-dir requires a path"; exit 1; }
+ BACKUP_DIR="$2"; shift 2
+ ;;
+ --retain)
+ [[ $# -lt 2 ]] && { log_error "--retain requires a number"; exit 1; }
+ RETAIN="$2"; shift 2
+ ;;
+ --help|-h) show_help ;;
+ *) log_error "Unknown option: $1"; show_help ;;
+ esac
+ done
+ if [[ -z "$MODE" ]]; then
+ log_error "No action specified. Use --backup, --restore, or --list."
+ show_help
+ fi
+}
+
+check_dependencies() {
+ if ! command -v docker &>/dev/null; then
+ log_error "docker is required but not installed"; exit 1
+ fi
+ if ! docker info &>/dev/null; then
+ log_error "Cannot connect to Docker daemon. Is it running?"; exit 1
+ fi
+}
+
+backup_volume() {
+ local volume_name="$1"
+ local timestamp archive_name final_path tmp_file size
+ timestamp=$(date +%Y%m%d_%H%M%S)
+ archive_name="${volume_name}_${timestamp}.tar.gz"
+ final_path="${BACKUP_DIR}/${archive_name}"
+
+ log_step "Backing up volume: ${volume_name}"
+
+ if ! docker volume inspect "$volume_name" &>/dev/null; then
+ log_error "Volume '$volume_name' does not exist"; return 1
+ fi
+
+ mkdir -p "$BACKUP_DIR"
+ tmp_file=$(mktemp "${BACKUP_DIR}/.backup_XXXXXX.tar.gz")
+
+ if docker run --rm \
+ -v "${volume_name}:/source:ro" \
+ -v "${BACKUP_DIR}:/backup" \
+ "$ALPINE_IMAGE" \
+ tar czf "/backup/$(basename "$tmp_file")" -C /source . 2>/dev/null; then
+ mv "$tmp_file" "$final_path"
+ size=$(du -h "$final_path" | cut -f1)
+ log_info "Created backup: ${final_path} (${size})"
+ else
+ rm -f "$tmp_file"
+ log_error "Failed to backup volume: ${volume_name}"; return 1
+ fi
+}
+
+do_backup() {
+ log_step "Starting Docker volume backup"
+ log_info "Backup directory: ${BACKUP_DIR}"
+
+ local volumes=()
+ if [[ -n "$TARGET_VOLUME" ]]; then
+ volumes=("$TARGET_VOLUME")
+ else
+ while IFS= read -r vol; do
+ [[ -n "$vol" ]] && volumes+=("$vol")
+ done < <(docker volume ls --format '{{.Name}}' | sort)
+ fi
+
+ if [[ ${#volumes[@]} -eq 0 ]]; then
+ log_warn "No Docker named volumes found"; return 0
+ fi
+ log_info "Found ${#volumes[@]} volume(s) to backup"
+
+ local success=0 failed=0
+ for vol in "${volumes[@]}"; do
+ if backup_volume "$vol"; then
+ success=$((success + 1))
+ else
+ failed=$((failed + 1))
+ fi
+ done
+
+ apply_retention
+ log_step "Backup complete: ${success} succeeded, ${failed} failed"
+ [[ $failed -gt 0 ]] && return 1
+ return 0
+}
+
+do_restore() {
+ if [[ ! -f "$RESTORE_ARCHIVE" ]]; then
+ log_error "Archive not found: ${RESTORE_ARCHIVE}"; exit 1
+ fi
+
+ local basename_archive volume_name
+ basename_archive=$(basename "$RESTORE_ARCHIVE")
+ volume_name=$(echo "$basename_archive" | sed 's/_[0-9]\{8\}_[0-9]\{6\}\.tar\.gz$//')
+
+ if [[ -z "$volume_name" || "$volume_name" == "$basename_archive" ]]; then
+ log_error "Cannot determine volume name from archive: ${basename_archive}"
+ log_error "Expected format: volumename_YYYYMMDD_HHMMSS.tar.gz"; exit 1
+ fi
+
+ log_step "Restoring volume: ${volume_name} from ${RESTORE_ARCHIVE}"
+
+ if ! docker volume inspect "$volume_name" &>/dev/null; then
+ log_info "Creating volume: ${volume_name}"
+ docker volume create "$volume_name" >/dev/null
+ else
+ log_warn "Volume '${volume_name}' already exists — contents will be overwritten"
+ fi
+
+ local archive_abs archive_dir archive_file
+ archive_abs=$(realpath "$RESTORE_ARCHIVE")
+ archive_dir=$(dirname "$archive_abs")
+ archive_file=$(basename "$archive_abs")
+
+ if docker run --rm \
+ -v "${volume_name}:/target" \
+ -v "${archive_dir}:/backup:ro" \
+ "$ALPINE_IMAGE" \
+ sh -c "rm -rf /target/* /target/..?* /target/.[!.]* 2>/dev/null; tar xzf /backup/${archive_file} -C /target" 2>/dev/null; then
+ log_info "Volume '${volume_name}' restored successfully"
+ else
+ log_error "Failed to restore volume: ${volume_name}"; exit 1
+ fi
+}
+
+do_list() {
+ if [[ ! -d "$BACKUP_DIR" ]]; then
+ log_info "No backups found (directory does not exist: ${BACKUP_DIR})"; return 0
+ fi
+
+ local count=0
+ log_step "Available backups in ${BACKUP_DIR}:"
+ printf "\n %-40s %-10s %s\n" "ARCHIVE" "SIZE" "VOLUME"
+ printf " %-40s %-10s %s\n" "-------" "----" "------"
+
+ for archive in "$BACKUP_DIR"/*.tar.gz; do
+ [[ -f "$archive" ]] || continue
+ count=$((count + 1))
+ local name size vol_name
+ name=$(basename "$archive")
+ size=$(du -h "$archive" | cut -f1)
+ vol_name=$(echo "$name" | sed 's/_[0-9]\{8\}_[0-9]\{6\}\.tar\.gz$//')
+ printf " %-40s %-10s %s\n" "$name" "$size" "$vol_name"
+ done
+
+ echo ""
+ if [[ $count -eq 0 ]]; then
+ log_info "No backup archives found in ${BACKUP_DIR}"
+ else
+ log_info "${count} backup(s) found"
+ fi
+}
+
+apply_retention() {
+ log_step "Applying retention policy (keep last ${RETAIN} per volume)"
+
+ local vol_names=()
+ for archive in "$BACKUP_DIR"/*.tar.gz; do
+ [[ -f "$archive" ]] || continue
+ local name vol_name
+ name=$(basename "$archive")
+ vol_name=$(echo "$name" | sed 's/_[0-9]\{8\}_[0-9]\{6\}\.tar\.gz$//')
+ vol_names+=("$vol_name")
+ done
+
+ local unique_vols
+ unique_vols=$(printf '%s\n' "${vol_names[@]}" 2>/dev/null | sort -u)
+
+ while IFS= read -r vol; do
+ [[ -z "$vol" ]] && continue
+ local old_archives=()
+ while IFS= read -r f; do
+ [[ -n "$f" ]] && old_archives+=("$f")
+ done < <(ls -1t "$BACKUP_DIR"/${vol}_[0-9]*_[0-9]*.tar.gz 2>/dev/null | tail -n +$((RETAIN + 1)))
+ for old_archive in "${old_archives[@]}"; do
+ log_info "Removing old backup: $(basename "$old_archive")"
+ rm -f "$old_archive"
+ done
+ done <<< "$unique_vols"
+}
+
+main() {
+ parse_args "$@"
+ check_dependencies
+ case "$MODE" in
+ backup) do_backup ;;
+ restore) do_restore ;;
+ list) do_list ;;
+ esac
+}
+
+main "$@"
diff --git a/dokku-exporter.sh b/dokku-exporter.sh
new file mode 100755
index 0000000..617d6fc
--- /dev/null
+++ b/dokku-exporter.sh
@@ -0,0 +1,410 @@
+#!/bin/bash
+################################################################################
+# Script Name: dokku-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for Dokku PaaS providing operational
+# metrics via the Dokku CLI — application status, plugin counts,
+# domain configuration, SSL status, and host health
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - Dokku installed on the local host
+# - Root or dokku user access to run dokku commands
+# - netcat (nc) for HTTP mode
+#
+# Usage:
+# # Output to stdout
+# ./dokku-exporter.sh
+#
+# # HTTP server mode
+# ./dokku-exporter.sh --http -p 9198
+#
+# # Textfile collector mode
+# ./dokku-exporter.sh --textfile
+#
+# Metrics Exported:
+# - dokku_up - Dokku reachability (1=up, 0=down)
+# - dokku_info{version} - Dokku version info
+# - dokku_apps_total - Total app count
+# - dokku_apps_running - Running apps
+# - dokku_apps_stopped - Stopped apps
+# - dokku_plugins_total - Installed plugin count
+# - dokku_domains_app_total - Total app domains configured
+# - dokku_ssl_enabled_total - Apps with SSL enabled
+# - dokku_exporter_duration_seconds - Script execution time
+# - dokku_exporter_last_run_timestamp - Last run timestamp
+#
+# Configuration:
+# Default HTTP port: 9198
+# Textfile directory: /var/lib/node_exporter
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9198
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Check prerequisites
+# Returns: 0 if OK, 1 if error
+check_prerequisites() {
+ if ! command -v dokku >/dev/null 2>&1; then
+ echo "ERROR: dokku not found" >&2
+ return 1
+ fi
+
+ return 0
+}
+
+# Escape special characters in Prometheus label values
+# Args: $1 - string to escape
+# Returns: escaped string safe for Prometheus labels
+prom_escape() {
+ local val="$1"
+ val="${val//\\/\\\\}"
+ val="${val//\"/\\\"}"
+ val="${val//$'\n'/}"
+ echo "$val"
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+# Generate all Prometheus metrics
+# Returns: Prometheus text format metrics on stdout
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s)
+
+ # Check prerequisites
+ if ! check_prerequisites; then
+ cat </dev/null)
+
+ if [ -z "$version_output" ]; then
+ cat < "0.38.0")
+ local dokku_version
+ dokku_version=$(echo "$version_output" | awk '{print $NF}')
+
+ if [ -z "$dokku_version" ]; then
+ cat </dev/null || true)
+
+ local total_apps=0
+ local running_apps=0
+ local stopped_apps=0
+
+ if [ -n "$apps_list" ]; then
+ total_apps=$(echo "$apps_list" | wc -l)
+ total_apps=${total_apps:-0}
+
+ # Count running apps by checking each app's process status
+ while IFS= read -r app; do
+ local ps_running
+ ps_running=$(dokku ps:report "$app" --ps-running 2>/dev/null || echo "false")
+ if [ "$ps_running" = "true" ]; then
+ running_apps=$((running_apps + 1))
+ fi
+ done <<< "$apps_list"
+
+ stopped_apps=$((total_apps - running_apps))
+ fi
+
+ cat </dev/null | wc -l)
+ plugins_count=${plugins_count:-0}
+
+ cat </dev/null || true)
+ if [ -n "$app_domains" ]; then
+ # Domains are space-separated; count words
+ local domain_count
+ domain_count=$(echo "$app_domains" | wc -w)
+ total_domains=$((total_domains + domain_count))
+ fi
+ done <<< "$apps_list"
+ fi
+
+ cat </dev/null || echo "false")
+ if [ "$ssl_enabled" = "true" ]; then
+ ssl_enabled_count=$((ssl_enabled_count + 1))
+ fi
+ done <<< "$apps_list"
+ fi
+
+ cat <&2
+
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+
+ # Infinite loop accepting HTTP requests
+ while true; do
+ {
+ read -r request
+ # Check if request is for /metrics endpoint
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else # Serve HTML landing page for other requests
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ cat <
+
+Dokku Exporter v1.0
+
+Dokku Prometheus Exporter v1.0
+Metrics
+Operational metrics from the Dokku CLI.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+# Main entry point - routes to appropriate output mode
+main() {
+ parse_args "$@"
+
+ if [ "$HTTP_MODE" = true ]; then
+ # Run HTTP server (blocks until killed)
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ # Textfile collector mode: write atomically using temp file
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ # Create temp file in SAME directory for atomic rename (same filesystem)
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.dokku_metrics.XXXXXX")
+
+ # Generate metrics to temp file
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ # Validate: file must exist, have content
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ # Set permissions before move
+ chmod 644 "$temp_file"
+
+ # Atomic rename - no gap where file is missing
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ # Default: output to stdout
+ generate_metrics
+ fi
+}
+
+# Execute main function with all script arguments
+main "$@"
diff --git a/dokku-smoke-tests.sh b/dokku-smoke-tests.sh
new file mode 100755
index 0000000..1114f91
--- /dev/null
+++ b/dokku-smoke-tests.sh
@@ -0,0 +1,455 @@
+#!/bin/bash
+################################################################################
+# Script Name: dokku-smoke-tests.sh
+# Version: 1.0
+# Description: Smoke test suite for Dokku PaaS — validates connectivity,
+# app deployment lifecycle, plugin health, SSL certificates,
+# and resource usage via the dokku CLI
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - bash 4+
+# - dokku binary (run on the Dokku host)
+# - Root or dokku user access
+#
+# Usage:
+# sudo ./dokku-smoke-tests.sh
+# sudo ./dokku-smoke-tests.sh --skip-app --skip-ssl
+# sudo ./dokku-smoke-tests.sh --format tap
+# sudo ./dokku-smoke-tests.sh --format junit --junit-file results.xml
+#
+################################################################################
+
+set -euo pipefail
+
+# --- Defaults ---
+SKIP_APP="${SKIP_APP_LIFECYCLE:-false}"
+SKIP_SSL="${SKIP_SSL:-false}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+JUNIT_FILE="${JUNIT_FILE:-smoke-results.xml}"
+DOKKU_DOMAIN="${DOKKU_DOMAIN:-}"
+VERBOSE=false
+USE_COLOR=true
+TEST_APP_NAME=""
+PASSED=0
+FAILED=0
+SKIPPED=0
+START_TIME=""
+JUNIT_RESULTS=()
+TAP_RESULTS=()
+TEST_NUM=0
+
+# --- Colors ---
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+usage() {
+ cat <&2
+ fi
+}
+
+pass() {
+ local suite="$1" msg="$2"
+ ((TEST_NUM++)) || true
+ ((PASSED++)) || true
+ case "$OUTPUT_FORMAT" in
+ tap) TAP_RESULTS+=("ok $TEST_NUM - [$suite] $msg") ;;
+ junit) JUNIT_RESULTS+=("") ;;
+ *) echo -e " ${GREEN}✓${NC} $msg" ;;
+ esac
+}
+
+fail() {
+ local suite="$1" msg="$2" detail="${3:-}"
+ ((TEST_NUM++)) || true
+ ((FAILED++)) || true
+ case "$OUTPUT_FORMAT" in
+ tap) TAP_RESULTS+=("not ok $TEST_NUM - [$suite] $msg") ;;
+ junit) JUNIT_RESULTS+=("$detail") ;;
+ *) echo -e " ${RED}✗${NC} $msg${detail:+ — $detail}" ;;
+ esac
+}
+
+skip() {
+ local suite="$1" msg="$2"
+ ((TEST_NUM++)) || true
+ ((SKIPPED++)) || true
+ case "$OUTPUT_FORMAT" in
+ tap) TAP_RESULTS+=("ok $TEST_NUM - [$suite] $msg # SKIP") ;;
+ junit) JUNIT_RESULTS+=("") ;;
+ *) echo -e " ${YELLOW}⊘${NC} $msg — skipped" ;;
+ esac
+}
+
+suite_header() {
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e "\n${BOLD}$1${NC}"
+ fi
+}
+
+# --- Cleanup ---
+cleanup() {
+ if [[ -n "$TEST_APP_NAME" ]]; then
+ debug "Cleaning up test app: $TEST_APP_NAME"
+ dokku apps:destroy "$TEST_APP_NAME" --force >/dev/null 2>&1 || true
+ TEST_APP_NAME=""
+ fi
+}
+trap cleanup EXIT INT TERM
+
+# --- Header ---
+START_TIME=$(date +%s)
+HOSTNAME_STR=$(hostname -f 2>/dev/null || hostname)
+
+if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e "${BOLD}Dokku Smoke Tests${NC}"
+ echo "Host: $HOSTNAME_STR"
+ echo "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+fi
+
+# =====================================================
+# Suite 1: Connectivity
+# =====================================================
+suite_header "Connectivity"
+
+# Check dokku binary
+DOKKU_BIN=$(command -v dokku 2>/dev/null || true)
+if [[ -n "$DOKKU_BIN" ]]; then
+ pass "Connectivity" "Dokku binary found — $DOKKU_BIN"
+else
+ fail "Connectivity" "Dokku binary not found" "dokku is not in PATH"
+ echo -e "\n${RED}Cannot continue without dokku binary. Aborting.${NC}" >&2
+ exit 1
+fi
+
+# Check Docker daemon
+if docker info >/dev/null 2>&1; then
+ pass "Connectivity" "Docker daemon running"
+else
+ fail "Connectivity" "Docker daemon not running" "docker info failed"
+fi
+
+# Check dokku version
+DOKKU_VERSION=$(dokku version 2>/dev/null | grep -oP 'dokku version \K[0-9]+\.[0-9]+\.[0-9]+' || true)
+if [[ -z "$DOKKU_VERSION" ]]; then
+ # Try alternate format
+ DOKKU_VERSION=$(dokku version 2>/dev/null | grep -oP '[0-9]+\.[0-9]+\.[0-9]+' || true)
+fi
+
+if [[ -n "$DOKKU_VERSION" ]]; then
+ pass "Connectivity" "Dokku version — $DOKKU_VERSION"
+else
+ fail "Connectivity" "Dokku version" "Could not parse version string"
+fi
+
+# Auto-detect global domain if not set
+if [[ -z "$DOKKU_DOMAIN" ]]; then
+ DOKKU_DOMAIN=$(dokku domains:report --global 2>/dev/null | grep -i "global vhosts" | awk '{print $NF}' || true)
+ if [[ -z "$DOKKU_DOMAIN" ]]; then
+ DOKKU_DOMAIN=$(dokku domains:report --global 2>/dev/null | tail -1 | awk '{print $NF}' || true)
+ fi
+ debug "Auto-detected domain: $DOKKU_DOMAIN"
+fi
+
+# =====================================================
+# Suite 2: App Lifecycle
+# =====================================================
+if [[ "$SKIP_APP" == "true" ]]; then
+ suite_header "App Lifecycle"
+ skip "App Lifecycle" "Create test app"
+ skip "App Lifecycle" "Deploy image"
+ skip "App Lifecycle" "App responding"
+ skip "App Lifecycle" "Delete test app"
+else
+ suite_header "App Lifecycle"
+
+ TEST_APP_NAME="dokku-smoke-$(date +%s)"
+ debug "Test app name: $TEST_APP_NAME"
+
+ # Create app
+ create_output=$(dokku apps:create "$TEST_APP_NAME" 2>&1) || true
+ debug "Create output: $create_output"
+
+ if dokku apps:exists "$TEST_APP_NAME" >/dev/null 2>&1; then
+ pass "App Lifecycle" "Create test app — $TEST_APP_NAME"
+ else
+ fail "App Lifecycle" "Create test app" "$create_output"
+ skip "App Lifecycle" "Deploy image"
+ skip "App Lifecycle" "App responding"
+ skip "App Lifecycle" "Delete test app"
+ TEST_APP_NAME=""
+ SKIP_APP=true
+ fi
+
+ if [[ "$SKIP_APP" != "true" ]]; then
+ # Deploy image via git:from-image
+ deploy_output=$(dokku git:from-image "$TEST_APP_NAME" nginxdemos/hello 2>&1) || true
+ debug "Deploy output: $deploy_output"
+
+ # Check if app is running
+ app_running=false
+ for i in $(seq 1 12); do
+ sleep 5
+ debug "Waiting for app to start... attempt $i/12"
+ ps_output=$(dokku ps:report "$TEST_APP_NAME" 2>/dev/null || true)
+ if echo "$ps_output" | grep -qi "running"; then
+ app_running=true
+ break
+ fi
+ # Also check container status directly
+ running_count=$(dokku ps:report "$TEST_APP_NAME" 2>/dev/null | grep -i "running" | wc -l || echo "0")
+ if [[ "$running_count" -gt 0 ]]; then
+ app_running=true
+ break
+ fi
+ done
+
+ if [[ "$app_running" == "true" ]]; then
+ pass "App Lifecycle" "Deploy image — nginxdemos/hello deployed"
+ else
+ fail "App Lifecycle" "Deploy image" "App not running after 60s"
+ fi
+
+ # Verify HTTP response
+ if [[ -n "$DOKKU_DOMAIN" ]]; then
+ app_url="http://${TEST_APP_NAME}.${DOKKU_DOMAIN}"
+ debug "App URL: $app_url"
+ sleep 3
+
+ app_http=$(curl -s -o /dev/null -w "%{http_code}" \
+ --connect-timeout 10 --max-time 30 \
+ "$app_url" 2>/dev/null || echo "000")
+
+ if [[ "$app_http" == "200" ]]; then
+ pass "App Lifecycle" "App responding — HTTP 200 at ${TEST_APP_NAME}.${DOKKU_DOMAIN}"
+ else
+ fail "App Lifecycle" "App responding" "HTTP $app_http at $app_url"
+ fi
+ else
+ # No domain configured — check container port directly
+ debug "No global domain — checking container directly"
+ port=$(dokku proxy:ports "$TEST_APP_NAME" 2>/dev/null | grep -oP ':\K[0-9]+$' | head -1 || true)
+ if [[ -n "$port" ]]; then
+ app_http=$(curl -s -o /dev/null -w "%{http_code}" \
+ --connect-timeout 10 --max-time 30 \
+ "http://localhost:$port" 2>/dev/null || echo "000")
+ if [[ "$app_http" == "200" ]]; then
+ pass "App Lifecycle" "App responding — HTTP 200 on port $port"
+ else
+ fail "App Lifecycle" "App responding" "HTTP $app_http on port $port"
+ fi
+ else
+ skip "App Lifecycle" "App responding"
+ fi
+ fi
+
+ # Delete test app
+ delete_output=$(dokku apps:destroy "$TEST_APP_NAME" --force 2>&1) || true
+ debug "Delete output: $delete_output"
+
+ if ! dokku apps:exists "$TEST_APP_NAME" >/dev/null 2>&1; then
+ pass "App Lifecycle" "Delete test app — cleaned up"
+ TEST_APP_NAME=""
+ else
+ fail "App Lifecycle" "Delete test app" "Manual cleanup may be required"
+ fi
+ fi
+fi
+
+# =====================================================
+# Suite 3: Plugin Health
+# =====================================================
+suite_header "Plugins"
+
+plugin_list=$(dokku plugin:list 2>/dev/null || true)
+debug "Plugin list: $plugin_list"
+
+if [[ -n "$plugin_list" ]]; then
+ plugin_count=$(echo "$plugin_list" | grep -c "enabled" || echo "0")
+ pass "Plugins" "Plugin list — $plugin_count plugins installed"
+else
+ fail "Plugins" "Plugin list" "dokku plugin:list failed"
+fi
+
+# Check core plugins
+CORE_PLUGINS=("nginx-vhosts" "apps" "config" "ps")
+for plugin in "${CORE_PLUGINS[@]}"; do
+ if echo "$plugin_list" | grep -q "$plugin"; then
+ pass "Plugins" "Core plugin present — $plugin"
+ else
+ fail "Plugins" "Core plugin present — $plugin" "Not found in plugin list"
+ fi
+done
+
+# =====================================================
+# Suite 4: SSL
+# =====================================================
+if [[ "$SKIP_SSL" == "true" ]]; then
+ suite_header "SSL"
+ skip "SSL" "Letsencrypt plugin installed"
+ skip "SSL" "TLS certificate valid"
+else
+ suite_header "SSL"
+
+ # Check if letsencrypt plugin is installed
+ le_installed=false
+ if echo "$plugin_list" | grep -qi "letsencrypt"; then
+ le_installed=true
+ pass "SSL" "Letsencrypt plugin installed"
+ else
+ skip "SSL" "Letsencrypt plugin installed"
+ fi
+
+ # Check global domain certificate
+ if [[ "$le_installed" == "true" && -n "$DOKKU_DOMAIN" ]]; then
+ # Check certificate via openssl if the domain resolves
+ cert_host="$DOKKU_DOMAIN"
+ cert_output=$(echo | openssl s_client -servername "$cert_host" -connect "${cert_host}:443" 2>/dev/null || true)
+ cert_enddate=$(echo "$cert_output" | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || true)
+
+ if [[ -n "$cert_enddate" ]]; then
+ expiry_epoch=$(date -d "$cert_enddate" +%s 2>/dev/null || echo "0")
+ now_epoch=$(date +%s)
+ days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
+
+ if [[ "$days_left" -gt 0 ]]; then
+ pass "SSL" "TLS certificate valid — $days_left days remaining"
+ else
+ fail "SSL" "TLS certificate expired" "$days_left days past expiry"
+ fi
+ else
+ skip "SSL" "TLS certificate valid"
+ fi
+ else
+ skip "SSL" "TLS certificate valid"
+ fi
+fi
+
+# =====================================================
+# Suite 5: Resources
+# =====================================================
+suite_header "Resources"
+
+# Disk usage
+disk_line=$(df -h / 2>/dev/null | tail -1 || true)
+if [[ -n "$disk_line" ]]; then
+ disk_pct=$(echo "$disk_line" | awk '{print $5}' | tr -d '%')
+ disk_used=$(echo "$disk_line" | awk '{print $3}')
+ disk_total=$(echo "$disk_line" | awk '{print $2}')
+ pass "Resources" "Disk usage — ${disk_pct}% (${disk_used} / ${disk_total})"
+else
+ fail "Resources" "Disk usage" "Could not read disk info"
+fi
+
+# Docker images
+image_count=$(docker images -q 2>/dev/null | wc -l || echo "0")
+pass "Resources" "Docker images — $image_count images"
+
+# Docker volumes
+volume_count=$(docker volume ls -q 2>/dev/null | wc -l || echo "0")
+pass "Resources" "Docker volumes — $volume_count volumes"
+
+# Docker containers
+container_count=$(docker ps -q 2>/dev/null | wc -l || echo "0")
+pass "Resources" "Docker containers — $container_count running"
+
+# =====================================================
+# Summary
+# =====================================================
+END_TIME=$(date +%s)
+DURATION=$((END_TIME - START_TIME))
+
+case "$OUTPUT_FORMAT" in
+ tap)
+ echo "TAP version 13"
+ echo "1..$TEST_NUM"
+ for line in "${TAP_RESULTS[@]}"; do
+ echo "$line"
+ done
+ echo "# passed: $PASSED"
+ echo "# failed: $FAILED"
+ echo "# skipped: $SKIPPED"
+ echo "# duration: ${DURATION}s"
+ ;;
+ junit)
+ {
+ echo ''
+ echo ""
+ echo " "
+ echo " "
+ echo " "
+ for result in "${JUNIT_RESULTS[@]}"; do
+ echo " $result"
+ done
+ echo ""
+ } > "$JUNIT_FILE"
+ echo "JUnit results written to $JUNIT_FILE"
+ ;;
+ *)
+ echo ""
+ echo "────────────────────────────────────────"
+ echo -e "Summary ${BOLD}$HOSTNAME_STR${NC}"
+ echo -e " ${GREEN}$PASSED passed${NC} ${RED}$FAILED failed${NC} ${YELLOW}$SKIPPED skipped${NC} (${DURATION}s)"
+ echo "────────────────────────────────────────"
+ if [[ "$FAILED" -eq 0 ]]; then
+ echo -e "${GREEN}All tests passed.${NC}"
+ else
+ echo -e "${RED}Some tests failed.${NC}"
+ fi
+ ;;
+esac
+
+exit $((FAILED > 0 ? 1 : 0))
diff --git a/dokploy-exporter.sh b/dokploy-exporter.sh
new file mode 100755
index 0000000..9f5f9c9
--- /dev/null
+++ b/dokploy-exporter.sh
@@ -0,0 +1,470 @@
+#!/bin/bash
+################################################################################
+# Script Name: dokploy-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for Dokploy PaaS providing operational
+# metrics via the Dokploy API — project counts, application status,
+# database breakdown by type, compose services, server info,
+# and API health
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - Dokploy instance running with API enabled
+# - Dokploy API key (generate in Settings → API)
+# - curl for API calls
+# - jq for JSON parsing
+# - netcat (nc) for HTTP mode
+#
+# Usage:
+# # Output to stdout
+# ./dokploy-exporter.sh
+#
+# # HTTP server mode
+# ./dokploy-exporter.sh --http -p 9197
+#
+# # Textfile collector mode
+# ./dokploy-exporter.sh --textfile
+#
+# # Custom API token and URL
+# ./dokploy-exporter.sh --api-url http://dokploy.local:3000 --api-token mytoken
+#
+# Metrics Exported:
+# - dokploy_up - API reachability (1=up, 0=down)
+# - dokploy_info{version} - Dokploy version info
+# - dokploy_projects_total - Total project count
+# - dokploy_applications_total - Total applications across all projects
+# - dokploy_applications_by_status{status} - Applications by status
+# - dokploy_compose_services_total - Total Docker Compose services
+# - dokploy_databases_total - Total managed databases
+# - dokploy_databases_by_type{type} - Databases by type
+# - dokploy_servers_total - Total servers managed
+# - dokploy_exporter_duration_seconds - Script execution time
+# - dokploy_exporter_last_run_timestamp - Last run timestamp
+#
+# Configuration:
+# Default HTTP port: 9197
+# Default API URL: http://localhost:3000
+# Textfile directory: /var/lib/node_exporter
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9197
+API_URL="http://localhost:3000"
+API_TOKEN=""
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Check prerequisites
+# Returns: 0 if OK, 1 if error
+check_prerequisites() {
+ if ! command -v curl >/dev/null 2>&1; then
+ echo "ERROR: curl not found" >&2
+ return 1
+ fi
+
+ if ! command -v jq >/dev/null 2>&1; then
+ echo "ERROR: jq not found (required for JSON parsing)" >&2
+ return 1
+ fi
+
+ if [ -z "$API_TOKEN" ]; then
+ echo "ERROR: --api-token is required" >&2
+ return 1
+ fi
+
+ return 0
+}
+
+# Escape special characters in Prometheus label values
+# Args: $1 - string to escape
+# Returns: escaped string safe for Prometheus labels
+prom_escape() {
+ local val="$1"
+ val="${val//\\/\\\\}"
+ val="${val//\"/\\\"}"
+ val="${val//$'\n'/}"
+ echo "$val"
+}
+
+# Make an authenticated API call
+# Args: $1 - API endpoint path (e.g., /api/project.all)
+# Returns: JSON response on stdout
+api_call() {
+ local endpoint="$1"
+ curl -s -X GET \
+ -H "x-api-key: ${API_TOKEN}" \
+ -H "Accept: application/json" \
+ "${API_URL}${endpoint}" 2>/dev/null
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+# Generate all Prometheus metrics
+# Returns: Prometheus text format metrics on stdout
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s)
+
+ # Check prerequisites
+ if ! check_prerequisites; then
+ cat </dev/null)
+
+ if [ -z "$health_response" ]; then
+ cat </dev/null)
+
+ if [ "$is_error" = "yes" ]; then
+ cat </dev/null)
+ dokploy_version="${dokploy_version:-unknown}"
+
+ cat </dev/null)
+ total_projects=${total_projects:-0}
+
+ # Count applications across all projects
+ total_apps=$(echo "$projects_response" | jq '[.[] | (.applications // []) | length] | add // 0' 2>/dev/null)
+ total_apps=${total_apps:-0}
+
+ # Count applications by status
+ done_apps=$(echo "$projects_response" | jq '[.[] | (.applications // [])[] | select(.applicationStatus == "done")] | length' 2>/dev/null)
+ idle_apps=$(echo "$projects_response" | jq '[.[] | (.applications // [])[] | select(.applicationStatus == "idle")] | length' 2>/dev/null)
+ running_apps=$(echo "$projects_response" | jq '[.[] | (.applications // [])[] | select(.applicationStatus == "running")] | length' 2>/dev/null)
+ error_apps=$(echo "$projects_response" | jq '[.[] | (.applications // [])[] | select(.applicationStatus == "error")] | length' 2>/dev/null)
+ done_apps=${done_apps:-0}
+ idle_apps=${idle_apps:-0}
+ running_apps=${running_apps:-0}
+ error_apps=${error_apps:-0}
+
+ # Count compose services across all projects
+ total_compose=$(echo "$projects_response" | jq '[.[] | (.compose // []) | length] | add // 0' 2>/dev/null)
+ total_compose=${total_compose:-0}
+
+ # Count databases by type across all projects
+ pg_count=$(echo "$projects_response" | jq '[.[] | (.postgres // []) | length] | add // 0' 2>/dev/null)
+ mysql_count=$(echo "$projects_response" | jq '[.[] | (.mysql // []) | length] | add // 0' 2>/dev/null)
+ mariadb_count=$(echo "$projects_response" | jq '[.[] | (.mariadb // []) | length] | add // 0' 2>/dev/null)
+ mongo_count=$(echo "$projects_response" | jq '[.[] | (.mongo // []) | length] | add // 0' 2>/dev/null)
+ redis_count=$(echo "$projects_response" | jq '[.[] | (.redis // []) | length] | add // 0' 2>/dev/null)
+ pg_count=${pg_count:-0}
+ mysql_count=${mysql_count:-0}
+ mariadb_count=${mariadb_count:-0}
+ mongo_count=${mongo_count:-0}
+ redis_count=${redis_count:-0}
+
+ total_databases=$((pg_count + mysql_count + mariadb_count + mongo_count + redis_count))
+ fi
+
+ cat </dev/null)
+ total_servers=${total_servers:-0}
+ fi
+
+ cat <&2
+
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+
+ # Infinite loop accepting HTTP requests
+ while true; do
+ {
+ read -r request
+ # Check if request is for /metrics endpoint
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else # Serve HTML landing page for other requests
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ cat <
+
+Dokploy Exporter v1.0
+
+Dokploy Prometheus Exporter v1.0
+Metrics
+Operational metrics from the Dokploy API.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+# Main entry point - routes to appropriate output mode
+main() {
+ parse_args "$@"
+
+ if [ "$HTTP_MODE" = true ]; then
+ # Run HTTP server (blocks until killed)
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ # Textfile collector mode: write atomically using temp file
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ # Create temp file in SAME directory for atomic rename (same filesystem)
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.dokploy_metrics.XXXXXX")
+
+ # Generate metrics to temp file
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ # Validate: file must exist, have content
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ # Set permissions before move
+ chmod 644 "$temp_file"
+
+ # Atomic rename - no gap where file is missing
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ # Default: output to stdout
+ generate_metrics
+ fi
+}
+
+# Execute main function with all script arguments
+main "$@"
diff --git a/dovecot-metrics-exporter.sh b/dovecot-metrics-exporter.sh
new file mode 100644
index 0000000..0ff0f32
--- /dev/null
+++ b/dovecot-metrics-exporter.sh
@@ -0,0 +1,372 @@
+#!/bin/bash
+################################################################################
+# Script Name: dovecot-metrics-exporter.sh
+# Description: Prometheus exporter for Dovecot IMAP/POP3 server metrics
+#
+# Collects connection counts, authentication stats, mailbox operations,
+# process info, and protocol-level metrics from doveadm and Dovecot stats.
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+# Version: 1.0
+#
+# Usage:
+# # Output to stdout
+# ./dovecot-metrics-exporter.sh
+#
+# # Textfile collector mode (atomic write)
+# ./dovecot-metrics-exporter.sh --textfile
+#
+# # Custom output file
+# ./dovecot-metrics-exporter.sh -o /path/to/metrics.prom
+#
+################################################################################
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HOSTNAME=$(hostname)
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Safe integer extraction — returns 0 on failure
+safe_int() {
+ local val="$1"
+ if [[ "$val" =~ ^[0-9]+$ ]]; then
+ echo "$val"
+ else
+ echo 0
+ fi
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+generate_metrics() {
+ local START_TIME
+ START_TIME=$(date +%s.%N)
+
+# --- Exporter info ---
+echo "# HELP dovecot_up Exporter status (1=up, 0=down)"
+echo "# TYPE dovecot_up gauge"
+
+# Check if Dovecot is running
+if systemctl is-active --quiet dovecot 2>/dev/null; then
+ echo "dovecot_up 1"
+else
+ echo "dovecot_up 0"
+fi
+
+echo ""
+echo "# HELP dovecot_exporter_info Exporter version information"
+echo "# TYPE dovecot_exporter_info gauge"
+echo 'dovecot_exporter_info{version="1.0"} 1'
+echo ""
+
+# --- Dovecot version ---
+echo "# HELP dovecot_version_info Dovecot version information"
+echo "# TYPE dovecot_version_info gauge"
+local dovecot_version
+dovecot_version=$(dovecot --version 2>/dev/null | awk '{print $1}') || dovecot_version="unknown"
+echo "dovecot_version_info{version=\"${dovecot_version}\"} 1"
+echo ""
+
+# --- Process counts ---
+echo "# HELP dovecot_processes Number of running Dovecot processes by type"
+echo "# TYPE dovecot_processes gauge"
+for proc_type in imap pop3 lmtp managesieve submission auth anvil; do
+ count=$(pgrep -c "dovecot/${proc_type}" 2>/dev/null) || count=0
+ echo "dovecot_processes{type=\"${proc_type}\"} ${count}"
+done
+local total_procs
+total_procs=$(pgrep -c dovecot 2>/dev/null) || total_procs=0
+echo "dovecot_processes{type=\"total\"} ${total_procs}"
+echo ""
+
+# --- Connected users (from doveadm) ---
+echo "# HELP dovecot_connected_users Number of currently connected users by protocol"
+echo "# TYPE dovecot_connected_users gauge"
+local imap_users=0 pop3_users=0 lmtp_users=0 managesieve_users=0
+if command -v doveadm >/dev/null 2>&1; then
+ imap_users=$(doveadm who -1 2>/dev/null | grep -c 'imap' 2>/dev/null) || imap_users=0
+ pop3_users=$(doveadm who -1 2>/dev/null | grep -c 'pop3' 2>/dev/null) || pop3_users=0
+ lmtp_users=$(doveadm who -1 2>/dev/null | grep -c 'lmtp' 2>/dev/null) || lmtp_users=0
+ managesieve_users=$(doveadm who -1 2>/dev/null | grep -c 'managesieve' 2>/dev/null) || managesieve_users=0
+fi
+echo "dovecot_connected_users{protocol=\"imap\"} ${imap_users}"
+echo "dovecot_connected_users{protocol=\"pop3\"} ${pop3_users}"
+echo "dovecot_connected_users{protocol=\"lmtp\"} ${lmtp_users}"
+echo "dovecot_connected_users{protocol=\"managesieve\"} ${managesieve_users}"
+echo ""
+
+# --- Total connections (from doveadm who) ---
+echo "# HELP dovecot_connections_total Total active connections by protocol"
+echo "# TYPE dovecot_connections_total gauge"
+local imap_conns=0 pop3_conns=0
+if command -v doveadm >/dev/null 2>&1; then
+ imap_conns=$(doveadm who -1 2>/dev/null | grep 'imap' | awk '{sum+=$3} END {print sum+0}' 2>/dev/null) || imap_conns=0
+ pop3_conns=$(doveadm who -1 2>/dev/null | grep 'pop3' | awk '{sum+=$3} END {print sum+0}' 2>/dev/null) || pop3_conns=0
+fi
+echo "dovecot_connections_total{protocol=\"imap\"} ${imap_conns}"
+echo "dovecot_connections_total{protocol=\"pop3\"} ${pop3_conns}"
+echo ""
+
+# --- Authentication stats from mail.log ---
+local LOG_FILE="/var/log/mail.log"
+if [[ ! -f "$LOG_FILE" ]]; then
+ LOG_FILE="/var/log/maillog"
+fi
+
+echo "# HELP dovecot_auth_success_total Successful authentication attempts by protocol"
+echo "# TYPE dovecot_auth_success_total counter"
+local imap_auth_ok=0 pop3_auth_ok=0
+if [[ -f "$LOG_FILE" ]]; then
+ imap_auth_ok=$(grep -c 'imap-login: Info: Login:' "$LOG_FILE" 2>/dev/null) || imap_auth_ok=0
+ pop3_auth_ok=$(grep -c 'pop3-login: Info: Login:' "$LOG_FILE" 2>/dev/null) || pop3_auth_ok=0
+fi
+echo "dovecot_auth_success_total{protocol=\"imap\"} ${imap_auth_ok}"
+echo "dovecot_auth_success_total{protocol=\"pop3\"} ${pop3_auth_ok}"
+echo ""
+
+echo "# HELP dovecot_auth_failed_total Failed authentication attempts by protocol"
+echo "# TYPE dovecot_auth_failed_total counter"
+local imap_auth_fail=0 pop3_auth_fail=0
+if [[ -f "$LOG_FILE" ]]; then
+ imap_auth_fail=$(grep -c 'imap-login:.*auth failed\|imap-login: Info: Aborted login' "$LOG_FILE" 2>/dev/null) || imap_auth_fail=0
+ pop3_auth_fail=$(grep -c 'pop3-login:.*auth failed\|pop3-login: Info: Aborted login' "$LOG_FILE" 2>/dev/null) || pop3_auth_fail=0
+fi
+echo "dovecot_auth_failed_total{protocol=\"imap\"} ${imap_auth_fail}"
+echo "dovecot_auth_failed_total{protocol=\"pop3\"} ${pop3_auth_fail}"
+echo ""
+
+# --- TLS connections ---
+echo "# HELP dovecot_tls_connections_total TLS connections by status"
+echo "# TYPE dovecot_tls_connections_total counter"
+local tls_yes=0 tls_no=0
+if [[ -f "$LOG_FILE" ]]; then
+ tls_yes=$(grep -c 'Login:.*TLS' "$LOG_FILE" 2>/dev/null) || tls_yes=0
+ tls_no=$(grep 'Login:' "$LOG_FILE" 2>/dev/null | grep -cv 'TLS' 2>/dev/null) || tls_no=0
+fi
+echo "dovecot_tls_connections_total{tls=\"yes\"} ${tls_yes}"
+echo "dovecot_tls_connections_total{tls=\"no\"} ${tls_no}"
+echo ""
+
+# --- Authentication methods ---
+echo "# HELP dovecot_auth_method_total Logins by authentication method"
+echo "# TYPE dovecot_auth_method_total counter"
+if [[ -f "$LOG_FILE" ]]; then
+ for method in PLAIN LOGIN CRAM-MD5 DIGEST-MD5; do
+ count=$(grep -c "Login:.*method=${method}" "$LOG_FILE" 2>/dev/null) || count=0
+ echo "dovecot_auth_method_total{method=\"${method}\"} ${count}"
+ done
+else
+ for method in PLAIN LOGIN CRAM-MD5 DIGEST-MD5; do
+ echo "dovecot_auth_method_total{method=\"${method}\"} 0"
+ done
+fi
+echo ""
+
+# --- Disconnections ---
+echo "# HELP dovecot_disconnections_total Client disconnections by reason"
+echo "# TYPE dovecot_disconnections_total counter"
+local dc_logout=0 dc_timeout=0 dc_closed=0 dc_internal=0
+if [[ -f "$LOG_FILE" ]]; then
+ dc_logout=$(grep -c 'Logged out' "$LOG_FILE" 2>/dev/null) || dc_logout=0
+ dc_timeout=$(grep -c 'Disconnected.*Timed out\|Connection timed out' "$LOG_FILE" 2>/dev/null) || dc_timeout=0
+ dc_closed=$(grep -c 'Disconnected.*Connection closed' "$LOG_FILE" 2>/dev/null) || dc_closed=0
+ dc_internal=$(grep -c 'Disconnected.*Internal error' "$LOG_FILE" 2>/dev/null) || dc_internal=0
+fi
+echo "dovecot_disconnections_total{reason=\"logout\"} ${dc_logout}"
+echo "dovecot_disconnections_total{reason=\"timeout\"} ${dc_timeout}"
+echo "dovecot_disconnections_total{reason=\"connection_closed\"} ${dc_closed}"
+echo "dovecot_disconnections_total{reason=\"internal_error\"} ${dc_internal}"
+echo ""
+
+# --- LMTP delivery stats ---
+echo "# HELP dovecot_lmtp_deliveries_total LMTP deliveries by status"
+echo "# TYPE dovecot_lmtp_deliveries_total counter"
+local lmtp_ok=0 lmtp_reject=0 lmtp_tempfail=0
+if [[ -f "$LOG_FILE" ]]; then
+ lmtp_ok=$(grep -c 'lmtp.*saved mail' "$LOG_FILE" 2>/dev/null) || lmtp_ok=0
+ lmtp_reject=$(grep -c 'lmtp.*rejected' "$LOG_FILE" 2>/dev/null) || lmtp_reject=0
+ lmtp_tempfail=$(grep -c 'lmtp.*temporary failure\|lmtp.*temp-fail' "$LOG_FILE" 2>/dev/null) || lmtp_tempfail=0
+fi
+echo "dovecot_lmtp_deliveries_total{status=\"delivered\"} ${lmtp_ok}"
+echo "dovecot_lmtp_deliveries_total{status=\"rejected\"} ${lmtp_reject}"
+echo "dovecot_lmtp_deliveries_total{status=\"tempfail\"} ${lmtp_tempfail}"
+echo ""
+
+# --- Sieve stats ---
+echo "# HELP dovecot_sieve_actions_total Sieve filter actions"
+echo "# TYPE dovecot_sieve_actions_total counter"
+local sieve_filed=0 sieve_discard=0 sieve_redirect=0 sieve_reject=0
+if [[ -f "$LOG_FILE" ]]; then
+ sieve_filed=$(grep -c 'sieve:.*stored mail\|sieve:.*fileinto' "$LOG_FILE" 2>/dev/null) || sieve_filed=0
+ sieve_discard=$(grep -c 'sieve:.*discard' "$LOG_FILE" 2>/dev/null) || sieve_discard=0
+ sieve_redirect=$(grep -c 'sieve:.*redirect' "$LOG_FILE" 2>/dev/null) || sieve_redirect=0
+ sieve_reject=$(grep -c 'sieve:.*reject' "$LOG_FILE" 2>/dev/null) || sieve_reject=0
+fi
+echo "dovecot_sieve_actions_total{action=\"filed\"} ${sieve_filed}"
+echo "dovecot_sieve_actions_total{action=\"discard\"} ${sieve_discard}"
+echo "dovecot_sieve_actions_total{action=\"redirect\"} ${sieve_redirect}"
+echo "dovecot_sieve_actions_total{action=\"reject\"} ${sieve_reject}"
+echo ""
+
+# --- Dovecot stats (if old_stats or stats plugin enabled) ---
+# Try doveadm stats dump for Dovecot 2.3+
+echo "# HELP dovecot_mail_commands_total Mail commands executed"
+echo "# TYPE dovecot_mail_commands_total counter"
+local cmds_select=0 cmds_fetch=0 cmds_store=0 cmds_search=0 cmds_copy=0 cmds_expunge=0
+if command -v doveadm >/dev/null 2>&1; then
+ local stats_output
+ stats_output=$(doveadm stats dump session 2>/dev/null | head -20)
+ if [[ -n "$stats_output" ]]; then
+ cmds_select=$(echo "$stats_output" | awk '{sum+=$4} END {print sum+0}') || cmds_select=0
+ cmds_fetch=$(echo "$stats_output" | awk '{sum+=$5} END {print sum+0}') || cmds_fetch=0
+ fi
+fi
+# Fallback: count from logs
+if [[ -f "$LOG_FILE" ]]; then
+ cmds_copy=$(grep -c 'Copy\|copy' "$LOG_FILE" 2>/dev/null | head -1) || cmds_copy=0
+ cmds_expunge=$(grep -c 'Expunged' "$LOG_FILE" 2>/dev/null) || cmds_expunge=0
+fi
+echo "dovecot_mail_commands_total{command=\"copy\"} ${cmds_copy}"
+echo "dovecot_mail_commands_total{command=\"expunge\"} ${cmds_expunge}"
+echo ""
+
+# --- Mail storage quota (top users if doveadm quota available) ---
+echo "# HELP dovecot_quota_usage_bytes User quota usage in bytes (top users)"
+echo "# TYPE dovecot_quota_usage_bytes gauge"
+echo "# HELP dovecot_quota_limit_bytes User quota limit in bytes"
+echo "# TYPE dovecot_quota_limit_bytes gauge"
+if command -v doveadm >/dev/null 2>&1; then
+ doveadm quota get -A 2>/dev/null | grep 'STORAGE' | head -20 | while IFS=$'\t' read -r user type value limit _; do
+ local usage_bytes=$((value * 1024))
+ local limit_bytes=$((limit * 1024))
+ echo "dovecot_quota_usage_bytes{user=\"${user}\"} ${usage_bytes}"
+ echo "dovecot_quota_limit_bytes{user=\"${user}\"} ${limit_bytes}"
+ done 2>/dev/null
+fi
+echo ""
+
+# --- Dovecot uptime ---
+echo "# HELP dovecot_uptime_seconds Dovecot process uptime in seconds"
+echo "# TYPE dovecot_uptime_seconds gauge"
+local dovecot_pid uptime_seconds=0
+dovecot_pid=$(pgrep -o dovecot 2>/dev/null) || dovecot_pid=""
+if [[ -n "$dovecot_pid" ]] && [[ -d "/proc/${dovecot_pid}" ]]; then
+ local start_time
+ start_time=$(stat -c %Y "/proc/${dovecot_pid}" 2>/dev/null) || start_time=0
+ if [[ "$start_time" -gt 0 ]]; then
+ uptime_seconds=$(( $(date +%s) - start_time ))
+ fi
+fi
+echo "dovecot_uptime_seconds ${uptime_seconds}"
+echo ""
+
+# --- Memory usage ---
+echo "# HELP dovecot_memory_bytes Total memory usage of all Dovecot processes"
+echo "# TYPE dovecot_memory_bytes gauge"
+local total_mem=0
+total_mem=$(pgrep dovecot 2>/dev/null | xargs -I {} cat /proc/{}/status 2>/dev/null | awk '/VmRSS/{sum+=$2} END {print sum*1024+0}') || total_mem=0
+echo "dovecot_memory_bytes ${total_mem}"
+echo ""
+
+# --- Script execution time ---
+local END_TIME
+END_TIME=$(date +%s.%N)
+local DURATION
+DURATION=$(echo "$END_TIME - $START_TIME" | bc)
+
+echo "# HELP dovecot_exporter_duration_seconds Time to generate all metrics"
+echo "# TYPE dovecot_exporter_duration_seconds gauge"
+echo "dovecot_exporter_duration_seconds ${DURATION}"
+echo ""
+
+echo "# HELP dovecot_exporter_last_run_timestamp Unix timestamp of last successful run"
+echo "# TYPE dovecot_exporter_last_run_timestamp gauge"
+echo "dovecot_exporter_last_run_timestamp $(date +%s)"
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+main() {
+ parse_args "$@"
+
+ if [ -n "$OUTPUT_FILE" ]; then
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.dovecot_metrics.XXXXXX")
+
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ chmod 644 "$temp_file"
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ generate_metrics
+ fi
+}
+
+main "$@"
diff --git a/duplicati-exporter.sh b/duplicati-exporter.sh
new file mode 100755
index 0000000..7027b3f
--- /dev/null
+++ b/duplicati-exporter.sh
@@ -0,0 +1,445 @@
+#!/bin/bash
+################################################################################
+# Script Name: duplicati-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for Duplicati backups — backup job status,
+# last run time, backup age, file counts, and size metrics
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - curl installed
+# - jq installed
+# - netcat (nc) for HTTP mode
+#
+# Usage:
+# ./duplicati-exporter.sh --textfile
+# ./duplicati-exporter.sh --http -p 9203
+# ./duplicati-exporter.sh --url http://myhost:8200 --password secret
+# DUPLICATI_PASSWORD=secret ./duplicati-exporter.sh --textfile
+#
+# Configuration:
+# Default HTTP port: 9203
+# Default Duplicati URL: http://localhost:8200
+# Textfile directory: /var/lib/node_exporter
+#
+################################################################################
+
+EXPORTER_VERSION="1.0"
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9203
+DUPLICATI_URL="http://localhost:8200"
+DUPLICATI_PASS="${DUPLICATI_PASSWORD:-}"
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+ # Strip trailing slash from URL
+ DUPLICATI_URL="${DUPLICATI_URL%/}"
+}
+
+check_dependencies() {
+ local missing=0
+ for cmd in curl jq; do
+ if ! command -v "$cmd" >/dev/null 2>&1; then
+ echo "ERROR: $cmd not found" >&2
+ missing=1
+ fi
+ done
+ return "$missing"
+}
+
+# Authenticate with Duplicati and store cookie jar
+duplicati_auth() {
+ COOKIE_JAR=$(mktemp /tmp/.duplicati_cookies.XXXXXX)
+ trap 'rm -f "$COOKIE_JAR"' EXIT
+
+ # If no password, try unauthenticated access
+ if [ -z "$DUPLICATI_PASS" ]; then
+ return 0
+ fi
+
+ # Get XSRF token first
+ local xsrf_token
+ xsrf_token=$(curl -s -c "$COOKIE_JAR" -b "$COOKIE_JAR" \
+ "${DUPLICATI_URL}/api/v1/auth/refresh" 2>/dev/null | jq -r '.Token // empty')
+
+ if [ -z "$xsrf_token" ]; then
+ # Try fetching the login page to get cookies
+ curl -s -c "$COOKIE_JAR" -b "$COOKIE_JAR" \
+ "${DUPLICATI_URL}/" >/dev/null 2>&1
+ xsrf_token=$(grep -i "xsrf" "$COOKIE_JAR" 2>/dev/null | awk '{print $NF}')
+ fi
+
+ # Authenticate with password
+ local auth_response
+ auth_response=$(curl -s -c "$COOKIE_JAR" -b "$COOKIE_JAR" \
+ -X POST "${DUPLICATI_URL}/api/v1/auth/login" \
+ -H "Content-Type: application/json" \
+ ${xsrf_token:+-H "X-XSRF-Token: $xsrf_token"} \
+ -d "{\"Password\":\"${DUPLICATI_PASS}\"}" 2>/dev/null)
+
+ if echo "$auth_response" | jq -e '.Token' >/dev/null 2>&1; then
+ AUTH_TOKEN=$(echo "$auth_response" | jq -r '.Token')
+ return 0
+ fi
+
+ return 1
+}
+
+# Make an authenticated API call
+api_call() {
+ local endpoint="$1"
+ curl -s -b "$COOKIE_JAR" \
+ ${AUTH_TOKEN:+-H "Authorization: Bearer $AUTH_TOKEN"} \
+ "${DUPLICATI_URL}/api/v1/${endpoint}" 2>/dev/null
+}
+
+# Map status string to numeric value
+status_to_number() {
+ case "$1" in
+ Success|Completed) echo 1 ;;
+ Warning) echo 2 ;;
+ Error|Failed) echo 3 ;;
+ Fatal) echo 4 ;;
+ *) echo 0 ;;
+ esac
+}
+
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s%N)
+
+ if ! check_dependencies; then
+ echo "# HELP duplicati_up Exporter status (1=up, 0=down)"
+ echo "# TYPE duplicati_up gauge"
+ echo "duplicati_up 0"
+ return
+ fi
+
+ # Test server reachability
+ local server_up=0
+ if curl -s --connect-timeout 5 "${DUPLICATI_URL}/api/v1/systeminfo" >/dev/null 2>&1; then
+ server_up=1
+ fi
+
+ # Authenticate
+ AUTH_TOKEN=""
+ if [ "$server_up" -eq 1 ]; then
+ duplicati_auth
+ fi
+
+ echo "# HELP duplicati_up Duplicati server reachable (1=up, 0=down)"
+ echo "# TYPE duplicati_up gauge"
+ echo "duplicati_up $server_up"
+
+ echo "# HELP duplicati_exporter_info Exporter version information"
+ echo "# TYPE duplicati_exporter_info gauge"
+ echo "duplicati_exporter_info{version=\"${EXPORTER_VERSION}\"} 1"
+
+ if [ "$server_up" -eq 0 ]; then
+ echo "# HELP duplicati_backup_count Total number of configured backup jobs"
+ echo "# TYPE duplicati_backup_count gauge"
+ echo "duplicati_backup_count 0"
+ local script_end
+ script_end=$(date +%s)
+ echo "# HELP duplicati_exporter_duration_seconds Script execution time"
+ echo "# TYPE duplicati_exporter_duration_seconds gauge"
+ echo "duplicati_exporter_duration_seconds 0"
+ echo "# HELP duplicati_exporter_last_run_timestamp Last successful run"
+ echo "# TYPE duplicati_exporter_last_run_timestamp gauge"
+ echo "duplicati_exporter_last_run_timestamp $script_end"
+ return
+ fi
+
+ # Fetch all backups
+ local backups_json
+ backups_json=$(api_call "backups")
+
+ if [ -z "$backups_json" ] || ! echo "$backups_json" | jq -e '.' >/dev/null 2>&1; then
+ echo "# HELP duplicati_backup_count Total number of configured backup jobs"
+ echo "# TYPE duplicati_backup_count gauge"
+ echo "duplicati_backup_count 0"
+ local script_end
+ script_end=$(date +%s)
+ echo "# HELP duplicati_exporter_duration_seconds Script execution time"
+ echo "# TYPE duplicati_exporter_duration_seconds gauge"
+ echo "duplicati_exporter_duration_seconds 0"
+ echo "# HELP duplicati_exporter_last_run_timestamp Last successful run"
+ echo "# TYPE duplicati_exporter_last_run_timestamp gauge"
+ echo "duplicati_exporter_last_run_timestamp $script_end"
+ return
+ fi
+
+ local backup_count
+ backup_count=$(echo "$backups_json" | jq 'length')
+ echo "# HELP duplicati_backup_count Total number of configured backup jobs"
+ echo "# TYPE duplicati_backup_count gauge"
+ echo "duplicati_backup_count ${backup_count:-0}"
+
+ local now
+ now=$(date +%s)
+
+ # Collect per-backup metrics into arrays so HELP/TYPE appears once per metric
+ local info_lines=()
+ local last_run_lines=()
+ local age_lines=()
+ local duration_lines=()
+ local status_lines=()
+ local files_lines=()
+ local size_lines=()
+ local uploaded_lines=()
+ local next_ts_lines=()
+ local next_sec_lines=()
+ local error_lines=()
+ local warning_lines=()
+
+ while IFS= read -r backup; do
+ local id name target_url
+ id=$(echo "$backup" | jq -r '.Backup.ID // empty')
+ name=$(echo "$backup" | jq -r '.Backup.Name // empty')
+ target_url=$(echo "$backup" | jq -r '.Backup.TargetURL // empty')
+
+ [ -z "$name" ] && continue
+
+ local safe_name="${name//\"/\\\"}"
+ local safe_target="${target_url//\"/\\\"}"
+
+ info_lines+=("duplicati_backup_info{id=\"${id}\",name=\"${safe_name}\",target_url=\"${safe_target}\"} 1")
+
+ # Last run timestamp
+ local last_run_ts=0
+ local last_run_raw
+ last_run_raw=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupDate" // empty')
+ if [ -n "$last_run_raw" ]; then
+ last_run_ts=$(date -d "$last_run_raw" +%s 2>/dev/null || echo 0)
+ fi
+ last_run_lines+=("duplicati_backup_last_run_timestamp{name=\"${safe_name}\"} $last_run_ts")
+
+ # Age since last run
+ local age=0
+ if [ "$last_run_ts" -gt 0 ]; then
+ age=$((now - last_run_ts))
+ fi
+ age_lines+=("duplicati_backup_last_run_age_seconds{name=\"${safe_name}\"} $age")
+
+ # Last duration
+ local duration
+ duration=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupDuration" // "0"')
+ local duration_seconds=0
+ if [[ "$duration" =~ ^([0-9]+):([0-9]+):([0-9]+) ]]; then
+ duration_seconds=$(( BASH_REMATCH[1] * 3600 + BASH_REMATCH[2] * 60 + BASH_REMATCH[3] ))
+ elif [[ "$duration" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
+ duration_seconds="${duration%%.*}"
+ fi
+ duration_lines+=("duplicati_backup_last_duration_seconds{name=\"${safe_name}\"} $duration_seconds")
+
+ # Last status
+ local status_raw status_num
+ status_raw=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupResult" // "Unknown"')
+ status_num=$(status_to_number "$status_raw")
+ status_lines+=("duplicati_backup_last_status{name=\"${safe_name}\",status=\"${status_raw}\"} $status_num")
+
+ # Files examined
+ local files_total
+ files_total=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupExaminedFiles" // "0"')
+ files_lines+=("duplicati_backup_files_total{name=\"${safe_name}\"} ${files_total:-0}")
+
+ # Files size
+ local files_size
+ files_size=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupSizeOfExaminedFiles" // "0"')
+ size_lines+=("duplicati_backup_files_size_bytes{name=\"${safe_name}\"} ${files_size:-0}")
+
+ # Uploaded bytes
+ local uploaded
+ uploaded=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupUploadedSize" // "0"')
+ uploaded_lines+=("duplicati_backup_uploaded_bytes{name=\"${safe_name}\"} ${uploaded:-0}")
+
+ # Next run timestamp
+ local next_run_ts=0
+ local next_run_raw
+ next_run_raw=$(echo "$backup" | jq -r '.Schedule.Time // empty')
+ if [ -n "$next_run_raw" ]; then
+ next_run_ts=$(date -d "$next_run_raw" +%s 2>/dev/null || echo 0)
+ fi
+ next_ts_lines+=("duplicati_backup_next_run_timestamp{name=\"${safe_name}\"} $next_run_ts")
+
+ # Seconds until next run
+ local next_run_seconds=0
+ if [ "$next_run_ts" -gt "$now" ]; then
+ next_run_seconds=$((next_run_ts - now))
+ fi
+ next_sec_lines+=("duplicati_backup_next_run_seconds{name=\"${safe_name}\"} $next_run_seconds")
+
+ # Error count
+ local error_count
+ error_count=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupErrors" // "0"')
+ error_lines+=("duplicati_backup_error_count{name=\"${safe_name}\"} ${error_count:-0}")
+
+ # Warning count
+ local warning_count
+ warning_count=$(echo "$backup" | jq -r '.Backup.Metadata."LastBackupWarnings" // "0"')
+ warning_lines+=("duplicati_backup_warning_count{name=\"${safe_name}\"} ${warning_count:-0}")
+ done < <(echo "$backups_json" | jq -c '.[]' 2>/dev/null)
+
+ # Output each metric group with HELP/TYPE immediately before values
+ if [ ${#info_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_info Backup job information"
+ echo "# TYPE duplicati_backup_info gauge"
+ printf '%s\n' "${info_lines[@]}"
+ fi
+ if [ ${#last_run_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_last_run_timestamp Unix timestamp of last backup run"
+ echo "# TYPE duplicati_backup_last_run_timestamp gauge"
+ printf '%s\n' "${last_run_lines[@]}"
+ fi
+ if [ ${#age_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_last_run_age_seconds Seconds since last backup run"
+ echo "# TYPE duplicati_backup_last_run_age_seconds gauge"
+ printf '%s\n' "${age_lines[@]}"
+ fi
+ if [ ${#duration_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_last_duration_seconds Duration of last backup run in seconds"
+ echo "# TYPE duplicati_backup_last_duration_seconds gauge"
+ printf '%s\n' "${duration_lines[@]}"
+ fi
+ if [ ${#status_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_last_status Last backup status (Success=1, Warning=2, Error=3, Fatal=4, Unknown=0)"
+ echo "# TYPE duplicati_backup_last_status gauge"
+ printf '%s\n' "${status_lines[@]}"
+ fi
+ if [ ${#files_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_files_total Total files examined in last backup"
+ echo "# TYPE duplicati_backup_files_total gauge"
+ printf '%s\n' "${files_lines[@]}"
+ fi
+ if [ ${#size_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_files_size_bytes Total size of examined files in bytes"
+ echo "# TYPE duplicati_backup_files_size_bytes gauge"
+ printf '%s\n' "${size_lines[@]}"
+ fi
+ if [ ${#uploaded_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_uploaded_bytes Bytes uploaded in last backup"
+ echo "# TYPE duplicati_backup_uploaded_bytes gauge"
+ printf '%s\n' "${uploaded_lines[@]}"
+ fi
+ if [ ${#next_ts_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_next_run_timestamp Next scheduled run unix timestamp"
+ echo "# TYPE duplicati_backup_next_run_timestamp gauge"
+ printf '%s\n' "${next_ts_lines[@]}"
+ fi
+ if [ ${#next_sec_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_next_run_seconds Seconds until next scheduled run"
+ echo "# TYPE duplicati_backup_next_run_seconds gauge"
+ printf '%s\n' "${next_sec_lines[@]}"
+ fi
+ if [ ${#error_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_error_count Number of errors in last backup run"
+ echo "# TYPE duplicati_backup_error_count gauge"
+ printf '%s\n' "${error_lines[@]}"
+ fi
+ if [ ${#warning_lines[@]} -gt 0 ]; then
+ echo "# HELP duplicati_backup_warning_count Number of warnings in last backup run"
+ echo "# TYPE duplicati_backup_warning_count gauge"
+ printf '%s\n' "${warning_lines[@]}"
+ fi
+
+ local script_end script_duration_ns script_duration
+ script_end=$(date +%s)
+ script_duration_ns=$(( $(date +%s%N) - script_start ))
+ script_duration=$(( script_duration_ns / 1000000000 ))
+
+ echo "# HELP duplicati_exporter_duration_seconds Script execution time"
+ echo "# TYPE duplicati_exporter_duration_seconds gauge"
+ echo "duplicati_exporter_duration_seconds $script_duration"
+ echo "# HELP duplicati_exporter_last_run_timestamp Last successful run"
+ echo "# TYPE duplicati_exporter_last_run_timestamp gauge"
+ echo "duplicati_exporter_last_run_timestamp $script_end"
+}
+
+run_http_server() {
+ echo "Starting Duplicati exporter on port $HTTP_PORT..." >&2
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+ while true; do
+ {
+ read -r request
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ echo "Duplicati ExporterDuplicati Prometheus Exporter
Metrics
"
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+main() {
+ parse_args "$@"
+ if [ "$HTTP_MODE" = true ]; then
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.duplicati_metrics.XXXXXX")
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+ chmod 644 "$temp_file"
+ mv -f "$temp_file" "$OUTPUT_FILE"
+ echo "Metrics written to $OUTPUT_FILE" >&2
+ else
+ generate_metrics
+ fi
+}
+
+main "$@"
diff --git a/ebs-snapshot-manager.sh b/ebs-snapshot-manager.sh
new file mode 100644
index 0000000..ba62a55
--- /dev/null
+++ b/ebs-snapshot-manager.sh
@@ -0,0 +1,813 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### ebs-snapshot-manager.sh — Create, manage, audit, and prune AWS EBS snapshots ####
+#### Supports automated creation, cross-region copy, retention, and orphan detection ####
+#### Requires: bash 4+, aws-cli v2, jq ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.01 ####
+#### ####
+#### Usage: ####
+#### export AWS_PROFILE="production" ####
+#### ./ebs-snapshot-manager.sh --snapshot ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+AWS_REGION="${AWS_REGION:-}"
+VOLUME_IDS="${VOLUME_IDS:-}"
+VOLUME_TAG_KEY="${VOLUME_TAG_KEY:-}"
+VOLUME_TAG_VALUE="${VOLUME_TAG_VALUE:-}"
+RETENTION_DAYS="${RETENTION_DAYS:-30}"
+COPY_TO_REGION="${COPY_TO_REGION:-}"
+SNAPSHOT_DESCRIPTION="${SNAPSHOT_DESCRIPTION:-Automated snapshot by ebs-snapshot-manager}"
+NO_WAIT="${NO_WAIT:-false}"
+DRY_RUN="${DRY_RUN:-true}"
+RESTORE_AZ="${RESTORE_AZ:-}"
+RESTORE_VOLUME_TYPE="${RESTORE_VOLUME_TYPE:-gp3}"
+RESTORE_IOPS="${RESTORE_IOPS:-}"
+RESTORE_THROUGHPUT="${RESTORE_THROUGHPUT:-}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+RUN_MODE=""
+TARGET_VOLUME=""
+TARGET_SNAPSHOT=""
+START_TIME=""
+WARNINGS=0
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; ((WARNINGS++)) || true; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
+
+# ── AWS CLI wrapper ───────────────────────────────────────────────────
+aws_cmd() {
+ local args=("$@")
+ [[ -n "$AWS_REGION" ]] && args+=(--region "$AWS_REGION")
+ verbose "aws ${args[*]}"
+ aws "${args[@]}"
+}
+
+# ── Dependency check ──────────────────────────────────────────────────
+check_deps() {
+ for cmd in aws jq; do
+ if ! command -v "$cmd" &>/dev/null; then
+ err "${cmd} is required but not installed"
+ exit 1
+ fi
+ done
+
+ # Verify AWS credentials
+ if ! aws sts get-caller-identity &>/dev/null; then
+ err "AWS credentials not configured or expired"
+ exit 1
+ fi
+
+ # Determine region
+ if [[ -z "$AWS_REGION" ]]; then
+ AWS_REGION=$(aws configure get region 2>/dev/null || echo "")
+ if [[ -z "$AWS_REGION" ]]; then
+ err "AWS_REGION is required (set via env var or aws configure)"
+ exit 1
+ fi
+ fi
+
+ verbose "Using region: ${AWS_REGION}"
+ verbose "Account: $(aws sts get-caller-identity --query 'Account' --output text 2>/dev/null)"
+}
+
+# ── Get volume list ───────────────────────────────────────────────────
+get_volumes() {
+ local filters=()
+
+ if [[ -n "$VOLUME_IDS" ]]; then
+ # Specific volumes requested
+ local vol_array
+ IFS=',' read -ra vol_array <<< "$VOLUME_IDS"
+ aws_cmd ec2 describe-volumes \
+ --volume-ids "${vol_array[@]}" \
+ --query 'Volumes[*].VolumeId' \
+ --output text | tr '\t' '\n'
+ return
+ fi
+
+ if [[ -n "$VOLUME_TAG_KEY" ]]; then
+ filters+=(--filters "Name=tag:${VOLUME_TAG_KEY},Values=${VOLUME_TAG_VALUE:-*}")
+ fi
+
+ aws_cmd ec2 describe-volumes \
+ "${filters[@]}" \
+ --query 'Volumes[*].VolumeId' \
+ --output text | tr '\t' '\n'
+}
+
+# ── Get account ID ───────────────────────────────────────────────────
+get_account_id() {
+ aws sts get-caller-identity --query 'Account' --output text
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SNAPSHOT MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_snapshot() {
+ log "Creating EBS snapshots..."
+ local volumes
+ volumes=$(get_volumes)
+
+ if [[ -z "$volumes" ]]; then
+ warn "No volumes found matching criteria"
+ return
+ fi
+
+ local vol_count
+ vol_count=$(echo "$volumes" | wc -l)
+ log "Found ${vol_count} volume(s) to snapshot"
+
+ local created=0
+ local failed=0
+ local snapshot_ids=()
+ local now
+ now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+ while IFS= read -r vol_id; do
+ [[ -z "$vol_id" ]] && continue
+ verbose "Snapshotting ${vol_id}..."
+
+ local vol_name
+ # shellcheck disable=SC2016
+ vol_name=$(aws_cmd ec2 describe-volumes \
+ --volume-ids "$vol_id" \
+ --query 'Volumes[0].Tags[?Key==`Name`].Value | [0]' \
+ --output text 2>/dev/null) || vol_name="N/A"
+ [[ "$vol_name" == "None" ]] && vol_name="N/A"
+
+ local snap_id
+ snap_id=$(aws_cmd ec2 create-snapshot \
+ --volume-id "$vol_id" \
+ --description "$SNAPSHOT_DESCRIPTION" \
+ --tag-specifications "ResourceType=snapshot,Tags=[
+ {Key=Name,Value=snap-${vol_id}-$(date +%Y%m%d)},
+ {Key=CreatedBy,Value=ebs-snapshot-manager},
+ {Key=CreatedAt,Value=${now}},
+ {Key=VolumeId,Value=${vol_id}},
+ {Key=VolumeName,Value=${vol_name}}
+ ]" \
+ --query 'SnapshotId' \
+ --output text 2>/dev/null) || snap_id=""
+
+ if [[ -n "$snap_id" ]]; then
+ echo -e " ${GREEN}✓${RESET} ${vol_id} → ${snap_id} (${vol_name})"
+ snapshot_ids+=("$snap_id")
+ ((created++)) || true
+ else
+ echo -e " ${RED}✗${RESET} ${vol_id} — snapshot creation failed"
+ ((failed++)) || true
+ fi
+ done <<< "$volumes"
+
+ # Wait for completion
+ if [[ "$NO_WAIT" != "true" && ${#snapshot_ids[@]} -gt 0 ]]; then
+ log "Waiting for ${#snapshot_ids[@]} snapshot(s) to complete..."
+ for snap_id in "${snapshot_ids[@]}"; do
+ if aws_cmd ec2 wait snapshot-completed --snapshot-ids "$snap_id" 2>/dev/null; then
+ local size
+ size=$(aws_cmd ec2 describe-snapshots \
+ --snapshot-ids "$snap_id" \
+ --query 'Snapshots[0].VolumeSize' \
+ --output text 2>/dev/null) || size="?"
+ verbose "${snap_id} completed (${size} GiB)"
+ else
+ warn "${snap_id} did not complete within timeout"
+ fi
+ done
+ fi
+
+ echo ""
+ log "Snapshots created: ${created}, failed: ${failed}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# PRUNE MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_prune() {
+ local cutoff_epoch
+ cutoff_epoch=$(date -d "-${RETENTION_DAYS} days" +%s 2>/dev/null) || \
+ cutoff_epoch=$(date -v-"${RETENTION_DAYS}"d +%s 2>/dev/null) || {
+ err "Could not calculate retention cutoff date"
+ exit 1
+ }
+
+ local cutoff_date
+ cutoff_date=$(date -d "@${cutoff_epoch}" +%Y-%m-%dT%H:%M:%S 2>/dev/null) || \
+ cutoff_date=$(date -r "${cutoff_epoch}" +%Y-%m-%dT%H:%M:%S 2>/dev/null)
+
+ log "Pruning snapshots older than ${RETENTION_DAYS} days (before ${cutoff_date})"
+ if [[ "$DRY_RUN" == "true" ]]; then
+ log "${YELLOW}DRY RUN${RESET} — no snapshots will be deleted. Use --force to delete."
+ fi
+
+ local owner_id
+ owner_id=$(get_account_id)
+
+ local snapshots_json
+ snapshots_json=$(aws_cmd ec2 describe-snapshots \
+ --owner-ids "$owner_id" \
+ --filters "Name=tag:CreatedBy,Values=ebs-snapshot-manager" \
+ --query 'Snapshots[*].{Id:SnapshotId,Start:StartTime,Size:VolumeSize,Vol:VolumeId}' \
+ --output json)
+
+ local total
+ total=$(echo "$snapshots_json" | jq 'length')
+ echo "$snapshots_json" | jq -c '.[]' | while IFS= read -r snap; do
+ local snap_id start_time size vol_id
+ snap_id=$(echo "$snap" | jq -r '.Id')
+ start_time=$(echo "$snap" | jq -r '.Start')
+ size=$(echo "$snap" | jq -r '.Size')
+ vol_id=$(echo "$snap" | jq -r '.Vol')
+
+ local snap_epoch
+ snap_epoch=$(date -d "$start_time" +%s 2>/dev/null) || \
+ snap_epoch=$(date -jf "%Y-%m-%dT%H:%M:%S" "${start_time%%.*}" +%s 2>/dev/null) || snap_epoch=0
+
+ if [[ $snap_epoch -lt $cutoff_epoch ]]; then
+ local age_days=$(( ($(date +%s) - snap_epoch) / 86400 ))
+
+ if [[ "$DRY_RUN" == "true" ]]; then
+ echo -e " ${YELLOW}⊘${RESET} ${snap_id} — ${age_days}d old, ${size} GiB, vol: ${vol_id} (would delete)"
+ else
+ if aws_cmd ec2 delete-snapshot --snapshot-id "$snap_id" 2>/dev/null; then
+ echo -e " ${GREEN}✓${RESET} ${snap_id} — deleted (${age_days}d old, ${size} GiB)"
+ else
+ echo -e " ${RED}✗${RESET} ${snap_id} — delete failed"
+ fi
+ fi
+ fi
+ done
+
+ log "Total managed snapshots: ${total}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# COPY-REGION MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_copy_region() {
+ if [[ -z "$COPY_TO_REGION" ]]; then
+ err "Target region required. Use --copy-region REGION or set COPY_TO_REGION"
+ exit 1
+ fi
+
+ log "Copying latest snapshots to ${COPY_TO_REGION}..."
+
+ local owner_id
+ owner_id=$(get_account_id)
+
+ # Get volumes to copy snapshots for
+ local volumes
+ if [[ -n "$TARGET_VOLUME" ]]; then
+ volumes="$TARGET_VOLUME"
+ else
+ volumes=$(get_volumes)
+ fi
+
+ if [[ -z "$volumes" ]]; then
+ warn "No volumes found"
+ return
+ fi
+
+ local copied=0
+ local failed=0
+
+ while IFS= read -r vol_id; do
+ [[ -z "$vol_id" ]] && continue
+
+ # Find latest snapshot for this volume
+ local latest_snap
+ latest_snap=$(aws_cmd ec2 describe-snapshots \
+ --owner-ids "$owner_id" \
+ --filters "Name=volume-id,Values=${vol_id}" "Name=status,Values=completed" \
+ --query 'sort_by(Snapshots, &StartTime)[-1].SnapshotId' \
+ --output text 2>/dev/null) || latest_snap=""
+
+ if [[ -z "$latest_snap" || "$latest_snap" == "None" ]]; then
+ echo -e " ${YELLOW}⊘${RESET} ${vol_id} — no completed snapshots found"
+ continue
+ fi
+
+ # Copy to target region
+ local copy_id
+ copy_id=$(aws ec2 copy-snapshot \
+ --region "$COPY_TO_REGION" \
+ --source-region "$AWS_REGION" \
+ --source-snapshot-id "$latest_snap" \
+ --description "DR copy of ${latest_snap} from ${AWS_REGION}" \
+ --tag-specifications "ResourceType=snapshot,Tags=[
+ {Key=Name,Value=dr-copy-${latest_snap}},
+ {Key=CreatedBy,Value=ebs-snapshot-manager},
+ {Key=SourceRegion,Value=${AWS_REGION}},
+ {Key=SourceSnapshotId,Value=${latest_snap}},
+ {Key=VolumeId,Value=${vol_id}}
+ ]" \
+ --query 'SnapshotId' \
+ --output text 2>/dev/null) || copy_id=""
+
+ if [[ -n "$copy_id" ]]; then
+ echo -e " ${GREEN}✓${RESET} ${latest_snap} → ${copy_id} (${AWS_REGION} → ${COPY_TO_REGION})"
+ ((copied++)) || true
+ else
+ echo -e " ${RED}✗${RESET} ${latest_snap} — copy failed"
+ ((failed++)) || true
+ fi
+ done <<< "$volumes"
+
+ echo ""
+ log "Copied: ${copied}, failed: ${failed}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# AUDIT MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_audit() {
+ log "Auditing EBS snapshots in ${AWS_REGION}..."
+
+ local owner_id
+ owner_id=$(get_account_id)
+
+ local snapshots_json
+ snapshots_json=$(aws_cmd ec2 describe-snapshots \
+ --owner-ids "$owner_id" \
+ --query 'Snapshots[*].{Id:SnapshotId,Vol:VolumeId,Size:VolumeSize,Status:State,Start:StartTime,Desc:Description,Tags:Tags}' \
+ --output json)
+
+ local total
+ total=$(echo "$snapshots_json" | jq 'length')
+
+ if [[ "$total" -eq 0 ]]; then
+ log "No snapshots found"
+ return
+ fi
+
+ # Get existing volumes for orphan detection
+ local existing_volumes
+ existing_volumes=$(aws_cmd ec2 describe-volumes \
+ --query 'Volumes[*].VolumeId' \
+ --output text | tr '\t' '\n' | sort)
+
+ local orphan_count=0
+ local untagged_count=0
+ local managed_count=0
+
+ echo ""
+ echo -e "${BOLD}Snapshot Inventory${RESET}"
+ printf " %-24s %-14s %8s %6s %s\n" "SNAPSHOT" "VOLUME" "SIZE" "AGE" "STATUS"
+ echo " $(printf '%.0s─' {1..70})"
+
+ echo "$snapshots_json" | jq -c '.[]' | while IFS= read -r snap; do
+ local snap_id vol_id size status start_time
+ snap_id=$(echo "$snap" | jq -r '.Id')
+ vol_id=$(echo "$snap" | jq -r '.Vol')
+ size=$(echo "$snap" | jq -r '.Size')
+ status=$(echo "$snap" | jq -r '.Status')
+ start_time=$(echo "$snap" | jq -r '.Start')
+
+ local snap_epoch
+ snap_epoch=$(date -d "$start_time" +%s 2>/dev/null) || snap_epoch=0
+ local age_days=$(( ($(date +%s) - snap_epoch) / 86400 ))
+
+ # Check if managed
+ local is_managed
+ is_managed=$(echo "$snap" | jq -r '.Tags // [] | map(select(.Key == "CreatedBy" and .Value == "ebs-snapshot-manager")) | length')
+
+ # Check if orphaned
+ local is_orphan="no"
+ if ! echo "$existing_volumes" | grep -q "^${vol_id}$" 2>/dev/null; then
+ is_orphan="yes"
+ fi
+
+ # Check if tagged
+ local tag_count
+ tag_count=$(echo "$snap" | jq '.Tags // [] | length')
+
+ local status_marker=""
+ if [[ "$is_orphan" == "yes" ]]; then
+ status_marker="${RED}orphan${RESET}"
+ elif [[ "$tag_count" -eq 0 ]]; then
+ status_marker="${YELLOW}untagged${RESET}"
+ elif [[ "$is_managed" -gt 0 ]]; then
+ status_marker="${GREEN}managed${RESET}"
+ else
+ status_marker="${status}"
+ fi
+
+ printf " %-24s %-14s %6s G %4sd %b\n" \
+ "$snap_id" "$vol_id" "$size" "$age_days" "$status_marker"
+ done
+
+ # Summary stats
+ local total_size
+ total_size=$(echo "$snapshots_json" | jq '[.[].Size] | add // 0')
+ orphan_count=$(echo "$snapshots_json" | jq --arg vols "$existing_volumes" '
+ [.[] | select(.Vol as $v | ($vols | split("\n") | map(select(. != "")) | index($v) == null))] | length
+ ')
+ untagged_count=$(echo "$snapshots_json" | jq '[.[] | select((.Tags // []) | length == 0)] | length')
+ managed_count=$(echo "$snapshots_json" | jq '[.[] | select((.Tags // []) | map(select(.Key == "CreatedBy" and .Value == "ebs-snapshot-manager")) | length > 0)] | length')
+
+ local monthly_cost
+ monthly_cost=$(echo "$total_size * 0.05" | bc 2>/dev/null || echo "?")
+
+ echo ""
+ echo -e "${BOLD}Summary${RESET}"
+ echo -e " Total snapshots: ${total}"
+ echo -e " Managed snapshots: ${managed_count}"
+ echo -e " Total storage: ${total_size} GiB"
+ echo -e " Est. monthly cost: \$${monthly_cost}"
+ echo -e " Orphaned: ${orphan_count}"
+ echo -e " Untagged: ${untagged_count}"
+
+ if [[ "$orphan_count" -gt 0 ]]; then
+ echo ""
+ warn "${orphan_count} orphaned snapshot(s) found — source volume no longer exists"
+ fi
+
+ # Check volumes without recent snapshots
+ echo ""
+ echo -e "${BOLD}Volumes Without Recent Snapshots (>${RETENTION_DAYS}d)${RESET}"
+
+ local volumes_json
+ volumes_json=$(aws_cmd ec2 describe-volumes \
+ --query 'Volumes[*].{Id:VolumeId,Size:Size,State:State}' \
+ --output json)
+
+ echo "$volumes_json" | jq -c '.[]' | while IFS= read -r vol; do
+ local v_id v_size
+ v_id=$(echo "$vol" | jq -r '.Id')
+ v_size=$(echo "$vol" | jq -r '.Size')
+
+ local latest_snap_time
+ latest_snap_time=$(aws_cmd ec2 describe-snapshots \
+ --owner-ids "$owner_id" \
+ --filters "Name=volume-id,Values=${v_id}" "Name=status,Values=completed" \
+ --query 'sort_by(Snapshots, &StartTime)[-1].StartTime' \
+ --output text 2>/dev/null) || latest_snap_time="None"
+
+ if [[ "$latest_snap_time" == "None" || -z "$latest_snap_time" ]]; then
+ echo -e " ${RED}✗${RESET} ${v_id} (${v_size} GiB) — ${RED}no snapshots${RESET}"
+ else
+ local snap_epoch
+ snap_epoch=$(date -d "$latest_snap_time" +%s 2>/dev/null) || snap_epoch=0
+ local cutoff_epoch
+ cutoff_epoch=$(date -d "-${RETENTION_DAYS} days" +%s 2>/dev/null) || cutoff_epoch=0
+
+ if [[ $snap_epoch -lt $cutoff_epoch ]]; then
+ local age=$(( ($(date +%s) - snap_epoch) / 86400 ))
+ echo -e " ${YELLOW}!${RESET} ${v_id} (${v_size} GiB) — last snapshot ${age}d ago"
+ fi
+ fi
+ done
+
+ # Prometheus output
+ if [[ "$OUTPUT_FORMAT" == "prometheus" ]]; then
+ echo ""
+ echo "# HELP ebs_snapshots_total Total EBS snapshots"
+ echo "# TYPE ebs_snapshots_total gauge"
+ echo "ebs_snapshots_total{region=\"${AWS_REGION}\"} ${total}"
+ echo "# HELP ebs_snapshots_managed_total Managed EBS snapshots"
+ echo "# TYPE ebs_snapshots_managed_total gauge"
+ echo "ebs_snapshots_managed_total{region=\"${AWS_REGION}\"} ${managed_count}"
+ echo "# HELP ebs_snapshots_orphaned_total Orphaned EBS snapshots"
+ echo "# TYPE ebs_snapshots_orphaned_total gauge"
+ echo "ebs_snapshots_orphaned_total{region=\"${AWS_REGION}\"} ${orphan_count}"
+ echo "# HELP ebs_snapshots_untagged_total Untagged EBS snapshots"
+ echo "# TYPE ebs_snapshots_untagged_total gauge"
+ echo "ebs_snapshots_untagged_total{region=\"${AWS_REGION}\"} ${untagged_count}"
+ echo "# HELP ebs_snapshots_size_gib_total Total snapshot storage in GiB"
+ echo "# TYPE ebs_snapshots_size_gib_total gauge"
+ echo "ebs_snapshots_size_gib_total{region=\"${AWS_REGION}\"} ${total_size}"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# RESTORE MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_restore() {
+ if [[ -z "$TARGET_SNAPSHOT" ]]; then
+ err "Snapshot ID required. Use --restore SNAP_ID"
+ exit 1
+ fi
+
+ log "Restoring volume from snapshot ${TARGET_SNAPSHOT}..."
+
+ # Verify snapshot exists and is completed
+ local snap_info
+ snap_info=$(aws_cmd ec2 describe-snapshots \
+ --snapshot-ids "$TARGET_SNAPSHOT" \
+ --query 'Snapshots[0].{State:State,Size:VolumeSize,Vol:VolumeId}' \
+ --output json 2>/dev/null) || {
+ err "Snapshot ${TARGET_SNAPSHOT} not found"
+ exit 1
+ }
+
+ local snap_state snap_size source_vol
+ snap_state=$(echo "$snap_info" | jq -r '.State')
+ snap_size=$(echo "$snap_info" | jq -r '.Size')
+ source_vol=$(echo "$snap_info" | jq -r '.Vol')
+
+ if [[ "$snap_state" != "completed" ]]; then
+ err "Snapshot state is '${snap_state}' — must be 'completed'"
+ exit 1
+ fi
+
+ # Determine AZ
+ if [[ -z "$RESTORE_AZ" ]]; then
+ RESTORE_AZ=$(aws_cmd ec2 describe-availability-zones \
+ --query 'AvailabilityZones[0].ZoneName' \
+ --output text)
+ log "No AZ specified, using ${RESTORE_AZ}"
+ fi
+
+ # Build create-volume args
+ local create_args=(
+ ec2 create-volume
+ --snapshot-id "$TARGET_SNAPSHOT"
+ --availability-zone "$RESTORE_AZ"
+ --volume-type "$RESTORE_VOLUME_TYPE"
+ --tag-specifications "ResourceType=volume,Tags=[
+ {Key=Name,Value=restored-from-${TARGET_SNAPSHOT}},
+ {Key=CreatedBy,Value=ebs-snapshot-manager},
+ {Key=RestoredFrom,Value=${TARGET_SNAPSHOT}},
+ {Key=SourceVolumeId,Value=${source_vol}}
+ ]"
+ )
+
+ [[ -n "$RESTORE_IOPS" ]] && create_args+=(--iops "$RESTORE_IOPS")
+ [[ -n "$RESTORE_THROUGHPUT" ]] && create_args+=(--throughput "$RESTORE_THROUGHPUT")
+
+ local vol_id
+ vol_id=$(aws_cmd "${create_args[@]}" \
+ --query 'VolumeId' \
+ --output text 2>/dev/null) || {
+ err "Failed to create volume from snapshot"
+ exit 1
+ }
+
+ echo -e " ${GREEN}✓${RESET} Created volume ${vol_id}"
+ echo -e " Source snapshot: ${TARGET_SNAPSHOT}"
+ echo -e " Size: ${snap_size} GiB"
+ echo -e " Type: ${RESTORE_VOLUME_TYPE}"
+ echo -e " AZ: ${RESTORE_AZ}"
+
+ # Wait for volume to become available
+ log "Waiting for volume to become available..."
+ if aws_cmd ec2 wait volume-available --volume-ids "$vol_id" 2>/dev/null; then
+ echo -e " ${GREEN}✓${RESET} Volume ${vol_id} is available"
+ else
+ warn "Volume did not become available within timeout"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# LIST MODE
+# ══════════════════════════════════════════════════════════════════════
+
+do_list() {
+ local owner_id
+ owner_id=$(get_account_id)
+
+ local filters=("Name=owner-id,Values=${owner_id}")
+
+ if [[ -n "$TARGET_VOLUME" ]]; then
+ filters+=("Name=volume-id,Values=${TARGET_VOLUME}")
+ fi
+
+ local snapshots_json
+ snapshots_json=$(aws_cmd ec2 describe-snapshots \
+ --owner-ids "$owner_id" \
+ ${TARGET_VOLUME:+--filters "Name=volume-id,Values=${TARGET_VOLUME}"} \
+ --query 'sort_by(Snapshots, &StartTime) | reverse(@) | [*].{Id:SnapshotId,Vol:VolumeId,Size:VolumeSize,Status:State,Start:StartTime,Desc:Description}' \
+ --output json)
+
+ local total
+ total=$(echo "$snapshots_json" | jq 'length')
+
+ if [[ "$total" -eq 0 ]]; then
+ log "No snapshots found"
+ return
+ fi
+
+ if [[ "$OUTPUT_FORMAT" == "json" ]]; then
+ echo "$snapshots_json" | jq '.'
+ return
+ fi
+
+ echo ""
+ printf " %-24s %-14s %8s %-12s %-22s %s\n" "SNAPSHOT" "VOLUME" "SIZE" "STATUS" "CREATED" "DESCRIPTION"
+ echo " $(printf '%.0s─' {1..100})"
+
+ echo "$snapshots_json" | jq -c '.[]' | while IFS= read -r snap; do
+ local snap_id vol_id size status start_time desc
+ snap_id=$(echo "$snap" | jq -r '.Id')
+ vol_id=$(echo "$snap" | jq -r '.Vol')
+ size=$(echo "$snap" | jq -r '.Size')
+ status=$(echo "$snap" | jq -r '.Status')
+ start_time=$(echo "$snap" | jq -r '.Start' | cut -c1-19)
+ desc=$(echo "$snap" | jq -r '.Desc' | cut -c1-40)
+
+ printf " %-24s %-14s %6s G %-12s %-22s %s\n" \
+ "$snap_id" "$vol_id" "$size" "$status" "$start_time" "$desc"
+ done
+
+ echo ""
+ log "Total: ${total} snapshot(s)"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+show_help() {
+ cat </dev/null || echo 'default')}"
+ echo -e "Mode: ${RUN_MODE}"
+ echo -e "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ echo ""
+
+ check_deps
+
+ case "$RUN_MODE" in
+ snapshot) do_snapshot ;;
+ prune) do_prune ;;
+ copy-region) do_copy_region ;;
+ audit) do_audit ;;
+ restore) do_restore ;;
+ list) do_list ;;
+ esac
+
+ local end_time
+ end_time=$(date +%s)
+ local duration=$(( end_time - START_TIME ))
+ echo ""
+ log "Completed in ${duration}s"
+
+ if [[ $WARNINGS -gt 0 ]]; then
+ exit 2
+ fi
+}
+
+main "$@"
diff --git a/ec2-inventory-reporter.sh b/ec2-inventory-reporter.sh
new file mode 100644
index 0000000..3029e75
--- /dev/null
+++ b/ec2-inventory-reporter.sh
@@ -0,0 +1,704 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### ec2-inventory-reporter.sh — AWS EC2 instance inventory and compliance report ####
+#### Instance metadata, uptime, cost estimates, tag compliance, SG audit ####
+#### Requires: bash 4+, aws-cli v2, jq ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./ec2-inventory-reporter.sh ####
+#### ./ec2-inventory-reporter.sh --all-regions --format csv ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+AWS_REGION="${AWS_REGION:-us-east-1}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+REQUIRED_TAGS="${REQUIRED_TAGS:-Name,Environment,Owner}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+SCAN_REGION="$AWS_REGION"
+ALL_REGIONS="false"
+FILTER_STATE=""
+FILTER_TAG_KEY=""
+FILTER_TAG_VALUE=""
+FILTER_TYPE=""
+TAG_CHECK="false"
+SG_AUDIT="false"
+START_TIME=""
+
+# ── Colors ────────────────────────────────────────────────────────────
+RED="" GREEN="" YELLOW="" BLUE="" BOLD="" DIM="" RESET=""
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ return
+ fi
+ if [[ "$COLOR" == "auto" && ! -t 1 ]]; then
+ return
+ fi
+ RED="\033[0;31m"
+ GREEN="\033[0;32m"
+ YELLOW="\033[0;33m"
+ BLUE="\033[0;34m"
+ BOLD="\033[1m"
+ DIM="\033[2m"
+ RESET="\033[0m"
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log_info() { printf "${GREEN}[INFO]${RESET} %s\n" "$*"; }
+log_warn() { printf "${YELLOW}[WARN]${RESET} %s\n" "$*" >&2; }
+log_error() { printf "${RED}[ERROR]${RESET} %s\n" "$*" >&2; }
+log_debug() { [[ "$VERBOSE" == "true" ]] && printf "${DIM}[DEBUG] %s${RESET}\n" "$*"; }
+
+# ── Helpers ───────────────────────────────────────────────────────────
+die() { log_error "$@"; exit 1; }
+
+check_deps() {
+ local missing=()
+ command -v aws >/dev/null 2>&1 || missing+=("aws-cli")
+ command -v jq >/dev/null 2>&1 || missing+=("jq")
+ if (( ${#missing[@]} > 0 )); then
+ die "Missing required tools: ${missing[*]}"
+ fi
+
+ local bash_major="${BASH_VERSINFO[0]}"
+ if (( bash_major < 4 )); then
+ die "Requires bash 4+, found ${BASH_VERSION}"
+ fi
+}
+
+# ── Pricing table (us-east-1 on-demand Linux, $/hr) ──────────────────
+declare -A PRICING=(
+ # General purpose
+ ["t3.nano"]=0.0052 ["t3.micro"]=0.0104 ["t3.small"]=0.0208
+ ["t3.medium"]=0.0416 ["t3.large"]=0.0832 ["t3.xlarge"]=0.1664
+ ["t3.2xlarge"]=0.3328
+ ["t3a.nano"]=0.0047 ["t3a.micro"]=0.0094 ["t3a.small"]=0.0188
+ ["t3a.medium"]=0.0376 ["t3a.large"]=0.0752 ["t3a.xlarge"]=0.1504
+ ["t3a.2xlarge"]=0.3008
+ ["m5.large"]=0.096 ["m5.xlarge"]=0.192 ["m5.2xlarge"]=0.384
+ ["m5.4xlarge"]=0.768 ["m5.8xlarge"]=1.536
+ ["m6i.large"]=0.096 ["m6i.xlarge"]=0.192 ["m6i.2xlarge"]=0.384
+ ["m6i.4xlarge"]=0.768
+ ["m7i.large"]=0.1008 ["m7i.xlarge"]=0.2016 ["m7i.2xlarge"]=0.4032
+ # Compute optimized
+ ["c5.large"]=0.085 ["c5.xlarge"]=0.17 ["c5.2xlarge"]=0.34
+ ["c5.4xlarge"]=0.68 ["c5.9xlarge"]=1.53
+ ["c6i.large"]=0.085 ["c6i.xlarge"]=0.17 ["c6i.2xlarge"]=0.34
+ # Memory optimized
+ ["r5.large"]=0.126 ["r5.xlarge"]=0.252 ["r5.2xlarge"]=0.504
+ ["r5.4xlarge"]=1.008
+ ["r6i.large"]=0.126 ["r6i.xlarge"]=0.252 ["r6i.2xlarge"]=0.504
+ # Storage optimized
+ ["i3.large"]=0.156 ["i3.xlarge"]=0.312 ["i3.2xlarge"]=0.624
+ # Accelerated
+ ["g4dn.xlarge"]=0.526 ["g4dn.2xlarge"]=0.752
+ # Burstable previous gen
+ ["t2.nano"]=0.0058 ["t2.micro"]=0.0116 ["t2.small"]=0.023
+ ["t2.medium"]=0.0464 ["t2.large"]=0.0928
+)
+
+# ── Cost estimation ──────────────────────────────────────────────────
+estimate_cost() {
+ local instance_type="$1" state="$2"
+ if [[ "$state" != "running" ]]; then
+ echo "0.00"
+ return
+ fi
+ local hourly="${PRICING[$instance_type]:-}"
+ if [[ -z "$hourly" ]]; then
+ echo "N/A"
+ return
+ fi
+ printf "%.2f" "$(echo "$hourly * 730" | bc -l)"
+}
+
+# ── Uptime calculation ───────────────────────────────────────────────
+format_uptime() {
+ local launch_time="$1" state="$2"
+ if [[ "$state" != "running" || -z "$launch_time" || "$launch_time" == "null" ]]; then
+ echo "—"
+ return
+ fi
+
+ local launch_epoch now_epoch diff_sec
+ if date --version >/dev/null 2>&1; then
+ launch_epoch=$(date -d "$launch_time" +%s 2>/dev/null) || { echo "—"; return; }
+ else
+ launch_epoch=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${launch_time%%.*}" +%s 2>/dev/null) || { echo "—"; return; }
+ fi
+ now_epoch=$(date -u +%s)
+ diff_sec=$(( now_epoch - launch_epoch ))
+
+ if (( diff_sec < 0 )); then
+ echo "—"
+ return
+ fi
+
+ local days=$(( diff_sec / 86400 ))
+ local hours=$(( (diff_sec % 86400) / 3600 ))
+ local mins=$(( (diff_sec % 3600) / 60 ))
+ printf "%dd %dh %dm" "$days" "$hours" "$mins"
+}
+
+# ── Tag compliance check ─────────────────────────────────────────────
+check_tag_compliance() {
+ local tags_json="$1"
+ local missing=()
+
+ IFS=',' read -ra required <<< "$REQUIRED_TAGS"
+ for tag in "${required[@]}"; do
+ tag=$(echo "$tag" | xargs)
+ local found
+ found=$(echo "$tags_json" | jq -r --arg key "$tag" '.[] | select(.Key == $key) | .Key' 2>/dev/null)
+ if [[ -z "$found" ]]; then
+ missing+=("$tag")
+ fi
+ done
+
+ if (( ${#missing[@]} == 0 )); then
+ echo "PASS"
+ else
+ echo "MISSING: ${missing[*]}"
+ fi
+}
+
+# ── Security group audit ─────────────────────────────────────────────
+audit_security_groups() {
+ local region="$1"
+ shift
+ local sg_ids=("$@")
+ local findings=()
+
+ if (( ${#sg_ids[@]} == 0 )); then
+ echo "—"
+ return
+ fi
+
+ local sg_data
+ sg_data=$(aws ec2 describe-security-groups \
+ --region "$region" \
+ --group-ids "${sg_ids[@]}" \
+ --output json 2>/dev/null) || { echo "ERROR"; return; }
+
+ local open_rules
+ open_rules=$(echo "$sg_data" | jq -r '
+ .SecurityGroups[].IpPermissions[] |
+ select(
+ (.IpRanges[]?.CidrIp == "0.0.0.0/0") or
+ (.Ipv6Ranges[]?.CidrIpv6 == "::/0")
+ ) |
+ select(
+ (.FromPort != 80 or .ToPort != 80) and
+ (.FromPort != 443 or .ToPort != 443)
+ ) |
+ if .FromPort == .ToPort then
+ "port \(.FromPort // "all")"
+ elif .FromPort == -1 then
+ "all ports"
+ else
+ "ports \(.FromPort)-\(.ToPort)"
+ end
+ ' 2>/dev/null)
+
+ if [[ -z "$open_rules" ]]; then
+ echo "OK"
+ else
+ local unique
+ unique=$(echo "$open_rules" | sort -u | paste -sd ", " -)
+ echo "OPEN: $unique"
+ fi
+}
+
+# ── Query EC2 instances ──────────────────────────────────────────────
+get_instances() {
+ local region="$1"
+ local filters=()
+
+ if [[ -n "$FILTER_STATE" ]]; then
+ filters+=("Name=instance-state-name,Values=$FILTER_STATE")
+ fi
+ if [[ -n "$FILTER_TAG_KEY" && -n "$FILTER_TAG_VALUE" ]]; then
+ filters+=("Name=tag:$FILTER_TAG_KEY,Values=$FILTER_TAG_VALUE")
+ fi
+ if [[ -n "$FILTER_TYPE" ]]; then
+ filters+=("Name=instance-type,Values=$FILTER_TYPE")
+ fi
+
+ local cmd=(
+ aws ec2 describe-instances
+ --region "$region"
+ --output json
+ )
+
+ if (( ${#filters[@]} > 0 )); then
+ cmd+=(--filters "${filters[@]}")
+ fi
+
+ log_debug "Running: ${cmd[*]}"
+
+ local result=""
+ local next_token=""
+
+ while true; do
+ local page_cmd=("${cmd[@]}")
+ if [[ -n "$next_token" ]]; then
+ page_cmd+=(--starting-token "$next_token")
+ fi
+
+ local page
+ page=$("${page_cmd[@]}" 2>/dev/null) || { log_warn "Failed to query EC2 in $region"; echo "[]"; return; }
+
+ local page_instances
+ page_instances=$(echo "$page" | jq '[.Reservations[].Instances[]]')
+
+ if [[ -z "$result" ]]; then
+ result="$page_instances"
+ else
+ result=$(echo "$result $page_instances" | jq -s 'add')
+ fi
+
+ next_token=$(echo "$page" | jq -r '.NextToken // empty')
+ if [[ -z "$next_token" ]]; then
+ break
+ fi
+ done
+
+ echo "$result"
+}
+
+# ── Get all enabled regions ──────────────────────────────────────────
+get_all_regions() {
+ aws ec2 describe-regions \
+ --region "$AWS_REGION" \
+ --query 'Regions[].RegionName' \
+ --output text 2>/dev/null | tr '\t' '\n' | sort
+}
+
+# ── Text table output ────────────────────────────────────────────────
+output_text() {
+ local region="$1" instances_json="$2"
+ local count
+ count=$(echo "$instances_json" | jq 'length')
+
+ local account_id
+ account_id=$(aws sts get-caller-identity --query Account --output text 2>/dev/null || echo "unknown")
+
+ echo "EC2 Inventory Reporter"
+ echo "Account: $account_id"
+ echo "Region: $region"
+ echo "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ echo "Instances: $count"
+ echo ""
+
+ if (( count == 0 )); then
+ echo " No instances found."
+ echo ""
+ return
+ fi
+
+ local divider="─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────"
+
+ printf " %-21s %-14s %-10s %-15s %-16s %-17s %-16s %s\n" \
+ "INSTANCE ID" "TYPE" "STATE" "AZ" "PRIVATE IP" "PUBLIC IP" "UPTIME" "EST \$/MO"
+ printf " %s\n" "$divider"
+
+ local total_cost=0 running=0 stopped=0 other=0
+ local compliance_issues=0 sg_issues=0
+
+ while IFS=$'\t' read -r iid itype istate iaz pip eip launch_time tags_json sg_json; do
+ local uptime
+ uptime=$(format_uptime "$launch_time" "$istate")
+
+ local cost
+ cost=$(estimate_cost "$itype" "$istate")
+
+ [[ -z "$eip" || "$eip" == "null" ]] && eip="—"
+
+ printf " %-21s %-14s %-10s %-15s %-16s %-17s %-16s %s\n" \
+ "$iid" "$itype" "$istate" "$iaz" "$pip" "$eip" "$uptime" "\$$cost"
+
+ if [[ "$cost" != "N/A" ]]; then
+ total_cost=$(echo "$total_cost + $cost" | bc -l)
+ fi
+
+ case "$istate" in
+ running) (( running++ )) ;;
+ stopped) (( stopped++ )) ;;
+ *) (( other++ )) ;;
+ esac
+
+ if [[ "$TAG_CHECK" == "true" ]]; then
+ local compliance
+ compliance=$(check_tag_compliance "$tags_json")
+ if [[ "$compliance" != "PASS" ]]; then
+ (( compliance_issues++ ))
+ printf " ${YELLOW} ↳ Tag compliance: %s${RESET}\n" "$compliance"
+ fi
+ fi
+
+ if [[ "$SG_AUDIT" == "true" && "$istate" == "running" ]]; then
+ local sg_ids_list
+ sg_ids_list=$(echo "$sg_json" | jq -r '.[].GroupId' 2>/dev/null)
+ if [[ -n "$sg_ids_list" ]]; then
+ local sg_arr=()
+ while IFS= read -r sg; do
+ [[ -n "$sg" ]] && sg_arr+=("$sg")
+ done <<< "$sg_ids_list"
+ local sg_result
+ sg_result=$(audit_security_groups "$region" "${sg_arr[@]}")
+ if [[ "$sg_result" != "OK" && "$sg_result" != "—" ]]; then
+ (( sg_issues++ ))
+ printf " ${RED} ↳ SG audit: %s${RESET}\n" "$sg_result"
+ fi
+ fi
+ fi
+
+ done < <(echo "$instances_json" | jq -r '
+ .[] |
+ [
+ .InstanceId,
+ .InstanceType,
+ (.State.Name),
+ (.Placement.AvailabilityZone),
+ (.PrivateIpAddress // "—"),
+ (.PublicIpAddress // "null"),
+ (.LaunchTime // "null"),
+ (.Tags // [] | tojson),
+ (.SecurityGroups // [] | tojson)
+ ] | @tsv
+ ')
+
+ printf " %s\n" "$divider"
+
+ local summary="TOTAL: $count instances"
+ local parts=()
+ (( running > 0 )) && parts+=("$running running")
+ (( stopped > 0 )) && parts+=("$stopped stopped")
+ (( other > 0 )) && parts+=("$other other")
+ if (( ${#parts[@]} > 0 )); then
+ local joined
+ joined=$(printf ", %s" "${parts[@]}")
+ summary+=" (${joined:2})"
+ fi
+ printf " %-70s Estimated monthly cost: \$%.2f\n" "$summary" "$total_cost"
+
+ if [[ "$TAG_CHECK" == "true" ]]; then
+ echo ""
+ if (( compliance_issues > 0 )); then
+ printf " ${YELLOW}Tag compliance issues: %d instance(s) missing required tags${RESET}\n" "$compliance_issues"
+ else
+ printf " ${GREEN}Tag compliance: all instances have required tags${RESET}\n"
+ fi
+ fi
+
+ if [[ "$SG_AUDIT" == "true" ]]; then
+ if (( sg_issues > 0 )); then
+ printf " ${RED}Security group issues: %d instance(s) with overly permissive rules${RESET}\n" "$sg_issues"
+ else
+ printf " ${GREEN}Security groups: no overly permissive rules found${RESET}\n"
+ fi
+ fi
+
+ echo ""
+}
+
+# ── CSV output ────────────────────────────────────────────────────────
+output_csv() {
+ local region="$1" instances_json="$2"
+ local count
+ count=$(echo "$instances_json" | jq 'length')
+
+ local header="instance_id,type,state,az,private_ip,public_ip,launch_time,uptime,est_monthly_cost"
+ if [[ "$TAG_CHECK" == "true" ]]; then
+ header+=",tag_compliance"
+ fi
+ if [[ "$SG_AUDIT" == "true" ]]; then
+ header+=",sg_audit"
+ fi
+ header+=",region"
+ echo "$header"
+
+ if (( count == 0 )); then
+ return
+ fi
+
+ while IFS=$'\t' read -r iid itype istate iaz pip eip launch_time tags_json sg_json; do
+ local uptime
+ uptime=$(format_uptime "$launch_time" "$istate")
+
+ local cost
+ cost=$(estimate_cost "$itype" "$istate")
+
+ [[ -z "$eip" || "$eip" == "null" ]] && eip=""
+
+ local line="$iid,$itype,$istate,$iaz,$pip,$eip,$launch_time,\"$uptime\",$cost"
+
+ if [[ "$TAG_CHECK" == "true" ]]; then
+ local compliance
+ compliance=$(check_tag_compliance "$tags_json")
+ line+=",\"$compliance\""
+ fi
+
+ if [[ "$SG_AUDIT" == "true" ]]; then
+ local sg_ids_list
+ sg_ids_list=$(echo "$sg_json" | jq -r '.[].GroupId' 2>/dev/null)
+ if [[ -n "$sg_ids_list" && "$istate" == "running" ]]; then
+ local sg_arr=()
+ while IFS= read -r sg; do
+ [[ -n "$sg" ]] && sg_arr+=("$sg")
+ done <<< "$sg_ids_list"
+ local sg_result
+ sg_result=$(audit_security_groups "$region" "${sg_arr[@]}")
+ line+=",\"$sg_result\""
+ else
+ line+=",\"—\""
+ fi
+ fi
+
+ line+=",$region"
+ echo "$line"
+
+ done < <(echo "$instances_json" | jq -r '
+ .[] |
+ [
+ .InstanceId,
+ .InstanceType,
+ (.State.Name),
+ (.Placement.AvailabilityZone),
+ (.PrivateIpAddress // "—"),
+ (.PublicIpAddress // "null"),
+ (.LaunchTime // "null"),
+ (.Tags // [] | tojson),
+ (.SecurityGroups // [] | tojson)
+ ] | @tsv
+ ')
+}
+
+# ── JSON output ───────────────────────────────────────────────────────
+output_json() {
+ local region="$1" instances_json="$2"
+ local count
+ count=$(echo "$instances_json" | jq 'length')
+
+ local items="[]"
+
+ if (( count > 0 )); then
+ items=$(echo "$instances_json" | jq --arg region "$region" '[
+ .[] | {
+ instance_id: .InstanceId,
+ type: .InstanceType,
+ state: .State.Name,
+ az: .Placement.AvailabilityZone,
+ private_ip: (.PrivateIpAddress // null),
+ public_ip: (.PublicIpAddress // null),
+ launch_time: (.LaunchTime // null),
+ ami_id: (.ImageId // null),
+ vpc_id: (.VpcId // null),
+ key_name: (.KeyName // null),
+ tags: (.Tags // []),
+ security_groups: (.SecurityGroups // []),
+ region: $region
+ }
+ ]')
+ fi
+
+ local account_id
+ account_id=$(aws sts get-caller-identity --query Account --output text 2>/dev/null || echo "unknown")
+
+ jq -n \
+ --arg account "$account_id" \
+ --arg region "$region" \
+ --arg time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+ --argjson count "$count" \
+ --argjson instances "$items" \
+ '{
+ account: $account,
+ region: $region,
+ time: $time,
+ instance_count: $count,
+ instances: $instances
+ }'
+}
+
+# ── Process a single region ──────────────────────────────────────────
+process_region() {
+ local region="$1"
+ local csv_header_printed="$2"
+
+ log_debug "Querying region: $region"
+
+ local instances_json
+ instances_json=$(get_instances "$region")
+
+ local count
+ count=$(echo "$instances_json" | jq 'length' 2>/dev/null || echo 0)
+
+ if (( count == 0 )) && [[ "$ALL_REGIONS" == "true" ]]; then
+ log_debug "No instances in $region, skipping"
+ return
+ fi
+
+ case "$OUTPUT_FORMAT" in
+ text)
+ output_text "$region" "$instances_json"
+ ;;
+ csv)
+ if [[ "$csv_header_printed" == "false" ]]; then
+ output_csv "$region" "$instances_json"
+ else
+ output_csv "$region" "$instances_json" | tail -n +2
+ fi
+ ;;
+ json)
+ output_json "$region" "$instances_json"
+ ;;
+ esac
+}
+
+# ── Usage ─────────────────────────────────────────────────────────────
+usage() {
+ cat < 0 )); do
+ case "$1" in
+ --region)
+ [[ $# -lt 2 ]] && die "--region requires a value"
+ SCAN_REGION="$2"; shift 2 ;;
+ --all-regions)
+ ALL_REGIONS="true"; shift ;;
+ --state)
+ [[ $# -lt 2 ]] && die "--state requires a value"
+ FILTER_STATE="$2"; shift 2 ;;
+ --tag)
+ [[ $# -lt 2 ]] && die "--tag requires KEY=VALUE"
+ [[ "$2" != *"="* ]] && die "--tag value must be KEY=VALUE"
+ FILTER_TAG_KEY="${2%%=*}"; FILTER_TAG_VALUE="${2#*=}"; shift 2 ;;
+ --type)
+ [[ $# -lt 2 ]] && die "--type requires a value"
+ FILTER_TYPE="$2"; shift 2 ;;
+ --format)
+ [[ $# -lt 2 ]] && die "--format requires a value"
+ OUTPUT_FORMAT="$2"; shift 2 ;;
+ --tag-check)
+ TAG_CHECK="true"; shift ;;
+ --sg-audit)
+ SG_AUDIT="true"; shift ;;
+ --verbose)
+ VERBOSE="true"; shift ;;
+ --no-color)
+ COLOR="never"; shift ;;
+ --help|-h)
+ usage ;;
+ *)
+ die "Unknown option: $1 (see --help)" ;;
+ esac
+ done
+
+ case "$OUTPUT_FORMAT" in
+ text|csv|json) ;;
+ *) die "Invalid --format: $OUTPUT_FORMAT (expected text, csv, json)" ;;
+ esac
+}
+
+# ── Main ──────────────────────────────────────────────────────────────
+main() {
+ parse_args "$@"
+ setup_colors
+ check_deps
+
+ START_TIME=$(date +%s)
+
+ log_debug "Validating AWS credentials..."
+ aws sts get-caller-identity --output text >/dev/null 2>&1 \
+ || die "AWS credentials not configured or expired"
+
+ if [[ "$ALL_REGIONS" == "true" ]]; then
+ log_info "Scanning all enabled regions..."
+ local regions
+ regions=$(get_all_regions)
+
+ if [[ -z "$regions" ]]; then
+ die "Failed to retrieve region list"
+ fi
+
+ local csv_header="false"
+ local json_first="true"
+
+ if [[ "$OUTPUT_FORMAT" == "json" ]]; then
+ echo "["
+ fi
+
+ while IFS= read -r region; do
+ [[ -z "$region" ]] && continue
+ log_info "Scanning $region..."
+
+ if [[ "$OUTPUT_FORMAT" == "json" ]]; then
+ if [[ "$json_first" == "true" ]]; then
+ json_first="false"
+ else
+ echo ","
+ fi
+ fi
+
+ process_region "$region" "$csv_header"
+ csv_header="true"
+ done <<< "$regions"
+
+ if [[ "$OUTPUT_FORMAT" == "json" ]]; then
+ echo "]"
+ fi
+ else
+ log_info "Scanning region: $SCAN_REGION"
+ process_region "$SCAN_REGION" "false"
+ fi
+
+ local elapsed=$(( $(date +%s) - START_TIME ))
+ log_info "Completed in ${elapsed}s"
+}
+
+main "$@"
diff --git a/elasticsearch-exporter.sh b/elasticsearch-exporter.sh
new file mode 100755
index 0000000..137a9a7
--- /dev/null
+++ b/elasticsearch-exporter.sh
@@ -0,0 +1,424 @@
+#!/usr/bin/env bash
+#
+# Elasticsearch Prometheus Metrics Exporter
+#
+# Prometheus textfile collector exporter for Elasticsearch.
+# Uses the Elasticsearch REST API to collect cluster health,
+# node statistics, index counts, JVM memory, search/indexing
+# throughput, circuit breaker state, and shard status.
+#
+# Usage:
+# ./elasticsearch-exporter.sh
+# ./elasticsearch-exporter.sh --textfile
+# ./elasticsearch-exporter.sh --install
+#
+# Parameters:
+# --textfile Write to textfile collector directory
+# --install Create cron job for automatic collection
+# --help Show usage
+#
+# Environment:
+# ES_URL Elasticsearch REST API URL (default: http://localhost:9200)
+# ES_USER Username for basic auth (optional)
+# ES_PASS Password for basic auth (optional)
+# TEXTFILE_DIR Textfile collector directory (default: /var/lib/node_exporter/textfile_collector)
+# CURL_TIMEOUT API request timeout in seconds (default: 10)
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+# Version: 1.0
+#
+# Metrics Exported:
+# Core:
+# - elasticsearch_up
+# - elasticsearch_exporter_info{version}
+#
+# Cluster Health:
+# - elasticsearch_cluster_health{status}
+# - elasticsearch_cluster_nodes_total
+# - elasticsearch_cluster_data_nodes
+# - elasticsearch_cluster_shards_active
+# - elasticsearch_cluster_shards_relocating
+# - elasticsearch_cluster_shards_initializing
+# - elasticsearch_cluster_shards_unassigned
+# - elasticsearch_cluster_pending_tasks
+#
+# Cluster Stats:
+# - elasticsearch_indices_total
+# - elasticsearch_documents_total
+# - elasticsearch_store_size_bytes
+#
+# Node Stats:
+# - elasticsearch_jvm_heap_used_bytes{node}
+# - elasticsearch_jvm_heap_max_bytes{node}
+# - elasticsearch_search_query_total{node}
+# - elasticsearch_indexing_index_total{node}
+# - elasticsearch_circuit_breaker_tripped{node,breaker}
+#
+# Exporter:
+# - elasticsearch_exporter_duration_seconds
+# - elasticsearch_exporter_last_run_timestamp
+
+set -euo pipefail
+
+# --- Configuration ---
+readonly VERSION="1.0"
+readonly SCRIPT_NAME="$(basename "$0")"
+ES_URL="${ES_URL:-http://localhost:9200}"
+ES_USER="${ES_USER:-}"
+ES_PASS="${ES_PASS:-}"
+TEXTFILE_DIR="${TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}"
+CURL_TIMEOUT="${CURL_TIMEOUT:-10}"
+TEXTFILE_MODE=false
+OUTPUT=""
+START_TIME=""
+
+# --- Functions ---
+
+usage() {
+ cat </dev/null; then
+ missing+=("$cmd")
+ fi
+ done
+ if [[ ${#missing[@]} -gt 0 ]]; then
+ echo "ERROR: Missing required commands: ${missing[*]}" >&2
+ echo "Install with: apt install ${missing[*]} OR dnf install ${missing[*]}" >&2
+ exit 1
+ fi
+}
+
+validate_config() {
+ # Strip trailing slash
+ ES_URL="${ES_URL%/}"
+}
+
+api_get() {
+ local endpoint="$1"
+ local curl_args=(-sf --max-time "$CURL_TIMEOUT")
+
+ if [[ -n "$ES_USER" && -n "$ES_PASS" ]]; then
+ curl_args+=(-u "${ES_USER}:${ES_PASS}")
+ fi
+
+ curl "${curl_args[@]}" "${ES_URL}${endpoint}" 2>/dev/null || echo ""
+}
+
+add_metric() {
+ local name="$1"
+ local type="$2"
+ local help="$3"
+ local value="$4"
+ local labels="${5:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name} ${value}
+"
+ fi
+}
+
+add_metric_value() {
+ local name="$1"
+ local value="$2"
+ local labels="${3:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="${name} ${value}
+"
+ fi
+}
+
+collect_cluster_health() {
+ local health_json
+ health_json=$(api_get "/_cluster/health")
+
+ if [[ -z "$health_json" ]]; then
+ add_metric "elasticsearch_up" "gauge" "Elasticsearch reachability (1=up, 0=down)" "0"
+ return 1
+ fi
+
+ add_metric "elasticsearch_up" "gauge" "Elasticsearch reachability (1=up, 0=down)" "1"
+
+ # Cluster health status (green=0, yellow=1, red=2)
+ local status
+ status=$(echo "$health_json" | jq -r '.status // "red"' 2>/dev/null)
+
+ local status_value
+ case "$status" in
+ green) status_value=0 ;;
+ yellow) status_value=1 ;;
+ red) status_value=2 ;;
+ *) status_value=2 ;;
+ esac
+
+ add_metric "elasticsearch_cluster_health" "gauge" "Cluster health status (green=0, yellow=1, red=2)" "$status_value" "status=\"${status}\""
+
+ # Node counts
+ local nodes_total data_nodes
+ nodes_total=$(echo "$health_json" | jq '.number_of_nodes // 0' 2>/dev/null)
+ data_nodes=$(echo "$health_json" | jq '.number_of_data_nodes // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_cluster_nodes_total" "gauge" "Total number of cluster nodes" "${nodes_total:-0}"
+ add_metric "elasticsearch_cluster_data_nodes" "gauge" "Number of data nodes" "${data_nodes:-0}"
+
+ # Shard counts
+ local active_shards relocating initializing unassigned
+ active_shards=$(echo "$health_json" | jq '.active_shards // 0' 2>/dev/null)
+ relocating=$(echo "$health_json" | jq '.relocating_shards // 0' 2>/dev/null)
+ initializing=$(echo "$health_json" | jq '.initializing_shards // 0' 2>/dev/null)
+ unassigned=$(echo "$health_json" | jq '.unassigned_shards // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_cluster_shards_active" "gauge" "Number of active shards" "${active_shards:-0}"
+ add_metric "elasticsearch_cluster_shards_relocating" "gauge" "Number of relocating shards" "${relocating:-0}"
+ add_metric "elasticsearch_cluster_shards_initializing" "gauge" "Number of initializing shards" "${initializing:-0}"
+ add_metric "elasticsearch_cluster_shards_unassigned" "gauge" "Number of unassigned shards" "${unassigned:-0}"
+
+ # Pending tasks
+ local pending_tasks
+ pending_tasks=$(echo "$health_json" | jq '.number_of_pending_tasks // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_cluster_pending_tasks" "gauge" "Number of pending cluster tasks" "${pending_tasks:-0}"
+
+ return 0
+}
+
+collect_cluster_stats() {
+ local stats_json
+ stats_json=$(api_get "/_cluster/stats")
+
+ if [[ -z "$stats_json" ]]; then
+ return
+ fi
+
+ # Indices count
+ local indices_count
+ indices_count=$(echo "$stats_json" | jq '.indices.count // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_indices_total" "gauge" "Total number of indices" "${indices_count:-0}"
+
+ # Document count
+ local doc_count
+ doc_count=$(echo "$stats_json" | jq '.indices.docs.count // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_documents_total" "gauge" "Total number of documents" "${doc_count:-0}"
+
+ # Store size
+ local store_size
+ store_size=$(echo "$stats_json" | jq '.indices.store.size_in_bytes // 0' 2>/dev/null)
+
+ add_metric "elasticsearch_store_size_bytes" "gauge" "Total store size in bytes" "${store_size:-0}"
+}
+
+collect_node_stats() {
+ local nodes_json
+ nodes_json=$(api_get "/_nodes/stats")
+
+ if [[ -z "$nodes_json" ]]; then
+ return
+ fi
+
+ local node_ids
+ node_ids=$(echo "$nodes_json" | jq -r '.nodes | keys[]' 2>/dev/null)
+
+ if [[ -z "$node_ids" ]]; then
+ return
+ fi
+
+ # JVM heap used per node
+ OUTPUT+="# HELP elasticsearch_jvm_heap_used_bytes JVM heap memory used per node
+# TYPE elasticsearch_jvm_heap_used_bytes gauge
+"
+
+ local node_id node_name heap_used
+ for node_id in $node_ids; do
+ node_name=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].name // \"${node_id}\"" 2>/dev/null)
+ heap_used=$(echo "$nodes_json" | jq ".nodes[\"${node_id}\"].jvm.mem.heap_used_in_bytes // 0" 2>/dev/null)
+
+ add_metric_value "elasticsearch_jvm_heap_used_bytes" "${heap_used:-0}" "node=\"${node_name}\""
+ done
+
+ # JVM heap max per node
+ OUTPUT+="# HELP elasticsearch_jvm_heap_max_bytes JVM heap memory max per node
+# TYPE elasticsearch_jvm_heap_max_bytes gauge
+"
+
+ local heap_max
+ for node_id in $node_ids; do
+ node_name=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].name // \"${node_id}\"" 2>/dev/null)
+ heap_max=$(echo "$nodes_json" | jq ".nodes[\"${node_id}\"].jvm.mem.heap_max_in_bytes // 0" 2>/dev/null)
+
+ add_metric_value "elasticsearch_jvm_heap_max_bytes" "${heap_max:-0}" "node=\"${node_name}\""
+ done
+
+ # Search query total per node
+ OUTPUT+="# HELP elasticsearch_search_query_total Total search queries per node
+# TYPE elasticsearch_search_query_total gauge
+"
+
+ local query_total
+ for node_id in $node_ids; do
+ node_name=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].name // \"${node_id}\"" 2>/dev/null)
+ query_total=$(echo "$nodes_json" | jq ".nodes[\"${node_id}\"].indices.search.query_total // 0" 2>/dev/null)
+
+ add_metric_value "elasticsearch_search_query_total" "${query_total:-0}" "node=\"${node_name}\""
+ done
+
+ # Indexing index total per node
+ OUTPUT+="# HELP elasticsearch_indexing_index_total Total indexing operations per node
+# TYPE elasticsearch_indexing_index_total gauge
+"
+
+ local index_total
+ for node_id in $node_ids; do
+ node_name=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].name // \"${node_id}\"" 2>/dev/null)
+ index_total=$(echo "$nodes_json" | jq ".nodes[\"${node_id}\"].indices.indexing.index_total // 0" 2>/dev/null)
+
+ add_metric_value "elasticsearch_indexing_index_total" "${index_total:-0}" "node=\"${node_name}\""
+ done
+
+ # Circuit breaker trips per node per breaker type
+ OUTPUT+="# HELP elasticsearch_circuit_breaker_tripped Circuit breaker trip count per node and breaker
+# TYPE elasticsearch_circuit_breaker_tripped gauge
+"
+
+ local breaker_names breaker_name tripped
+ for node_id in $node_ids; do
+ node_name=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].name // \"${node_id}\"" 2>/dev/null)
+ breaker_names=$(echo "$nodes_json" | jq -r ".nodes[\"${node_id}\"].breakers | keys[]" 2>/dev/null)
+
+ for breaker_name in $breaker_names; do
+ tripped=$(echo "$nodes_json" | jq ".nodes[\"${node_id}\"].breakers[\"${breaker_name}\"].tripped // 0" 2>/dev/null)
+
+ add_metric_value "elasticsearch_circuit_breaker_tripped" "${tripped:-0}" "node=\"${node_name}\",breaker=\"${breaker_name}\""
+ done
+ done
+}
+
+write_output() {
+ if [[ "$TEXTFILE_MODE" == true ]]; then
+ local output_file="${TEXTFILE_DIR}/elasticsearch.prom"
+ local temp_file="${output_file}.$$"
+
+ mkdir -p "$TEXTFILE_DIR"
+ echo "$OUTPUT" > "$temp_file"
+ mv "$temp_file" "$output_file"
+ else
+ echo "$OUTPUT"
+ fi
+}
+
+install_cron() {
+ if [[ $EUID -ne 0 ]]; then
+ echo "ERROR: --install requires root" >&2
+ exit 1
+ fi
+
+ local script_path
+ script_path=$(readlink -f "$0")
+
+ local env_vars=""
+ env_vars+="ES_URL=${ES_URL}\n"
+ if [[ -n "$ES_USER" ]]; then
+ env_vars+="ES_USER=${ES_USER}\n"
+ fi
+ if [[ -n "$ES_PASS" ]]; then
+ env_vars+="ES_PASS=${ES_PASS}\n"
+ fi
+ env_vars+="TEXTFILE_DIR=${TEXTFILE_DIR}"
+
+ cat > /etc/cron.d/elasticsearch-exporter </dev/null
+EOF
+
+ chmod 644 /etc/cron.d/elasticsearch-exporter
+ echo "Installed cron job: /etc/cron.d/elasticsearch-exporter"
+ echo "Metrics will be written to: ${TEXTFILE_DIR}/elasticsearch.prom"
+}
+
+# --- Main ---
+
+main() {
+ # Parse arguments
+ for arg in "$@"; do
+ case "$arg" in
+ --textfile) TEXTFILE_MODE=true ;;
+ --install)
+ check_dependencies
+ validate_config
+ install_cron
+ exit 0
+ ;;
+ --help|-h) usage ;;
+ *) echo "Unknown option: $arg" >&2; usage ;;
+ esac
+ done
+
+ check_dependencies
+ validate_config
+
+ START_TIME=$(date +%s%N)
+
+ # Exporter info
+ add_metric "elasticsearch_exporter_info" "gauge" "Exporter version information" "1" "version=\"${VERSION}\""
+
+ # Collect metrics
+ if collect_cluster_health; then
+ collect_cluster_stats
+ collect_node_stats
+ fi
+
+ # Exporter performance
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
+ add_metric "elasticsearch_exporter_duration_seconds" "gauge" "Time to generate all metrics" "$duration"
+ add_metric "elasticsearch_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run" "$(date +%s)"
+
+ write_output
+}
+
+main "$@"
diff --git a/exchange-metrics.ps1 b/exchange-metrics.ps1
index 98acaef..e7cd223 100644
--- a/exchange-metrics.ps1
+++ b/exchange-metrics.ps1
@@ -1,6 +1,43 @@
-# Exchange Metrics Collector - Outputs Prometheus-compatible metrics
+# Exchange Metrics Collector - Outputs Prometheus-compatible metrics
# Requires Exchange Management Shell and appropriate permissions
+param(
+ [switch]$InstallScheduledTask,
+ [int]$TaskIntervalMinutes = 5
+)
+
+if ($InstallScheduledTask) {
+ $taskName = "ExchangeMetricsExporter"
+ $existingTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+
+ if (-not $existingTask) {
+ $taskAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$($MyInvocation.MyCommand.Path)`""
+
+ if (-not $TaskIntervalMinutes -or $TaskIntervalMinutes -le 0) {
+ throw "TaskIntervalMinutes must be a positive integer"
+ }
+
+ $taskTrigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes $TaskIntervalMinutes) -RepetitionDuration (New-TimeSpan -Days 365)
+ $taskPrincipal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest
+
+ try {
+ Write-Host "Creating scheduled task: $taskName"
+ Register-ScheduledTask -TaskName $taskName -Action $taskAction -Trigger $taskTrigger -Principal $taskPrincipal -Description "Exports Exchange metrics for Prometheus every $TaskIntervalMinutes minutes"
+
+ $createdTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+ if (-not $createdTask) {
+ throw "Failed to verify scheduled task creation"
+ }
+ Write-Host "Successfully created scheduled task: $taskName" -ForegroundColor Green
+ } catch {
+ Write-Error "Failed to create auto-start task: $($_.Exception.Message)"
+ throw
+ }
+ } else {
+ Write-Host "Scheduled task '$taskName' already exists, skipping creation"
+ }
+}
+
$StartTime = Get-Date
$Hostname = $env:COMPUTERNAME
diff --git a/expand-drive.sh b/expand-drive.sh
index f85c145..d699e3a 100755
--- a/expand-drive.sh
+++ b/expand-drive.sh
@@ -7,10 +7,13 @@
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
#### License: MIT ####
-#### Version: 2.3 ####
+#### Version: 2.4 ####
#### ####
#### Usage: sudo ./expand-drive.sh ####
#############################################################
+# v2.4 changes:
+# - Fixed: grep in pipeline crashes under set -euo pipefail when no matches found. Added || true guard
+#############################################################
# Set strict error handling:
# -e: Exit immediately if a command exits with a non-zero status
@@ -189,7 +192,7 @@ process_partition() {
# Extract partition number from device path (e.g., extract "1" from "/dev/sda1")
local part_num
- part_num=$(echo "$partition" | grep -o '[0-9]\+$' | tail -1)
+ part_num=$(echo "$partition" | { grep -o '[0-9]\+$' || true; } | tail -1)
if [ -z "$part_num" ]; then
log_error "Could not extract partition number from $partition"
return 1
@@ -293,7 +296,7 @@ main() {
# Get list of all disk devices in the system using lsblk
# Filter for disk type and extract device names
local devices
- devices=$($LSBLK_PATH -pln -o NAME,TYPE | grep "disk" | cut -d' ' -f1)
+ devices=$($LSBLK_PATH -pln -o NAME,TYPE | { grep "disk" || true; } | cut -d' ' -f1)
# Verify we found at least one disk device
if [ -z "$devices" ]; then
diff --git a/fapolicyd-log-analyzer.sh b/fapolicyd-log-analyzer.sh
new file mode 100644
index 0000000..d6f30c5
--- /dev/null
+++ b/fapolicyd-log-analyzer.sh
@@ -0,0 +1,387 @@
+#!/bin/bash
+
+#############################################################
+#### fapolicyd Log Analyzer Script ####
+#### Parses denial logs and suggests fix commands ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### To use this script chmod it to 755 ####
+#### or simply type bash ####
+#############################################################
+
+# ── Colors ────────────────────────────────────────────────
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m' # No Color
+
+# ── Defaults ──────────────────────────────────────────────
+MODE="recent"
+OUTPUT_FILE=""
+QUIET=0
+TOTAL_DENIALS=0
+UNIQUE_FILES=0
+SUGGESTED_FIXES=0
+
+# ── Functions ─────────────────────────────────────────────
+
+usage() {
+ echo -e "${BOLD}fapolicyd Log Analyzer${NC}"
+ echo ""
+ echo "Usage: $0 [OPTIONS]"
+ echo ""
+ echo "Options:"
+ echo " --help Show this help message"
+ echo " --recent Analyze denials from the last hour only (default)"
+ echo " --all Analyze all denials in the log"
+ echo " --output FILE Save suggested fixes to FILE"
+ echo " --quiet Show suggestions only, suppress raw denial lines"
+ echo ""
+ echo "Examples:"
+ echo " sudo bash $0 --recent"
+ echo " sudo bash $0 --all --output fixes.txt"
+ echo " sudo bash $0 --quiet --output /tmp/fixes.txt"
+ exit 0
+}
+
+check_root() {
+ if [[ $EUID -ne 0 ]]; then
+ echo -e "${RED}Error: This script must be run as root.${NC}"
+ echo "Please run with: sudo bash $0"
+ exit 1
+ fi
+}
+
+check_fapolicyd() {
+ if ! command -v fapolicyd-cli &>/dev/null; then
+ echo -e "${RED}Error: fapolicyd does not appear to be installed.${NC}"
+ echo -e "${YELLOW}Install with: dnf install fapolicyd${NC}"
+ exit 1
+ fi
+
+ if ! systemctl is-active --quiet fapolicyd 2>/dev/null; then
+ echo -e "${YELLOW}Warning: fapolicyd service is not currently running.${NC}"
+ echo -e "${CYAN}Continuing to analyze existing log entries...${NC}"
+ echo ""
+ fi
+}
+
+output_line() {
+ local line="$1"
+ echo -e "$line"
+ if [[ -n "$OUTPUT_FILE" ]]; then
+ echo -e "$line" | sed 's/\x1b\[[0-9;]*m//g' >> "$OUTPUT_FILE"
+ fi
+}
+
+# ── fapolicyd Analysis ───────────────────────────────────
+
+parse_fapolicyd_denial() {
+ local line="$1"
+
+ local dec perm fname exe trust pid
+ dec=$(echo "$line" | grep -oP 'dec=\K[^ ]+')
+ perm=$(echo "$line" | grep -oP 'perm=\K[^ ]+')
+ fname=$(echo "$line" | grep -oP 'fname=\K[^ ]+')
+ exe=$(echo "$line" | grep -oP 'exe=\K[^ ]+')
+ trust=$(echo "$line" | grep -oP 'trust=\K[^ ]+')
+ pid=$(echo "$line" | grep -oP 'pid=\K[^ ]+')
+
+ if [[ $QUIET -eq 0 ]]; then
+ output_line "${RED}DENIAL:${NC} $line"
+ fi
+
+ [[ -n "$dec" ]] && output_line "${CYAN} Decision:${NC} $dec"
+ [[ -n "$perm" ]] && output_line "${CYAN} Permission:${NC} $perm"
+ [[ -n "$fname" ]] && output_line "${CYAN} File:${NC} $fname"
+ [[ -n "$exe" ]] && output_line "${CYAN} Executable:${NC} $exe"
+ [[ -n "$trust" ]] && output_line "${CYAN} Trust status:${NC} $trust"
+ [[ -n "$pid" ]] && output_line "${CYAN} PID:${NC} $pid"
+
+ suggest_fapolicyd_fix "$fname" "$exe" "$perm" "$trust"
+ output_line ""
+}
+
+suggest_fapolicyd_fix() {
+ local fname="$1" exe="$2" perm="$3" trust="$4"
+
+ ((SUGGESTED_FIXES++))
+
+ if [[ -n "$fname" ]]; then
+ # Check current trust status
+ output_line "${GREEN} Suggested fixes:${NC}"
+
+ # Trust the file
+ output_line "${GREEN} 1. Add file to trust database:${NC}"
+ output_line "${GREEN} fapolicyd-cli --file add ${fname}${NC}"
+ output_line "${GREEN} fapolicyd-cli --update${NC}"
+
+ # Check trust
+ output_line "${GREEN} 2. Verify trust status:${NC}"
+ output_line "${GREEN} fapolicyd-cli --check-path ${fname}${NC}"
+
+ # If the file is a script or binary from a known package
+ if command -v rpm &>/dev/null; then
+ local pkg
+ pkg=$(rpm -qf "$fname" 2>/dev/null)
+ if [[ $? -eq 0 && -n "$pkg" ]]; then
+ output_line "${CYAN} Note: File belongs to package: ${pkg}${NC}"
+ output_line "${YELLOW} If the file was modified after install, consider:${NC}"
+ output_line "${GREEN} rpm --restore ${pkg}${NC}"
+ output_line "${GREEN} fapolicyd-cli --update${NC}"
+ fi
+ fi
+
+ # If it looks like a shared library
+ if [[ "$fname" == *.so* ]]; then
+ output_line "${YELLOW} Library denial — also check:${NC}"
+ output_line "${GREEN} ldconfig${NC}"
+ output_line "${GREEN} fapolicyd-cli --update${NC}"
+ fi
+ fi
+
+ # Suggest rule-based approach
+ if [[ -n "$exe" && -n "$perm" ]]; then
+ output_line "${GREEN} 3. Or add a custom rule in /etc/fapolicyd/rules.d/:${NC}"
+ output_line "${GREEN} allow ${perm} exe=${exe} : all${NC}"
+ fi
+}
+
+categorize_fapolicyd_denial() {
+ local line="$1"
+ local perm fname
+
+ perm=$(echo "$line" | grep -oP 'perm=\K[^ ]+')
+ fname=$(echo "$line" | grep -oP 'fname=\K[^ ]+')
+
+ case "$perm" in
+ execute)
+ if [[ "$fname" == *.so* ]]; then
+ echo "library"
+ else
+ echo "execute"
+ fi
+ ;;
+ open)
+ echo "open"
+ ;;
+ *)
+ echo "other"
+ ;;
+ esac
+}
+
+analyze_fapolicyd() {
+ output_line "${BOLD}═══════════════════════════════════════════════════${NC}"
+ output_line "${BOLD} fapolicyd Log Analysis${NC}"
+ output_line "${BOLD}═══════════════════════════════════════════════════${NC}"
+ output_line ""
+
+ # Show daemon status
+ local daemon_status
+ daemon_status=$(systemctl is-active fapolicyd 2>/dev/null)
+ output_line "${CYAN}fapolicyd status:${NC} $daemon_status"
+
+ # Show integrity setting
+ if [[ -f /etc/fapolicyd/fapolicyd.conf ]]; then
+ local integrity
+ integrity=$(grep -oP '^\s*integrity\s*=\s*\K.*' /etc/fapolicyd/fapolicyd.conf 2>/dev/null)
+ [[ -n "$integrity" ]] && output_line "${CYAN}Integrity mode:${NC} $integrity"
+ fi
+
+ # Show trust database stats
+ if command -v fapolicyd-cli &>/dev/null; then
+ local trust_count
+ trust_count=$(fapolicyd-cli --dump-db 2>/dev/null | wc -l)
+ [[ -n "$trust_count" ]] && output_line "${CYAN}Trusted files:${NC} $trust_count"
+ fi
+ output_line ""
+
+ # Gather denials from audit log
+ local denials=""
+
+ if [[ ! -f /var/log/audit/audit.log ]]; then
+ output_line "${RED}Error: Cannot find /var/log/audit/audit.log${NC}"
+ output_line "${YELLOW}Ensure auditd is running: systemctl start auditd${NC}"
+ return
+ fi
+
+ if [[ "$MODE" == "recent" ]]; then
+ if command -v ausearch &>/dev/null; then
+ denials=$(ausearch -m FANOTIFY -ts recent 2>/dev/null | grep "type=FANOTIFY")
+ fi
+ # Fallback to manual log parsing
+ if [[ -z "$denials" ]]; then
+ local one_hour_ago
+ one_hour_ago=$(date -d '1 hour ago' '+%s' 2>/dev/null)
+ if [[ -n "$one_hour_ago" ]]; then
+ denials=$(awk -v cutoff="$one_hour_ago" '
+ /type=FANOTIFY/ && /dec=deny/ {
+ match($0, /msg=audit\(([0-9]+)\./, arr)
+ if (arr[1] >= cutoff) print
+ }
+ ' /var/log/audit/audit.log)
+ fi
+ fi
+ else
+ denials=$(grep "type=FANOTIFY" /var/log/audit/audit.log | grep "dec=deny")
+ fi
+
+ if [[ -z "$denials" ]]; then
+ output_line "${GREEN}No fapolicyd denials found.${NC}"
+ output_line ""
+ return
+ fi
+
+ # Group denials by category
+ declare -A categories
+ local denial_count=0
+ local -A seen_files
+
+ while IFS= read -r line; do
+ [[ -z "$line" ]] && continue
+ ((denial_count++))
+ local category
+ category=$(categorize_fapolicyd_denial "$line")
+ categories["$category"]+="$line"$'\n'
+
+ local f
+ f=$(echo "$line" | grep -oP 'fname=\K[^ ]+')
+ [[ -n "$f" ]] && seen_files["$f"]=1
+ done <<< "$denials"
+
+ TOTAL_DENIALS=$denial_count
+ UNIQUE_FILES=${#seen_files[@]}
+
+ # Display grouped results
+ for category in "execute" "library" "open" "other"; do
+ if [[ -n "${categories[$category]}" ]]; then
+ local label
+ case "$category" in
+ execute) label="Execution Denials" ;;
+ library) label="Library Load Denials" ;;
+ open) label="File Open Denials" ;;
+ other) label="Other Denials" ;;
+ esac
+
+ output_line "${BOLD}── ${label} ──────────────────────────────────${NC}"
+ output_line ""
+
+ while IFS= read -r denial_line; do
+ [[ -z "$denial_line" ]] && continue
+ parse_fapolicyd_denial "$denial_line"
+ done <<< "${categories[$category]}"
+ fi
+ done
+
+ # Show bulk fix suggestions
+ if [[ ${#seen_files[@]} -gt 0 ]]; then
+ output_line "${BOLD}── Bulk Fix Commands ────────────────────────────${NC}"
+ output_line ""
+ output_line "${YELLOW}To trust all denied files at once:${NC}"
+ for f in "${!seen_files[@]}"; do
+ output_line "${GREEN} fapolicyd-cli --file add ${f}${NC}"
+ done
+ output_line "${GREEN} fapolicyd-cli --update${NC}"
+ output_line ""
+ fi
+
+ # Rule file reference
+ output_line "${BOLD}── Rule File Reference ──────────────────────────${NC}"
+ output_line ""
+ output_line "${CYAN}Rules are loaded from:${NC}"
+ if [[ -d /etc/fapolicyd/rules.d ]]; then
+ output_line " /etc/fapolicyd/rules.d/ (drop-in directory)"
+ local rule_files
+ rule_files=$(ls /etc/fapolicyd/rules.d/ 2>/dev/null)
+ if [[ -n "$rule_files" ]]; then
+ output_line "${CYAN} Current rule files:${NC}"
+ while IFS= read -r rf; do
+ output_line " $rf"
+ done <<< "$rule_files"
+ fi
+ fi
+ if [[ -f /etc/fapolicyd/fapolicyd.rules ]]; then
+ output_line " /etc/fapolicyd/fapolicyd.rules (compiled rules)"
+ fi
+ output_line ""
+ output_line "${YELLOW}After making changes, restart the daemon:${NC}"
+ output_line "${GREEN} systemctl restart fapolicyd${NC}"
+ output_line ""
+}
+
+# ── Summary ───────────────────────────────────────────────
+
+print_summary() {
+ output_line "${BOLD}═══════════════════════════════════════════════════${NC}"
+ output_line "${BOLD} Summary${NC}"
+ output_line "${BOLD}═══════════════════════════════════════════════════${NC}"
+ output_line ""
+ output_line " Total denials found: ${BOLD}${TOTAL_DENIALS}${NC}"
+ output_line " Unique files denied: ${BOLD}${UNIQUE_FILES}${NC}"
+ output_line " Suggested fixes: ${BOLD}${SUGGESTED_FIXES}${NC}"
+ output_line ""
+
+ if [[ -n "$OUTPUT_FILE" ]]; then
+ output_line "${GREEN}Suggestions saved to: ${OUTPUT_FILE}${NC}"
+ output_line ""
+ fi
+}
+
+# ── Parse Arguments ───────────────────────────────────────
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --help|-h)
+ usage
+ ;;
+ --recent)
+ MODE="recent"
+ shift
+ ;;
+ --all)
+ MODE="all"
+ shift
+ ;;
+ --output)
+ if [[ -z "$2" || "$2" == --* ]]; then
+ echo -e "${RED}Error: --output requires a filename argument.${NC}"
+ exit 1
+ fi
+ OUTPUT_FILE="$2"
+ shift 2
+ ;;
+ --quiet|-q)
+ QUIET=1
+ shift
+ ;;
+ *)
+ echo -e "${RED}Unknown option: $1${NC}"
+ echo "Use --help for usage information."
+ exit 1
+ ;;
+ esac
+done
+
+# ── Main ──────────────────────────────────────────────────
+
+check_root
+
+# Clear output file if specified
+if [[ -n "$OUTPUT_FILE" ]]; then
+ > "$OUTPUT_FILE"
+fi
+
+echo -e "${BOLD}fapolicyd Log Analyzer v1.00${NC}"
+echo -e "${CYAN}Mode: ${MODE}${NC}"
+echo ""
+
+check_fapolicyd
+analyze_fapolicyd
+print_summary
diff --git a/file-permissions-audit.sh b/file-permissions-audit.sh
new file mode 100644
index 0000000..81ab5fb
--- /dev/null
+++ b/file-permissions-audit.sh
@@ -0,0 +1,354 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### file-permissions-audit.sh — Find world-writable files, SUID/SGID binaries, ####
+#### and files owned by nobody or with no valid owner ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./file-permissions-audit.sh ####
+#### ./file-permissions-audit.sh --scan-dirs /usr /bin /home ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+SCAN_DIRS="${SCAN_DIRS:-/usr /bin /sbin /var /opt /home /tmp}"
+EXCLUDE_PATHS=()
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+COUNT_WORLD_WRITABLE=0
+COUNT_SUID=0
+COUNT_SGID=0
+COUNT_NOBODY=0
+COUNT_UNOWNED=0
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ YELLOW='\033[0;33m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ else
+ RED="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${CYAN}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+
+# ── Helpers ───────────────────────────────────────────────────────────
+section_header() {
+ echo ""
+ echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
+ echo ""
+}
+
+build_exclude_args() {
+ local args=()
+ for path in "${EXCLUDE_PATHS[@]+"${EXCLUDE_PATHS[@]}"}"; do
+ args+=(-not -path "${path}/*")
+ done
+ # Always exclude /proc and /sys
+ args+=(-not -path "/proc/*" -not -path "/sys/*")
+ echo "${args[@]}"
+}
+
+get_file_info() {
+ local file="$1"
+ local octal symbolic owner group ftype
+
+ octal=$(stat -c '%a' "$file" 2>/dev/null || echo "????")
+ symbolic=$(stat -c '%A' "$file" 2>/dev/null || echo "??????????")
+ owner=$(stat -c '%U' "$file" 2>/dev/null || echo "UNKNOWN")
+ group=$(stat -c '%G' "$file" 2>/dev/null || echo "UNKNOWN")
+
+ if [[ -d "$file" ]]; then
+ ftype="dir"
+ elif [[ -L "$file" ]]; then
+ ftype="link"
+ else
+ ftype="file"
+ fi
+
+ echo "${octal} ${symbolic} ${owner}:${group} ${ftype}"
+}
+
+print_file_entry() {
+ local color="$1"
+ local file="$2"
+ local info
+ info=$(get_file_info "$file")
+ local octal symbolic ownership ftype
+ octal=$(echo "$info" | awk '{print $1}')
+ symbolic=$(echo "$info" | awk '{print $2}')
+ ownership=$(echo "$info" | awk '{print $3}')
+ ftype=$(echo "$info" | awk '{print $4}')
+
+ printf " %b%-4s %-11s %-20s %-6s%b %s\n" "$color" "$octal" "$symbolic" "$ownership" "$ftype" "$RESET" "$file"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SCAN FUNCTIONS
+# ══════════════════════════════════════════════════════════════════════
+
+scan_world_writable() {
+ section_header "World-Writable Files & Directories"
+
+ printf " ${BOLD}%-4s %-11s %-20s %-6s${RESET} %s\n" "PERM" "MODE" "OWNER:GROUP" "TYPE" "PATH"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=0
+ local exclude_args
+ exclude_args=$(build_exclude_args)
+
+ for dir in $SCAN_DIRS; do
+ [[ -d "$dir" ]] || continue
+ # shellcheck disable=SC2086
+ while IFS= read -r file; do
+ [[ -z "$file" ]] && continue
+ print_file_entry "$CYAN" "$file"
+ count=$((count + 1))
+ done < <(find "$dir" -xdev -perm -0002 -not -type l $exclude_args 2>/dev/null)
+ done
+
+ COUNT_WORLD_WRITABLE=$count
+ echo ""
+ log "Found ${count} world-writable entries"
+}
+
+scan_suid() {
+ section_header "SUID Binaries"
+
+ printf " ${BOLD}%-4s %-11s %-20s %-6s${RESET} %s\n" "PERM" "MODE" "OWNER:GROUP" "TYPE" "PATH"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=0
+ local exclude_args
+ exclude_args=$(build_exclude_args)
+
+ for dir in $SCAN_DIRS; do
+ [[ -d "$dir" ]] || continue
+ # shellcheck disable=SC2086
+ while IFS= read -r file; do
+ [[ -z "$file" ]] && continue
+ local owner
+ owner=$(stat -c '%U' "$file" 2>/dev/null || echo "UNKNOWN")
+ local color="$YELLOW"
+ if [[ "$owner" == "root" ]]; then
+ color="$RED"
+ fi
+ print_file_entry "$color" "$file"
+ count=$((count + 1))
+ done < <(find "$dir" -xdev -type f -perm -4000 $exclude_args 2>/dev/null)
+ done
+
+ COUNT_SUID=$count
+ echo ""
+ log "Found ${count} SUID binaries"
+}
+
+scan_sgid() {
+ section_header "SGID Binaries"
+
+ printf " ${BOLD}%-4s %-11s %-20s %-6s${RESET} %s\n" "PERM" "MODE" "OWNER:GROUP" "TYPE" "PATH"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=0
+ local exclude_args
+ exclude_args=$(build_exclude_args)
+
+ for dir in $SCAN_DIRS; do
+ [[ -d "$dir" ]] || continue
+ # shellcheck disable=SC2086
+ while IFS= read -r file; do
+ [[ -z "$file" ]] && continue
+ print_file_entry "$YELLOW" "$file"
+ count=$((count + 1))
+ done < <(find "$dir" -xdev -type f -perm -2000 $exclude_args 2>/dev/null)
+ done
+
+ COUNT_SGID=$count
+ echo ""
+ log "Found ${count} SGID binaries"
+}
+
+scan_nobody() {
+ section_header "Files Owned by nobody/nogroup"
+
+ printf " ${BOLD}%-4s %-11s %-20s %-6s${RESET} %s\n" "PERM" "MODE" "OWNER:GROUP" "TYPE" "PATH"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=0
+ local exclude_args
+ exclude_args=$(build_exclude_args)
+
+ for dir in $SCAN_DIRS; do
+ [[ -d "$dir" ]] || continue
+ # shellcheck disable=SC2086
+ while IFS= read -r file; do
+ [[ -z "$file" ]] && continue
+ print_file_entry "$YELLOW" "$file"
+ count=$((count + 1))
+ done < <(find "$dir" -xdev \( -user nobody -o -group nogroup \) $exclude_args 2>/dev/null)
+ done
+
+ COUNT_NOBODY=$count
+ echo ""
+ log "Found ${count} files owned by nobody/nogroup"
+}
+
+scan_unowned() {
+ section_header "Files With No Valid Owner"
+
+ printf " ${BOLD}%-4s %-11s %-20s %-6s${RESET} %s\n" "PERM" "MODE" "OWNER:GROUP" "TYPE" "PATH"
+ printf " %s\n" "$(printf '%.0s─' {1..78})"
+
+ local count=0
+ local exclude_args
+ exclude_args=$(build_exclude_args)
+
+ for dir in $SCAN_DIRS; do
+ [[ -d "$dir" ]] || continue
+ # shellcheck disable=SC2086
+ while IFS= read -r file; do
+ [[ -z "$file" ]] && continue
+ print_file_entry "$RED" "$file"
+ count=$((count + 1))
+ done < <(find "$dir" -xdev \( -nouser -o -nogroup \) $exclude_args 2>/dev/null)
+ done
+
+ COUNT_UNOWNED=$count
+ echo ""
+ log "Found ${count} files with no valid owner"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SUMMARY
+# ══════════════════════════════════════════════════════════════════════
+
+print_summary() {
+ echo ""
+ echo -e " ${BOLD}══════════════════════════════════════════${RESET}"
+ echo -e " ${BOLD}Permissions Audit Summary${RESET}"
+ echo -e " ${BOLD}══════════════════════════════════════════${RESET}"
+ echo ""
+
+ printf " %-30s %b\n" "World-writable:" "${CYAN}${COUNT_WORLD_WRITABLE}${RESET}"
+ printf " %-30s %b\n" "SUID binaries:" "${RED}${COUNT_SUID}${RESET}"
+ printf " %-30s %b\n" "SGID binaries:" "${YELLOW}${COUNT_SGID}${RESET}"
+ printf " %-30s %b\n" "Owned by nobody/nogroup:" "${YELLOW}${COUNT_NOBODY}${RESET}"
+ printf " %-30s %b\n" "No valid owner:" "${RED}${COUNT_UNOWNED}${RESET}"
+
+ local total=$((COUNT_WORLD_WRITABLE + COUNT_SUID + COUNT_SGID + COUNT_NOBODY + COUNT_UNOWNED))
+ echo ""
+ printf " %-30s %d\n" "Total findings:" "$total"
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# USAGE
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2
+ exit 1 ;;
+ esac
+ done
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+main() {
+ parse_args "$@"
+ setup_colors
+
+ echo ""
+ echo -e "${BOLD}File Permissions Audit — $(hostname -f 2>/dev/null || hostname)${RESET}"
+ echo -e "${DIM}$(date '+%Y-%m-%d %H:%M:%S %Z')${RESET}"
+ echo -e "${DIM}Scanning: ${SCAN_DIRS}${RESET}"
+
+ scan_world_writable
+ scan_suid
+ scan_sgid
+ scan_nobody
+ scan_unowned
+ print_summary
+}
+
+main "$@"
diff --git a/firewall-rule-diff.sh b/firewall-rule-diff.sh
new file mode 100644
index 0000000..8c52884
--- /dev/null
+++ b/firewall-rule-diff.sh
@@ -0,0 +1,620 @@
+#!/usr/bin/env bash
+
+######################################################################################
+#### firewall-rule-diff.sh — Detect firewall rule drift against a saved baseline ####
+#### Supports UFW, iptables, and nftables. Saves snapshots, diffs against ####
+#### baseline, exports Prometheus metrics via textfile collector. ####
+#### Requires: bash 4+, diff, coreutils ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### sudo ./firewall-rule-diff.sh --save ####
+#### sudo ./firewall-rule-diff.sh --check ####
+#### ####
+#### See --help for all options. ####
+######################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+MODE="" # save or check
+BACKEND="" # auto-detect: ufw, iptables, nftables
+BASELINE_DIR="/etc/firewall-baseline"
+MAX_AGE_DAYS=30
+TEXTFILE_MODE=false
+PROM_FILE="/var/lib/node_exporter/firewall_drift.prom"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+JUNIT_FILE="${JUNIT_FILE:-firewall-drift-results.xml}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+PASS=0
+FAIL=0
+WARN=0
+TOTAL=0
+RESULTS=()
+START_TIME=""
+RULES_ADDED=0
+RULES_REMOVED=0
+RULES_TOTAL=0
+DRIFT_DETECTED=0
+BASELINE_AGE=0
+DETECTED_BACKEND=""
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
+
+# ── Test Result Recording ─────────────────────────────────────────────
+record_pass() {
+ local name="$1"
+ local detail="${2:-}"
+ ((PASS++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("PASS|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok ${TOTAL} - ${name}"
+ elif [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e " ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}"
+ fi
+}
+
+record_fail() {
+ local name="$1"
+ local detail="${2:-}"
+ ((FAIL++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("FAIL|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "not ok ${TOTAL} - ${name}"
+ [[ -n "$detail" ]] && echo " # ${detail}"
+ elif [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e " ${RED}✗${RESET} ${name}${detail:+ — ${detail}}"
+ fi
+}
+
+record_warn() {
+ local name="$1"
+ local detail="${2:-}"
+ ((WARN++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("WARN|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok ${TOTAL} - ${name} # SKIP ${detail}"
+ elif [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e " ${YELLOW}⊘${RESET} ${name}${detail:+ — ${detail}}"
+ fi
+}
+
+# ── Help ──────────────────────────────────────────────────────────────
+show_help() {
+ cat <<'EOF'
+Usage: firewall-rule-diff.sh [OPTIONS]
+
+Detect firewall rule drift by comparing current state against a saved baseline.
+Supports UFW, iptables, and nftables with auto-detection.
+
+Modes:
+ --save Save current firewall rules as new baseline
+ --check Compare current rules against baseline (default)
+
+Options:
+ --backend BACKEND Force backend: ufw, iptables, nftables (default: auto-detect)
+ --baseline-dir PATH Baseline storage directory (default: /etc/firewall-baseline/)
+ --max-age DAYS Warn if baseline older than N days (default: 30)
+ --textfile Write Prometheus metrics to textfile collector
+ --prom-file PATH Textfile path (default: /var/lib/node_exporter/firewall_drift.prom)
+ --format FORMAT Output: text (default), tap, junit
+ --junit-file FILE JUnit output path (default: firewall-drift-results.xml)
+ --verbose Show debug output
+ --no-color Disable colored output
+ -h, --help Show this help
+
+Examples:
+ sudo ./firewall-rule-diff.sh --save
+ sudo ./firewall-rule-diff.sh --check
+ sudo ./firewall-rule-diff.sh --check --textfile
+ sudo ./firewall-rule-diff.sh --backend iptables --check
+ sudo ./firewall-rule-diff.sh --check --max-age 7
+EOF
+ exit 0
+}
+
+# ── Parse Arguments ───────────────────────────────────────────────────
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --save) MODE="save"; shift ;;
+ --check) MODE="check"; shift ;;
+ --backend) BACKEND="$2"; shift 2 ;;
+ --baseline-dir) BASELINE_DIR="$2"; shift 2 ;;
+ --max-age) MAX_AGE_DAYS="$2"; shift 2 ;;
+ --textfile) TEXTFILE_MODE=true; shift ;;
+ --prom-file) PROM_FILE="$2"; shift 2 ;;
+ --format) OUTPUT_FORMAT="$2"; shift 2 ;;
+ --junit-file) JUNIT_FILE="$2"; shift 2 ;;
+ --verbose) VERBOSE=true; shift ;;
+ --no-color) COLOR="never"; shift ;;
+ -h|--help) show_help ;;
+ *) err "Unknown option: $1"; echo "Run with --help for usage."; exit 1 ;;
+ esac
+ done
+
+ if [[ -z "$MODE" ]]; then
+ MODE="check"
+ fi
+}
+
+# ── Detect Backend ────────────────────────────────────────────────────
+detect_backend() {
+ if [[ -n "$BACKEND" ]]; then
+ DETECTED_BACKEND="$BACKEND"
+ verbose "Backend forced: ${DETECTED_BACKEND}"
+ return
+ fi
+
+ if command -v ufw &>/dev/null && ufw status &>/dev/null; then
+ local ufw_status
+ ufw_status=$(ufw status 2>/dev/null | head -1)
+ if [[ "$ufw_status" == *"active"* ]]; then
+ DETECTED_BACKEND="ufw"
+ verbose "Detected active UFW"
+ return
+ fi
+ fi
+
+ if command -v nft &>/dev/null; then
+ local nft_rules
+ nft_rules=$(nft list ruleset 2>/dev/null | wc -l)
+ if [[ "$nft_rules" -gt 0 ]]; then
+ DETECTED_BACKEND="nftables"
+ verbose "Detected nftables with ${nft_rules} lines"
+ return
+ fi
+ fi
+
+ if command -v iptables-save &>/dev/null; then
+ DETECTED_BACKEND="iptables"
+ verbose "Detected iptables"
+ return
+ fi
+
+ err "No supported firewall backend found (ufw, nftables, iptables)"
+ exit 1
+}
+
+# ── Snapshot Functions ────────────────────────────────────────────────
+snapshot_ufw() {
+ local dir="$1"
+ ufw status numbered > "${dir}/ufw-status.txt" 2>/dev/null || true
+ ufw status verbose > "${dir}/ufw-verbose.txt" 2>/dev/null || true
+ if [[ -f /etc/ufw/user.rules ]]; then
+ cp /etc/ufw/user.rules "${dir}/user.rules"
+ fi
+ if [[ -f /etc/ufw/user6.rules ]]; then
+ cp /etc/ufw/user6.rules "${dir}/user6.rules"
+ fi
+ # count rules from numbered output (skip header lines)
+ RULES_TOTAL=$(grep -cE '^\[' "${dir}/ufw-status.txt" 2>/dev/null) || RULES_TOTAL=0
+ verbose "UFW snapshot: ${RULES_TOTAL} rules"
+}
+
+snapshot_iptables() {
+ local dir="$1"
+ iptables-save > "${dir}/iptables-v4.rules" 2>/dev/null || true
+ if command -v ip6tables-save &>/dev/null; then
+ ip6tables-save > "${dir}/iptables-v6.rules" 2>/dev/null || true
+ fi
+ # count non-comment, non-empty lines
+ RULES_TOTAL=$(grep -cvE '^(#|$|\*|COMMIT|:)' "${dir}/iptables-v4.rules" 2>/dev/null) || RULES_TOTAL=0
+ if [[ -f "${dir}/iptables-v6.rules" ]]; then
+ local v6_count
+ v6_count=$(grep -cvE '^(#|$|\*|COMMIT|:)' "${dir}/iptables-v6.rules" 2>/dev/null) || v6_count=0
+ RULES_TOTAL=$((RULES_TOTAL + v6_count))
+ fi
+ verbose "iptables snapshot: ${RULES_TOTAL} rules"
+}
+
+snapshot_nftables() {
+ local dir="$1"
+ nft list ruleset > "${dir}/nftables.rules" 2>/dev/null || true
+ RULES_TOTAL=$(grep -cE '^\s+(rule|chain|table)' "${dir}/nftables.rules" 2>/dev/null) || RULES_TOTAL=0
+ verbose "nftables snapshot: ${RULES_TOTAL} lines"
+}
+
+take_snapshot() {
+ local dir="$1"
+ case "$DETECTED_BACKEND" in
+ ufw) snapshot_ufw "$dir" ;;
+ iptables) snapshot_iptables "$dir" ;;
+ nftables) snapshot_nftables "$dir" ;;
+ esac
+ echo "$DETECTED_BACKEND" > "${dir}/backend.txt"
+ date +%s > "${dir}/timestamp.txt"
+ date -Is > "${dir}/timestamp-human.txt"
+}
+
+# ── Save Mode ─────────────────────────────────────────────────────────
+do_save() {
+ mkdir -p "${BASELINE_DIR}"
+ local snapshot_dir="${BASELINE_DIR}/baseline"
+
+ # clean up any previous baseline
+ if [[ -d "$snapshot_dir" ]]; then
+ local prev_ts
+ prev_ts=$(cat "${snapshot_dir}/timestamp.txt" 2>/dev/null || echo "unknown")
+ local archive_dir="${BASELINE_DIR}/archive-${prev_ts}"
+ mv "$snapshot_dir" "$archive_dir" 2>/dev/null || rm -rf "$snapshot_dir"
+ verbose "Archived previous baseline"
+ fi
+
+ mkdir -p "$snapshot_dir"
+
+ verbose "Taking ${DETECTED_BACKEND} snapshot to ${snapshot_dir}"
+ take_snapshot "$snapshot_dir"
+
+ if [[ ! -f "${snapshot_dir}/backend.txt" ]]; then
+ err "Snapshot failed — ${snapshot_dir}/backend.txt not created"
+ exit 1
+ fi
+
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e "${BOLD}Firewall Rule Diff${RESET}"
+ echo "Backend: ${DETECTED_BACKEND}"
+ echo "Baseline: ${snapshot_dir}"
+ echo "Time: $(cat "${snapshot_dir}/timestamp-human.txt")"
+ echo "Rules: ${RULES_TOTAL}"
+ echo ""
+ echo -e " ${GREEN}✓${RESET} Baseline saved — ${RULES_TOTAL} rules (${DETECTED_BACKEND})"
+ elif [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "1..1"
+ echo "ok 1 - Baseline saved (${RULES_TOTAL} rules, ${DETECTED_BACKEND})"
+ fi
+}
+
+# ── Check Mode ────────────────────────────────────────────────────────
+do_check() {
+ START_TIME=$(date +%s)
+ local baseline_dir="${BASELINE_DIR}/baseline"
+
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo -e "${BOLD}Firewall Rule Diff${RESET}"
+ echo "Backend: ${DETECTED_BACKEND}"
+ echo "Baseline: ${baseline_dir}"
+ echo "Time: $(date -Is)"
+ echo ""
+ elif [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "TAP version 13"
+ fi
+
+ # check baseline exists
+ if [[ ! -d "$baseline_dir" ]]; then
+ record_fail "Baseline exists" "no baseline found — run with --save first"
+ DRIFT_DETECTED=1
+ print_summary
+ return
+ fi
+
+ # check backend matches
+ local baseline_backend
+ baseline_backend=$(cat "${baseline_dir}/backend.txt" 2>/dev/null || echo "unknown")
+ if [[ "$baseline_backend" != "$DETECTED_BACKEND" ]]; then
+ record_fail "Backend match" "baseline uses ${baseline_backend}, current is ${DETECTED_BACKEND}"
+ DRIFT_DETECTED=1
+ else
+ record_pass "Backend match" "${DETECTED_BACKEND}"
+ fi
+
+ # check baseline age
+ local baseline_ts
+ baseline_ts=$(cat "${baseline_dir}/timestamp.txt" 2>/dev/null || echo "0")
+ local now
+ now=$(date +%s)
+ BASELINE_AGE=$((now - baseline_ts))
+ local age_days=$((BASELINE_AGE / 86400))
+
+ if [[ $age_days -gt $MAX_AGE_DAYS ]]; then
+ record_warn "Baseline age" "${age_days} days old (threshold: ${MAX_AGE_DAYS})"
+ else
+ record_pass "Baseline age" "${age_days} days old"
+ fi
+
+ # take current snapshot to temp dir
+ local tmp_dir
+ tmp_dir=$(mktemp -d)
+ trap 'rm -rf "'"$tmp_dir"'"' EXIT
+ take_snapshot "$tmp_dir"
+
+ if [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo ""
+ echo -e "${BOLD}Rule Comparison${RESET}"
+ fi
+
+ # diff based on backend
+ case "$DETECTED_BACKEND" in
+ ufw) diff_ufw "$baseline_dir" "$tmp_dir" ;;
+ iptables) diff_iptables "$baseline_dir" "$tmp_dir" ;;
+ nftables) diff_nftables "$baseline_dir" "$tmp_dir" ;;
+ esac
+
+ print_summary
+}
+
+# ── Diff Functions ────────────────────────────────────────────────────
+diff_rules_file() {
+ local label="$1"
+ local baseline_file="$2"
+ local current_file="$3"
+
+ if [[ ! -f "$baseline_file" ]] && [[ ! -f "$current_file" ]]; then
+ verbose "Both files missing for ${label} — skipping"
+ return
+ fi
+
+ if [[ ! -f "$baseline_file" ]]; then
+ record_fail "${label}" "file missing from baseline but present now"
+ DRIFT_DETECTED=1
+ return
+ fi
+
+ if [[ ! -f "$current_file" ]]; then
+ record_fail "${label}" "file present in baseline but missing now"
+ DRIFT_DETECTED=1
+ return
+ fi
+
+ local diff_output
+ diff_output=$(diff --unified=0 "$baseline_file" "$current_file" 2>/dev/null) || true
+
+ if [[ -z "$diff_output" ]]; then
+ record_pass "${label}" "no changes"
+ return
+ fi
+
+ DRIFT_DETECTED=1
+
+ local added removed
+ added=$(echo "$diff_output" | grep -c '^+[^+]' 2>/dev/null) || added=0
+ removed=$(echo "$diff_output" | grep -c '^-[^-]' 2>/dev/null) || removed=0
+
+ RULES_ADDED=$((RULES_ADDED + added))
+ RULES_REMOVED=$((RULES_REMOVED + removed))
+
+ record_fail "${label}" "${added} added, ${removed} removed"
+
+ if [[ "$VERBOSE" == "true" || "$OUTPUT_FORMAT" == "text" ]]; then
+ # show the actual diff lines (limit to 20 lines)
+ local count=0
+ while IFS= read -r line; do
+ if [[ "$line" == +* && "$line" != +++* ]]; then
+ echo -e " ${GREEN}${line}${RESET}"
+ ((count++)) || true
+ elif [[ "$line" == -* && "$line" != ---* ]]; then
+ echo -e " ${RED}${line}${RESET}"
+ ((count++)) || true
+ fi
+ [[ $count -ge 20 ]] && { echo " ... (truncated)"; break; }
+ done <<< "$diff_output"
+ fi
+}
+
+diff_ufw() {
+ local baseline="$1"
+ local current="$2"
+
+ diff_rules_file "UFW status" "${baseline}/ufw-status.txt" "${current}/ufw-status.txt"
+ diff_rules_file "UFW IPv4 rules" "${baseline}/user.rules" "${current}/user.rules"
+ diff_rules_file "UFW IPv6 rules" "${baseline}/user6.rules" "${current}/user6.rules"
+
+ # rule count comparison
+ local baseline_count current_count
+ baseline_count=$(grep -cE '^\[' "${baseline}/ufw-status.txt" 2>/dev/null) || baseline_count=0
+ current_count=$(grep -cE '^\[' "${current}/ufw-status.txt" 2>/dev/null) || current_count=0
+
+ if [[ $baseline_count -ne $current_count ]]; then
+ record_fail "Rule count" "baseline: ${baseline_count}, current: ${current_count}"
+ DRIFT_DETECTED=1
+ else
+ record_pass "Rule count" "${current_count} rules"
+ fi
+}
+
+diff_iptables() {
+ local baseline="$1"
+ local current="$2"
+
+ diff_rules_file "iptables IPv4 rules" "${baseline}/iptables-v4.rules" "${current}/iptables-v4.rules"
+ diff_rules_file "iptables IPv6 rules" "${baseline}/iptables-v6.rules" "${current}/iptables-v6.rules"
+
+ # chain count comparison
+ local baseline_chains current_chains
+ baseline_chains=$(grep -cE '^:' "${baseline}/iptables-v4.rules" 2>/dev/null) || baseline_chains=0
+ current_chains=$(grep -cE '^:' "${current}/iptables-v4.rules" 2>/dev/null) || current_chains=0
+
+ if [[ $baseline_chains -ne $current_chains ]]; then
+ record_fail "Chain count (IPv4)" "baseline: ${baseline_chains}, current: ${current_chains}"
+ DRIFT_DETECTED=1
+ else
+ record_pass "Chain count (IPv4)" "${current_chains} chains"
+ fi
+}
+
+diff_nftables() {
+ local baseline="$1"
+ local current="$2"
+
+ diff_rules_file "nftables ruleset" "${baseline}/nftables.rules" "${current}/nftables.rules"
+
+ # table count comparison
+ local baseline_tables current_tables
+ baseline_tables=$(grep -c '^table' "${baseline}/nftables.rules" 2>/dev/null) || baseline_tables=0
+ current_tables=$(grep -c '^table' "${current}/nftables.rules" 2>/dev/null) || current_tables=0
+
+ if [[ $baseline_tables -ne $current_tables ]]; then
+ record_fail "Table count" "baseline: ${baseline_tables}, current: ${current_tables}"
+ DRIFT_DETECTED=1
+ else
+ record_pass "Table count" "${current_tables} tables"
+ fi
+}
+
+# ── Summary ───────────────────────────────────────────────────────────
+print_summary() {
+ local end_time
+ end_time=$(date +%s)
+ local elapsed=$((end_time - START_TIME))
+
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "1..${TOTAL}"
+ elif [[ "$OUTPUT_FORMAT" == "text" ]]; then
+ echo ""
+ echo "────────────────────────────────────────"
+ echo -e "${BOLD}Summary${RESET} ${DETECTED_BACKEND}"
+ echo -e " ${PASS} passed ${FAIL} failed ${WARN} skipped (${elapsed}s)"
+ if [[ $DRIFT_DETECTED -eq 1 ]]; then
+ echo -e " Rules added: ${RULES_ADDED} removed: ${RULES_REMOVED}"
+ echo -e " ${RED}Drift detected.${RESET}"
+ else
+ echo -e " ${GREEN}No drift detected.${RESET}"
+ fi
+ echo "────────────────────────────────────────"
+ fi
+
+ if [[ "$OUTPUT_FORMAT" == "junit" ]]; then
+ write_junit
+ fi
+
+ if [[ "$TEXTFILE_MODE" == "true" ]]; then
+ write_prometheus
+ fi
+}
+
+# ── JUnit Output ──────────────────────────────────────────────────────
+write_junit() {
+ local end_time
+ end_time=$(date +%s)
+ local elapsed=$((end_time - START_TIME))
+
+ {
+ echo ''
+ echo ""
+ echo " "
+
+ for result in "${RESULTS[@]}"; do
+ local status name detail
+ status=$(echo "$result" | cut -d'|' -f1)
+ name=$(echo "$result" | cut -d'|' -f2)
+ detail=$(echo "$result" | cut -d'|' -f3)
+
+ # escape XML
+ name="${name//&/&}"
+ name="${name///>}"
+ detail="${detail//&/&}"
+ detail="${detail///>}"
+
+ echo " "
+ if [[ "$status" == "FAIL" ]]; then
+ echo " "
+ elif [[ "$status" == "WARN" ]]; then
+ echo " "
+ fi
+ echo " "
+ done
+
+ echo " "
+ echo ""
+ } > "$JUNIT_FILE"
+
+ verbose "JUnit report written to ${JUNIT_FILE}"
+}
+
+# ── Prometheus Output ─────────────────────────────────────────────────
+write_prometheus() {
+ local prom_dir
+ prom_dir=$(dirname "$PROM_FILE")
+ if [[ ! -d "$prom_dir" ]]; then
+ warn "Prometheus textfile directory does not exist: ${prom_dir}"
+ return
+ fi
+
+ local tmp_file="${PROM_FILE}.$$"
+ {
+ echo "# HELP firewall_drift_detected Whether firewall rules differ from baseline"
+ echo "# TYPE firewall_drift_detected gauge"
+ echo "firewall_drift_detected ${DRIFT_DETECTED}"
+ echo "# HELP firewall_rules_added Rules added since baseline"
+ echo "# TYPE firewall_rules_added gauge"
+ echo "firewall_rules_added ${RULES_ADDED}"
+ echo "# HELP firewall_rules_removed Rules removed since baseline"
+ echo "# TYPE firewall_rules_removed gauge"
+ echo "firewall_rules_removed ${RULES_REMOVED}"
+ echo "# HELP firewall_rules_total Current total firewall rules"
+ echo "# TYPE firewall_rules_total gauge"
+ echo "firewall_rules_total ${RULES_TOTAL}"
+ echo "# HELP firewall_baseline_age_seconds Seconds since baseline was saved"
+ echo "# TYPE firewall_baseline_age_seconds gauge"
+ echo "firewall_baseline_age_seconds ${BASELINE_AGE}"
+ echo "# HELP firewall_scan_timestamp Unix timestamp of last scan"
+ echo "# TYPE firewall_scan_timestamp gauge"
+ echo "firewall_scan_timestamp $(date +%s)"
+ echo "# HELP firewall_backend Active firewall backend"
+ echo "# TYPE firewall_backend gauge"
+ echo "firewall_backend{backend=\"${DETECTED_BACKEND}\"} 1"
+ } > "$tmp_file"
+
+ mv "$tmp_file" "$PROM_FILE"
+ verbose "Prometheus metrics written to ${PROM_FILE}"
+}
+
+# ── Main ──────────────────────────────────────────────────────────────
+main() {
+ setup_colors
+ parse_args "$@"
+ setup_colors # re-apply after --no-color
+
+ if [[ $EUID -ne 0 ]]; then
+ err "This script must be run as root."
+ exit 1
+ fi
+
+ detect_backend
+
+ case "$MODE" in
+ save) do_save ;;
+ check) do_check ;;
+ esac
+
+ if [[ $FAIL -gt 0 ]]; then
+ exit 1
+ fi
+ exit 0
+}
+
+main "$@"
diff --git a/fix-code-server-nginx.sh b/fix-code-server-nginx.sh
new file mode 100755
index 0000000..9b4c7fe
--- /dev/null
+++ b/fix-code-server-nginx.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+###############################################################
+#### Fix code-server Nginx Config ####
+#### Applies X-Frame-Options + query-filter fixes ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version: 1.0 ####
+#### ####
+#### Usage: sudo ./fix-code-server-nginx.sh [domain] ####
+###############################################################
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+ echo "ERROR: This script must be run as root (sudo)."
+ exit 1
+fi
+
+DOMAIN="${1:-}"
+if [[ -z "$DOMAIN" ]]; then
+ read -rp "Enter the code-server domain (e.g. code.mydomain.com): " DOMAIN
+fi
+
+HEADERS_FILE="/etc/nginx/snippets/security-headers-${DOMAIN}.conf"
+SITE_CONF="/etc/nginx/conf.d/code-server.conf"
+
+echo "=== Fix 1: X-Frame-Options DENY -> SAMEORIGIN ==="
+if [[ -f "$HEADERS_FILE" ]]; then
+ if grep -q 'X-Frame-Options "DENY"' "$HEADERS_FILE"; then
+ sed -i 's/X-Frame-Options "DENY"/X-Frame-Options "SAMEORIGIN"/' "$HEADERS_FILE"
+ echo " Updated: $HEADERS_FILE"
+ else
+ echo " Already set to SAMEORIGIN (or not found) in $HEADERS_FILE — skipping."
+ fi
+else
+ echo " WARNING: $HEADERS_FILE not found. Skipping."
+fi
+
+echo ""
+echo "=== Fix 2: Disable query-filter snippet ==="
+QUERY_FILTER="snippets/query-filter-${DOMAIN}.conf"
+if [[ -f "$SITE_CONF" ]]; then
+ if grep -qE "^\s*include\s+${QUERY_FILTER}" "$SITE_CONF"; then
+ sed -i "s|^\(\s*\)include ${QUERY_FILTER};|\1# Disabled: breaks VS Code extensions\n\1# include ${QUERY_FILTER};|" "$SITE_CONF"
+ echo " Commented out query filter in: $SITE_CONF"
+ else
+ echo " Query filter already disabled (or not found) in $SITE_CONF — skipping."
+ fi
+else
+ echo " WARNING: $SITE_CONF not found. Skipping."
+fi
+
+echo ""
+echo "=== Testing and reloading nginx ==="
+if nginx -t; then
+ systemctl reload nginx
+ echo " Nginx reloaded successfully."
+else
+ echo " ERROR: nginx config test failed. Check the files manually."
+ exit 1
+fi
+
+echo ""
+echo "Done! Reload your code-server browser tab to verify."
diff --git a/freeradius-exporter.sh b/freeradius-exporter.sh
new file mode 100644
index 0000000..783c5a5
--- /dev/null
+++ b/freeradius-exporter.sh
@@ -0,0 +1,395 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### freeradius-exporter.sh — Prometheus metrics exporter for FreeRADIUS ####
+#### Exports authentication, accounting, and proxy statistics from the ####
+#### FreeRADIUS status server as Prometheus metrics ####
+#### Requires: bash 4+, radclient ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./freeradius-exporter.sh --http --port 9620 ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -uo pipefail
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+RADIUS_HOST="${RADIUS_HOST:-localhost}"
+RADIUS_STATUS_PORT="${RADIUS_STATUS_PORT:-18121}"
+RADIUS_SECRET="${RADIUS_SECRET:-adminsecret}"
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9620
+
+EXPORTER_VERSION="1.00"
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Query the FreeRADIUS status server via radclient
+# Returns: raw attribute-value pair output, or empty on failure
+query_status_server() {
+ echo "Message-Authenticator = 0x00, FreeRADIUS-Statistics-Type = All" \
+ | radclient "${RADIUS_HOST}:${RADIUS_STATUS_PORT}" status "${RADIUS_SECRET}" 2>/dev/null
+}
+
+# Extract a numeric value from radclient output
+# Args: $1 - attribute name, $2 - radclient output
+# Returns: numeric value or 0 if not found
+extract_value() {
+ local attr="$1"
+ local data="$2"
+ local val
+ val=$(echo "$data" | grep -F "$attr" | awk -F'= ' '{print $2}' | tr -d '[:space:]')
+ echo "${val:-0}"
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+# Generate all Prometheus metrics
+# Returns: Prometheus text format metrics on stdout
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s)
+
+ # Check radclient is available
+ if ! command -v radclient >/dev/null 2>&1; then
+ echo "ERROR: radclient command not found" >&2
+ cat <&2
+
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+
+ while true; do
+ {
+ read -r request
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ cat <
+
+FreeRADIUS Exporter v${EXPORTER_VERSION}
+
+FreeRADIUS Prometheus Exporter v${EXPORTER_VERSION}
+Metrics
+Authentication, accounting, and proxy statistics from the FreeRADIUS status server.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+main() {
+ parse_args "$@"
+
+ if [[ "$HTTP_MODE" = true ]]; then
+ run_http_server
+ elif [[ -n "$OUTPUT_FILE" ]]; then
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.freeradius_metrics.XXXXXX")
+
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [[ "$file_lines" -lt 5 ]]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ chmod 644 "$temp_file"
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ generate_metrics
+ fi
+}
+
+main "$@"
diff --git a/game-server-exporter.sh b/game-server-exporter.sh
new file mode 100755
index 0000000..1f66285
--- /dev/null
+++ b/game-server-exporter.sh
@@ -0,0 +1,624 @@
+#!/bin/bash
+################################################################################
+# Script Name: game-server-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for game servers providing operational
+# metrics — Minecraft, Valheim, and Palworld player counts,
+# server status, TPS, query response times, and server version info
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - nmap-ncat (nc) for network queries
+# - curl for REST API queries (Palworld)
+# - python3 with mcstatus (optional, enhanced Minecraft metrics)
+# - netcat (nc) for HTTP mode
+#
+# Usage:
+# # Output to stdout
+# ./game-server-exporter.sh
+#
+# # HTTP server mode
+# ./game-server-exporter.sh --http -p 9195
+#
+# # Textfile collector mode
+# ./game-server-exporter.sh --textfile
+#
+# # Custom server addresses
+# ./game-server-exporter.sh --minecraft-host mc.example.com
+#
+# Metrics Exported:
+# - game_server_up{game,server} - Server reachability (1=up, 0=down)
+# - game_server_players_online{game,server} - Online player count
+# - game_server_players_max{game,server} - Maximum player slots
+# - game_server_info{game,server,version,motd} - Server version info
+# - game_server_tps{game="minecraft",server} - Ticks per second (Minecraft)
+# - game_server_query_duration_seconds{game,server} - Query time per server
+# - game_server_exporter_duration_seconds - Total script execution time
+# - game_server_exporter_last_run_timestamp - Last run timestamp
+#
+# Configuration:
+# Default HTTP port: 9195
+# Textfile directory: /var/lib/node_exporter
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9195
+
+# Server configuration
+MINECRAFT_HOST=""
+MINECRAFT_QUERY_PORT=25565
+MINECRAFT_RCON_PORT=25575
+MINECRAFT_RCON_PASS=""
+VALHEIM_HOST=""
+VALHEIM_QUERY_PORT=2457
+PALWORLD_HOST=""
+PALWORLD_QUERY_PORT=8212
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Check prerequisites
+# Returns: 0 if OK, 1 if error
+check_prerequisites() {
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: nc (nmap-ncat) not found" >&2
+ return 1
+ fi
+
+ if [ -n "$PALWORLD_HOST" ] && ! command -v curl >/dev/null 2>&1; then
+ echo "ERROR: curl not found (required for Palworld REST API)" >&2
+ return 1
+ fi
+
+ return 0
+}
+
+# Escape special characters in Prometheus label values
+# Args: $1 - string to escape
+# Returns: escaped string safe for Prometheus labels
+prom_escape() {
+ local val="$1"
+ val="${val//\\/\\\\}"
+ val="${val//\"/\\\"}"
+ val="${val//$'\n'/}"
+ echo "$val"
+}
+
+# ============================================================================
+# GAME SERVER QUERY FUNCTIONS
+# ============================================================================
+
+# Query Minecraft server using python3 mcstatus or basic TCP check
+# Args: $1 - host, $2 - port
+# Sets global variables: mc_up, mc_players_online, mc_players_max, mc_version, mc_motd, mc_tps, mc_query_duration
+query_minecraft() {
+ local host="$1"
+ local port="$2"
+ local query_start query_end
+
+ mc_up=0
+ mc_players_online=0
+ mc_players_max=0
+ mc_version="unknown"
+ mc_motd="unknown"
+ mc_tps=""
+ mc_query_duration=0
+
+ query_start=$(date +%s%N)
+
+ # Try python3 mcstatus first (most reliable)
+ if command -v python3 >/dev/null 2>&1; then
+ local py_result
+ py_result=$(python3 -c "
+import sys
+try:
+ from mcstatus import JavaServer
+ server = JavaServer.lookup('${host}:${port}', timeout=5)
+ status = server.status()
+ print('UP')
+ print(status.players.online)
+ print(status.players.max)
+ print(status.version.name)
+ desc = status.description
+ if isinstance(desc, dict):
+ desc = desc.get('text', 'unknown')
+ print(str(desc).replace(chr(10), ' '))
+except ImportError:
+ print('NO_MCSTATUS')
+except Exception as e:
+ print('DOWN')
+" 2>/dev/null) || true
+
+ local first_line
+ first_line=$(echo "$py_result" | head -1)
+
+ if [ "$first_line" = "UP" ]; then
+ mc_up=1
+ mc_players_online=$(echo "$py_result" | sed -n '2p')
+ mc_players_max=$(echo "$py_result" | sed -n '3p')
+ mc_version=$(echo "$py_result" | sed -n '4p')
+ mc_motd=$(echo "$py_result" | sed -n '5p')
+ elif [ "$first_line" != "NO_MCSTATUS" ]; then
+ # mcstatus available but server is down
+ query_end=$(date +%s%N)
+ mc_query_duration=$(( (query_end - query_start) / 1000000000 ))
+ return
+ fi
+ fi
+
+ # Fallback: basic TCP check if mcstatus not available or not tried yet
+ if [ "$mc_up" -eq 0 ] && [ -z "${py_result:-}" ] || { [ -n "${first_line:-}" ] && [ "$first_line" = "NO_MCSTATUS" ]; }; then
+ if nc -z -w 3 "$host" "$port" 2>/dev/null; then
+ mc_up=1
+ # Try to read SLP response for basic info
+ local slp_response
+ slp_response=$(printf '\xfe\x01' | nc -w 3 "$host" "$port" 2>/dev/null | strings 2>/dev/null) || true
+ if [ -n "$slp_response" ]; then
+ # Legacy SLP response: §1\0\0\0\0\0
+ mc_version=$(echo "$slp_response" | tr '\0' '\n' | sed -n '4p' 2>/dev/null) || mc_version="unknown"
+ mc_motd=$(echo "$slp_response" | tr '\0' '\n' | sed -n '5p' 2>/dev/null) || mc_motd="unknown"
+ mc_players_online=$(echo "$slp_response" | tr '\0' '\n' | sed -n '6p' 2>/dev/null) || mc_players_online=0
+ mc_players_max=$(echo "$slp_response" | tr '\0' '\n' | sed -n '7p' 2>/dev/null) || mc_players_max=0
+ # Sanitize numeric values
+ [[ "$mc_players_online" =~ ^[0-9]+$ ]] || mc_players_online=0
+ [[ "$mc_players_max" =~ ^[0-9]+$ ]] || mc_players_max=0
+ fi
+ fi
+ fi
+
+ # Try RCON for TPS if credentials are provided and server is up
+ if [ "$mc_up" -eq 1 ] && [ -n "$MINECRAFT_RCON_PASS" ]; then
+ local tps_result
+ tps_result=$(python3 -c "
+import sys
+try:
+ from mcrcon import MCRcon
+ with MCRcon('${host}', '${MINECRAFT_RCON_PASS}', port=${MINECRAFT_RCON_PORT}) as mcr:
+ resp = mcr.command('tps')
+ # Parse TPS from response (e.g., '§6TPS from last 1m, 5m, 15m: §a20.0, §a20.0, §a20.0')
+ import re
+ nums = re.findall(r'[\d.]+', resp)
+ if nums:
+ print(nums[-1]) # Last TPS value (15m average)
+except Exception:
+ pass
+" 2>/dev/null) || true
+ if [ -n "$tps_result" ]; then
+ mc_tps="$tps_result"
+ fi
+ fi
+
+ query_end=$(date +%s%N)
+ mc_query_duration=$(( (query_end - query_start) / 1000000000 ))
+}
+
+# Query Valheim server using Steam A2S protocol or TCP fallback
+# Args: $1 - host, $2 - port
+# Sets global variables: vh_up, vh_players_online, vh_players_max, vh_version, vh_motd, vh_query_duration
+query_valheim() {
+ local host="$1"
+ local port="$2"
+ local query_start query_end
+
+ vh_up=0
+ vh_players_online=0
+ vh_players_max=0
+ vh_version="unknown"
+ vh_motd="unknown"
+ vh_query_duration=0
+
+ query_start=$(date +%s%N)
+
+ # Try python3 A2S query first (Steam query protocol)
+ if command -v python3 >/dev/null 2>&1; then
+ local py_result
+ py_result=$(python3 -c "
+import sys
+try:
+ import a2s
+ address = ('${host}', ${port})
+ info = a2s.info(address, timeout=5)
+ print('UP')
+ print(info.player_count)
+ print(info.max_players)
+ print(info.version)
+ print(info.server_name.replace(chr(10), ' '))
+except ImportError:
+ print('NO_A2S')
+except Exception:
+ print('DOWN')
+" 2>/dev/null) || true
+
+ local first_line
+ first_line=$(echo "$py_result" | head -1)
+
+ if [ "$first_line" = "UP" ]; then
+ vh_up=1
+ vh_players_online=$(echo "$py_result" | sed -n '2p')
+ vh_players_max=$(echo "$py_result" | sed -n '3p')
+ vh_version=$(echo "$py_result" | sed -n '4p')
+ vh_motd=$(echo "$py_result" | sed -n '5p')
+ elif [ "$first_line" != "NO_A2S" ]; then
+ query_end=$(date +%s%N)
+ vh_query_duration=$(( (query_end - query_start) / 1000000000 ))
+ return
+ fi
+ fi
+
+ # Fallback: TCP port check on game port (query port - 1 is typically the game port)
+ if [ "$vh_up" -eq 0 ]; then
+ local game_port=$((port - 1))
+ if nc -z -w 3 "$host" "$game_port" 2>/dev/null || nc -z -w 3 "$host" "$port" 2>/dev/null; then
+ vh_up=1
+ fi
+ fi
+
+ query_end=$(date +%s%N)
+ vh_query_duration=$(( (query_end - query_start) / 1000000000 ))
+}
+
+# Query Palworld server using REST API or TCP fallback
+# Args: $1 - host, $2 - port
+# Sets global variables: pw_up, pw_players_online, pw_players_max, pw_version, pw_motd, pw_query_duration
+query_palworld() {
+ local host="$1"
+ local port="$2"
+ local query_start query_end
+
+ pw_up=0
+ pw_players_online=0
+ pw_players_max=0
+ pw_version="unknown"
+ pw_motd="unknown"
+ pw_query_duration=0
+
+ query_start=$(date +%s%N)
+
+ # Try REST API query first
+ if command -v curl >/dev/null 2>&1; then
+ local api_response
+ api_response=$(curl -s -m 5 "http://${host}:${port}/v1/api/info" 2>/dev/null) || true
+
+ if [ -n "$api_response" ] && command -v python3 >/dev/null 2>&1; then
+ local parse_result
+ parse_result=$(python3 -c "
+import json, sys
+try:
+ data = json.loads('''${api_response}''')
+ print('UP')
+ print(data.get('currentPlayerNum', 0))
+ print(data.get('maxPlayerNum', 0))
+ print(data.get('version', 'unknown'))
+ print(data.get('serverName', 'unknown').replace(chr(10), ' '))
+except Exception:
+ print('PARSE_FAIL')
+" 2>/dev/null) || true
+
+ local first_line
+ first_line=$(echo "$parse_result" | head -1)
+
+ if [ "$first_line" = "UP" ]; then
+ pw_up=1
+ pw_players_online=$(echo "$parse_result" | sed -n '2p')
+ pw_players_max=$(echo "$parse_result" | sed -n '3p')
+ pw_version=$(echo "$parse_result" | sed -n '4p')
+ pw_motd=$(echo "$parse_result" | sed -n '5p')
+ fi
+ elif [ -n "$api_response" ]; then
+ # curl got a response but no python3 to parse JSON
+ pw_up=1
+ fi
+ fi
+
+ # Fallback: TCP port check
+ if [ "$pw_up" -eq 0 ]; then
+ if nc -z -w 3 "$host" "$port" 2>/dev/null; then
+ pw_up=1
+ fi
+ fi
+
+ query_end=$(date +%s%N)
+ pw_query_duration=$(( (query_end - query_start) / 1000000000 ))
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+# Generate all Prometheus metrics
+# Returns: Prometheus text format metrics on stdout
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s)
+
+ # Check prerequisites
+ if ! check_prerequisites; then
+ return
+ fi
+
+ # Check that at least one server is configured
+ if [ -z "$MINECRAFT_HOST" ] && [ -z "$VALHEIM_HOST" ] && [ -z "$PALWORLD_HOST" ]; then
+ echo "# No game servers configured. Use --minecraft-host, --valheim-host, or --palworld-host" >&2
+ return
+ fi
+
+ cat <&2
+
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+
+ # Infinite loop accepting HTTP requests
+ while true; do
+ {
+ read -r request
+ # Check if request is for /metrics endpoint
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else # Serve HTML landing page for other requests
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ cat <
+
+Game Server Exporter v1.0
+
+Game Server Prometheus Exporter v1.0
+Metrics
+Operational metrics from Minecraft, Valheim, and Palworld servers.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+# Main entry point - routes to appropriate output mode
+main() {
+ parse_args "$@"
+
+ if [ "$HTTP_MODE" = true ]; then
+ # Run HTTP server (blocks until killed)
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ # Textfile collector mode: write atomically using temp file
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ # Create temp file in SAME directory for atomic rename (same filesystem)
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.game_server_metrics.XXXXXX")
+
+ # Generate metrics to temp file
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ # Validate: file must exist, have content
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ # Set permissions before move
+ chmod 644 "$temp_file"
+
+ # Atomic rename - no gap where file is missing
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ # Default: output to stdout
+ generate_metrics
+ fi
+}
+
+# Execute main function with all script arguments
+main "$@"
diff --git a/gcp-cost-reporter.sh b/gcp-cost-reporter.sh
new file mode 100755
index 0000000..872790d
--- /dev/null
+++ b/gcp-cost-reporter.sh
@@ -0,0 +1,563 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### gcp-cost-reporter.sh — GCP cost breakdown by service, project, or label. ####
+#### Queries BigQuery billing export for spend data with period comparison ####
+#### Requires: bash 4+, gcloud CLI (bq), jq ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./gcp-cost-reporter.sh --daily ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+BQ_BILLING_TABLE="${BQ_BILLING_TABLE:-}"
+GCP_PROJECT="${GCP_PROJECT:-}"
+GROUP_BY="${GROUP_BY:-SERVICE}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
+SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-}"
+LABEL_FILTER_KEY="${LABEL_FILTER_KEY:-}"
+LABEL_FILTER_VALUE="${LABEL_FILTER_VALUE:-}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+RUN_MODE=""
+CUSTOM_START=""
+CUSTOM_END=""
+SLACK_URL=""
+START_TIME=""
+
+# ── Colors ────────────────────────────────────────────────────────────
+RED="" GREEN="" YELLOW="" BLUE="" BOLD="" DIM="" RESET=""
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "auto" && ! -t 1 ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ RED="\033[0;31m"
+ GREEN="\033[0;32m"
+ YELLOW="\033[0;33m"
+ # shellcheck disable=SC2034
+ BLUE="\033[0;34m"
+ # shellcheck disable=SC2034
+ BOLD="\033[1m"
+ DIM="\033[2m"
+ RESET="\033[0m"
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log_info() { printf "${GREEN}[INFO]${RESET} %s\n" "$*"; }
+log_warn() { printf "${YELLOW}[WARN]${RESET} %s\n" "$*" >&2; }
+log_error() { printf "${RED}[ERROR]${RESET} %s\n" "$*" >&2; }
+log_debug() { [[ "$VERBOSE" == "true" ]] && printf "${DIM}[DEBUG] %s${RESET}\n" "$*"; }
+
+# ── Helpers ───────────────────────────────────────────────────────────
+die() { log_error "$@"; exit 1; }
+
+check_deps() {
+ local missing=()
+ command -v gcloud >/dev/null 2>&1 || missing+=("gcloud")
+ command -v bq >/dev/null 2>&1 || missing+=("bq")
+ command -v jq >/dev/null 2>&1 || missing+=("jq")
+ command -v curl >/dev/null 2>&1 || missing+=("curl")
+ if (( ${#missing[@]} > 0 )); then
+ die "Missing required tools: ${missing[*]}"
+ fi
+
+ local bash_major="${BASH_VERSINFO[0]}"
+ if (( bash_major < 4 )); then
+ die "Requires bash 4+, found ${BASH_VERSION}"
+ fi
+}
+
+validate_date() {
+ local d="$1"
+ if [[ ! "$d" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
+ die "Invalid date format: $d (expected YYYY-MM-DD)"
+ fi
+}
+
+# ── Date math (portable) ─────────────────────────────────────────────
+date_offset() {
+ local base="$1" offset="$2"
+ if date --version >/dev/null 2>&1; then
+ date -d "${base} ${offset} days" +%Y-%m-%d
+ else
+ date -j -v"${offset}d" -f "%Y-%m-%d" "$base" +%Y-%m-%d
+ fi
+}
+
+today_utc() { date -u +%Y-%m-%d; }
+
+first_of_month() {
+ local d="$1"
+ echo "${d:0:8}01"
+}
+
+first_of_prev_month() {
+ local d="$1"
+ local year="${d:0:4}"
+ local month="${d:5:2}"
+ month=$((10#$month - 1))
+ if (( month == 0 )); then
+ month=12
+ year=$((year - 1))
+ fi
+ printf "%04d-%02d-01" "$year" "$month"
+}
+
+days_between() {
+ local s="$1" e="$2"
+ local ss se
+ if date --version >/dev/null 2>&1; then
+ ss=$(date -d "$s" +%s)
+ se=$(date -d "$e" +%s)
+ else
+ ss=$(date -j -f "%Y-%m-%d" "$s" +%s)
+ se=$(date -j -f "%Y-%m-%d" "$e" +%s)
+ fi
+ echo $(( (se - ss) / 86400 ))
+}
+
+# ── Compute date ranges ──────────────────────────────────────────────
+compute_ranges() {
+ local today
+ today="$(today_utc)"
+
+ case "$RUN_MODE" in
+ daily)
+ PERIOD_START="$(date_offset "$today" -1)"
+ PERIOD_END="$today"
+ PREV_START="$(date_offset "$today" -2)"
+ PREV_END="$(date_offset "$today" -1)"
+ ;;
+ weekly)
+ PERIOD_START="$(date_offset "$today" -7)"
+ PERIOD_END="$today"
+ PREV_START="$(date_offset "$today" -14)"
+ PREV_END="$(date_offset "$today" -7)"
+ ;;
+ monthly)
+ PERIOD_START="$(first_of_month "$today")"
+ PERIOD_END="$today"
+ local prev_first
+ prev_first="$(first_of_prev_month "$today")"
+ PREV_START="$prev_first"
+ PREV_END="$PERIOD_START"
+ ;;
+ custom)
+ PERIOD_START="$CUSTOM_START"
+ PERIOD_END="$CUSTOM_END"
+ local span
+ span="$(days_between "$CUSTOM_START" "$CUSTOM_END")"
+ PREV_START="$(date_offset "$CUSTOM_START" "-$span")"
+ PREV_END="$CUSTOM_START"
+ ;;
+ *)
+ die "Unknown mode: $RUN_MODE"
+ ;;
+ esac
+
+ log_debug "Current period: $PERIOD_START → $PERIOD_END"
+ log_debug "Previous period: $PREV_START → $PREV_END"
+}
+
+# ── Build BigQuery SQL ────────────────────────────────────────────────
+build_select_column() {
+ case "$GROUP_BY" in
+ SERVICE) echo "service.description AS group_key" ;;
+ PROJECT) echo "project.id AS group_key" ;;
+ LABEL)
+ if [[ -z "$LABEL_FILTER_KEY" ]]; then
+ die "--group-by LABEL requires --label KEY=VALUE"
+ fi
+ echo "( SELECT value FROM UNNEST(labels) WHERE key = '${LABEL_FILTER_KEY}' ) AS group_key"
+ ;;
+ *)
+ die "Invalid --group-by value: $GROUP_BY (expected SERVICE, PROJECT, or LABEL)"
+ ;;
+ esac
+}
+
+build_where_clause() {
+ local start="$1" end="$2"
+ local where="usage_start_time >= TIMESTAMP('${start}') AND usage_start_time < TIMESTAMP('${end}')"
+
+ if [[ -n "$LABEL_FILTER_KEY" && -n "$LABEL_FILTER_VALUE" ]]; then
+ where="${where} AND EXISTS( SELECT 1 FROM UNNEST(labels) l WHERE l.key = '${LABEL_FILTER_KEY}' AND l.value = '${LABEL_FILTER_VALUE}' )"
+ fi
+
+ echo "$where"
+}
+
+build_query() {
+ local start="$1" end="$2"
+ local select_col where_clause
+
+ select_col="$(build_select_column)"
+ where_clause="$(build_where_clause "$start" "$end")"
+
+ cat </dev/null
+}
+
+# ── Parse cost data ──────────────────────────────────────────────────
+parse_costs() {
+ local raw="$1"
+ echo "$raw" | jq -r '
+ .[] |
+ select(.group_key != null and .group_key != "") |
+ "\(.group_key)\t\(.total_cost)"
+ ' 2>/dev/null || echo ""
+}
+
+# ── Format helpers ────────────────────────────────────────────────────
+fmt_currency() {
+ printf "$%.2f" "$1"
+}
+
+fmt_delta() {
+ local curr="$1" prev="$2"
+ if (( $(echo "$prev == 0" | bc -l) )); then
+ echo "N/A"
+ return
+ fi
+ local pct
+ pct=$(echo "scale=1; (($curr - $prev) / $prev) * 100" | bc -l)
+ local sign=""
+ if (( $(echo "$pct > 0" | bc -l) )); then
+ sign="+"
+ fi
+ echo "${sign}${pct}%"
+}
+
+print_header() {
+ local account_id
+ account_id=$(gcloud config get-value account 2>/dev/null || echo "unknown")
+ local project_id
+ project_id="${GCP_PROJECT:-$(gcloud config get-value project 2>/dev/null || echo "unknown")}"
+
+ echo "GCP Cost Reporter"
+ echo "Account: $account_id"
+ echo "Project: $project_id"
+ echo "Table: $BQ_BILLING_TABLE"
+ echo "Mode: $RUN_MODE"
+ echo "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+ if [[ "$RUN_MODE" == "custom" ]]; then
+ echo "Period: $PERIOD_START → $PERIOD_END"
+ fi
+ echo ""
+}
+
+# ── Text table output ────────────────────────────────────────────────
+output_text_table() {
+ local -n curr_data=$1
+ local -n prev_data=$2
+ local label="SERVICE"
+ case "$GROUP_BY" in
+ PROJECT) label="PROJECT" ;;
+ LABEL) label="LABEL" ;;
+ esac
+ local divider="──────────────────────────────────────────────────────────────────────"
+ printf " %-38s %-12s %-12s %s\n" "$label" "COST" "PREV" "DELTA"
+ printf " %s\n" "$divider"
+ local total_curr=0 total_prev=0
+ for key in "${!curr_data[@]}"; do
+ local cost="${curr_data[$key]}" prev_cost="${prev_data[$key]:-0}"
+ printf " %-38s %-12s %-12s %s\n" \
+ "$key" "$(fmt_currency "$cost")" "$(fmt_currency "$prev_cost")" "$(fmt_delta "$cost" "$prev_cost")"
+ total_curr=$(echo "$total_curr + $cost" | bc -l)
+ total_prev=$(echo "$total_prev + $prev_cost" | bc -l)
+ done
+ printf " %s\n" "$divider"
+ printf " %-38s %-12s %-12s %s\n" \
+ "TOTAL" "$(fmt_currency "$total_curr")" "$(fmt_currency "$total_prev")" "$(fmt_delta "$total_curr" "$total_prev")"
+}
+
+# ── CSV output ────────────────────────────────────────────────────────
+output_csv() {
+ local -n curr_data=$1
+ local -n prev_data=$2
+ local label="service"
+ case "$GROUP_BY" in
+ PROJECT) label="project" ;;
+ LABEL) label="label" ;;
+ esac
+ echo "${label},cost,previous_cost,delta_pct"
+ for key in "${!curr_data[@]}"; do
+ local cost="${curr_data[$key]}" prev_cost="${prev_data[$key]:-0}" pct="0"
+ if (( $(echo "$prev_cost != 0" | bc -l) )); then
+ pct=$(echo "scale=2; (($cost - $prev_cost) / $prev_cost) * 100" | bc -l)
+ fi
+ echo "\"$key\",$cost,$prev_cost,$pct"
+ done
+}
+
+# ── JSON output ───────────────────────────────────────────────────────
+output_json() {
+ local -n curr_data=$1
+ local -n prev_data=$2
+ local label="service"
+ case "$GROUP_BY" in
+ PROJECT) label="project" ;;
+ LABEL) label="label" ;;
+ esac
+ local items=()
+ for key in "${!curr_data[@]}"; do
+ items+=("{\"${label}\": \"${key}\", \"cost\": ${curr_data[$key]}, \"previous_cost\": ${prev_data[$key]:-0}}")
+ done
+ local joined
+ joined=$(printf ",%s" "${items[@]}")
+ joined="${joined:1}"
+ printf '{"mode":"%s","period_start":"%s","period_end":"%s","previous_start":"%s","previous_end":"%s","group_by":"%s","items":[%s]}\n' \
+ "$RUN_MODE" "$PERIOD_START" "$PERIOD_END" "$PREV_START" "$PREV_END" "$GROUP_BY" "$joined"
+}
+
+# ── Render report ─────────────────────────────────────────────────────
+render_report() {
+ local curr_raw="$1" prev_raw="$2"
+
+ declare -A curr_costs
+ declare -A prev_costs
+
+ while IFS=$'\t' read -r key amount; do
+ [[ -z "$key" ]] && continue
+ curr_costs["$key"]="$amount"
+ done <<< "$(parse_costs "$curr_raw")"
+
+ while IFS=$'\t' read -r key amount; do
+ [[ -z "$key" ]] && continue
+ prev_costs["$key"]="$amount"
+ done <<< "$(parse_costs "$prev_raw")"
+
+ for key in "${!prev_costs[@]}"; do
+ if [[ -z "${curr_costs[$key]+x}" ]]; then
+ curr_costs["$key"]="0"
+ fi
+ done
+
+ case "$OUTPUT_FORMAT" in
+ text)
+ print_header
+ local title="Cost Breakdown — ${PERIOD_START} → ${PERIOD_END}"
+ echo "$title"
+ output_text_table curr_costs prev_costs
+ echo ""
+ ;;
+ csv)
+ output_csv curr_costs prev_costs
+ ;;
+ json)
+ output_json curr_costs prev_costs
+ ;;
+ *)
+ die "Unknown format: $OUTPUT_FORMAT"
+ ;;
+ esac
+}
+
+# ── Slack webhook ─────────────────────────────────────────────────────
+send_slack() {
+ local report="$1" webhook="$2"
+
+ log_info "Posting report to Slack..."
+
+ local max_len=3000
+ local body="$report"
+ if (( ${#body} > max_len )); then
+ body="${body:0:$max_len}
+
+... (truncated — full report exceeds Slack message limit)"
+ fi
+
+ local payload
+ payload=$(jq -n --arg text "\`\`\`${body}\`\`\`" '{ text: $text }')
+
+ local http_code
+ http_code=$(curl -s -o /dev/null -w "%{http_code}" \
+ -X POST \
+ -H "Content-Type: application/json" \
+ -d "$payload" \
+ "$webhook")
+
+ if [[ "$http_code" != "200" ]]; then
+ log_error "Slack webhook returned HTTP $http_code"
+ return 1
+ fi
+
+ log_info "Slack message posted"
+}
+
+# ── Usage ─────────────────────────────────────────────────────────────
+usage() {
+ cat < 0 )); do
+ case "$1" in
+ --daily|--weekly|--monthly)
+ RUN_MODE="${1#--}"; shift ;;
+ --custom)
+ RUN_MODE="custom"
+ [[ $# -lt 3 ]] && die "--custom requires START and END dates"
+ CUSTOM_START="$2"; CUSTOM_END="$3"
+ validate_date "$CUSTOM_START"; validate_date "$CUSTOM_END"
+ shift 3 ;;
+ --group-by)
+ [[ $# -lt 2 ]] && die "--group-by requires a value"
+ GROUP_BY="$2"; shift 2 ;;
+ --label)
+ [[ $# -lt 2 ]] && die "--label requires KEY=VALUE"
+ [[ "$2" != *"="* ]] && die "--label value must be KEY=VALUE"
+ LABEL_FILTER_KEY="${2%%=*}"; LABEL_FILTER_VALUE="${2#*=}"; shift 2 ;;
+ --format)
+ [[ $# -lt 2 ]] && die "--format requires a value"
+ OUTPUT_FORMAT="$2"; shift 2 ;;
+ --slack)
+ [[ $# -lt 2 ]] && die "--slack requires a webhook URL"
+ SLACK_URL="$2"; shift 2 ;;
+ --project)
+ [[ $# -lt 2 ]] && die "--project requires a project ID"
+ GCP_PROJECT="$2"; shift 2 ;;
+ --verbose) VERBOSE="true"; shift ;;
+ --no-color) COLOR="never"; shift ;;
+ --help|-h) usage ;;
+ *) die "Unknown option: $1 (see --help)" ;;
+ esac
+ done
+
+ if [[ -z "$RUN_MODE" ]]; then log_error "No mode specified"; echo ""; usage; exit 1; fi
+ [[ -z "$BQ_BILLING_TABLE" ]] && die "BQ_BILLING_TABLE is required (e.g., project.dataset.gcp_billing_export_v1_XXXXXX)"
+ [[ -z "$SLACK_URL" && -n "$SLACK_WEBHOOK_URL" ]] && SLACK_URL="$SLACK_WEBHOOK_URL"
+
+ case "$GROUP_BY" in
+ SERVICE|PROJECT|LABEL) ;;
+ *) die "Invalid --group-by: $GROUP_BY (expected SERVICE, PROJECT, or LABEL)" ;;
+ esac
+ case "$OUTPUT_FORMAT" in
+ text|csv|json) ;;
+ *) die "Invalid --format: $OUTPUT_FORMAT" ;;
+ esac
+}
+
+# ── Main ──────────────────────────────────────────────────────────────
+main() {
+ parse_args "$@"
+ setup_colors
+ check_deps
+
+ START_TIME=$(date +%s)
+
+ log_debug "Validating GCP credentials..."
+ gcloud auth print-access-token >/dev/null 2>&1 \
+ || die "GCP credentials not configured or expired (run gcloud auth login)"
+
+ compute_ranges
+
+ log_info "Querying BigQuery billing data ($RUN_MODE, group by $GROUP_BY)..."
+
+ local curr_raw prev_raw
+ curr_raw="$(query_costs "$PERIOD_START" "$PERIOD_END")"
+ prev_raw="$(query_costs "$PREV_START" "$PREV_END")"
+
+ if [[ -z "$curr_raw" || "$curr_raw" == "[]" ]]; then
+ die "No cost data returned for $PERIOD_START → $PERIOD_END"
+ fi
+
+ local report
+ report="$(render_report "$curr_raw" "$prev_raw")"
+
+ echo "$report"
+
+ if [[ -n "$SLACK_URL" ]]; then
+ send_slack "$report" "$SLACK_URL"
+ fi
+
+ local elapsed=$(( $(date +%s) - START_TIME ))
+ log_info "Completed in ${elapsed}s"
+}
+
+main "$@"
diff --git a/gcp-firewall-auditor.sh b/gcp-firewall-auditor.sh
new file mode 100644
index 0000000..970d528
--- /dev/null
+++ b/gcp-firewall-auditor.sh
@@ -0,0 +1,635 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### gcp-firewall-auditor.sh — Audit GCP VPC firewall rules for risky configs ####
+#### Finds 0.0.0.0/0 rules, dangerous ports, overly permissive access, unused rules ####
+#### Requires: bash 4+, gcloud CLI, jq ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.01 ####
+#### ####
+#### Usage: ####
+#### ./gcp-firewall-auditor.sh --full ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Colors (pre-initialized) ─────────────────────────────────────────
+RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
+
+setup_colors() {
+ if [[ "${COLOR:-auto}" == "never" ]]; then
+ return
+ fi
+ if [[ "${COLOR:-auto}" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+die() { err "$*"; exit 1; }
+
+# ── Severity counters ────────────────────────────────────────────────
+TOTAL_CRIT=0
+TOTAL_WARN=0
+TOTAL_INFO=0
+TOTAL_OK=0
+
+flag_crit() { ((TOTAL_CRIT++)) || true; }
+flag_warn() { ((TOTAL_WARN++)) || true; }
+flag_info() { ((TOTAL_INFO++)) || true; }
+flag_ok() { ((TOTAL_OK++)) || true; }
+
+# ── Defaults ──────────────────────────────────────────────────────────
+RUN_MODE=""
+DANGEROUS_PORTS="${DANGEROUS_PORTS:-22,3389,3306,5432,1433,6379,27017,9200,8080,8443}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+GCP_PROJECT=""
+VPC_NETWORK=""
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+START_TIME=""
+
+# ── Dependency and credential checks ────────────────────────────────
+check_deps() {
+ command -v gcloud &>/dev/null || die "gcloud CLI is required (install: https://cloud.google.com/sdk/docs/install)"
+ command -v jq &>/dev/null || die "jq is required"
+}
+
+check_credentials() {
+ local account
+ account=$(gcloud auth list --filter="status:ACTIVE" --format="value(account)" 2>/dev/null)
+ [[ -z "$account" ]] && die "No active gcloud credentials — run 'gcloud auth login'"
+
+ if [[ -n "$GCP_PROJECT" ]]; then
+ gcloud config set project "$GCP_PROJECT" --quiet 2>/dev/null \
+ || die "Cannot set project: ${GCP_PROJECT}"
+ else
+ GCP_PROJECT=$(gcloud config get-value project 2>/dev/null)
+ [[ -z "$GCP_PROJECT" || "$GCP_PROJECT" == "(unset)" ]] && die "No project set — use --project or 'gcloud config set project'"
+ fi
+
+ verbose "Account: ${account}"
+ log "Project: ${GCP_PROJECT}"
+}
+
+# ── gcloud wrapper ───────────────────────────────────────────────────
+gc_cmd() {
+ local args=("$@")
+ [[ -n "$GCP_PROJECT" ]] && args+=(--project "$GCP_PROJECT")
+ verbose "gcloud ${args[*]}"
+ gcloud "${args[@]}"
+}
+
+# ── Port-to-service mapping ─────────────────────────────────────────
+port_to_service() {
+ local port="$1"
+ case "$port" in
+ 22) echo "SSH" ;;
+ 80) echo "HTTP" ;;
+ 443) echo "HTTPS" ;;
+ 3306) echo "MySQL" ;;
+ 5432) echo "PostgreSQL" ;;
+ 1433) echo "MSSQL" ;;
+ 3389) echo "RDP" ;;
+ 6379) echo "Redis" ;;
+ 27017) echo "MongoDB" ;;
+ 9200) echo "Elasticsearch" ;;
+ 8080) echo "HTTP-Alt" ;;
+ 8443) echo "HTTPS-Alt" ;;
+ 53) echo "DNS" ;;
+ 25) echo "SMTP" ;;
+ 5900) echo "VNC" ;;
+ 11211) echo "Memcached" ;;
+ 2379) echo "etcd" ;;
+ 9090) echo "Prometheus" ;;
+ *) echo "" ;;
+ esac
+}
+
+# ── Check if port is in dangerous list ───────────────────────────────
+is_dangerous_port() {
+ local port="$1"
+ local IFS=','
+ for dp in $DANGEROUS_PORTS; do
+ if [[ "$port" == "$dp" ]]; then
+ return 0
+ fi
+ done
+ return 1
+}
+
+# ── Check if port falls in a range ───────────────────────────────────
+port_in_range() {
+ local port="$1" range="$2"
+ if [[ "$range" == *-* ]]; then
+ local start="${range%-*}"
+ local end="${range#*-}"
+ [[ "$port" -ge "$start" && "$port" -le "$end" ]]
+ else
+ [[ "$port" == "$range" ]]
+ fi
+}
+
+# ── Fetch firewall rules ────────────────────────────────────────────
+fetch_rules() {
+ local args=(compute firewall-rules list --format=json)
+ if [[ -n "$VPC_NETWORK" ]]; then
+ args+=(--filter="network~${VPC_NETWORK}")
+ fi
+ gc_cmd "${args[@]}" 2>/dev/null
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OPEN PORTS AUDIT
+# ══════════════════════════════════════════════════════════════════════
+audit_open_ports() {
+ log "Auditing firewall rules for dangerous open ports..."
+ log "Dangerous ports: ${DANGEROUS_PORTS}"
+ echo ""
+
+ printf " %-28s %-14s %-8s %-8s %-18s %s\n" \
+ "RULE_NAME" "NETWORK" "PROTO" "PORT" "SOURCE" "SEVERITY"
+ printf " %s\n" "$(printf '%.0s─' {1..95})"
+
+ local rules_json
+ rules_json=$(fetch_rules)
+
+ echo "$rules_json" | jq -c '.[] | select(.direction == "INGRESS" and .disabled != true)' 2>/dev/null | while IFS= read -r rule; do
+ local rule_name network
+ rule_name=$(echo "$rule" | jq -r '.name')
+ network=$(echo "$rule" | jq -r '.network' | rev | cut -d/ -f1 | rev)
+
+ local has_open="false"
+ while IFS= read -r src; do
+ if [[ "$src" == "0.0.0.0/0" ]]; then
+ has_open="true"
+ break
+ fi
+ done < <(echo "$rule" | jq -r '.sourceRanges[]? // empty' 2>/dev/null)
+
+ [[ "$has_open" != "true" ]] && continue
+
+ echo "$rule" | jq -c '.allowed[]? // empty' 2>/dev/null | while IFS= read -r allowed; do
+ local protocol
+ protocol=$(echo "$allowed" | jq -r '.IPProtocol')
+
+ local ports
+ ports=$(echo "$allowed" | jq -r '.ports[]? // empty' 2>/dev/null)
+
+ if [[ -z "$ports" ]]; then
+ if [[ "$protocol" == "all" ]]; then
+ printf " %-28s %-14s %-8s %-8s %-18s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "all" "all" \
+ "0.0.0.0/0" "$RED" "CRITICAL" "$RESET"
+ flag_crit
+ else
+ local IFS=','
+ for dp in $DANGEROUS_PORTS; do
+ local svc
+ svc=$(port_to_service "$dp")
+ printf " %-28s %-14s %-8s %-8s %-18s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "$protocol" "$dp" \
+ "0.0.0.0/0" "$RED" "CRITICAL" "$RESET"
+ flag_crit
+ done
+ fi
+ continue
+ fi
+
+ while IFS= read -r port_spec; do
+ [[ -z "$port_spec" ]] && continue
+
+ local IFS=','
+ for dp in $DANGEROUS_PORTS; do
+ if port_in_range "$dp" "$port_spec"; then
+ local svc severity color
+ svc=$(port_to_service "$dp")
+ if [[ "$dp" == "80" || "$dp" == "443" ]]; then
+ severity="INFO"; color="$CYAN"; flag_info
+ else
+ severity="CRITICAL"; color="$RED"; flag_crit
+ fi
+ printf " %-28s %-14s %-8s %-8s %-18s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "$protocol" \
+ "${dp} (${svc})" "0.0.0.0/0" "$color" "$severity" "$RESET"
+ fi
+ done
+ done <<< "$ports"
+ done
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# PERMISSIVE RULES AUDIT
+# ══════════════════════════════════════════════════════════════════════
+audit_permissive() {
+ log "Auditing overly permissive firewall rules..."
+ echo ""
+
+ printf " %-28s %-14s %-14s %-18s %s\n" \
+ "RULE_NAME" "NETWORK" "PROTOCOLS" "SOURCE" "SEVERITY"
+ printf " %s\n" "$(printf '%.0s─' {1..85})"
+
+ local rules_json
+ rules_json=$(fetch_rules)
+
+ echo "$rules_json" | jq -c '.[] | select(.direction == "INGRESS" and .disabled != true)' 2>/dev/null | while IFS= read -r rule; do
+ local rule_name network
+ rule_name=$(echo "$rule" | jq -r '.name')
+ network=$(echo "$rule" | jq -r '.network' | rev | cut -d/ -f1 | rev)
+
+ local has_open="false"
+ while IFS= read -r src; do
+ if [[ "$src" == "0.0.0.0/0" ]]; then
+ has_open="true"
+ break
+ fi
+ done < <(echo "$rule" | jq -r '.sourceRanges[]? // empty' 2>/dev/null)
+
+ [[ "$has_open" != "true" ]] && continue
+
+ local has_all_traffic="false"
+ while IFS= read -r allowed; do
+ local proto
+ proto=$(echo "$allowed" | jq -r '.IPProtocol')
+ local port_count
+ port_count=$(echo "$allowed" | jq '.ports // [] | length')
+
+ if [[ "$proto" == "all" ]]; then
+ has_all_traffic="true"
+ elif [[ "$port_count" -eq 0 ]]; then
+ has_all_traffic="true"
+ fi
+ done < <(echo "$rule" | jq -c '.allowed[]? // empty' 2>/dev/null)
+
+ if [[ "$has_all_traffic" == "true" ]]; then
+ local proto_list
+ proto_list=$(echo "$rule" | jq -r '[.allowed[]?.IPProtocol] | join(",")' 2>/dev/null)
+ printf " %-28s %-14s %-14s %-18s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "${proto_list:0:13}" \
+ "0.0.0.0/0" "$RED" "CRITICAL" "$RESET"
+ flag_crit
+ fi
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# EGRESS AUDIT
+# ══════════════════════════════════════════════════════════════════════
+audit_egress() {
+ log "Auditing egress firewall rules..."
+ echo ""
+
+ printf " %-28s %-14s %-14s %-18s %s\n" \
+ "RULE_NAME" "NETWORK" "PROTOCOLS" "DESTINATION" "SEVERITY"
+ printf " %s\n" "$(printf '%.0s─' {1..85})"
+
+ local rules_json
+ rules_json=$(fetch_rules)
+
+ echo "$rules_json" | jq -c '.[] | select(.direction == "EGRESS" and .disabled != true)' 2>/dev/null | while IFS= read -r rule; do
+ local rule_name network
+ rule_name=$(echo "$rule" | jq -r '.name')
+ network=$(echo "$rule" | jq -r '.network' | rev | cut -d/ -f1 | rev)
+
+ local has_wide="false"
+ while IFS= read -r dest; do
+ if [[ "$dest" == "0.0.0.0/0" ]]; then
+ has_wide="true"
+ break
+ fi
+ done < <(echo "$rule" | jq -r '.destinationRanges[]? // empty' 2>/dev/null)
+
+ [[ "$has_wide" != "true" ]] && continue
+
+ local proto_list
+ proto_list=$(echo "$rule" | jq -r '[.allowed[]?.IPProtocol] | join(",")' 2>/dev/null)
+
+ local severity="WARN" color="$YELLOW"
+ if [[ "$proto_list" == "all" ]]; then
+ severity="WARN"; color="$YELLOW"; flag_warn
+ else
+ severity="INFO"; color="$CYAN"; flag_info
+ fi
+
+ printf " %-28s %-14s %-14s %-18s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "${proto_list:0:13}" \
+ "0.0.0.0/0" "$color" "$severity" "$RESET"
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# UNUSED RULES AUDIT
+# ══════════════════════════════════════════════════════════════════════
+audit_unused() {
+ log "Checking for disabled or potentially unused firewall rules..."
+ echo ""
+
+ printf " %-28s %-14s %-10s %-10s %s\n" \
+ "RULE_NAME" "NETWORK" "DIRECTION" "DISABLED" "SEVERITY"
+ printf " %s\n" "$(printf '%.0s─' {1..80})"
+
+ local rules_json
+ rules_json=$(fetch_rules)
+
+ echo "$rules_json" | jq -c '.[]' 2>/dev/null | while IFS= read -r rule; do
+ local rule_name network direction disabled
+ rule_name=$(echo "$rule" | jq -r '.name')
+ network=$(echo "$rule" | jq -r '.network' | rev | cut -d/ -f1 | rev)
+ direction=$(echo "$rule" | jq -r '.direction')
+ disabled=$(echo "$rule" | jq -r '.disabled // false')
+
+ if [[ "$disabled" == "true" ]]; then
+ printf " %-28s %-14s %-10s %-10s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "$direction" "YES" \
+ "$YELLOW" "WARN — disabled" "$RESET"
+ flag_warn
+ continue
+ fi
+
+ local target_tags
+ target_tags=$(echo "$rule" | jq -r '.targetTags // [] | join(",")' 2>/dev/null)
+
+ if [[ -n "$target_tags" && "$target_tags" != "null" ]]; then
+ local first_tag="${target_tags%%,*}"
+ local instance_count
+ instance_count=$(gcloud compute instances list \
+ --filter="tags.items=${first_tag}" \
+ --format="value(name)" 2>/dev/null | wc -l)
+
+ if [[ "$instance_count" -eq 0 ]]; then
+ printf " %-28s %-14s %-10s %-10s %b%s%b\n" \
+ "${rule_name:0:27}" "${network:0:13}" "$direction" "NO" \
+ "$YELLOW" "WARN — no targets" "$RESET"
+ flag_warn
+ else
+ verbose "Rule ${rule_name}: ${instance_count} matching instance(s)"
+ flag_ok
+ fi
+ else
+ flag_ok
+ fi
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# LIST ALL RULES
+# ══════════════════════════════════════════════════════════════════════
+list_rules() {
+ log "Listing all firewall rules..."
+ echo ""
+
+ printf " %-28s %-14s %-10s %-8s %-12s %-18s %s\n" \
+ "RULE_NAME" "NETWORK" "DIR" "PROTO" "PORTS" "SOURCE/DEST" "PRIORITY"
+ printf " %s\n" "$(printf '%.0s─' {1..105})"
+
+ local rules_json
+ rules_json=$(fetch_rules)
+
+ echo "$rules_json" | jq -c '.[]' 2>/dev/null | while IFS= read -r rule; do
+ local rule_name network direction priority
+ rule_name=$(echo "$rule" | jq -r '.name')
+ network=$(echo "$rule" | jq -r '.network' | rev | cut -d/ -f1 | rev)
+ direction=$(echo "$rule" | jq -r '.direction')
+ priority=$(echo "$rule" | jq -r '.priority')
+
+ local cidr_list
+ if [[ "$direction" == "INGRESS" ]]; then
+ cidr_list=$(echo "$rule" | jq -r '.sourceRanges[0]? // "any"' 2>/dev/null)
+ else
+ cidr_list=$(echo "$rule" | jq -r '.destinationRanges[0]? // "any"' 2>/dev/null)
+ fi
+
+ echo "$rule" | jq -c '.allowed[]? // empty' 2>/dev/null | while IFS= read -r allowed; do
+ local proto port_str
+ proto=$(echo "$allowed" | jq -r '.IPProtocol')
+ port_str=$(echo "$allowed" | jq -r '.ports // ["all"] | join(",")' 2>/dev/null)
+ [[ "$port_str" == "null" ]] && port_str="all"
+
+ local dir_color="$CYAN"
+ [[ "$direction" == "EGRESS" ]] && dir_color="$YELLOW"
+
+ printf " %-28s %-14s %b%-10s%b %-8s %-12s %-18s %s\n" \
+ "${rule_name:0:27}" "${network:0:13}" "$dir_color" "$direction" "$RESET" \
+ "$proto" "${port_str:0:11}" "${cidr_list:0:17}" "$priority"
+ done
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SUMMARY
+# ══════════════════════════════════════════════════════════════════════
+print_summary() {
+ local elapsed
+ elapsed=$(( $(date +%s) - START_TIME ))
+
+ echo ""
+ echo " ══════════════════════════════════════════"
+ echo " Firewall Audit Summary"
+ echo " ══════════════════════════════════════════"
+ printf " %-20s %b%d%b\n" "CRITICAL:" "$RED" "$TOTAL_CRIT" "$RESET"
+ printf " %-20s %b%d%b\n" "WARN:" "$YELLOW" "$TOTAL_WARN" "$RESET"
+ printf " %-20s %b%d%b\n" "INFO:" "$CYAN" "$TOTAL_INFO" "$RESET"
+ printf " %-20s %b%d%b\n" "OK:" "$GREEN" "$TOTAL_OK" "$RESET"
+ echo " ──────────────────────────────────────────"
+ printf " Completed in %ds\n" "$elapsed"
+ echo ""
+
+ if [[ "$TOTAL_CRIT" -gt 0 ]]; then
+ echo -e " ${RED}${BOLD}Action required:${RESET} ${TOTAL_CRIT} critical finding(s)"
+ echo ""
+ echo " Top recommendations:"
+ echo " • Close 0.0.0.0/0 rules on SSH (22), RDP (3389), and database ports"
+ echo " • Replace all-protocol allow rules with specific port lists"
+ echo " • Use target tags or service accounts to scope rules"
+ echo " • Delete disabled rules that are no longer needed"
+ echo ""
+ elif [[ "$TOTAL_WARN" -gt 0 ]]; then
+ echo -e " ${YELLOW}Review recommended:${RESET} ${TOTAL_WARN} warning(s)"
+ echo ""
+ echo " Suggestions:"
+ echo " • Review disabled rules for deletion"
+ echo " • Check rules with no matching target instances"
+ echo " • Restrict egress where applicable"
+ echo ""
+ else
+ echo -e " ${GREEN}All checks passed${RESET}"
+ echo ""
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# USAGE
+# ══════════════════════════════════════════════════════════════════════
+show_help() {
+ cat <&2
+ exit 1
+ fi
+
+ RUN_MODE="${modes[*]}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+main() {
+ parse_args "$@"
+ setup_colors
+ check_deps
+ check_credentials
+
+ START_TIME=$(date +%s)
+
+ echo ""
+ echo -e "${BOLD}GCP Firewall Auditor${RESET}"
+ echo -e "Project: ${GCP_PROJECT}"
+ echo -e "Mode: ${RUN_MODE}"
+ if [[ -n "$VPC_NETWORK" ]]; then
+ echo -e "Network: ${VPC_NETWORK}"
+ fi
+ echo -e "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ echo ""
+
+ for mode in $RUN_MODE; do
+ case "$mode" in
+ open-ports) audit_open_ports ;;
+ permissive) audit_permissive ;;
+ unused) audit_unused ;;
+ egress) audit_egress ;;
+ rules) list_rules ;;
+ esac
+ done
+
+ print_summary
+
+ if [[ "$TOTAL_CRIT" -gt 0 ]]; then
+ exit 2
+ elif [[ "$TOTAL_WARN" -gt 0 ]]; then
+ exit 1
+ fi
+ exit 0
+}
+
+main "$@"
diff --git a/gcp-snapshot-manager.sh b/gcp-snapshot-manager.sh
new file mode 100644
index 0000000..054dbb6
--- /dev/null
+++ b/gcp-snapshot-manager.sh
@@ -0,0 +1,708 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### gcp-snapshot-manager.sh — Create, rotate, list, audit, and restore GCP ####
+#### persistent disk snapshots via gcloud CLI. Automated retention and fleet ops ####
+#### Requires: bash 4+, gcloud CLI, jq ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.01 ####
+#### ####
+#### Usage: ####
+#### ./gcp-snapshot-manager.sh --snapshot --all ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Colors (pre-initialized) ─────────────────────────────────────────
+RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
+
+setup_colors() {
+ if [[ "${COLOR:-auto}" == "never" ]]; then
+ return
+ fi
+ if [[ "${COLOR:-auto}" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+die() { err "$*"; exit 1; }
+
+section_header() {
+ echo ""
+ echo -e " ${BOLD}${CYAN}── $1 ──${RESET}"
+ echo ""
+}
+
+field() {
+ printf " ${BOLD}%-22s${RESET} %s\n" "$1" "$2"
+}
+
+field_color() {
+ printf " ${BOLD}%-22s${RESET} %b\n" "$1" "$2"
+}
+
+elapsed() {
+ local end_time
+ end_time=$(date +%s)
+ echo "$(( end_time - START_TIME ))s"
+}
+
+# ── Defaults ──────────────────────────────────────────────────────────
+RUN_MODE=""
+ALSO_ROTATE="false"
+INSTANCE_NAME=""
+ZONE=""
+TARGET_ALL="false"
+SNAPSHOT_NAME=""
+KEEP="${GSM_KEEP:-3}"
+PREFIX="${GSM_PREFIX:-auto}"
+MAX_AGE="${GSM_MAX_AGE:-7}"
+OUTPUT_FORMAT="${GSM_FORMAT:-text}"
+DRY_RUN="true"
+FORCE="false"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+GCP_PROJECT=""
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+START_TIME=""
+SNAP_CREATED=0
+SNAP_DELETED=0
+SNAP_ERRORS=0
+
+# ── Dependency and credential checks ────────────────────────────────
+check_deps() {
+ command -v gcloud &>/dev/null || die "gcloud CLI is required"
+ command -v jq &>/dev/null || die "jq is required"
+}
+
+check_credentials() {
+ local account
+ account=$(gcloud auth list --filter="status:ACTIVE" --format="value(account)" 2>/dev/null)
+ [[ -z "$account" ]] && die "No active gcloud credentials — run 'gcloud auth login'"
+
+ if [[ -n "$GCP_PROJECT" ]]; then
+ gcloud config set project "$GCP_PROJECT" --quiet 2>/dev/null \
+ || die "Cannot set project: ${GCP_PROJECT}"
+ else
+ GCP_PROJECT=$(gcloud config get-value project 2>/dev/null)
+ [[ -z "$GCP_PROJECT" || "$GCP_PROJECT" == "(unset)" ]] && die "No project set — use --project or 'gcloud config set project'"
+ fi
+
+ log "Project: ${GCP_PROJECT}"
+}
+
+# ── Instance helpers ─────────────────────────────────────────────────
+get_all_instances() {
+ gcloud compute instances list --project "$GCP_PROJECT" --format=json 2>/dev/null
+}
+
+get_boot_disk() {
+ local instance="$1" zone="$2"
+ gcloud compute instances describe "$instance" --zone "$zone" --project "$GCP_PROJECT" \
+ --format='json(disks)' 2>/dev/null \
+ | jq -r '.disks[] | select(.boot == true) | .source' 2>/dev/null \
+ | rev | cut -d/ -f1 | rev
+}
+
+get_instance_zone() {
+ local instance_json="$1"
+ echo "$instance_json" | jq -r '.zone' | rev | cut -d/ -f1 | rev
+}
+
+# ── Snapshot helpers ─────────────────────────────────────────────────
+list_snapshots() {
+ gcloud compute snapshots list --project "$GCP_PROJECT" --format=json 2>/dev/null
+}
+
+managed_snapshots() {
+ list_snapshots | jq --arg pfx "$PREFIX" \
+ '[.[] | select(.name | startswith($pfx))]'
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# SNAPSHOT
+# ══════════════════════════════════════════════════════════════════════
+do_snapshot() {
+ local instances_json
+ instances_json=$(get_all_instances)
+
+ local instances
+ if [[ "$TARGET_ALL" == "true" ]]; then
+ instances="$instances_json"
+ elif [[ -n "$INSTANCE_NAME" ]]; then
+ instances=$(echo "$instances_json" | jq --arg n "$INSTANCE_NAME" '[.[] | select(.name == $n)]')
+ else
+ die "Specify --instance NAME or --all"
+ fi
+
+ local count
+ count=$(echo "$instances" | jq 'length')
+ [[ "$count" -eq 0 ]] && die "No instances found"
+
+ local target_label="$INSTANCE_NAME"
+ [[ "$TARGET_ALL" == "true" ]] && target_label="all (${count} instances)"
+
+ section_header "Creating Snapshots"
+ field "Target:" "$target_label"
+ field "Prefix:" "$PREFIX"
+ echo ""
+
+ echo "$instances" | jq -c '.[]' | while IFS= read -r inst; do
+ local name zone disk_name snap_name
+ name=$(echo "$inst" | jq -r '.name')
+ zone=$(get_instance_zone "$inst")
+ disk_name=$(get_boot_disk "$name" "$zone")
+ snap_name="${PREFIX}-${name}-$(date +%Y%m%d-%H%M%S)"
+
+ if [[ -z "$disk_name" ]]; then
+ echo -e " ${RED}✗${RESET} ${name} (${zone}) no boot disk found"
+ ((SNAP_ERRORS++)) || true
+ continue
+ fi
+
+ verbose "Snapshotting ${name} disk ${disk_name} in ${zone}"
+
+ if gcloud compute snapshots create "$snap_name" \
+ --source-disk="$disk_name" \
+ --source-disk-zone="$zone" \
+ --project "$GCP_PROJECT" \
+ --labels="managed-by=gcp-snapshot-manager,source-instance=${name}" \
+ --quiet 2>/dev/null; then
+ echo -e " ${GREEN}✓${RESET} ${name} (${zone}) ${snap_name}"
+ ((SNAP_CREATED++)) || true
+ else
+ echo -e " ${RED}✗${RESET} ${name} (${zone}) failed"
+ ((SNAP_ERRORS++)) || true
+ fi
+
+ sleep 1
+ done
+
+ echo ""
+ field_color "Created:" "${GREEN}${SNAP_CREATED}${RESET}"
+ if [[ "$SNAP_ERRORS" -gt 0 ]]; then
+ field_color "Errors:" "${RED}${SNAP_ERRORS}${RESET}"
+ fi
+
+ if [[ "$ALSO_ROTATE" == "true" ]]; then
+ do_rotate
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# ROTATE
+# ══════════════════════════════════════════════════════════════════════
+do_rotate() {
+ section_header "Rotating Snapshots"
+ field "Keep:" "$KEEP per instance"
+ field "Prefix:" "$PREFIX"
+ if [[ "$DRY_RUN" == "true" && "$FORCE" != "true" ]]; then
+ field "Mode:" "DRY RUN (use --force to delete)"
+ else
+ field "Mode:" "LIVE — deletions are permanent"
+ fi
+ echo ""
+
+ local snaps
+ snaps=$(managed_snapshots)
+
+ local instance_names
+ instance_names=$(echo "$snaps" | jq -r '.[].labels["source-instance"] // empty' | sort -u)
+
+ if [[ -z "$instance_names" ]]; then
+ log "No managed snapshots found matching prefix '${PREFIX}'"
+ return
+ fi
+
+ while IFS= read -r inst; do
+ [[ -z "$inst" ]] && continue
+ local inst_snaps
+ inst_snaps=$(echo "$snaps" | jq --arg inst "$inst" \
+ '[.[] | select(.labels["source-instance"] == $inst)] | sort_by(.creationTimestamp) | reverse')
+ local total
+ total=$(echo "$inst_snaps" | jq 'length')
+
+ if (( total <= KEEP )); then
+ verbose "${inst}: ${total} snapshots, keeping all"
+ continue
+ fi
+
+ local to_delete
+ to_delete=$(echo "$inst_snaps" | jq --argjson k "$KEEP" '.[$k:]')
+ local del_count
+ del_count=$(echo "$to_delete" | jq 'length')
+
+ echo "$to_delete" | jq -c '.[]' | while IFS= read -r snap; do
+ local sname
+ sname=$(echo "$snap" | jq -r '.name')
+
+ if [[ "$DRY_RUN" == "true" && "$FORCE" != "true" ]]; then
+ echo -e " ${DIM}[DRY RUN]${RESET} would delete ${sname}"
+ else
+ if gcloud compute snapshots delete "$sname" \
+ --project "$GCP_PROJECT" --quiet 2>/dev/null; then
+ echo -e " ${YELLOW}✓${RESET} deleted ${sname}"
+ ((SNAP_DELETED++)) || true
+ else
+ echo -e " ${RED}✗${RESET} failed to delete ${sname}"
+ ((SNAP_ERRORS++)) || true
+ fi
+ fi
+ done
+
+ log "${inst}: ${total} total, keeping ${KEEP}, removing ${del_count}"
+ done <<< "$instance_names"
+
+ echo ""
+ field_color "Deleted:" "${YELLOW}${SNAP_DELETED}${RESET}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# LIST
+# ══════════════════════════════════════════════════════════════════════
+do_list() {
+ section_header "All Snapshots"
+
+ local snaps
+ snaps=$(list_snapshots)
+ local count
+ count=$(echo "$snaps" | jq 'length')
+
+ if [[ "$count" -eq 0 ]]; then
+ log "No snapshots found"
+ return
+ fi
+
+ printf " %-40s %-10s %-12s %-16s %s\n" \
+ "NAME" "SIZE_GB" "AGE" "SOURCE_DISK" "SOURCE_INSTANCE"
+ printf " %s\n" "$(printf '%.0s─' {1..100})"
+
+ local now
+ now=$(date +%s)
+
+ echo "$snaps" | jq -c '.[]' | while IFS= read -r snap; do
+ local name size_gb created source_disk source_inst age_str
+ name=$(echo "$snap" | jq -r '.name')
+ size_gb=$(echo "$snap" | jq -r '.diskSizeGb // 0')
+ created=$(echo "$snap" | jq -r '.creationTimestamp // ""')
+ source_disk=$(echo "$snap" | jq -r '.sourceDisk // ""' | rev | cut -d/ -f1 | rev)
+ source_inst=$(echo "$snap" | jq -r '.labels["source-instance"] // "manual"')
+
+ if [[ -n "$created" ]]; then
+ local snap_epoch
+ snap_epoch=$(date -d "$created" +%s 2>/dev/null || echo 0)
+ if [[ "$snap_epoch" -gt 0 ]]; then
+ local age_days=$(( (now - snap_epoch) / 86400 ))
+ age_str="${age_days}d"
+ else
+ age_str="unknown"
+ fi
+ else
+ age_str="unknown"
+ fi
+
+ printf " %-40s %-10s %-12s %-16s %s\n" \
+ "${name:0:39}" "$size_gb" "$age_str" "${source_disk:0:15}" "${source_inst:0:20}"
+ done
+
+ echo ""
+ field "Total snapshots:" "$count"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# AUDIT
+# ══════════════════════════════════════════════════════════════════════
+do_audit() {
+ section_header "Snapshot Audit"
+
+ local instances_json
+ instances_json=$(get_all_instances)
+ local snaps
+ snaps=$(list_snapshots)
+ local now
+ now=$(date +%s)
+
+ printf " %-24s %-14s %-24s %-8s %-8s %s\n" \
+ "INSTANCE" "ZONE" "LATEST_SNAPSHOT" "AGE" "COUNT" "STATUS"
+ printf " %s\n" "$(printf '%.0s─' {1..100})"
+
+ echo "$instances_json" | jq -c '.[]' | while IFS= read -r inst; do
+ local name zone
+ name=$(echo "$inst" | jq -r '.name')
+ zone=$(get_instance_zone "$inst")
+
+ local inst_snaps snap_count
+ inst_snaps=$(echo "$snaps" | jq --arg inst "$name" \
+ '[.[] | select(.labels["source-instance"] == $inst)]')
+ snap_count=$(echo "$inst_snaps" | jq 'length')
+
+ if [[ "$snap_count" -eq 0 ]]; then
+ printf " %-24s %-14s %-24s %-8s %-8s %b%s%b\n" \
+ "${name:0:23}" "${zone:0:13}" "(none)" "—" "0" \
+ "$RED" "✗ Unprotected" "$RESET"
+ continue
+ fi
+
+ local latest_name latest_date age_str status color
+ latest_name=$(echo "$inst_snaps" | jq -r 'sort_by(.creationTimestamp) | last | .name // ""')
+ latest_date=$(echo "$inst_snaps" | jq -r 'sort_by(.creationTimestamp) | last | .creationTimestamp // ""')
+
+ if [[ -n "$latest_date" ]]; then
+ local snap_epoch
+ snap_epoch=$(date -d "$latest_date" +%s 2>/dev/null || echo 0)
+ if [[ "$snap_epoch" -gt 0 ]]; then
+ local age_days=$(( (now - snap_epoch) / 86400 ))
+ age_str="${age_days}d"
+ if (( age_days > MAX_AGE )); then
+ status="⚠ Stale"; color="$YELLOW"
+ else
+ status="✓ OK"; color="$GREEN"
+ fi
+ else
+ age_str="unknown"; status="✓ OK"; color="$GREEN"
+ fi
+ else
+ age_str="unknown"; status="✓ OK"; color="$GREEN"
+ fi
+
+ printf " %-24s %-14s %-24s %-8s %-8s %b%s%b\n" \
+ "${name:0:23}" "${zone:0:13}" "${latest_name:0:23}" \
+ "$age_str" "$snap_count" "$color" "$status" "$RESET"
+ done
+
+ echo ""
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# RESTORE
+# ══════════════════════════════════════════════════════════════════════
+do_restore() {
+ [[ -z "$INSTANCE_NAME" ]] && die "--restore requires --instance NAME"
+ [[ -z "$SNAPSHOT_NAME" ]] && die "--restore requires --snapshot-name NAME"
+ [[ -z "$ZONE" ]] && die "--restore requires --zone ZONE"
+
+ section_header "Restore from Snapshot"
+ field "Instance:" "$INSTANCE_NAME"
+ field "Snapshot:" "$SNAPSHOT_NAME"
+ field "Zone:" "$ZONE"
+ echo ""
+
+ if [[ "$FORCE" != "true" ]]; then
+ warn "This will stop the instance and replace its boot disk. Use --force to confirm."
+ return
+ fi
+
+ log "Creating disk from snapshot..."
+ local disk_name="restored-${INSTANCE_NAME}-$(date +%Y%m%d-%H%M%S)"
+
+ if gcloud compute disks create "$disk_name" \
+ --source-snapshot="$SNAPSHOT_NAME" \
+ --zone="$ZONE" \
+ --project "$GCP_PROJECT" \
+ --quiet 2>/dev/null; then
+ echo -e " ${GREEN}✓${RESET} Disk created: ${disk_name}"
+ else
+ die "Failed to create disk from snapshot"
+ fi
+
+ log "Stopping instance..."
+ gcloud compute instances stop "$INSTANCE_NAME" \
+ --zone="$ZONE" --project "$GCP_PROJECT" --quiet 2>/dev/null \
+ || die "Failed to stop instance"
+
+ local old_disk
+ old_disk=$(get_boot_disk "$INSTANCE_NAME" "$ZONE")
+
+ log "Detaching old boot disk..."
+ gcloud compute instances detach-disk "$INSTANCE_NAME" \
+ --disk="$old_disk" --zone="$ZONE" --project "$GCP_PROJECT" \
+ --quiet 2>/dev/null || die "Failed to detach old disk"
+
+ log "Attaching restored disk..."
+ gcloud compute instances attach-disk "$INSTANCE_NAME" \
+ --disk="$disk_name" --zone="$ZONE" --boot \
+ --project "$GCP_PROJECT" --quiet 2>/dev/null \
+ || die "Failed to attach restored disk"
+
+ log "Starting instance..."
+ gcloud compute instances start "$INSTANCE_NAME" \
+ --zone="$ZONE" --project "$GCP_PROJECT" --quiet 2>/dev/null
+ echo -e " ${GREEN}✓${RESET} Instance started with restored disk"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# STATUS
+# ══════════════════════════════════════════════════════════════════════
+do_status() {
+ local instances_json
+ instances_json=$(get_all_instances)
+ local snaps
+ snaps=$(list_snapshots)
+ local now
+ now=$(date +%s)
+
+ local total_instances=0 total_snaps=0 total_gb=0
+ local protected=0 stale=0 unprotected=0
+
+ while IFS= read -r inst; do
+ [[ -z "$inst" ]] && continue
+ ((total_instances++)) || true
+
+ local name
+ name=$(echo "$inst" | jq -r '.name')
+
+ local inst_snaps snap_count
+ inst_snaps=$(echo "$snaps" | jq --arg inst "$name" \
+ '[.[] | select(.labels["source-instance"] == $inst)]')
+ snap_count=$(echo "$inst_snaps" | jq 'length')
+ total_snaps=$(( total_snaps + snap_count ))
+
+ local gb
+ gb=$(echo "$inst_snaps" | jq '[.[].diskSizeGb // 0 | tonumber] | add // 0')
+ total_gb=$(( total_gb + gb ))
+
+ if [[ "$snap_count" -eq 0 ]]; then
+ ((unprotected++)) || true
+ continue
+ fi
+
+ local latest_date
+ latest_date=$(echo "$inst_snaps" | jq -r \
+ 'sort_by(.creationTimestamp) | last | .creationTimestamp // ""')
+
+ if [[ -n "$latest_date" ]]; then
+ local snap_epoch
+ snap_epoch=$(date -d "$latest_date" +%s 2>/dev/null || echo 0)
+ if [[ "$snap_epoch" -gt 0 ]]; then
+ local age_days=$(( (now - snap_epoch) / 86400 ))
+ if (( age_days > MAX_AGE )); then
+ ((stale++)) || true
+ else
+ ((protected++)) || true
+ fi
+ else
+ ((protected++)) || true
+ fi
+ else
+ ((protected++)) || true
+ fi
+ done < <(echo "$instances_json" | jq -c '.[]')
+
+ if [[ "$OUTPUT_FORMAT" == "prometheus" ]]; then
+ cat <${MAX_AGE}d):" "${YELLOW}${stale}${RESET}"
+ else
+ field_color "Stale (>${MAX_AGE}d):" "${GREEN}0${RESET}"
+ fi
+ if [[ "$unprotected" -gt 0 ]]; then
+ field_color "Unprotected:" "${RED}${unprotected}${RESET}"
+ else
+ field_color "Unprotected:" "${GREEN}0${RESET}"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# HELP
+# ══════════════════════════════════════════════════════════════════════
+show_help() {
+ cat </dev/null; then
+ missing+=("$cmd")
+ fi
+ done
+ if [[ ${#missing[@]} -gt 0 ]]; then
+ echo "ERROR: Missing required commands: ${missing[*]}" >&2
+ echo "Install with: apt install ${missing[*]} OR dnf install ${missing[*]}" >&2
+ exit 1
+ fi
+}
+
+validate_config() {
+ if [[ -z "$GITEA_URL" ]]; then
+ echo "ERROR: GITEA_URL environment variable is required" >&2
+ exit 1
+ fi
+ if [[ -z "$GITEA_TOKEN" ]]; then
+ echo "ERROR: GITEA_TOKEN environment variable is required" >&2
+ exit 1
+ fi
+ # Strip trailing slash
+ GITEA_URL="${GITEA_URL%/}"
+}
+
+api_get() {
+ local endpoint="$1"
+ curl -sf --max-time "$CURL_TIMEOUT" \
+ -H "Authorization: token ${GITEA_TOKEN}" \
+ "${GITEA_URL}${endpoint}" 2>/dev/null || echo ""
+}
+
+api_get_with_headers() {
+ local endpoint="$1"
+ local response
+ response=$(curl -sD - --max-time "$CURL_TIMEOUT" \
+ -H "Authorization: token ${GITEA_TOKEN}" \
+ "${GITEA_URL}${endpoint}" 2>/dev/null) || { echo ""; return; }
+
+ local headers body
+ headers=$(echo "$response" | sed '/^\r$/q')
+ body=$(echo "$response" | sed '1,/^\r$/d')
+
+ local total_count
+ total_count=$(echo "$headers" | grep -i '^X-Total-Count:' | tr -d '\r' | awk '{print $2}')
+
+ echo "${total_count:-0}"
+ echo "$body"
+}
+
+sanitize_label() {
+ local value="$1"
+ echo "$value" | sed 's/[^a-zA-Z0-9_\/.-]/_/g'
+}
+
+add_metric() {
+ local name="$1"
+ local type="$2"
+ local help="$3"
+ local value="$4"
+ local labels="${5:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name} ${value}
+"
+ fi
+}
+
+add_metric_value() {
+ local name="$1"
+ local value="$2"
+ local labels="${3:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="${name} ${value}
+"
+ fi
+}
+
+collect_version() {
+ local version_json
+ version_json=$(api_get "/api/v1/version")
+
+ if [[ -z "$version_json" ]]; then
+ add_metric "gitea_up" "gauge" "Gitea reachability (1=up, 0=down)" "0"
+ return 1
+ fi
+
+ add_metric "gitea_up" "gauge" "Gitea reachability (1=up, 0=down)" "1"
+
+ local version
+ version=$(echo "$version_json" | jq -r '.version // empty' 2>/dev/null)
+
+ if [[ -n "$version" ]]; then
+ add_metric "gitea_version_info" "gauge" "Gitea/Forgejo version" "1" "version=\"${version}\""
+ fi
+
+ return 0
+}
+
+collect_users() {
+ local response
+ response=$(api_get_with_headers "/api/v1/admin/users?limit=1")
+
+ if [[ -z "$response" ]]; then
+ return
+ fi
+
+ local total_count
+ total_count=$(echo "$response" | head -1)
+
+ if [[ -n "$total_count" && "$total_count" != "0" ]]; then
+ add_metric "gitea_users_total" "gauge" "Total number of users" "$total_count"
+ fi
+}
+
+collect_organizations() {
+ local response
+ response=$(api_get_with_headers "/api/v1/admin/orgs?limit=1")
+
+ if [[ -z "$response" ]]; then
+ return
+ fi
+
+ local total_count
+ total_count=$(echo "$response" | head -1)
+
+ if [[ -n "$total_count" ]]; then
+ add_metric "gitea_organizations_total" "gauge" "Total number of organizations" "$total_count"
+ fi
+}
+
+collect_repositories() {
+ local response
+ response=$(api_get_with_headers "/api/v1/repos/search?limit=1")
+
+ if [[ -z "$response" ]]; then
+ return
+ fi
+
+ local total_count
+ total_count=$(echo "$response" | head -1)
+
+ if [[ -n "$total_count" ]]; then
+ add_metric "gitea_repositories_total" "gauge" "Total number of repositories" "$total_count"
+ fi
+}
+
+collect_repo_details() {
+ local page=1
+ local per_page=50
+ local collected=0
+ local first_page=true
+
+ # Add HELP/TYPE lines for per-repo metrics
+ OUTPUT+="# HELP gitea_repo_stars Number of stars for the repository
+# TYPE gitea_repo_stars gauge
+# HELP gitea_repo_forks Number of forks for the repository
+# TYPE gitea_repo_forks gauge
+# HELP gitea_repo_open_issues Number of open issues for the repository
+# TYPE gitea_repo_open_issues gauge
+# HELP gitea_repo_open_pull_requests Number of open pull requests for the repository
+# TYPE gitea_repo_open_pull_requests gauge
+# HELP gitea_repo_size_bytes Repository size in bytes
+# TYPE gitea_repo_size_bytes gauge
+# HELP gitea_repo_is_mirror Whether the repository is a mirror (1=yes, 0=no)
+# TYPE gitea_repo_is_mirror gauge
+"
+
+ while [[ $collected -lt $MAX_REPOS ]]; do
+ local remaining=$((MAX_REPOS - collected))
+ local fetch_count=$((remaining < per_page ? remaining : per_page))
+
+ local repos_json
+ repos_json=$(api_get "/api/v1/repos/search?limit=${fetch_count}&page=${page}")
+
+ if [[ -z "$repos_json" ]]; then
+ break
+ fi
+
+ local repo_count
+ repo_count=$(echo "$repos_json" | jq -r '.data | length // 0' 2>/dev/null)
+
+ if [[ "$repo_count" == "0" || -z "$repo_count" ]]; then
+ break
+ fi
+
+ local i
+ for ((i = 0; i < repo_count && collected < MAX_REPOS; i++)); do
+ local full_name stars forks open_issues size mirror has_pull_requests
+ full_name=$(echo "$repos_json" | jq -r ".data[$i].full_name // empty" 2>/dev/null)
+ stars=$(echo "$repos_json" | jq -r ".data[$i].stars_count // 0" 2>/dev/null)
+ forks=$(echo "$repos_json" | jq -r ".data[$i].forks_count // 0" 2>/dev/null)
+ open_issues=$(echo "$repos_json" | jq -r ".data[$i].open_issues_count // 0" 2>/dev/null)
+ size=$(echo "$repos_json" | jq -r ".data[$i].size // 0" 2>/dev/null)
+ mirror=$(echo "$repos_json" | jq -r ".data[$i].mirror // false" 2>/dev/null)
+ has_pull_requests=$(echo "$repos_json" | jq -r ".data[$i].has_pull_requests // true" 2>/dev/null)
+
+ if [[ -z "$full_name" ]]; then
+ continue
+ fi
+
+ local safe_name
+ safe_name=$(sanitize_label "$full_name")
+ local label="repo=\"${safe_name}\""
+
+ # Size: API returns KB, convert to bytes
+ local size_bytes=$((size * 1024))
+
+ # Mirror: convert bool to 0/1
+ local mirror_val=0
+ if [[ "$mirror" == "true" ]]; then
+ mirror_val=1
+ fi
+
+ # Open PRs: fetch from repo API if pull requests are enabled
+ local open_prs=0
+ if [[ "$has_pull_requests" == "true" ]]; then
+ local owner repo_name
+ owner=$(echo "$repos_json" | jq -r ".data[$i].owner.login // empty" 2>/dev/null)
+ repo_name=$(echo "$repos_json" | jq -r ".data[$i].name // empty" 2>/dev/null)
+ if [[ -n "$owner" && -n "$repo_name" ]]; then
+ local pr_response
+ pr_response=$(api_get_with_headers "/api/v1/repos/${owner}/${repo_name}/pulls?state=open&limit=1")
+ if [[ -n "$pr_response" ]]; then
+ open_prs=$(echo "$pr_response" | head -1)
+ fi
+ fi
+ fi
+
+ add_metric_value "gitea_repo_stars" "$stars" "$label"
+ add_metric_value "gitea_repo_forks" "$forks" "$label"
+ add_metric_value "gitea_repo_open_issues" "$open_issues" "$label"
+ add_metric_value "gitea_repo_open_pull_requests" "${open_prs:-0}" "$label"
+ add_metric_value "gitea_repo_size_bytes" "$size_bytes" "$label"
+ add_metric_value "gitea_repo_is_mirror" "$mirror_val" "$label"
+
+ collected=$((collected + 1))
+ done
+
+ # If we got fewer than requested, we've reached the end
+ if [[ $repo_count -lt $fetch_count ]]; then
+ break
+ fi
+
+ page=$((page + 1))
+ done
+}
+
+collect_runners() {
+ local runners_json
+ runners_json=$(api_get "/api/v1/admin/runners")
+
+ # Runner endpoint may 404 if Actions is not enabled — skip gracefully
+ if [[ -z "$runners_json" ]]; then
+ return
+ fi
+
+ # Validate we got a JSON array
+ local is_array
+ is_array=$(echo "$runners_json" | jq -r 'if type == "array" then "yes" else "no" end' 2>/dev/null)
+
+ if [[ "$is_array" != "yes" ]]; then
+ return
+ fi
+
+ local total online offline
+ total=$(echo "$runners_json" | jq 'length' 2>/dev/null)
+ online=$(echo "$runners_json" | jq '[.[] | select(.status == "online")] | length' 2>/dev/null)
+ offline=$(echo "$runners_json" | jq '[.[] | select(.status != "online")] | length' 2>/dev/null)
+
+ add_metric "gitea_runners_total" "gauge" "Total number of registered runners" "${total:-0}"
+ add_metric "gitea_runners_online" "gauge" "Number of online runners" "${online:-0}"
+ add_metric "gitea_runners_offline" "gauge" "Number of offline runners" "${offline:-0}"
+}
+
+write_output() {
+ if [[ "$TEXTFILE_MODE" == true ]]; then
+ local output_file="${TEXTFILE_DIR}/gitea.prom"
+ local temp_file="${output_file}.$$"
+
+ mkdir -p "$TEXTFILE_DIR"
+ echo "$OUTPUT" > "$temp_file"
+ mv "$temp_file" "$output_file"
+ else
+ echo "$OUTPUT"
+ fi
+}
+
+install_cron() {
+ if [[ $EUID -ne 0 ]]; then
+ echo "ERROR: --install requires root" >&2
+ exit 1
+ fi
+
+ local script_path
+ script_path=$(readlink -f "$0")
+
+ cat > /etc/cron.d/gitea-exporter </dev/null
+EOF
+
+ chmod 644 /etc/cron.d/gitea-exporter
+ echo "Installed cron job: /etc/cron.d/gitea-exporter"
+ echo "Metrics will be written to: ${TEXTFILE_DIR}/gitea.prom"
+}
+
+# --- Main ---
+
+main() {
+ # Parse arguments
+ for arg in "$@"; do
+ case "$arg" in
+ --textfile) TEXTFILE_MODE=true ;;
+ --install)
+ check_dependencies
+ validate_config
+ install_cron
+ exit 0
+ ;;
+ --help|-h) usage ;;
+ *) echo "Unknown option: $arg" >&2; usage ;;
+ esac
+ done
+
+ check_dependencies
+ validate_config
+
+ START_TIME=$(date +%s%N)
+
+ # Exporter info
+ add_metric "gitea_exporter_info" "gauge" "Exporter version information" "1" "version=\"${VERSION}\""
+
+ # Collect metrics
+ if collect_version; then
+ collect_users
+ collect_organizations
+ collect_repositories
+ collect_repo_details
+ collect_runners
+ fi
+
+ # Exporter performance
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
+ add_metric "gitea_exporter_duration_seconds" "gauge" "Time to generate all metrics" "$duration"
+ add_metric "gitea_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run" "$(date +%s)"
+
+ write_output
+}
+
+main "$@"
diff --git a/gitlab-docker-register-runner.sh b/gitlab-docker-register-runner.sh
new file mode 100644
index 0000000..d46ff87
--- /dev/null
+++ b/gitlab-docker-register-runner.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# register-runner.sh — GitLab Runner registration helper
+#
+# Registers the Docker-based GitLab Runner against your GitLab CE instance.
+# Designed to work with the gitlab-docker-compose.yml stack.
+#
+# Usage:
+# ./register-runner.sh
+# ./register-runner.sh glrt-xxxxxxxxxxxxxxxxxxxx
+#
+# The runner token is obtained from:
+# Admin Area → CI/CD → Runners → New instance runner → Create runner
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# License: MIT
+###############################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+GITLAB_HOSTNAME="${GITLAB_HOSTNAME:-gitlab.local}"
+RUNNER_CONTAINER="${RUNNER_CONTAINER:-gitlab-runner}"
+RUNNER_EXECUTOR="${RUNNER_EXECUTOR:-docker}"
+RUNNER_IMAGE="${RUNNER_IMAGE:-alpine:latest}"
+RUNNER_DESCRIPTION="${RUNNER_DESCRIPTION:-docker-runner}"
+RUNNER_TAGS="${RUNNER_TAGS:-docker,linux}"
+
+# ── Colors ────────────────────────────────────────────────────────────
+if [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+else
+ RED="" GREEN="" YELLOW="" BOLD="" RESET=""
+fi
+
+log() { echo -e "${GREEN}[OK]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+
+# ── Validation ────────────────────────────────────────────────────────
+if [[ $# -lt 1 ]]; then
+ echo -e "${BOLD}Usage:${RESET} $(basename "$0") "
+ echo ""
+ echo "Get the token from: Admin Area → CI/CD → Runners → New instance runner"
+ echo ""
+ echo "Environment variables:"
+ echo " GITLAB_HOSTNAME GitLab server hostname (default: gitlab.local)"
+ echo " RUNNER_CONTAINER Runner container name (default: gitlab-runner)"
+ echo " RUNNER_EXECUTOR Executor type (default: docker)"
+ echo " RUNNER_IMAGE Default CI image (default: alpine:latest)"
+ echo " RUNNER_DESCRIPTION Runner description (default: docker-runner)"
+ echo " RUNNER_TAGS Comma-separated tags (default: docker,linux)"
+ exit 1
+fi
+
+RUNNER_TOKEN="$1"
+
+# Verify the runner container is running
+if ! docker inspect "$RUNNER_CONTAINER" &>/dev/null; then
+ err "Container '${RUNNER_CONTAINER}' not found. Is the stack running?"
+ err "Run: docker compose up -d"
+ exit 1
+fi
+
+if [[ "$(docker inspect -f '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null)" != "true" ]]; then
+ err "Container '${RUNNER_CONTAINER}' is not running."
+ exit 1
+fi
+
+# ── Register ──────────────────────────────────────────────────────────
+echo -e "${BOLD}Registering GitLab Runner...${RESET}"
+echo " GitLab URL: https://${GITLAB_HOSTNAME}"
+echo " Executor: ${RUNNER_EXECUTOR}"
+echo " Default image: ${RUNNER_IMAGE}"
+echo " Tags: ${RUNNER_TAGS}"
+echo " Description: ${RUNNER_DESCRIPTION}"
+echo ""
+
+docker exec "$RUNNER_CONTAINER" gitlab-runner register \
+ --non-interactive \
+ --url "https://${GITLAB_HOSTNAME}" \
+ --token "$RUNNER_TOKEN" \
+ --executor "$RUNNER_EXECUTOR" \
+ --docker-image "$RUNNER_IMAGE" \
+ --description "$RUNNER_DESCRIPTION" \
+ --tag-list "$RUNNER_TAGS" \
+ --docker-network-mode "gitlab-net" \
+ --docker-volumes "/var/run/docker.sock:/var/run/docker.sock"
+
+echo ""
+
+# ── Verify ────────────────────────────────────────────────────────────
+if docker exec "$RUNNER_CONTAINER" gitlab-runner list 2>&1 | grep -q "$RUNNER_DESCRIPTION"; then
+ log "Runner registered successfully."
+ echo ""
+ docker exec "$RUNNER_CONTAINER" gitlab-runner list
+else
+ warn "Registration completed but runner not found in list. Check logs:"
+ echo " docker compose logs gitlab-runner"
+fi
diff --git a/gitlab-metrics-exporter.sh b/gitlab-metrics-exporter.sh
index 5b47027..b2e7c3b 100755
--- a/gitlab-metrics-exporter.sh
+++ b/gitlab-metrics-exporter.sh
@@ -6,7 +6,7 @@
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
-#### Version: 1.00-030426 ####
+#### Version: 1.01-210426 ####
################################################
set -o pipefail
@@ -557,13 +557,13 @@ collect_local_metrics() {
# GitLab version info
local version_patterns="^gitlab_version_info[{ ]"
local version_help="^# (HELP|TYPE) gitlab_version_info"
- metrics+=$(echo "$raw_metrics" | grep -E "$version_help|$version_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$version_help|$version_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Puma metrics
local puma_patterns="^puma_workers[{ ]|^puma_running_workers[{ ]|^puma_running[{ ]|^puma_queued_connections[{ ]|^puma_active_connections[{ ]|^puma_pool_capacity[{ ]|^puma_max_threads[{ ]|^puma_idle_threads[{ ]"
local puma_help="^# (HELP|TYPE) puma_"
- metrics+=$(echo "$raw_metrics" | grep -E "$puma_help|$puma_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$puma_help|$puma_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Sidekiq metrics (served by separate Sidekiq exporter, default localhost:8082)
@@ -574,37 +574,37 @@ collect_local_metrics() {
# Core Sidekiq job metrics
local sidekiq_patterns="^sidekiq_running_jobs[{ ]|^sidekiq_concurrency[{ ]|^sidekiq_mem_total_bytes[{ ]|^sidekiq_jobs_failed_total[{ ]|^sidekiq_jobs_dead_total[{ ]|^sidekiq_enqueued_jobs_total[{ ]|^sidekiq_jobs_completion_seconds[_{ ]|^sidekiq_jobs_queue_duration_seconds[_{ ]|^sidekiq_jobs_cpu_seconds[_{ ]|^sidekiq_jobs_db_seconds[_{ ]|^sidekiq_jobs_gitaly_seconds[_{ ]|^sidekiq_redis_requests_total[{ ]|^sidekiq_redis_requests_duration_seconds[_{ ]"
local sidekiq_help="^# (HELP|TYPE) sidekiq_(running_jobs|concurrency|mem_total_bytes|jobs_failed_total|jobs_dead_total|enqueued_jobs_total|jobs_completion_seconds|jobs_queue_duration_seconds|jobs_cpu_seconds|jobs_db_seconds|jobs_gitaly_seconds|redis_requests_total|redis_requests_duration_seconds)"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$sidekiq_help|$sidekiq_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$sidekiq_help|$sidekiq_patterns" 2>/dev/null || true)
metrics+=$'\n'
# CI/CD pipeline internals
local ci_patterns="^pipelines_created_total[{ ]|^deployments[{ ]|^gitlab_ci_pipeline_creation_duration_seconds[_{ ]|^gitlab_ci_pipeline_failure_reasons[{ ]|^gitlab_ci_active_jobs[_{ ]"
local ci_help="^# (HELP|TYPE) (pipelines_created_total|deployments|gitlab_ci_pipeline_creation_duration_seconds|gitlab_ci_pipeline_failure_reasons|gitlab_ci_active_jobs)"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$ci_help|$ci_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$ci_help|$ci_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Email delivery metrics
local email_patterns="^gitlab_emails_delivered_total[{ ]|^gitlab_emails_delivery_attempts_total[{ ]"
local email_help="^# (HELP|TYPE) gitlab_emails_(delivered_total|delivery_attempts_total)"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$email_help|$email_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$email_help|$email_patterns" 2>/dev/null || true)
metrics+=$'\n'
# External HTTP (webhooks, integrations)
local ext_http_patterns="^gitlab_external_http_total[{ ]|^gitlab_external_http_duration_seconds[_{ ]"
local ext_http_help="^# (HELP|TYPE) gitlab_external_http_(total|duration_seconds)"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$ext_http_help|$ext_http_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$ext_http_help|$ext_http_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Sidekiq SLI apdex/errors
local sli_patterns="^gitlab_sli_sidekiq_execution_apdex_success_total[{ ]|^gitlab_sli_sidekiq_execution_apdex_total[{ ]|^gitlab_sli_sidekiq_execution_error_total[{ ]|^gitlab_sli_sidekiq_execution_total[{ ]"
local sli_help="^# (HELP|TYPE) gitlab_sli_sidekiq_execution"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$sli_help|$sli_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$sli_help|$sli_patterns" 2>/dev/null || true)
metrics+=$'\n'
# DB transaction duration, primary SQL, threads, cache, workers
local extra_patterns="^gitlab_database_transaction_seconds[_{ ]|^gitlab_sql_primary_duration_seconds[_{ ]|^gitlab_ruby_threads_running_threads[{ ]|^gitlab_ruby_threads_max_expected_threads[{ ]|^limited_capacity_worker_running_jobs[{ ]|^limited_capacity_worker_max_running_jobs[{ ]|^limited_capacity_worker_remaining_work_count[{ ]|^redis_hit_miss_operations_total[{ ]"
local extra_help="^# (HELP|TYPE) (gitlab_database_transaction_seconds|gitlab_sql_primary_duration_seconds|gitlab_ruby_threads_running_threads|gitlab_ruby_threads_max_expected_threads|limited_capacity_worker_running_jobs|limited_capacity_worker_max_running_jobs|limited_capacity_worker_remaining_work_count|redis_hit_miss_operations_total)"
- metrics+=$(echo "$sidekiq_raw" | grep -E "$extra_help|$extra_patterns" 2>/dev/null)
+ metrics+=$(echo "$sidekiq_raw" | grep -E "$extra_help|$extra_patterns" 2>/dev/null || true)
metrics+=$'\n'
else
debug_echo "Warning: Could not scrape Sidekiq exporter at $GITLAB_SIDEKIQ_URL (is sidekiq_exporter enabled?)"
@@ -613,31 +613,31 @@ collect_local_metrics() {
# Redis metrics
local redis_patterns="^gitlab_redis_client_requests_total[{ ]|^gitlab_redis_client_exceptions_total[{ ]|^gitlab_redis_client_requests_duration_seconds[_{ ]|^gitlab_redis_client_requests_duration_seconds_sum[{ ]|^gitlab_redis_client_requests_duration_seconds_count[{ ]"
local redis_help="^# (HELP|TYPE) gitlab_redis_client_(requests_total|exceptions_total|requests_duration_seconds)"
- metrics+=$(echo "$raw_metrics" | grep -E "$redis_help|$redis_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$redis_help|$redis_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Database connection pool metrics
local db_patterns="^gitlab_database_connection_pool_"
local db_help="^# (HELP|TYPE) gitlab_database_connection_pool_"
- metrics+=$(echo "$raw_metrics" | grep -E "$db_help|$db_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$db_help|$db_patterns" 2>/dev/null || true)
metrics+=$'\n'
# Process metrics (CPU, memory, file descriptors)
local process_patterns="^ruby_process_resident_memory_bytes[{ ]|^ruby_process_cpu_seconds_total[{ ]|^process_open_fds[{ ]|^process_max_fds[{ ]|^ruby_gc_stat_heap_live_slots[{ ]|^ruby_gc_stat_heap_free_slots[{ ]"
local process_help="^# (HELP|TYPE) (ruby_process_resident_memory_bytes|ruby_process_cpu_seconds_total|process_open_fds|process_max_fds|ruby_gc_stat_heap_live_slots|ruby_gc_stat_heap_free_slots)"
- metrics+=$(echo "$raw_metrics" | grep -E "$process_help|$process_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$process_help|$process_patterns" 2>/dev/null || true)
metrics+=$'\n'
# GitLab transaction/request metrics
local txn_patterns="^gitlab_transaction_duration_seconds[{ _]|^gitlab_sql_duration_seconds[{ _]|^gitlab_cache_operation_duration_seconds[{ _]"
local txn_help="^# (HELP|TYPE) (gitlab_transaction_duration_seconds|gitlab_sql_duration_seconds|gitlab_cache_operation_duration_seconds)"
- metrics+=$(echo "$raw_metrics" | grep -E "$txn_help|$txn_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$txn_help|$txn_patterns" 2>/dev/null || true)
metrics+=$'\n'
# User session and ActionCable metrics
local session_patterns="^user_session_logins_total[{ ]|^action_cable_active_connections[{ ]|^action_cable_pool_current_size[{ ]"
local session_help="^# (HELP|TYPE) (user_session_logins_total|action_cable_active_connections|action_cable_pool_current_size)"
- metrics+=$(echo "$raw_metrics" | grep -E "$session_help|$session_patterns" 2>/dev/null)
+ metrics+=$(echo "$raw_metrics" | grep -E "$session_help|$session_patterns" 2>/dev/null || true)
metrics+=$'\n'
local metric_count
diff --git a/gitlab-smoke-tests.ps1 b/gitlab-smoke-tests.ps1
new file mode 100644
index 0000000..735b1d7
--- /dev/null
+++ b/gitlab-smoke-tests.ps1
@@ -0,0 +1,864 @@
+###############################################################################
+# gitlab-smoke-tests.ps1 - Verify GitLab instance health after upgrades
+#
+# PowerShell port of gitlab-smoke-tests.sh. Zero external dependencies
+# beyond PowerShell 5.1+ and git. Runs on Windows, Linux, and macOS.
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# License: MIT
+# Version 1.00
+#
+# Usage:
+# $env:GITLAB_URL = "https://gitlab.example.com"
+# $env:GITLAB_TOKEN = "glpat-xxxxxxxxxxxx"
+# .\gitlab-smoke-tests.ps1
+# .\gitlab-smoke-tests.ps1 -SkipGit -SkipRegistry
+# .\gitlab-smoke-tests.ps1 -Insecure -Format junit
+# .\gitlab-smoke-tests.ps1 -Format tap
+###############################################################################
+
+[CmdletBinding()]
+param(
+ [string]$GitLabUrl = $env:GITLAB_URL,
+ [string]$GitLabToken = $env:GITLAB_TOKEN,
+ [string]$GitLabUser = $(if ($env:GITLAB_USER) { $env:GITLAB_USER } else { "root" }),
+ [string]$HealthToken = $env:GITLAB_HEALTH_TOKEN,
+ [string]$ProjectPrefix = $(if ($env:SMOKE_PROJECT_PREFIX) { $env:SMOKE_PROJECT_PREFIX } else { "smoke-test" }),
+ [int]$Timeout = $(if ($env:CURL_TIMEOUT) { [int]$env:CURL_TIMEOUT } else { 10 }),
+ [switch]$Insecure,
+ [switch]$SkipGit,
+ [switch]$SkipRegistry,
+ [switch]$SkipCleanup,
+ [ValidateSet("text","tap","junit")]
+ [string]$Format = "text",
+ [string]$JunitFile = "smoke-results.xml",
+ [switch]$NoColor
+)
+
+$ErrorActionPreference = "Continue"
+
+# ============================================================================
+# STATE
+# ============================================================================
+
+$script:Pass = 0
+$script:Fail = 0
+$script:Skip = 0
+$script:Total = 0
+$script:Results = @()
+$script:CleanupProjectId = ""
+$script:TmpDir = ""
+$script:StartTime = $null
+$script:GitCloneOk = $false
+
+# ============================================================================
+# COLORS
+# ============================================================================
+
+function Write-Color {
+ param([string]$Text, [string]$Color = "White")
+ if ($NoColor) {
+ Write-Host $Text
+ } else {
+ Write-Host $Text -ForegroundColor $Color
+ }
+}
+
+function Write-Log { param([string]$Msg) Write-Color "[INFO] $Msg" "Cyan" }
+function Write-Warn { param([string]$Msg) Write-Color "[WARN] $Msg" "Yellow" }
+function Write-Err { param([string]$Msg) Write-Color "[ERROR] $Msg" "Red" }
+
+# ============================================================================
+# TEST RESULT RECORDING
+# ============================================================================
+
+function Record-Pass {
+ param([string]$Name, [string]$Detail = "")
+ $script:Pass++
+ $script:Total++
+ $script:Results += [PSCustomObject]@{ Status="PASS"; Name=$Name; Detail=$Detail }
+ if ($Format -eq "tap") {
+ Write-Host "ok $($script:Total) - $Name"
+ } else {
+ $msg = " $(if($NoColor){'[PASS]'}else{[char]0x2713}) $Name"
+ if ($Detail) { $msg += " - $Detail" }
+ Write-Color $msg "Green"
+ }
+}
+
+function Record-Fail {
+ param([string]$Name, [string]$Detail = "")
+ $script:Fail++
+ $script:Total++
+ $script:Results += [PSCustomObject]@{ Status="FAIL"; Name=$Name; Detail=$Detail }
+ if ($Format -eq "tap") {
+ Write-Host "not ok $($script:Total) - $Name"
+ if ($Detail) { Write-Host " # $Detail" }
+ } else {
+ $msg = " $(if($NoColor){'[FAIL]'}else{[char]0x2717}) $Name"
+ if ($Detail) { $msg += " - $Detail" }
+ Write-Color $msg "Red"
+ }
+}
+
+function Record-Skip {
+ param([string]$Name, [string]$Reason = "")
+ $script:Skip++
+ $script:Total++
+ $script:Results += [PSCustomObject]@{ Status="SKIP"; Name=$Name; Detail=$Reason }
+ if ($Format -eq "tap") {
+ Write-Host "ok $($script:Total) - $Name # SKIP $Reason"
+ } else {
+ $msg = " $(if($NoColor){'[SKIP]'}else{[char]0x2298}) $Name"
+ if ($Reason) { $msg += " - $Reason" }
+ Write-Color $msg "Yellow"
+ }
+}
+
+# ============================================================================
+# HTTP HELPERS
+# ============================================================================
+
+function Invoke-GitLabApi {
+ param(
+ [string]$Method,
+ [string]$Endpoint,
+ [string]$Body = $null,
+ [switch]$StatusOnly
+ )
+
+ $uri = "$GitLabUrl/api/v4$Endpoint"
+ $headers = @{ "Content-Type" = "application/json" }
+ if ($GitLabToken) { $headers["PRIVATE-TOKEN"] = $GitLabToken }
+
+ $params = @{
+ Uri = $uri
+ Method = $Method
+ Headers = $headers
+ TimeoutSec = $Timeout
+ UseBasicParsing = $true
+ ErrorAction = "Stop"
+ }
+
+ if ($Insecure -and $PSVersionTable.PSVersion.Major -ge 7) {
+ $params["SkipCertificateCheck"] = $true
+ }
+
+ if ($Body) {
+ $params["Body"] = $Body
+ }
+
+ try {
+ if ($StatusOnly) {
+ $response = Invoke-WebRequest @params
+ return [int]$response.StatusCode
+ } else {
+ return Invoke-RestMethod @params
+ }
+ } catch {
+ if ($StatusOnly) {
+ if ($_.Exception.Response) {
+ return [int]$_.Exception.Response.StatusCode
+ }
+ return 0
+ }
+ return $null
+ }
+}
+
+function Invoke-HealthCheck {
+ param([string]$Path)
+
+ $uri = "$GitLabUrl$Path"
+ if ($HealthToken) { $uri += "?token=$HealthToken" }
+
+ $params = @{
+ Uri = $uri
+ Method = "GET"
+ TimeoutSec = $Timeout
+ UseBasicParsing = $true
+ ErrorAction = "Stop"
+ }
+
+ if ($Insecure -and $PSVersionTable.PSVersion.Major -ge 7) {
+ $params["SkipCertificateCheck"] = $true
+ }
+
+ try {
+ $response = Invoke-WebRequest @params
+ return [int]$response.StatusCode
+ } catch {
+ if ($_.Exception.Response) {
+ return [int]$_.Exception.Response.StatusCode
+ }
+ return 0
+ }
+}
+
+# ============================================================================
+# TLS HELPER
+# ============================================================================
+
+function Get-TlsCertExpiry {
+ param([string]$HostName, [int]$Port = 443)
+
+ try {
+ $tcpClient = New-Object System.Net.Sockets.TcpClient
+ $tcpClient.ReceiveTimeout = $Timeout * 1000
+ $tcpClient.SendTimeout = $Timeout * 1000
+ $tcpClient.Connect($HostName, $Port)
+
+ $sslStream = New-Object System.Net.Security.SslStream(
+ $tcpClient.GetStream(), $false,
+ { param($s,$c,$ch,$e) return $true }
+ )
+ $sslStream.AuthenticateAsClient($HostName)
+
+ $cert = $sslStream.RemoteCertificate
+ $expiry = [DateTime]$cert.GetExpirationDateString()
+
+ $sslStream.Dispose()
+ $tcpClient.Dispose()
+
+ return $expiry
+ } catch {
+ return $null
+ }
+}
+
+# ============================================================================
+# TEST SUITES
+# ============================================================================
+
+# -- 1. Connectivity --------------------------------------------------------
+
+function Test-Connectivity {
+ Write-Host ""
+ Write-Color "Connectivity" "White"
+
+ # 1a. Health endpoint
+ $code = Invoke-HealthCheck "/-/health"
+ if ($code -eq 200) {
+ Record-Pass "GitLab health endpoint reachable" "HTTP $code"
+ } else {
+ Record-Fail "GitLab health endpoint reachable" "HTTP $code"
+ }
+
+ # 1b. Readiness
+ $code = Invoke-HealthCheck "/-/readiness"
+ if ($code -eq 200) {
+ Record-Pass "GitLab readiness check" "HTTP $code"
+ } else {
+ Record-Fail "GitLab readiness check" "HTTP $code"
+ }
+
+ # 1c. Liveness
+ $code = Invoke-HealthCheck "/-/liveness"
+ if ($code -eq 200) {
+ Record-Pass "GitLab liveness check" "HTTP $code"
+ } else {
+ Record-Fail "GitLab liveness check" "HTTP $code"
+ }
+
+ # 1d. TLS certificate
+ if ($GitLabUrl -match "^https://") {
+ $hostPart = $GitLabUrl -replace "^https://", "" -replace "/.*", "" -replace ":.*", ""
+ $portPart = 443
+ if ($GitLabUrl -match ":(\d+)") { $portPart = [int]$Matches[1] }
+
+ $expiry = Get-TlsCertExpiry -HostName $hostPart -Port $portPart
+ if ($expiry) {
+ $daysLeft = [math]::Floor(($expiry - (Get-Date)).TotalDays)
+ if ($daysLeft -gt 30) {
+ Record-Pass "TLS certificate valid" "$daysLeft days remaining"
+ } elseif ($daysLeft -gt 0) {
+ Record-Pass "TLS certificate valid" "$daysLeft days remaining (renew soon)"
+ } else {
+ Record-Fail "TLS certificate valid" "expired or expiring in $daysLeft days"
+ }
+ } else {
+ Record-Skip "TLS certificate check" "could not retrieve certificate"
+ }
+ } else {
+ Record-Skip "TLS certificate check" "not using HTTPS"
+ }
+}
+
+# -- 2. API ----------------------------------------------------------------
+
+function Test-Api {
+ Write-Host ""
+ Write-Color "API" "White"
+
+ # 2a. Version
+ $versionData = Invoke-GitLabApi -Method GET -Endpoint "/version"
+ if ($versionData -and $versionData.version) {
+ Record-Pass "API version endpoint" "GitLab $($versionData.version) ($($versionData.revision))"
+ } else {
+ Record-Fail "API version endpoint" "no version returned"
+ }
+
+ # 2b. Authentication
+ $authStatus = Invoke-GitLabApi -Method GET -Endpoint "/user" -StatusOnly
+ if ($authStatus -eq 200) {
+ $userData = Invoke-GitLabApi -Method GET -Endpoint "/user"
+ Record-Pass "API authentication" "authenticated as $($userData.username)"
+ } elseif ($authStatus -eq 401) {
+ Record-Fail "API authentication" "token rejected (HTTP 401)"
+ } else {
+ Record-Fail "API authentication" "HTTP $authStatus"
+ }
+
+ # 2c. List projects
+ $projStatus = Invoke-GitLabApi -Method GET -Endpoint "/projects?per_page=1" -StatusOnly
+ if ($projStatus -eq 200) {
+ Record-Pass "API list projects" "database responding"
+ } else {
+ Record-Fail "API list projects" "HTTP $projStatus"
+ }
+
+ # 2d. List users
+ $userStatus = Invoke-GitLabApi -Method GET -Endpoint "/users?per_page=1" -StatusOnly
+ if ($userStatus -eq 200) {
+ Record-Pass "API list users" "user directory accessible"
+ } else {
+ Record-Fail "API list users" "HTTP $userStatus"
+ }
+
+ # 2e. Sidekiq
+ $sidekiq = Invoke-GitLabApi -Method GET -Endpoint "/sidekiq/compound_metrics"
+ if ($sidekiq -and -not $sidekiq.error) {
+ $procCount = 0
+ if ($sidekiq.processes) { $procCount = @($sidekiq.processes).Count }
+ Record-Pass "Sidekiq running" "$procCount process(es) responding"
+ } else {
+ Record-Fail "Sidekiq running" "could not query Sidekiq metrics"
+ }
+
+ # 2f. Runners
+ $runnerStatus = Invoke-GitLabApi -Method GET -Endpoint "/runners/all?per_page=1" -StatusOnly
+ if ($runnerStatus -eq 200) {
+ Record-Pass "API runners endpoint" "runner management accessible"
+ } elseif ($runnerStatus -eq 403) {
+ Record-Skip "API runners endpoint" "token lacks admin scope"
+ } else {
+ Record-Fail "API runners endpoint" "HTTP $runnerStatus"
+ }
+
+ # 2g. Search
+ $searchStatus = Invoke-GitLabApi -Method GET -Endpoint "/search?scope=projects&search=test" -StatusOnly
+ if ($searchStatus -eq 200) {
+ Record-Pass "API search" "search index responding"
+ } elseif ($searchStatus -eq 403) {
+ Record-Skip "API search" "search disabled or token lacks scope"
+ } else {
+ Record-Fail "API search" "HTTP $searchStatus"
+ }
+}
+
+# -- 3. Git Operations -----------------------------------------------------
+
+function Test-Git {
+ if ($SkipGit) {
+ Write-Host ""
+ Write-Color "Git Operations" "White"
+ Record-Skip "Git clone" "SkipGit specified"
+ Record-Skip "Git push" "SkipGit specified"
+ return
+ }
+
+ Write-Host ""
+ Write-Color "Git Operations" "White"
+
+ # Create test project
+ $projectName = "$ProjectPrefix-$([DateTimeOffset]::UtcNow.ToUnixTimeSeconds())"
+ $body = @{ name = $projectName; visibility = "private"; initialize_with_readme = $true } | ConvertTo-Json
+ $project = Invoke-GitLabApi -Method POST -Endpoint "/projects" -Body $body
+
+ if (-not $project -or -not $project.id) {
+ Record-Fail "Create test project" "API returned no project ID"
+ Record-Skip "Git clone" "no test project"
+ Record-Skip "Git push" "no test project"
+ return
+ }
+
+ $script:CleanupProjectId = $project.id
+ Record-Pass "Create test project" "$projectName (ID: $($project.id))"
+
+ # Build clone URL
+ $httpUrl = $project.http_url_to_repo
+ if (-not $httpUrl) {
+ $httpUrl = "$GitLabUrl/$GitLabUser/$projectName.git"
+ }
+
+ # Rewrite origin if API returns an internal hostname
+ $apiOrigin = if ($httpUrl -match "^(https?://[^/]+)") { $Matches[1] } else { "" }
+ if ($apiOrigin -and $apiOrigin -ne $GitLabUrl) {
+ $httpUrl = $httpUrl -replace [regex]::Escape($apiOrigin), $GitLabUrl
+ }
+
+ # Inject token
+ if ($httpUrl -match "^https://") {
+ $cloneUrl = $httpUrl -replace "^https://", "https://oauth2:${GitLabToken}@"
+ } elseif ($httpUrl -match "^http://") {
+ $cloneUrl = $httpUrl -replace "^http://", "http://oauth2:${GitLabToken}@"
+ } else {
+ $cloneUrl = $httpUrl
+ }
+
+ # Temp directory
+ $script:TmpDir = Join-Path ([System.IO.Path]::GetTempPath()) "gitlab-smoke-$([guid]::NewGuid().ToString('N').Substring(0,8))"
+ New-Item -ItemType Directory -Path $script:TmpDir -Force | Out-Null
+
+ # Wait for repo init
+ Start-Sleep -Seconds 2
+
+ # Clone
+ $gitArgs = @("clone")
+ if ($Insecure) { $env:GIT_SSL_NO_VERIFY = "true" }
+
+ $repoDir = Join-Path $script:TmpDir "repo"
+ $cloneOutput = & git clone $cloneUrl $repoDir 2>&1
+ $cloneRc = $LASTEXITCODE
+
+ if ($cloneRc -eq 0) {
+ $script:GitCloneOk = $true
+ Record-Pass "Git clone (HTTPS)" "Gitaly responding"
+ } else {
+ $shortErr = ($cloneOutput | Select-String -Pattern "fatal|error" | Select-Object -First 1) -replace [regex]::Escape($GitLabToken), "[REDACTED]"
+ Record-Fail "Git clone (HTTPS)" "$shortErr"
+ return
+ }
+
+ # Push
+ Push-Location $repoDir
+ try {
+ & git config user.email "smoke-test@example.com"
+ & git config user.name "Smoke Test"
+ "smoke test $(Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ')" | Out-File -FilePath "smoke-test.txt" -Encoding utf8
+
+ & git add smoke-test.txt
+ & git commit -m "smoke test commit" 2>&1 | Out-Null
+
+ $pushOutput = & git push origin main 2>&1
+ $pushRc = $LASTEXITCODE
+ if ($pushRc -ne 0) {
+ $pushOutput = & git push origin master 2>&1
+ $pushRc = $LASTEXITCODE
+ }
+
+ if ($pushRc -eq 0) {
+ Record-Pass "Git push (HTTPS)" "write to Gitaly succeeded"
+ } else {
+ Record-Fail "Git push (HTTPS)" "push failed"
+ }
+ } finally {
+ Pop-Location
+ }
+}
+
+# -- 4. Container Registry -------------------------------------------------
+
+function Test-Registry {
+ if ($SkipRegistry) {
+ Write-Host ""
+ Write-Color "Container Registry" "White"
+ Record-Skip "Registry API" "SkipRegistry specified"
+ return
+ }
+
+ Write-Host ""
+ Write-Color "Container Registry" "White"
+
+ # Check if registry is enabled
+ $registryEnabled = ""
+ $settings = Invoke-GitLabApi -Method GET -Endpoint "/application/settings"
+ if ($settings) {
+ $registryEnabled = $settings.container_registry_enabled
+ }
+
+ if ($registryEnabled -eq $false) {
+ Record-Skip "Registry API reachable" "container registry disabled in application settings"
+ Record-Skip "Registry project endpoint" "container registry disabled in application settings"
+ return
+ }
+
+ # Try registry v2 API
+ $hostPart = $GitLabUrl -replace "^https?://", "" -replace "/.*", ""
+ $registryStatus = 0
+
+ $registryUrls = @(
+ "$GitLabUrl`:5050/v2/",
+ "https://${hostPart}:5050/v2/",
+ "https://registry.${hostPart}/v2/"
+ )
+
+ foreach ($regUrl in $registryUrls) {
+ try {
+ $params = @{
+ Uri = $regUrl
+ Method = "GET"
+ TimeoutSec = $Timeout
+ UseBasicParsing = $true
+ ErrorAction = "Stop"
+ }
+ if ($Insecure -and $PSVersionTable.PSVersion.Major -ge 7) {
+ $params["SkipCertificateCheck"] = $true
+ }
+ $response = Invoke-WebRequest @params
+ $registryStatus = [int]$response.StatusCode
+ break
+ } catch {
+ if ($_.Exception.Response) {
+ $registryStatus = [int]$_.Exception.Response.StatusCode
+ if ($registryStatus -eq 401) { break }
+ }
+ }
+ }
+
+ if ($registryStatus -eq 200 -or $registryStatus -eq 401) {
+ Record-Pass "Registry API reachable" "HTTP $registryStatus"
+ } elseif ($registryStatus -eq 0) {
+ if ($registryEnabled -eq $true) {
+ Record-Fail "Registry API reachable" "enabled in settings but not reachable at standard ports/hosts"
+ } else {
+ Record-Skip "Registry API reachable" "not found at standard ports/hosts (settings unreadable - may need admin token)"
+ }
+ } else {
+ Record-Fail "Registry API reachable" "HTTP $registryStatus"
+ }
+
+ # Project-level registry
+ if ($script:CleanupProjectId) {
+ $regStatus = Invoke-GitLabApi -Method GET -Endpoint "/projects/$($script:CleanupProjectId)/registry/repositories" -StatusOnly
+ if ($regStatus -eq 200) {
+ Record-Pass "Registry project endpoint" "project registry accessible"
+ } elseif ($regStatus -eq 404) {
+ Record-Skip "Registry project endpoint" "container registry not enabled for project"
+ } else {
+ Record-Fail "Registry project endpoint" "HTTP $regStatus"
+ }
+ }
+}
+
+# -- 5. CI/CD --------------------------------------------------------------
+
+function Test-CICD {
+ Write-Host ""
+ Write-Color "CI/CD" "White"
+
+ # Runners
+ $runners = Invoke-GitLabApi -Method GET -Endpoint "/runners/all?per_page=100"
+ if ($runners -is [array]) {
+ $runnerCount = $runners.Count
+ $onlineCount = @($runners | Where-Object { $_.status -eq "online" }).Count
+
+ if ($onlineCount -gt 0) {
+ Record-Pass "CI/CD runners online" "$onlineCount/$runnerCount runners online"
+ } elseif ($runnerCount -gt 0) {
+ Record-Fail "CI/CD runners online" "0/$runnerCount runners online"
+ } else {
+ Record-Skip "CI/CD runners online" "no runners registered"
+ }
+ } else {
+ Record-Skip "CI/CD runners" "could not query runners (admin token required)"
+ }
+
+ # CI/CD settings
+ $cicdStatus = Invoke-GitLabApi -Method GET -Endpoint "/application/settings" -StatusOnly
+ if ($cicdStatus -eq 200) {
+ Record-Pass "CI/CD settings accessible" "application settings readable"
+ } elseif ($cicdStatus -eq 403) {
+ Record-Skip "CI/CD settings accessible" "admin token required"
+ } else {
+ Record-Fail "CI/CD settings accessible" "HTTP $cicdStatus"
+ }
+}
+
+# -- 6. Background Migrations ----------------------------------------------
+
+function Test-Migrations {
+ Write-Host ""
+ Write-Color "Background Migrations" "White"
+
+ $migrations = Invoke-GitLabApi -Method GET -Endpoint "/admin/batched_background_migrations?database=main"
+
+ if ($migrations -is [array]) {
+ $totalMig = $migrations.Count
+ $failedMig = @($migrations | Where-Object { $_.status -eq "failed" }).Count
+ $activeMig = @($migrations | Where-Object { $_.status -eq "active" }).Count
+ $pausedMig = @($migrations | Where-Object { $_.status -eq "paused" }).Count
+ $finishedMig = @($migrations | Where-Object { $_.status -eq "finished" }).Count
+
+ if ($failedMig -gt 0) {
+ Record-Fail "Background migrations" "$failedMig failed, $activeMig active, $pausedMig paused, $finishedMig finished of $totalMig"
+ } elseif ($pausedMig -gt 0) {
+ Record-Fail "Background migrations" "$pausedMig paused, $activeMig active, $finishedMig finished of $totalMig"
+ } elseif ($activeMig -gt 0) {
+ Record-Pass "Background migrations" "$activeMig active, $finishedMig finished of $totalMig (in progress)"
+ } else {
+ Record-Pass "Background migrations" "all $totalMig finished"
+ }
+ } else {
+ $migStatus = Invoke-GitLabApi -Method GET -Endpoint "/admin/batched_background_migrations?database=main" -StatusOnly
+ if ($migStatus -eq 403) {
+ Record-Skip "Background migrations" "admin token required"
+ } else {
+ Record-Skip "Background migrations" "could not query (HTTP $migStatus)"
+ }
+ }
+}
+
+# -- 7. Components ---------------------------------------------------------
+
+function Test-Components {
+ Write-Host ""
+ Write-Color "Components" "White"
+
+ # Metadata
+ $metadata = Invoke-GitLabApi -Method GET -Endpoint "/metadata"
+ if ($metadata -and $metadata.version) {
+ $edition = if ($metadata.enterprise -eq $true) { "EE" } else { "CE" }
+ Record-Pass "GitLab metadata" "$($metadata.version) $edition"
+ } elseif ($metadata) {
+ Record-Pass "GitLab metadata" "endpoint reachable"
+ } else {
+ Record-Skip "GitLab metadata" "metadata endpoint not available"
+ }
+
+ # Statistics
+ $stats = Invoke-GitLabApi -Method GET -Endpoint "/application/statistics"
+ if ($stats -and $stats.active_users) {
+ Record-Pass "Instance statistics" "$($stats.active_users) users, $($stats.projects) projects, $($stats.groups) groups"
+ } elseif ($stats) {
+ Record-Pass "Instance statistics" "endpoint reachable"
+ } else {
+ Record-Skip "Instance statistics" "admin token required"
+ }
+
+ # Gitaly (inferred)
+ if ($script:GitCloneOk) {
+ Record-Pass "Gitaly storage" "project created and cloned successfully"
+ } elseif ($script:CleanupProjectId) {
+ Record-Skip "Gitaly storage" "project created but clone was not tested or failed"
+ }
+
+ # PostgreSQL (inferred)
+ $pgStatus = Invoke-GitLabApi -Method GET -Endpoint "/projects?per_page=1&order_by=updated_at" -StatusOnly
+ if ($pgStatus -eq 200) {
+ Record-Pass "PostgreSQL" "database queries succeeding"
+ } else {
+ Record-Fail "PostgreSQL" "sorted query failed (HTTP $pgStatus)"
+ }
+
+ # Redis (inferred)
+ $redisStatus = Invoke-GitLabApi -Method GET -Endpoint "/user" -StatusOnly
+ if ($redisStatus -eq 200) {
+ Record-Pass "Redis" "session/cache operational (auth succeeded)"
+ } else {
+ Record-Skip "Redis" "cannot verify independently"
+ }
+}
+
+# ============================================================================
+# OUTPUT
+# ============================================================================
+
+function Write-Summary {
+ $duration = [math]::Floor(((Get-Date) - $script:StartTime).TotalSeconds)
+
+ Write-Host ""
+ $separator = [string]::new([char]0x2500, 40)
+ Write-Color $separator "White"
+ Write-Color "Summary $GitLabUrl" "White"
+
+ $summaryLine = " $($script:Pass) passed $($script:Fail) failed $($script:Skip) skipped (${duration}s)"
+ Write-Host $summaryLine
+ Write-Color $separator "White"
+
+ if ($script:Fail -eq 0) {
+ Write-Color "All tests passed." "Green"
+ } else {
+ Write-Color "$($script:Fail) test(s) failed." "Red"
+ }
+}
+
+function Write-TapHeader {
+ Write-Host "TAP version 13"
+}
+
+function Write-TapFooter {
+ Write-Host "1..$($script:Total)"
+ Write-Host "# pass $($script:Pass)"
+ Write-Host "# fail $($script:Fail)"
+ Write-Host "# skip $($script:Skip)"
+}
+
+function Write-JunitReport {
+ $duration = [math]::Floor(((Get-Date) - $script:StartTime).TotalSeconds)
+
+ $xml = @"
+
+
+
+"@
+
+ foreach ($r in $script:Results) {
+ $safeName = $r.Name -replace '&','&' -replace '<','<' -replace '>','>' -replace '"','"'
+ $safeDetail = $r.Detail -replace '&','&' -replace '<','<' -replace '>','>' -replace '"','"'
+
+ switch ($r.Status) {
+ "PASS" {
+ $xml += "`n "
+ if ($r.Detail) { $xml += "`n $safeDetail" }
+ $xml += "`n "
+ }
+ "FAIL" {
+ $xml += "`n "
+ $xml += "`n FAILED: $safeName - $safeDetail"
+ $xml += "`n "
+ }
+ "SKIP" {
+ $xml += "`n "
+ $xml += "`n "
+ $xml += "`n "
+ }
+ }
+ }
+
+ $xml += "`n "
+ $xml += "`n"
+
+ $xml | Out-File -FilePath $JunitFile -Encoding utf8
+ Write-Log "JUnit report written to $JunitFile"
+}
+
+# ============================================================================
+# CLEANUP
+# ============================================================================
+
+function Invoke-Cleanup {
+ if ($script:CleanupProjectId -and -not $SkipCleanup) {
+ try {
+ Invoke-GitLabApi -Method DELETE -Endpoint "/projects/$($script:CleanupProjectId)" | Out-Null
+ } catch { }
+ }
+
+ if ($script:TmpDir -and (Test-Path $script:TmpDir)) {
+ Remove-Item -Recurse -Force $script:TmpDir -ErrorAction SilentlyContinue
+ }
+
+ if ($env:GIT_SSL_NO_VERIFY) {
+ Remove-Item Env:\GIT_SSL_NO_VERIFY -ErrorAction SilentlyContinue
+ }
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+function Show-Usage {
+ @"
+Usage: .\gitlab-smoke-tests.ps1 [OPTIONS]
+
+Smoke-test a GitLab instance. PowerShell 5.1+, git only.
+Designed for air-gapped environments.
+
+Required environment variables:
+ GITLAB_URL GitLab base URL (https://gitlab.example.com)
+ GITLAB_TOKEN Personal access token (api scope; admin for full coverage)
+
+Optional environment variables:
+ GITLAB_HEALTH_TOKEN Health check access token
+ GITLAB_USER Username for git operations (default: root)
+
+Parameters:
+ -SkipGit Skip git clone/push tests
+ -SkipRegistry Skip container registry tests
+ -SkipCleanup Don't delete the test project after run
+ -Insecure Allow self-signed TLS certificates
+ -Timeout N HTTP timeout in seconds (default: 10)
+ -Format FORMAT Output: text (default), tap, junit
+ -JunitFile FILE JUnit output path (default: smoke-results.xml)
+ -NoColor Disable colored output
+ -Verbose Show debug output
+
+Examples:
+ `$env:GITLAB_URL = "https://gitlab.example.com"
+ `$env:GITLAB_TOKEN = "glpat-xxxxxxxxxxxx"
+ .\gitlab-smoke-tests.ps1
+
+ .\gitlab-smoke-tests.ps1 -Insecure -Format junit
+ .\gitlab-smoke-tests.ps1 -SkipGit -SkipRegistry
+ .\gitlab-smoke-tests.ps1 -Format tap
+"@
+}
+
+# Handle PS 5.1 TLS and self-signed certs
+if ($PSVersionTable.PSVersion.Major -lt 7) {
+ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+ if ($Insecure) {
+ Add-Type @"
+using System.Net;
+using System.Security.Cryptography.X509Certificates;
+public class TrustAll : ICertificatePolicy {
+ public bool CheckValidationResult(ServicePoint sp, X509Certificate cert,
+ WebRequest req, int problem) { return true; }
+}
+"@ -ErrorAction SilentlyContinue
+ [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAll
+ }
+}
+
+# Validate
+if (-not $GitLabUrl) {
+ Write-Err "GITLAB_URL is required"
+ Write-Host ""
+ Show-Usage
+ exit 1
+}
+
+if (-not $GitLabToken) {
+ Write-Err "GITLAB_TOKEN is required"
+ Write-Host ""
+ Show-Usage
+ exit 1
+}
+
+$GitLabUrl = $GitLabUrl.TrimEnd("/")
+$script:StartTime = Get-Date
+
+if ($Format -eq "tap") {
+ Write-TapHeader
+} else {
+ Write-Host ""
+ Write-Color "GitLab Smoke Tests" "White"
+ Write-Host "Target: $GitLabUrl"
+ Write-Host "Time: $(Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ')"
+ Write-Host ""
+}
+
+try {
+ Test-Connectivity
+ Test-Api
+ Test-Git
+ Test-Registry
+ Test-CICD
+ Test-Migrations
+ Test-Components
+} finally {
+ Invoke-Cleanup
+}
+
+if ($Format -eq "tap") {
+ Write-TapFooter
+} elseif ($Format -eq "junit") {
+ Write-Summary
+ Write-JunitReport
+} else {
+ Write-Summary
+}
+
+if ($script:Fail -eq 0) { exit 0 } else { exit 1 }
diff --git a/gitlab-smoke-tests.sh b/gitlab-smoke-tests.sh
new file mode 100644
index 0000000..cffd7b3
--- /dev/null
+++ b/gitlab-smoke-tests.sh
@@ -0,0 +1,862 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### gitlab-smoke-tests.sh — Verify GitLab instance health after upgrades or changes ####
+#### Zero external dependencies. Runs in air-gapped environments. ####
+#### Requires: bash 4+, curl, git, openssl (optional) ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.02 ####
+#### ####
+#### Usage: ####
+#### export GITLAB_URL="https://gitlab.example.com" ####
+#### export GITLAB_TOKEN="glpat-xxxxxxxxxxxxxxxxxxxx" ####
+#### export GITLAB_HEALTH_TOKEN="your-health-token" # optional ####
+#### ./gitlab-smoke-tests.sh ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+GITLAB_URL="${GITLAB_URL:-}"
+GITLAB_TOKEN="${GITLAB_TOKEN:-}"
+GITLAB_USER="${GITLAB_USER:-root}"
+SMOKE_PROJECT_PREFIX="${SMOKE_PROJECT_PREFIX:-smoke-test}"
+CURL_TIMEOUT="${CURL_TIMEOUT:-10}"
+CURL_INSECURE="${CURL_INSECURE:-false}"
+SKIP_GIT="${SKIP_GIT:-false}"
+SKIP_REGISTRY="${SKIP_REGISTRY:-false}"
+SKIP_CLEANUP="${SKIP_CLEANUP:-false}"
+GITLAB_HEALTH_TOKEN="${GITLAB_HEALTH_TOKEN:-}"
+OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}" # text, tap, junit
+JUNIT_FILE="${JUNIT_FILE:-smoke-results.xml}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+PASS=0
+FAIL=0
+SKIP=0
+TOTAL=0
+RESULTS=()
+CLEANUP_PROJECT_ID=""
+TMPDIR_SMOKE=""
+START_TIME=""
+GIT_CLONE_OK="false"
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[0;33m'
+ BLUE='\033[0;34m'
+ BOLD='\033[1m'
+ RESET='\033[0m'
+ else
+ RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
+ fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
+
+# ── Test Result Recording ─────────────────────────────────────────────
+record_pass() {
+ local name="$1"
+ local detail="${2:-}"
+ ((PASS++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("PASS|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok ${TOTAL} - ${name}"
+ else
+ echo -e " ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}"
+ fi
+}
+
+record_fail() {
+ local name="$1"
+ local detail="${2:-}"
+ ((FAIL++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("FAIL|${name}|${detail}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "not ok ${TOTAL} - ${name}"
+ [[ -n "$detail" ]] && echo " # ${detail}"
+ else
+ echo -e " ${RED}✗${RESET} ${name}${detail:+ — ${detail}}"
+ fi
+}
+
+record_skip() {
+ local name="$1"
+ local reason="${2:-}"
+ ((SKIP++)) || true
+ ((TOTAL++)) || true
+ RESULTS+=("SKIP|${name}|${reason}")
+ if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
+ echo "ok ${TOTAL} - ${name} # SKIP ${reason}"
+ else
+ echo -e " ${YELLOW}⊘${RESET} ${name}${reason:+ — ${reason}}"
+ fi
+}
+
+# ── curl wrapper ──────────────────────────────────────────────────────
+api_curl() {
+ local method="$1"
+ local endpoint="$2"
+ shift 2
+ local curl_opts=(-s -S --max-time "$CURL_TIMEOUT" -X "$method")
+
+ [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
+ [[ -n "$GITLAB_TOKEN" ]] && curl_opts+=(-H "PRIVATE-TOKEN: ${GITLAB_TOKEN}")
+ curl_opts+=(-H "Content-Type: application/json")
+
+ local url="${GITLAB_URL}/api/v4${endpoint}"
+ verbose "curl ${method} ${url} $*"
+
+ curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null
+}
+
+api_curl_status() {
+ local method="$1"
+ local endpoint="$2"
+ shift 2
+ local curl_opts=(-s -S -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT" -X "$method")
+
+ [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
+ [[ -n "$GITLAB_TOKEN" ]] && curl_opts+=(-H "PRIVATE-TOKEN: ${GITLAB_TOKEN}")
+ curl_opts+=(-H "Content-Type: application/json")
+
+ local url="${GITLAB_URL}/api/v4${endpoint}"
+ curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null
+}
+
+# ── JSON parsing (no jq required) ────────────────────────────────────
+# Extract a top-level string/number value from flat JSON
+json_value() {
+ local key="$1"
+ local json="$2"
+ echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"?\K[^\",}]+" || true; } | head -1
+}
+
+json_value_string() {
+ local key="$1"
+ local json="$2"
+ echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"\K[^\"]*" || true; } | head -1
+}
+
+# ── Cleanup ───────────────────────────────────────────────────────────
+cleanup() {
+ if [[ -n "$CLEANUP_PROJECT_ID" && "$SKIP_CLEANUP" != "true" ]]; then
+ verbose "Cleaning up smoke test project (ID: ${CLEANUP_PROJECT_ID})"
+ api_curl DELETE "/projects/${CLEANUP_PROJECT_ID}" >/dev/null 2>&1 || true
+ fi
+ if [[ -n "$TMPDIR_SMOKE" && -d "$TMPDIR_SMOKE" ]]; then
+ rm -rf "$TMPDIR_SMOKE"
+ fi
+}
+
+trap cleanup EXIT
+
+# ══════════════════════════════════════════════════════════════════════
+# TEST SUITES
+# ══════════════════════════════════════════════════════════════════════
+
+# ── 1. Connectivity ──────────────────────────────────────────────────
+test_connectivity() {
+ echo ""
+ echo -e "${BOLD}Connectivity${RESET}"
+
+ # 1a. HTTP(S) reachable
+ local curl_opts=(-s -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT")
+ [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
+
+ local health_qs=""
+ [[ -n "$GITLAB_HEALTH_TOKEN" ]] && health_qs="?token=${GITLAB_HEALTH_TOKEN}"
+
+ local http_code
+ http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/health${health_qs}" 2>/dev/null) || http_code="000"
+
+ if [[ "$http_code" == "200" ]]; then
+ record_pass "GitLab health endpoint reachable" "HTTP ${http_code}"
+ else
+ record_fail "GitLab health endpoint reachable" "HTTP ${http_code}"
+ fi
+
+ # 1b. Readiness check
+ http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/readiness${health_qs}" 2>/dev/null) || http_code="000"
+ if [[ "$http_code" == "200" ]]; then
+ record_pass "GitLab readiness check" "HTTP ${http_code}"
+ else
+ record_fail "GitLab readiness check" "HTTP ${http_code}"
+ fi
+
+ # 1c. Liveness check
+ http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/liveness${health_qs}" 2>/dev/null) || http_code="000"
+ if [[ "$http_code" == "200" ]]; then
+ record_pass "GitLab liveness check" "HTTP ${http_code}"
+ else
+ record_fail "GitLab liveness check" "HTTP ${http_code}"
+ fi
+
+ # 1d. TLS certificate validity (if HTTPS)
+ if [[ "$GITLAB_URL" == https://* ]]; then
+ local host
+ host=$(echo "$GITLAB_URL" | sed 's|https://||' | cut -d/ -f1 | cut -d: -f1)
+ local port
+ port=$(echo "$GITLAB_URL" | grep -oP ':\K[0-9]+$' || echo "443")
+
+ local expiry
+ expiry=$(echo | openssl s_client -servername "$host" -connect "${host}:${port}" 2>/dev/null | \
+ openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) || expiry=""
+
+ if [[ -n "$expiry" ]]; then
+ local expiry_epoch
+ expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null) || expiry_epoch=0
+ local now_epoch
+ now_epoch=$(date +%s)
+ local days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
+
+ if [[ $days_left -gt 30 ]]; then
+ record_pass "TLS certificate valid" "${days_left} days remaining"
+ elif [[ $days_left -gt 0 ]]; then
+ record_pass "TLS certificate valid" "${days_left} days remaining (renew soon)"
+ else
+ record_fail "TLS certificate valid" "expired or expiring in ${days_left} days"
+ fi
+ else
+ record_skip "TLS certificate check" "could not retrieve certificate"
+ fi
+ else
+ record_skip "TLS certificate check" "not using HTTPS"
+ fi
+}
+
+# ── 2. API ────────────────────────────────────────────────────────────
+test_api() {
+ echo ""
+ echo -e "${BOLD}API${RESET}"
+
+ # 2a. Version endpoint
+ local version_json
+ version_json=$(api_curl GET "/version" 2>/dev/null) || version_json=""
+
+ local gl_version
+ gl_version=$(json_value_string "version" "$version_json")
+ local gl_revision
+ gl_revision=$(json_value_string "revision" "$version_json")
+
+ if [[ -n "$gl_version" ]]; then
+ record_pass "API version endpoint" "GitLab ${gl_version} (${gl_revision})"
+ else
+ record_fail "API version endpoint" "no version returned"
+ fi
+
+ # 2b. Authentication
+ local auth_status
+ auth_status=$(api_curl_status GET "/user")
+ if [[ "$auth_status" == "200" ]]; then
+ local user_json
+ user_json=$(api_curl GET "/user")
+ local username
+ username=$(json_value_string "username" "$user_json")
+ record_pass "API authentication" "authenticated as ${username}"
+ elif [[ "$auth_status" == "401" ]]; then
+ record_fail "API authentication" "token rejected (HTTP 401)"
+ else
+ record_fail "API authentication" "HTTP ${auth_status}"
+ fi
+
+ # 2c. List projects (verify database queries work)
+ local projects_status
+ projects_status=$(api_curl_status GET "/projects?per_page=1")
+ if [[ "$projects_status" == "200" ]]; then
+ record_pass "API list projects" "database responding"
+ else
+ record_fail "API list projects" "HTTP ${projects_status}"
+ fi
+
+ # 2d. List users
+ local users_status
+ users_status=$(api_curl_status GET "/users?per_page=1")
+ if [[ "$users_status" == "200" ]]; then
+ record_pass "API list users" "user directory accessible"
+ else
+ record_fail "API list users" "HTTP ${users_status}"
+ fi
+
+ # 2e. Sidekiq health (job processing)
+ local sidekiq_json
+ sidekiq_json=$(api_curl GET "/sidekiq/compound_metrics" 2>/dev/null) || sidekiq_json=""
+
+ if [[ -n "$sidekiq_json" && "$sidekiq_json" != *"error"* ]]; then
+ local processes
+ processes=$(echo "$sidekiq_json" | { grep -oP '"hostname"\s*:' || true; } | wc -l)
+ record_pass "Sidekiq running" "${processes} process(es) responding"
+ else
+ record_fail "Sidekiq running" "could not query Sidekiq metrics"
+ fi
+
+ # 2f. Runners endpoint
+ local runners_status
+ runners_status=$(api_curl_status GET "/runners/all?per_page=1")
+ if [[ "$runners_status" == "200" ]]; then
+ record_pass "API runners endpoint" "runner management accessible"
+ elif [[ "$runners_status" == "403" ]]; then
+ record_skip "API runners endpoint" "token lacks admin scope"
+ else
+ record_fail "API runners endpoint" "HTTP ${runners_status}"
+ fi
+
+ # 2g. Search endpoint
+ local search_status
+ search_status=$(api_curl_status GET "/search?scope=projects&search=test")
+ if [[ "$search_status" == "200" ]]; then
+ record_pass "API search" "search index responding"
+ elif [[ "$search_status" == "403" ]]; then
+ record_skip "API search" "search disabled or token lacks scope"
+ else
+ record_fail "API search" "HTTP ${search_status}"
+ fi
+}
+
+# ── 3. Git Operations ────────────────────────────────────────────────
+test_git() {
+ if [[ "$SKIP_GIT" == "true" ]]; then
+ echo ""
+ echo -e "${BOLD}Git Operations${RESET}"
+ record_skip "Git clone" "SKIP_GIT=true"
+ record_skip "Git push" "SKIP_GIT=true"
+ return
+ fi
+
+ echo ""
+ echo -e "${BOLD}Git Operations${RESET}"
+
+ # Create a test project via API
+ local project_name
+ project_name="${SMOKE_PROJECT_PREFIX}-$(date +%s)"
+ local create_json
+ create_json=$(api_curl POST "/projects" -d "{\"name\":\"${project_name}\",\"visibility\":\"private\",\"initialize_with_readme\":true}")
+
+ local project_id
+ project_id=$(json_value "id" "$create_json")
+ local http_url
+ http_url=$(json_value_string "http_url_to_repo" "$create_json")
+
+ if [[ -z "$project_id" || "$project_id" == "null" ]]; then
+ record_fail "Create test project" "API returned: $(echo "$create_json" | head -c 200)"
+ record_skip "Git clone" "no test project"
+ record_skip "Git push" "no test project"
+ return
+ fi
+
+ CLEANUP_PROJECT_ID="$project_id"
+ record_pass "Create test project" "${project_name} (ID: ${project_id})"
+
+ # Clone
+ TMPDIR_SMOKE=$(mktemp -d)
+
+ # Fallback: if http_url_to_repo wasn't parsed, construct it
+ if [[ -z "$http_url" ]]; then
+ http_url="${GITLAB_URL}/${GITLAB_USER}/${project_name}.git"
+ verbose "http_url_to_repo not found in API response, constructed: ${http_url}"
+ fi
+ verbose "Clone URL (from API): ${http_url}"
+
+ # Replace the hostname in the API-returned URL with GITLAB_URL
+ # (the API may return an internal hostname that's unreachable remotely)
+ local api_origin
+ api_origin=$(echo "$http_url" | grep -oP 'https?://[^/]+')
+ if [[ -n "$api_origin" && "$api_origin" != "$GITLAB_URL" ]]; then
+ http_url="${http_url/$api_origin/$GITLAB_URL}"
+ verbose "Rewrote clone URL to: ${http_url}"
+ fi
+
+ local clone_url
+ # Inject token into URL for HTTPS clone
+ if [[ "$http_url" == https://* ]]; then
+ clone_url="https://oauth2:${GITLAB_TOKEN}@${http_url#https://}"
+ elif [[ "$http_url" == http://* ]]; then
+ clone_url="http://oauth2:${GITLAB_TOKEN}@${http_url#http://}"
+ else
+ clone_url="$http_url"
+ fi
+
+ local git_opts=()
+ [[ "$CURL_INSECURE" == "true" ]] && git_opts+=(-c http.sslVerify=false)
+
+ # Brief wait for repository initialization (initialize_with_readme is async)
+ sleep 2
+
+ verbose "Running: git clone ${TMPDIR_SMOKE}/repo"
+ local clone_err clone_rc
+ clone_err=$(git ${git_opts[@]+"${git_opts[@]}"} clone "$clone_url" "${TMPDIR_SMOKE}/repo" 2>&1) && clone_rc=0 || clone_rc=$?
+ if [[ "$clone_rc" -eq 0 ]]; then
+ GIT_CLONE_OK="true"
+ record_pass "Git clone (HTTPS)" "Gitaly responding"
+ else
+ local short_err redacted_url
+ short_err=$(echo "$clone_err" | grep -i -E 'fatal|error' | head -1 | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
+ redacted_url=$(echo "$http_url" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
+ verbose "Full clone output: $(echo "$clone_err" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")"
+ local redacted_clone
+ redacted_clone=$(echo "$clone_url" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
+ verbose "Attempted URL: ${redacted_clone}"
+ record_fail "Git clone (HTTPS)" "${short_err:-clone failed (exit $clone_rc)}"
+ return
+ fi
+
+ # Push a commit
+ pushd "${TMPDIR_SMOKE}/repo" >/dev/null
+ git config user.email "smoke-test@example.com"
+ git config user.name "Smoke Test"
+ echo "smoke test $(date -u +%Y-%m-%dT%H:%M:%SZ)" > smoke-test.txt
+ git add smoke-test.txt
+ git commit -m "smoke test commit" >/dev/null 2>&1
+
+ if git ${git_opts[@]+"${git_opts[@]}"} push origin main >/dev/null 2>&1 || \
+ git ${git_opts[@]+"${git_opts[@]}"} push origin master >/dev/null 2>&1; then
+ record_pass "Git push (HTTPS)" "write to Gitaly succeeded"
+ else
+ record_fail "Git push (HTTPS)" "push failed"
+ fi
+ popd >/dev/null
+}
+
+# ── 4. Container Registry ────────────────────────────────────────────
+test_registry() {
+ if [[ "$SKIP_REGISTRY" == "true" ]]; then
+ echo ""
+ echo -e "${BOLD}Container Registry${RESET}"
+ record_skip "Registry API" "SKIP_REGISTRY=true"
+ return
+ fi
+
+ echo ""
+ echo -e "${BOLD}Container Registry${RESET}"
+
+ # Check if registry is enabled via application settings API
+ local registry_enabled=""
+ local settings_json
+ settings_json=$(api_curl GET "/application/settings" 2>/dev/null) || settings_json=""
+
+ if [[ -n "$settings_json" ]]; then
+ registry_enabled=$(json_value "container_registry_enabled" "$settings_json" 2>/dev/null || echo "")
+ fi
+
+ if [[ "$registry_enabled" == "false" ]]; then
+ record_skip "Registry API reachable" "container registry disabled in application settings"
+ record_skip "Registry project endpoint" "container registry disabled in application settings"
+ return
+ fi
+
+ # Try the registry v2 API endpoint
+ local host
+ host=$(echo "$GITLAB_URL" | sed 's|https\?://||' | cut -d/ -f1)
+
+ local curl_opts=(-s -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT")
+ [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
+
+ local registry_status
+ registry_status=$(curl "${curl_opts[@]}" "${GITLAB_URL}:5050/v2/" 2>/dev/null) || \
+ registry_status=$(curl "${curl_opts[@]}" "https://${host}:5050/v2/" 2>/dev/null) || \
+ registry_status=$(curl "${curl_opts[@]}" "https://registry.${host}/v2/" 2>/dev/null) || \
+ registry_status="000"
+
+ if [[ "$registry_status" == "200" || "$registry_status" == "401" ]]; then
+ record_pass "Registry API reachable" "HTTP ${registry_status}"
+ elif [[ "$registry_status" == "000" ]]; then
+ if [[ "$registry_enabled" == "true" ]]; then
+ record_fail "Registry API reachable" "enabled in settings but not reachable at standard ports/hosts"
+ else
+ record_skip "Registry API reachable" "not found at standard ports/hosts (settings unreadable — may need admin token)"
+ fi
+ else
+ record_fail "Registry API reachable" "HTTP ${registry_status}"
+ fi
+
+ # Check registry via GitLab API (project-level)
+ if [[ -n "$CLEANUP_PROJECT_ID" ]]; then
+ local reg_status
+ reg_status=$(api_curl_status GET "/projects/${CLEANUP_PROJECT_ID}/registry/repositories")
+ if [[ "$reg_status" == "200" ]]; then
+ record_pass "Registry project endpoint" "project registry accessible"
+ elif [[ "$reg_status" == "404" ]]; then
+ record_skip "Registry project endpoint" "container registry not enabled for project"
+ else
+ record_fail "Registry project endpoint" "HTTP ${reg_status}"
+ fi
+ fi
+}
+
+# ── 5. CI/CD ──────────────────────────────────────────────────────────
+test_cicd() {
+ echo ""
+ echo -e "${BOLD}CI/CD${RESET}"
+
+ # Check runners
+ local runners_json
+ runners_json=$(api_curl GET "/runners/all?per_page=100" 2>/dev/null) || runners_json=""
+
+ if [[ "$runners_json" == "["* ]]; then
+ local runner_count
+ runner_count=$(echo "$runners_json" | { grep -oP '"id"\s*:' || true; } | wc -l)
+ local online_count
+ online_count=$(echo "$runners_json" | { grep -oP '"status"\s*:\s*"online"' || true; } | wc -l)
+
+ if [[ $online_count -gt 0 ]]; then
+ record_pass "CI/CD runners online" "${online_count}/${runner_count} runners online"
+ elif [[ $runner_count -gt 0 ]]; then
+ record_fail "CI/CD runners online" "0/${runner_count} runners online"
+ else
+ record_skip "CI/CD runners online" "no runners registered"
+ fi
+ else
+ record_skip "CI/CD runners" "could not query runners (admin token required)"
+ fi
+
+ # Check CI/CD settings via API
+ local cicd_status
+ cicd_status=$(api_curl_status GET "/application/settings")
+ if [[ "$cicd_status" == "200" ]]; then
+ record_pass "CI/CD settings accessible" "application settings readable"
+ elif [[ "$cicd_status" == "403" ]]; then
+ record_skip "CI/CD settings accessible" "admin token required"
+ else
+ record_fail "CI/CD settings accessible" "HTTP ${cicd_status}"
+ fi
+}
+
+# ── 6. Background Migrations ─────────────────────────────────────────
+test_migrations() {
+ echo ""
+ echo -e "${BOLD}Background Migrations${RESET}"
+
+ # Batched background migrations (admin only)
+ local migrations_json
+ migrations_json=$(api_curl GET "/admin/batched_background_migrations?database=main" 2>/dev/null) || migrations_json=""
+
+ if [[ "$migrations_json" == "["* ]]; then
+ local total_mig
+ total_mig=$(echo "$migrations_json" | { grep -oP '"id"\s*:' || true; } | wc -l)
+
+ local failed_mig
+ failed_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"failed"' || true; } | wc -l)
+ local active_mig
+ active_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"active"' || true; } | wc -l)
+ local paused_mig
+ paused_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"paused"' || true; } | wc -l)
+ local finalized_mig
+ finalized_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"finished"' || true; } | wc -l)
+
+ if [[ $failed_mig -gt 0 ]]; then
+ record_fail "Background migrations" "${failed_mig} failed, ${active_mig} active, ${paused_mig} paused, ${finalized_mig} finished of ${total_mig}"
+ elif [[ $paused_mig -gt 0 ]]; then
+ record_fail "Background migrations" "${paused_mig} paused, ${active_mig} active, ${finalized_mig} finished of ${total_mig}"
+ elif [[ $active_mig -gt 0 ]]; then
+ record_pass "Background migrations" "${active_mig} active, ${finalized_mig} finished of ${total_mig} (in progress)"
+ else
+ record_pass "Background migrations" "all ${total_mig} finished"
+ fi
+ else
+ local mig_status
+ mig_status=$(api_curl_status GET "/admin/batched_background_migrations?database=main")
+ if [[ "$mig_status" == "403" ]]; then
+ record_skip "Background migrations" "admin token required"
+ else
+ record_skip "Background migrations" "could not query (HTTP ${mig_status})"
+ fi
+ fi
+}
+
+# ── 7. Storage & Components ──────────────────────────────────────────
+test_components() {
+ echo ""
+ echo -e "${BOLD}Components${RESET}"
+
+ # Metadata endpoint
+ local metadata_json
+ metadata_json=$(api_curl GET "/metadata" 2>/dev/null) || metadata_json=""
+
+ if [[ -n "$metadata_json" ]]; then
+ local gl_version
+ gl_version=$(json_value_string "version" "$metadata_json")
+ local enterprise
+ enterprise=$(json_value "enterprise" "$metadata_json")
+
+ if [[ -n "$gl_version" ]]; then
+ local edition="CE"
+ [[ "$enterprise" == "true" ]] && edition="EE"
+ record_pass "GitLab metadata" "${gl_version} ${edition}"
+ else
+ record_pass "GitLab metadata" "endpoint reachable"
+ fi
+ else
+ record_skip "GitLab metadata" "metadata endpoint not available"
+ fi
+
+ # Statistics (admin)
+ local stats_json
+ stats_json=$(api_curl GET "/application/statistics" 2>/dev/null) || stats_json=""
+
+ if [[ -n "$stats_json" && "$stats_json" != *"error"* && "$stats_json" != *"403"* ]]; then
+ local active_users
+ active_users=$(json_value "active_users" "$stats_json")
+ local projects
+ projects=$(json_value "projects" "$stats_json")
+ local groups
+ groups=$(json_value "groups" "$stats_json")
+
+ if [[ -n "$active_users" ]]; then
+ record_pass "Instance statistics" "${active_users} users, ${projects} projects, ${groups} groups"
+ else
+ record_pass "Instance statistics" "endpoint reachable"
+ fi
+ else
+ record_skip "Instance statistics" "admin token required"
+ fi
+
+ # Gitaly check — only report pass if clone actually succeeded
+ if [[ "$GIT_CLONE_OK" == "true" ]]; then
+ record_pass "Gitaly storage" "project created and cloned successfully"
+ elif [[ -n "$CLEANUP_PROJECT_ID" ]]; then
+ record_skip "Gitaly storage" "project created but clone was not tested or failed"
+ fi
+
+ # PostgreSQL (inferred from API responsiveness)
+ local pg_test
+ pg_test=$(api_curl_status GET "/projects?per_page=1&order_by=updated_at")
+ if [[ "$pg_test" == "200" ]]; then
+ record_pass "PostgreSQL" "database queries succeeding"
+ else
+ record_fail "PostgreSQL" "sorted query failed (HTTP ${pg_test})"
+ fi
+
+ # Redis (inferred from session/cache)
+ local redis_test
+ redis_test=$(api_curl_status GET "/user")
+ if [[ "$redis_test" == "200" ]]; then
+ record_pass "Redis" "session/cache operational (auth succeeded)"
+ else
+ record_skip "Redis" "cannot verify independently"
+ fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# OUTPUT
+# ══════════════════════════════════════════════════════════════════════
+
+print_summary() {
+ local end_time
+ end_time=$(date +%s)
+ local duration=$(( end_time - START_TIME ))
+
+ echo ""
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+ echo -e "${BOLD}Summary${RESET} ${GITLAB_URL}"
+ echo -e " ${GREEN}${PASS} passed${RESET} ${RED}${FAIL} failed${RESET} ${YELLOW}${SKIP} skipped${RESET} (${duration}s)"
+ echo -e "${BOLD}────────────────────────────────────────${RESET}"
+
+ if [[ $FAIL -eq 0 ]]; then
+ echo -e "${GREEN}${BOLD}All tests passed.${RESET}"
+ else
+ echo -e "${RED}${BOLD}${FAIL} test(s) failed.${RESET}"
+ fi
+}
+
+print_tap_header() {
+ echo "TAP version 13"
+}
+
+print_tap_footer() {
+ echo "1..${TOTAL}"
+ echo "# pass ${PASS}"
+ echo "# fail ${FAIL}"
+ echo "# skip ${SKIP}"
+}
+
+write_junit() {
+ local end_time
+ end_time=$(date +%s)
+ local duration=$(( end_time - START_TIME ))
+
+ cat > "$JUNIT_FILE" <
+
+
+JUNIT_EOF
+
+ for result in "${RESULTS[@]}"; do
+ local status name detail
+ status=$(echo "$result" | cut -d'|' -f1)
+ name=$(echo "$result" | cut -d'|' -f2)
+ detail=$(echo "$result" | cut -d'|' -f3)
+
+ # XML-escape the values
+ name=$(echo "$name" | sed 's/&/\&/g; s//\>/g; s/"/\"/g')
+ detail=$(echo "$detail" | sed 's/&/\&/g; s//\>/g; s/"/\"/g')
+
+ case "$status" in
+ PASS)
+ echo " " >> "$JUNIT_FILE"
+ [[ -n "$detail" ]] && echo " ${detail}" >> "$JUNIT_FILE"
+ echo " " >> "$JUNIT_FILE"
+ ;;
+ FAIL)
+ echo " " >> "$JUNIT_FILE"
+ echo " FAILED: ${name} — ${detail}" >> "$JUNIT_FILE"
+ echo " " >> "$JUNIT_FILE"
+ ;;
+ SKIP)
+ echo " " >> "$JUNIT_FILE"
+ echo " " >> "$JUNIT_FILE"
+ echo " " >> "$JUNIT_FILE"
+ ;;
+ esac
+ done
+
+ echo " " >> "$JUNIT_FILE"
+ echo "" >> "$JUNIT_FILE"
+
+ log "JUnit report written to ${JUNIT_FILE}"
+}
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+ cat <&2; }
+err() { printf "${RED}✗ %s${RESET}\n" "$*" >&2; }
+verbose() { [[ "$VERBOSE" == "true" ]] && printf "${DIM} %s${RESET}\n" "$*" >&2 || true; }
+die() { err "$*"; exit 1; }
+
+# ── Help ──────────────────────────────────────────────────────────────────────
+
+show_help() {
+ cat </dev/null 2>&1; then
+ ver=$(gitlab-ctl version 2>/dev/null | grep -oP '\d+\.\d+\.\d+' | head -1 || true)
+ fi
+ if [[ -z "$ver" ]] && command -v dpkg >/dev/null 2>&1; then
+ ver=$(dpkg -l gitlab-ce gitlab-ee 2>/dev/null | awk '/^ii/{print $3}' | grep -oP '\d+\.\d+\.\d+' | head -1 || true)
+ fi
+ if [[ -z "$ver" ]] && command -v rpm >/dev/null 2>&1; then
+ ver=$(rpm -q gitlab-ce gitlab-ee 2>/dev/null | grep -oP '\d+\.\d+\.\d+' | head -1 || true)
+ fi
+ echo "$ver"
+}
+
+detect_pg_version() {
+ local ver=""
+ if command -v gitlab-psql >/dev/null 2>&1; then
+ ver=$(gitlab-psql --version 2>/dev/null | grep -oP '\d+' | head -1 || true)
+ fi
+ if [[ -z "$ver" ]] && command -v psql >/dev/null 2>&1; then
+ ver=$(psql --version 2>/dev/null | grep -oP '\d+' | head -1 || true)
+ fi
+ echo "$ver"
+}
+
+# ── Core Logic ────────────────────────────────────────────────────────────────
+
+get_pg_req() {
+ local gl_major="$1"
+ local entry
+ for entry in "${PG_REQS[@]}"; do
+ IFS='|' read -r req_gl req_min req_max <<< "$entry"
+ if [[ "$req_gl" == "$gl_major" ]]; then
+ echo "${req_min}|${req_max}"
+ return
+ fi
+ done
+ echo "unknown|unknown"
+}
+
+build_upgrade_path() {
+ local from_int to_int
+ from_int=$(version_to_int "$FROM_VERSION")
+ to_int=$(version_to_int "$TO_VERSION")
+
+ UPGRADE_PATH=()
+ local entry ver ver_int conditional notes
+ for entry in "${STOPS[@]}"; do
+ IFS='|' read -r ver conditional notes <<< "$entry"
+ ver_int=$(version_to_int "$ver")
+
+ if (( ver_int <= from_int )); then
+ continue
+ fi
+ if (( ver_int > to_int )); then
+ continue
+ fi
+ if [[ "$SKIP_CONDITIONAL" == "true" && "$conditional" == "1" ]]; then
+ verbose "Skipping conditional stop: $ver"
+ continue
+ fi
+
+ UPGRADE_PATH+=("$entry")
+ done
+
+ # Add target if it's not already in the path
+ local last_ver=""
+ if [[ ${#UPGRADE_PATH[@]} -gt 0 ]]; then
+ IFS='|' read -r last_ver _ _ <<< "${UPGRADE_PATH[-1]}"
+ fi
+ if [[ "$last_ver" != "$TO_VERSION" ]]; then
+ UPGRADE_PATH+=("${TO_VERSION}|0|Target version")
+ fi
+}
+
+get_pg_warnings() {
+ PG_WARNINGS=()
+ if [[ -z "$PG_VERSION" ]]; then
+ return
+ fi
+
+ local from_major to_major
+ from_major=$(version_major "$FROM_VERSION")
+ to_major=$(version_major "$TO_VERSION")
+
+ local gl_major
+ for (( gl_major = from_major; gl_major <= to_major; gl_major++ )); do
+ local req
+ req=$(get_pg_req "$gl_major")
+ IFS='|' read -r pg_min pg_max <<< "$req"
+ if [[ "$pg_min" == "unknown" ]]; then
+ continue
+ fi
+ if (( PG_VERSION < pg_min )); then
+ # Find the last stop before this major version
+ local boundary_stop=""
+ local prev_major=$(( gl_major - 1 ))
+ local entry ver
+ for entry in "${STOPS[@]}"; do
+ IFS='|' read -r ver _ _ <<< "$entry"
+ if [[ "$(version_major "$ver")" == "$prev_major" ]]; then
+ boundary_stop="$ver"
+ fi
+ done
+ PG_WARNINGS+=("PostgreSQL ${PG_VERSION} is below minimum for GitLab ${gl_major}.x (requires ${pg_min}+)|Upgrade PostgreSQL to ${pg_min}+ before upgrading past GitLab ${boundary_stop:-${prev_major}.x}|${gl_major}|${pg_min}|${pg_max}")
+ fi
+ done
+}
+
+estimate_downtime() {
+ local stop_count=${#UPGRADE_PATH[@]}
+ local pg_upgrade_count=${#PG_WARNINGS[@]}
+
+ # Software time: package install + gitlab-ctl reconfigure per stop
+ DT_SW_LOW=$(( stop_count * 5 ))
+ DT_SW_HIGH=$(( stop_count * 15 ))
+
+ # Background migration time per stop, based on database size
+ # These run between stops and must complete before proceeding
+ local mig_low=0 mig_high=0
+ case "$DB_SIZE" in
+ small) mig_low=2; mig_high=10 ;; # <10GB: minutes
+ medium) mig_low=10; mig_high=30 ;; # 10-50GB: tens of minutes
+ large) mig_low=30; mig_high=90 ;; # 50-200GB: up to hours
+ xlarge) mig_low=60; mig_high=240 ;; # 200GB+: hours per stop
+ *) mig_low=0; mig_high=0 ;; # unknown: show software only
+ esac
+ DT_MIG_LOW=$(( stop_count * mig_low ))
+ DT_MIG_HIGH=$(( stop_count * mig_high ))
+
+ # PostgreSQL major upgrade time
+ DT_PG_LOW=$(( pg_upgrade_count * 15 ))
+ DT_PG_HIGH=$(( pg_upgrade_count * 60 ))
+
+ DT_GL_LOW=$(( DT_SW_LOW + DT_MIG_LOW ))
+ DT_GL_HIGH=$(( DT_SW_HIGH + DT_MIG_HIGH ))
+ DT_TOTAL_LOW=$(( DT_GL_LOW + DT_PG_LOW ))
+ DT_TOTAL_HIGH=$(( DT_GL_HIGH + DT_PG_HIGH ))
+}
+
+# ── Text Output ───────────────────────────────────────────────────────────────
+
+print_header() {
+ printf "\n${BOLD}GitLab Upgrade Path Calculator${RESET}\n"
+ printf "══════════════════════════════════════════════════════════════\n\n"
+}
+
+format_path_text() {
+ print_header
+
+ printf " ${BOLD}From:${RESET} %s\n" "$FROM_VERSION"
+ printf " ${BOLD}To:${RESET} %s\n" "$TO_VERSION"
+ printf " ${BOLD}Stops:${RESET} %d\n" "${#UPGRADE_PATH[@]}"
+
+ if [[ -n "$PG_VERSION" ]]; then
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ local last_warn="${PG_WARNINGS[-1]}"
+ IFS='|' read -r _ _ _ final_pg_min _ <<< "$last_warn"
+ printf "\n ${BOLD}PostgreSQL:${RESET} ${YELLOW}Currently ${PG_VERSION} → Must upgrade to ${final_pg_min}+ before GitLab ${TO_VERSION}${RESET}\n"
+ else
+ printf "\n ${BOLD}PostgreSQL:${RESET} ${GREEN}${PG_VERSION} — compatible with target${RESET}\n"
+ fi
+ fi
+
+ printf "\n ── Upgrade Path ──────────────────────────────────────────\n\n"
+ printf " ${DIM}Step Version Notes PG Required${RESET}\n"
+ printf " ${DIM}──── ──────────── ─────────────────────────────────────── ──────────${RESET}\n"
+
+ local step=0 entry ver conditional notes
+ for entry in "${UPGRADE_PATH[@]}"; do
+ IFS='|' read -r ver conditional notes <<< "$entry"
+ step=$((step + 1))
+
+ local gl_major pg_range
+ gl_major=$(version_major "$ver")
+ local req
+ req=$(get_pg_req "$gl_major")
+ IFS='|' read -r pg_min pg_max <<< "$req"
+ if [[ "$pg_min" != "unknown" ]]; then
+ pg_range="${pg_min}-${pg_max}"
+ else
+ pg_range="—"
+ fi
+
+ local cond_marker=""
+ if [[ "$conditional" == "1" ]]; then
+ cond_marker=" ⓘ"
+ fi
+
+ local ver_color="$RESET"
+ if [[ "$notes" == "Target version" ]]; then
+ ver_color="$GREEN"
+ fi
+
+ printf " %3d ${ver_color}%-12s${RESET} %-39s %s\n" "$step" "$ver" "${notes}${cond_marker}" "$pg_range"
+ done
+
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ printf "\n ── PostgreSQL Upgrade Required ───────────────────────────\n\n"
+ local warning
+ for warning in "${PG_WARNINGS[@]}"; do
+ IFS='|' read -r msg action gl_major pg_min pg_max <<< "$warning"
+ printf " ${YELLOW}⚠ %s${RESET}\n" "$msg"
+ printf " → %s\n\n" "$action"
+ done
+ fi
+
+ estimate_downtime
+ printf " ── Estimated Downtime ────────────────────────────────────\n\n"
+ printf " Software: %d stops × 5-15 min = %d-%d min\n" "${#UPGRADE_PATH[@]}" "$DT_SW_LOW" "$DT_SW_HIGH"
+ printf " ${DIM}(package install + gitlab-ctl reconfigure)${RESET}\n"
+ if [[ -n "$DB_SIZE" ]]; then
+ printf " Migrations: %d stops × %s db = %d-%d min\n" "${#UPGRADE_PATH[@]}" "$DB_SIZE" "$DT_MIG_LOW" "$DT_MIG_HIGH"
+ printf " ${DIM}(background migrations must complete per stop)${RESET}\n"
+ else
+ printf " Migrations: ${DIM}use --db-size (small/medium/large/xlarge) for estimates${RESET}\n"
+ fi
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ printf " PG upgrades: %d × 15-60 min = %d-%d min\n" "${#PG_WARNINGS[@]}" "$DT_PG_LOW" "$DT_PG_HIGH"
+ fi
+ printf "\n ${BOLD}Total estimate: %d-%d min${RESET}" "$DT_TOTAL_LOW" "$DT_TOTAL_HIGH"
+ if (( DT_TOTAL_HIGH >= 120 )); then
+ local hours_low=$(( DT_TOTAL_LOW / 60 ))
+ local hours_high=$(( DT_TOTAL_HIGH / 60 ))
+ printf " (%d-%d hrs — plan a full maintenance window)" "$hours_low" "$hours_high"
+ fi
+ printf "\n\n"
+
+ if [[ "$SKIP_CONDITIONAL" == "false" ]]; then
+ local has_conditional=false
+ for entry in "${UPGRADE_PATH[@]}"; do
+ IFS='|' read -r _ conditional _ <<< "$entry"
+ if [[ "$conditional" == "1" ]]; then
+ has_conditional=true
+ break
+ fi
+ done
+ if [[ "$has_conditional" == "true" ]]; then
+ printf " ${DIM}ⓘ = conditional stop (may be skippable — use --skip-conditional)${RESET}\n\n"
+ fi
+ fi
+}
+
+format_path_json() {
+ local steps_json="["
+ local step=0 first=true entry ver conditional notes
+ for entry in "${UPGRADE_PATH[@]}"; do
+ IFS='|' read -r ver conditional notes <<< "$entry"
+ step=$((step + 1))
+ local gl_major req pg_min pg_max
+ gl_major=$(version_major "$ver")
+ req=$(get_pg_req "$gl_major")
+ IFS='|' read -r pg_min pg_max <<< "$req"
+
+ [[ "$first" == "true" ]] || steps_json+=","
+ first=false
+ steps_json+=$(printf '{"step":%d,"version":"%s","conditional":%s,"notes":"%s","pg_min":"%s","pg_max":"%s"}' \
+ "$step" "$ver" "$( [[ "$conditional" == "1" ]] && echo "true" || echo "false" )" "$notes" "$pg_min" "$pg_max")
+ done
+ steps_json+="]"
+
+ local pg_upgrades_json="["
+ first=true
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ local warning
+ for warning in "${PG_WARNINGS[@]}"; do
+ IFS='|' read -r msg action gl_major pg_min pg_max <<< "$warning"
+ [[ "$first" == "true" ]] || pg_upgrades_json+=","
+ first=false
+ pg_upgrades_json+=$(printf '{"before_gitlab":"%s.0.0","min_pg":"%s","max_pg":"%s"}' "$gl_major" "$pg_min" "$pg_max")
+ done
+ fi
+ pg_upgrades_json+="]"
+
+ estimate_downtime
+
+ printf '{\n'
+ printf ' "from": "%s",\n' "$FROM_VERSION"
+ printf ' "to": "%s",\n' "$TO_VERSION"
+ printf ' "total_stops": %d,\n' "${#UPGRADE_PATH[@]}"
+ printf ' "pg_current": "%s",\n' "${PG_VERSION:-null}"
+ printf ' "pg_upgrades_needed": %s,\n' "$pg_upgrades_json"
+ printf ' "steps": %s,\n' "$steps_json"
+ printf ' "db_size": "%s",\n' "${DB_SIZE:-unknown}"
+ printf ' "estimated_downtime_min": {"software": {"low": %d, "high": %d}, "migrations": {"low": %d, "high": %d}, "pg_upgrades": {"low": %d, "high": %d}, "total": {"low": %d, "high": %d}}\n' \
+ "$DT_SW_LOW" "$DT_SW_HIGH" "$DT_MIG_LOW" "$DT_MIG_HIGH" "$DT_PG_LOW" "$DT_PG_HIGH" "$DT_TOTAL_LOW" "$DT_TOTAL_HIGH"
+ printf '}\n'
+}
+
+# ── Mode: --path ──────────────────────────────────────────────────────────────
+
+run_path() {
+ if [[ -z "$FROM_VERSION" ]]; then
+ verbose "Detecting installed GitLab version..."
+ FROM_VERSION=$(detect_gitlab_version)
+ if [[ -z "$FROM_VERSION" ]]; then
+ die "Could not detect installed GitLab version. Use --from VERSION."
+ fi
+ log "Detected GitLab version: $FROM_VERSION"
+ fi
+ if [[ -z "$TO_VERSION" || "$TO_VERSION" == "latest" ]]; then
+ TO_VERSION="$LATEST_VERSION"
+ fi
+
+ validate_version "$FROM_VERSION"
+ validate_version "$TO_VERSION"
+
+ local from_int to_int
+ from_int=$(version_to_int "$FROM_VERSION")
+ to_int=$(version_to_int "$TO_VERSION")
+ if (( from_int >= to_int )); then
+ die "Target version ($TO_VERSION) must be higher than current version ($FROM_VERSION)"
+ fi
+
+ build_upgrade_path
+ get_pg_warnings
+
+ if [[ "$FORMAT" == "json" ]]; then
+ format_path_json
+ else
+ format_path_text
+ fi
+}
+
+# ── Mode: --check ─────────────────────────────────────────────────────────────
+
+run_check() {
+ verbose "Detecting installed GitLab version..."
+ FROM_VERSION=$(detect_gitlab_version)
+
+ if [[ -z "$FROM_VERSION" ]]; then
+ die "Could not detect installed GitLab version. Use --path --from VERSION instead."
+ fi
+
+ log "Detected GitLab version: $FROM_VERSION"
+
+ if [[ -z "$PG_VERSION" ]]; then
+ verbose "Detecting PostgreSQL version..."
+ PG_VERSION=$(detect_pg_version)
+ if [[ -n "$PG_VERSION" ]]; then
+ log "Detected PostgreSQL version: $PG_VERSION"
+ fi
+ fi
+
+ if [[ -z "$TO_VERSION" || "$TO_VERSION" == "latest" ]]; then
+ TO_VERSION="$LATEST_VERSION"
+ fi
+
+ validate_version "$FROM_VERSION"
+ validate_version "$TO_VERSION"
+
+ build_upgrade_path
+ get_pg_warnings
+
+ if [[ "$FORMAT" == "json" ]]; then
+ format_path_json
+ else
+ format_path_text
+ fi
+}
+
+# ── Mode: --list-stops ────────────────────────────────────────────────────────
+
+run_list_stops() {
+ print_header
+ printf " ── All Known Required Upgrade Stops ──────────────────────\n\n"
+ printf " ${DIM}Version Type Notes PG Required${RESET}\n"
+ printf " ${DIM}──────────── ──────────── ─────────────────────────────────────── ──────────${RESET}\n"
+
+ local entry ver conditional notes
+ for entry in "${STOPS[@]}"; do
+ IFS='|' read -r ver conditional notes <<< "$entry"
+
+ local gl_major req pg_min pg_max pg_range type_label
+ gl_major=$(version_major "$ver")
+ req=$(get_pg_req "$gl_major")
+ IFS='|' read -r pg_min pg_max <<< "$req"
+ if [[ "$pg_min" != "unknown" ]]; then
+ pg_range="${pg_min}-${pg_max}"
+ else
+ pg_range="—"
+ fi
+
+ if [[ "$conditional" == "1" ]]; then
+ type_label="${YELLOW}conditional${RESET} "
+ else
+ type_label="${GREEN}required${RESET} "
+ fi
+
+ printf " %-12s %b %-39s %s\n" "$ver" "$type_label" "$notes" "$pg_range"
+ done
+
+ printf "\n ── PostgreSQL Version Requirements ───────────────────────\n\n"
+ printf " ${DIM}GitLab Min PG Max PG${RESET}\n"
+ printf " ${DIM}───────── ──────── ────────${RESET}\n"
+ local pg_entry
+ for pg_entry in "${PG_REQS[@]}"; do
+ IFS='|' read -r gl_major pg_min pg_max <<< "$pg_entry"
+ printf " %-9s %-8s %s\n" "${gl_major}.x" "$pg_min" "$pg_max"
+ done
+ printf "\n"
+}
+
+# ── Mode: --db-check ──────────────────────────────────────────────────────────
+
+run_db_check() {
+ if [[ -z "$PG_VERSION" ]]; then
+ verbose "Detecting PostgreSQL version..."
+ PG_VERSION=$(detect_pg_version)
+ if [[ -z "$PG_VERSION" ]]; then
+ die "Could not detect PostgreSQL version. Use --pg-version VERSION."
+ fi
+ log "Detected PostgreSQL version: $PG_VERSION"
+ fi
+
+ if [[ -z "$FROM_VERSION" ]]; then
+ FROM_VERSION=$(detect_gitlab_version)
+ if [[ -z "$FROM_VERSION" ]]; then
+ die "Could not detect GitLab version. Use --from VERSION."
+ fi
+ log "Detected GitLab version: $FROM_VERSION"
+ fi
+
+ if [[ -z "$TO_VERSION" || "$TO_VERSION" == "latest" ]]; then
+ TO_VERSION="$LATEST_VERSION"
+ fi
+
+ validate_version "$FROM_VERSION"
+ validate_version "$TO_VERSION"
+
+ build_upgrade_path
+ get_pg_warnings
+
+ if [[ "$FORMAT" == "json" ]]; then
+ local pg_json="["
+ local first=true
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ local warning
+ for warning in "${PG_WARNINGS[@]}"; do
+ IFS='|' read -r msg action gl_major pg_min pg_max <<< "$warning"
+ [[ "$first" == "true" ]] || pg_json+=","
+ first=false
+ pg_json+=$(printf '{"message":"%s","action":"%s","gitlab_major":"%s","pg_min":"%s","pg_max":"%s"}' \
+ "$msg" "$action" "$gl_major" "$pg_min" "$pg_max")
+ done
+ fi
+ pg_json+="]"
+ printf '{"pg_current":"%s","from":"%s","to":"%s","compatible":%s,"warnings":%s}\n' \
+ "$PG_VERSION" "$FROM_VERSION" "$TO_VERSION" \
+ "$( [[ ${#PG_WARNINGS[@]} -eq 0 ]] && echo "true" || echo "false" )" "$pg_json"
+ return
+ fi
+
+ print_header
+ printf " ${BOLD}PostgreSQL Compatibility Check${RESET}\n\n"
+ printf " Current GitLab: %s\n" "$FROM_VERSION"
+ printf " Target GitLab: %s\n" "$TO_VERSION"
+ printf " Current PostgreSQL: %s\n\n" "$PG_VERSION"
+
+ local from_major to_major
+ from_major=$(version_major "$FROM_VERSION")
+ to_major=$(version_major "$TO_VERSION")
+
+ printf " ── Requirements by GitLab Version ────────────────────────\n\n"
+ printf " ${DIM}GitLab Min PG Max PG Your PG %s Status${RESET}\n" "$PG_VERSION"
+ printf " ${DIM}───────── ──────── ──────── ────────── ──────────${RESET}\n"
+
+ local gl_major
+ for (( gl_major = from_major; gl_major <= to_major; gl_major++ )); do
+ local req pg_min pg_max status
+ req=$(get_pg_req "$gl_major")
+ IFS='|' read -r pg_min pg_max <<< "$req"
+
+ if (( PG_VERSION < pg_min )); then
+ status="${RED}✗ Too low${RESET}"
+ elif (( PG_VERSION > pg_max )); then
+ status="${YELLOW}⚠ Too high${RESET}"
+ else
+ status="${GREEN}✓ OK${RESET}"
+ fi
+ printf " %-9s %-8s %-8s %-10s %b\n" "${gl_major}.x" "$pg_min" "$pg_max" "$PG_VERSION" "$status"
+ done
+
+ if [[ ${#PG_WARNINGS[@]} -gt 0 ]]; then
+ printf "\n ── Action Required ───────────────────────────────────────\n\n"
+ local warning
+ for warning in "${PG_WARNINGS[@]}"; do
+ IFS='|' read -r msg action gl_major pg_min pg_max <<< "$warning"
+ printf " ${YELLOW}⚠ %s${RESET}\n" "$msg"
+ printf " → %s\n\n" "$action"
+ done
+ else
+ printf "\n ${GREEN}✓ PostgreSQL %s is compatible with the full upgrade path.${RESET}\n\n" "$PG_VERSION"
+ fi
+}
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+main() {
+ setup_colors
+ parse_args "$@"
+ setup_colors
+
+ case "$RUN_MODE" in
+ path) run_path ;;
+ check) run_check ;;
+ list-stops) run_list_stops ;;
+ db-check) run_db_check ;;
+ *) die "Unknown mode: $RUN_MODE" ;;
+ esac
+}
+
+main "$@"
diff --git a/gitlab-upgrade.sh b/gitlab-upgrade.sh
index e0ac0ff..d6620d0 100644
--- a/gitlab-upgrade.sh
+++ b/gitlab-upgrade.sh
@@ -7,7 +7,7 @@
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
-#### Version: 1.00-030526 ####
+#### Version: 1.01-051326 ####
################################################
set -o pipefail
@@ -15,7 +15,8 @@ set -o pipefail
SCRIPT_NAME=$(basename "$0")
readonly SCRIPT_NAME
-# Required version stops (as of 2026)
+# Required version stops (as of May 2026)
+# Source: https://docs.gitlab.com/update/upgrade_paths/
readonly VERSION_STOPS=(
"14.0.12"
"14.3.6"
@@ -24,15 +25,19 @@ readonly VERSION_STOPS=(
"15.0.5"
"15.4.6"
"15.11.13"
- "16.0.9"
- "16.3.8"
- "16.7.9"
+ "16.0.10"
+ "16.3.9"
+ "16.7.10"
"16.11.10"
- "17.0.8"
"17.3.7"
"17.5.5"
"17.8.7"
- "18.0.1"
+ "17.11.7"
+ "18.0.2"
+ "18.2.6"
+ "18.5.2"
+ "18.8.7"
+ "18.11.0"
)
# Default configuration
diff --git a/gitops-bootstrap.sh b/gitops-bootstrap.sh
new file mode 100644
index 0000000..8bfb40f
--- /dev/null
+++ b/gitops-bootstrap.sh
@@ -0,0 +1,652 @@
+#!/usr/bin/env bash
+#########################################################################################
+#### gitops-bootstrap.sh — Bootstrap GitOps on Kubernetes with Flux or ArgoCD ####
+#### Install, configure git source, sync applications, and validate deployments ####
+#### Requires: bash 4+, kubectl, git, flux CLI or argocd CLI ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### License: MIT ####
+#### Version 1.00 ####
+#### ####
+#### Usage: ####
+#### ./gitops-bootstrap.sh --install flux --repo git@github.com:org/infra.git ####
+#### ####
+#### See --help for all options. ####
+#########################################################################################
+set -euo pipefail
+
+VERSION="1.00"
+
+# --- ANSI color variables (pre-initialized) ---
+RED=""
+GREEN=""
+YELLOW=""
+BLUE=""
+CYAN=""
+BOLD=""
+DIM=""
+RESET=""
+
+# --- Defaults ---
+RUN_MODE=""
+GITOPS_TOOL="${GITOPS_TOOL:-flux}"
+GIT_REPO="${GITOPS_REPO:-}"
+GIT_BRANCH="${GITOPS_BRANCH:-main}"
+GIT_PATH="${GITOPS_PATH:-./clusters/default}"
+NAMESPACE="${GITOPS_NAMESPACE:-}"
+KUBECONFIG_FILE="${KUBECONFIG:-}"
+KUBE_CTX="${KUBE_CONTEXT:-}"
+CONFIRM_YES=false
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# --- State ---
+readonly SCRIPT_NAME="${0##*/}"
+START_TIME=$(date +%s)
+
+# --- Source name used for flux commands ---
+SOURCE_NAME="main"
+KUSTOMIZATION_NAME="default"
+APP_NAME=""
+
+# --- Color setup ---
+setup_colors() {
+ if [[ "$COLOR" == "never" ]]; then
+ RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
+ return
+ fi
+ if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+ RED='\033[0;31m'
+ GREEN='\033[0;32m'
+ YELLOW='\033[1;33m'
+ BLUE='\033[0;34m'
+ CYAN='\033[0;36m'
+ BOLD='\033[1m'
+ DIM='\033[2m'
+ RESET='\033[0m'
+ fi
+}
+
+# --- Logging ---
+log() { printf "%b\n" "${GREEN}✔${RESET} $*"; }
+warn() { printf "%b\n" "${YELLOW}⚠${RESET} $*" >&2; }
+err() { printf "%b\n" "${RED}✖${RESET} $*" >&2; }
+verbose() { [[ "$VERBOSE" == "true" ]] && printf "%b\n" "${DIM}▸ $*${RESET}" >&2; return 0; }
+die() { err "$*"; exit 1; }
+
+section_header() {
+ printf "\n%b━━━ %s ━━━%b\n" "${BOLD}${BLUE}" "$1" "${RESET}"
+}
+
+field() {
+ printf " %-22s %s\n" "$1:" "$2"
+}
+
+field_color() {
+ printf " %-22s %b%s%b\n" "$1:" "$2" "$3" "${RESET}"
+}
+
+# --- Resolve namespace default based on tool ---
+resolve_namespace() {
+ if [[ -z "$NAMESPACE" ]]; then
+ if [[ "$GITOPS_TOOL" == "argocd" ]]; then
+ NAMESPACE="argocd"
+ else
+ NAMESPACE="flux-system"
+ fi
+ fi
+}
+
+# --- kubectl wrapper ---
+kubectl_cmd() {
+ local -a args=("kubectl")
+ [[ -n "$KUBECONFIG_FILE" ]] && args+=("--kubeconfig" "$KUBECONFIG_FILE")
+ [[ -n "$KUBE_CTX" ]] && args+=("--context" "$KUBE_CTX")
+ "${args[@]}" "$@"
+}
+
+# --- Dependency checks ---
+require_kubectl() {
+ command -v kubectl >/dev/null 2>&1 || die "kubectl is required but not found in PATH"
+}
+
+require_flux() {
+ command -v flux >/dev/null 2>&1 || die "flux CLI is required but not found in PATH"
+}
+
+require_argocd() {
+ command -v argocd >/dev/null 2>&1 || die "argocd CLI is required but not found in PATH"
+}
+
+require_git() {
+ command -v git >/dev/null 2>&1 || die "git is required but not found in PATH"
+}
+
+# --- Confirm prompt ---
+confirm_action() {
+ local prompt="${1:-Continue?}"
+ if [[ "$CONFIRM_YES" == "true" ]]; then
+ return 0
+ fi
+ printf "%s [y/N] " "$prompt"
+ read -r answer
+ [[ "$answer" =~ ^[Yy]$ ]] || die "Aborted"
+}
+
+# --- Wait for pods ready in namespace ---
+wait_for_pods() {
+ local ns="$1"
+ local timeout="${2:-120}"
+ log "Waiting for pods in namespace ${CYAN}${ns}${RESET} (timeout ${timeout}s)"
+
+ local deadline=$(($(date +%s) + timeout))
+ while true; do
+ local not_ready
+ not_ready=$(kubectl_cmd get pods -n "$ns" --no-headers 2>/dev/null \
+ | grep -cvE 'Running|Completed|Succeeded' || true)
+ if [[ "$not_ready" -eq 0 ]]; then
+ local total
+ total=$(kubectl_cmd get pods -n "$ns" --no-headers 2>/dev/null | wc -l)
+ if [[ "$total" -gt 0 ]]; then
+ log "All ${total} pod(s) ready in ${ns}"
+ return 0
+ fi
+ fi
+ if [[ $(date +%s) -ge $deadline ]]; then
+ warn "Timeout waiting for pods in ${ns}"
+ kubectl_cmd get pods -n "$ns" --no-headers 2>/dev/null || true
+ return 1
+ fi
+ sleep 5
+ done
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Install
+# ─────────────────────────────────────────────────────────────────────
+
+do_install_flux() {
+ section_header "Installing Flux"
+ require_kubectl
+ require_flux
+
+ log "Running pre-flight checks"
+ verbose "flux check --pre"
+ flux check --pre || die "Flux pre-flight checks failed"
+
+ log "Installing Flux components into namespace ${CYAN}${NAMESPACE}${RESET}"
+ verbose "flux install --namespace=${NAMESPACE}"
+ flux install --namespace="$NAMESPACE"
+
+ wait_for_pods "$NAMESPACE"
+
+ if [[ -n "$GIT_REPO" ]]; then
+ log "Configuring GitRepository source"
+ verbose "flux create source git ${SOURCE_NAME} --url=${GIT_REPO} --branch=${GIT_BRANCH} --namespace=${NAMESPACE}"
+ flux create source git "$SOURCE_NAME" \
+ --url="$GIT_REPO" \
+ --branch="$GIT_BRANCH" \
+ --namespace="$NAMESPACE"
+
+ log "Creating Kustomization"
+ verbose "flux create kustomization ${KUSTOMIZATION_NAME} --source=${SOURCE_NAME} --path=${GIT_PATH} --namespace=${NAMESPACE} --prune=true"
+ flux create kustomization "$KUSTOMIZATION_NAME" \
+ --source="$SOURCE_NAME" \
+ --path="$GIT_PATH" \
+ --namespace="$NAMESPACE" \
+ --prune=true
+ fi
+
+ section_header "Flux Installation Summary"
+ field "Namespace" "$NAMESPACE"
+ field "Git repository" "${GIT_REPO:-not configured}"
+ field "Branch" "$GIT_BRANCH"
+ field "Path" "$GIT_PATH"
+ log "Flux installation complete"
+}
+
+do_install_argocd() {
+ section_header "Installing Argo CD"
+ require_kubectl
+
+ local manifests_url="https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml"
+
+ log "Creating namespace ${CYAN}${NAMESPACE}${RESET}"
+ kubectl_cmd create namespace "$NAMESPACE" --dry-run=client -o yaml \
+ | kubectl_cmd apply -f -
+
+ log "Applying Argo CD manifests"
+ verbose "kubectl apply -n ${NAMESPACE} -f ${manifests_url}"
+ kubectl_cmd apply -n "$NAMESPACE" -f "$manifests_url"
+
+ wait_for_pods "$NAMESPACE"
+
+ section_header "Argo CD Installation Summary"
+ field "Namespace" "$NAMESPACE"
+ field "Manifests" "$manifests_url"
+ printf "\n"
+ log "Retrieve initial admin password:"
+ printf " %bkubectl -n %s get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d%b\n" \
+ "${CYAN}" "$NAMESPACE" "${RESET}"
+ log "Argo CD installation complete"
+}
+
+do_install() {
+ case "$GITOPS_TOOL" in
+ flux) do_install_flux ;;
+ argocd) do_install_argocd ;;
+ *) die "Unknown GitOps tool: ${GITOPS_TOOL}. Use 'flux' or 'argocd'." ;;
+ esac
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Status
+# ─────────────────────────────────────────────────────────────────────
+
+do_status_flux() {
+ section_header "Flux Status"
+ require_kubectl
+ require_flux
+
+ log "Sources"
+ flux get sources all --namespace="$NAMESPACE" 2>/dev/null || warn "No sources found"
+
+ printf "\n"
+ log "Kustomizations"
+ flux get kustomizations --namespace="$NAMESPACE" 2>/dev/null || warn "No kustomizations found"
+
+ printf "\n"
+ log "Helm releases"
+ flux get helmreleases --all-namespaces 2>/dev/null || verbose "No helm releases found"
+}
+
+do_status_argocd() {
+ section_header "Argo CD Status"
+ require_kubectl
+
+ log "Applications"
+ kubectl_cmd get applications -n "$NAMESPACE" -o wide 2>/dev/null || warn "No applications found"
+
+ printf "\n"
+ log "App Projects"
+ kubectl_cmd get appprojects -n "$NAMESPACE" -o wide 2>/dev/null || verbose "No app projects found"
+}
+
+do_status() {
+ case "$GITOPS_TOOL" in
+ flux) do_status_flux ;;
+ argocd) do_status_argocd ;;
+ *) die "Unknown GitOps tool: ${GITOPS_TOOL}" ;;
+ esac
+
+ section_header "Pod Status (${NAMESPACE})"
+ kubectl_cmd get pods -n "$NAMESPACE" -o wide 2>/dev/null || warn "No pods found"
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Add Source
+# ─────────────────────────────────────────────────────────────────────
+
+do_add_source_flux() {
+ section_header "Adding Git Source (Flux)"
+ require_flux
+
+ [[ -z "$GIT_REPO" ]] && die "--repo is required to add a source"
+
+ log "Creating GitRepository source ${CYAN}${SOURCE_NAME}${RESET}"
+ verbose "flux create source git ${SOURCE_NAME} --url=${GIT_REPO} --branch=${GIT_BRANCH} --namespace=${NAMESPACE}"
+ flux create source git "$SOURCE_NAME" \
+ --url="$GIT_REPO" \
+ --branch="$GIT_BRANCH" \
+ --namespace="$NAMESPACE"
+
+ log "Source added successfully"
+ flux get sources git --namespace="$NAMESPACE"
+}
+
+do_add_source_argocd() {
+ section_header "Adding Git Source (Argo CD)"
+ require_argocd
+
+ [[ -z "$GIT_REPO" ]] && die "--repo is required to add a source"
+
+ log "Adding repository ${CYAN}${GIT_REPO}${RESET}"
+ verbose "argocd repo add ${GIT_REPO}"
+ argocd repo add "$GIT_REPO" || die "Failed to add repository"
+
+ log "Repository added successfully"
+}
+
+do_add_source() {
+ case "$GITOPS_TOOL" in
+ flux) do_add_source_flux ;;
+ argocd) do_add_source_argocd ;;
+ *) die "Unknown GitOps tool: ${GITOPS_TOOL}" ;;
+ esac
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Sync / Reconcile
+# ─────────────────────────────────────────────────────────────────────
+
+do_sync_flux() {
+ section_header "Reconciling (Flux)"
+ require_flux
+
+ log "Reconciling source git/${SOURCE_NAME}"
+ verbose "flux reconcile source git ${SOURCE_NAME} --namespace=${NAMESPACE}"
+ flux reconcile source git "$SOURCE_NAME" --namespace="$NAMESPACE"
+
+ log "Reconciling kustomization ${KUSTOMIZATION_NAME}"
+ verbose "flux reconcile kustomization ${KUSTOMIZATION_NAME} --namespace=${NAMESPACE}"
+ flux reconcile kustomization "$KUSTOMIZATION_NAME" --namespace="$NAMESPACE"
+
+ log "Reconciliation triggered"
+}
+
+do_sync_argocd() {
+ section_header "Syncing (Argo CD)"
+ require_argocd
+
+ if [[ -n "$APP_NAME" ]]; then
+ log "Syncing application ${CYAN}${APP_NAME}${RESET}"
+ verbose "argocd app sync ${APP_NAME}"
+ argocd app sync "$APP_NAME"
+ else
+ log "Syncing all applications"
+ local apps
+ apps=$(kubectl_cmd get applications -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+ if [[ -z "$apps" ]]; then
+ warn "No applications found to sync"
+ return
+ fi
+ while IFS= read -r app; do
+ [[ -z "$app" ]] && continue
+ log "Syncing ${app}"
+ argocd app sync "$app" || warn "Failed to sync ${app}"
+ done <<< "$apps"
+ fi
+
+ log "Sync triggered"
+}
+
+do_sync() {
+ case "$GITOPS_TOOL" in
+ flux) do_sync_flux ;;
+ argocd) do_sync_argocd ;;
+ *) die "Unknown GitOps tool: ${GITOPS_TOOL}" ;;
+ esac
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Validate (pre-flight)
+# ─────────────────────────────────────────────────────────────────────
+
+do_validate() {
+ section_header "Pre-flight Validation"
+ require_kubectl
+
+ local checks_passed=0
+ local checks_failed=0
+
+ # Check kubectl connectivity
+ log "Checking kubectl connectivity"
+ if kubectl_cmd cluster-info >/dev/null 2>&1; then
+ field_color "Cluster access" "${GREEN}" "OK"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "Cluster access" "${RED}" "FAILED"
+ checks_failed=$((checks_failed + 1))
+ fi
+
+ # Check namespace exists
+ log "Checking namespace ${CYAN}${NAMESPACE}${RESET}"
+ if kubectl_cmd get namespace "$NAMESPACE" >/dev/null 2>&1; then
+ field_color "Namespace" "${GREEN}" "${NAMESPACE} exists"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "Namespace" "${YELLOW}" "${NAMESPACE} does not exist (will be created)"
+ checks_passed=$((checks_passed + 1))
+ fi
+
+ # Check CRDs installed
+ log "Checking CRDs"
+ if [[ "$GITOPS_TOOL" == "flux" ]]; then
+ if kubectl_cmd get crd gitrepositories.source.toolkit.fluxcd.io >/dev/null 2>&1; then
+ field_color "Flux CRDs" "${GREEN}" "installed"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "Flux CRDs" "${YELLOW}" "not installed"
+ checks_passed=$((checks_passed + 1))
+ fi
+ elif [[ "$GITOPS_TOOL" == "argocd" ]]; then
+ if kubectl_cmd get crd applications.argoproj.io >/dev/null 2>&1; then
+ field_color "ArgoCD CRDs" "${GREEN}" "installed"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "ArgoCD CRDs" "${YELLOW}" "not installed"
+ checks_passed=$((checks_passed + 1))
+ fi
+ fi
+
+ # Check git repo accessible
+ if [[ -n "$GIT_REPO" ]]; then
+ log "Checking git repository accessibility"
+ require_git
+ if git ls-remote "$GIT_REPO" HEAD >/dev/null 2>&1; then
+ field_color "Git repository" "${GREEN}" "accessible"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "Git repository" "${RED}" "not accessible"
+ checks_failed=$((checks_failed + 1))
+ fi
+ else
+ verbose "No git repository specified, skipping connectivity check"
+ fi
+
+ # Check tool CLI available
+ log "Checking CLI tools"
+ if [[ "$GITOPS_TOOL" == "flux" ]]; then
+ if command -v flux >/dev/null 2>&1; then
+ local flux_ver
+ flux_ver=$(flux version --client 2>/dev/null | head -1 || echo "unknown")
+ field_color "flux CLI" "${GREEN}" "$flux_ver"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "flux CLI" "${RED}" "not found"
+ checks_failed=$((checks_failed + 1))
+ fi
+ elif [[ "$GITOPS_TOOL" == "argocd" ]]; then
+ if command -v argocd >/dev/null 2>&1; then
+ local argocd_ver
+ argocd_ver=$(argocd version --client --short 2>/dev/null || echo "unknown")
+ field_color "argocd CLI" "${GREEN}" "$argocd_ver"
+ checks_passed=$((checks_passed + 1))
+ else
+ field_color "argocd CLI" "${RED}" "not found"
+ checks_failed=$((checks_failed + 1))
+ fi
+ fi
+
+ section_header "Validation Summary"
+ field_color "Passed" "${GREEN}" "$checks_passed"
+ if [[ "$checks_failed" -gt 0 ]]; then
+ field_color "Failed" "${RED}" "$checks_failed"
+ die "Validation failed with ${checks_failed} error(s)"
+ else
+ field_color "Failed" "${GREEN}" "0"
+ log "All pre-flight checks passed"
+ fi
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Teardown
+# ─────────────────────────────────────────────────────────────────────
+
+do_teardown_flux() {
+ section_header "Tearing Down Flux"
+ require_flux
+
+ confirm_action "Remove Flux from the cluster?"
+
+ log "Uninstalling Flux"
+ verbose "flux uninstall --namespace=${NAMESPACE} --silent"
+ flux uninstall --namespace="$NAMESPACE" --silent
+
+ log "Flux has been removed from the cluster"
+}
+
+do_teardown_argocd() {
+ section_header "Tearing Down Argo CD"
+ require_kubectl
+
+ confirm_action "Remove Argo CD from the cluster (delete namespace ${NAMESPACE})?"
+
+ log "Deleting namespace ${CYAN}${NAMESPACE}${RESET}"
+ kubectl_cmd delete namespace "$NAMESPACE" --wait=true
+
+ log "Argo CD has been removed from the cluster"
+}
+
+do_teardown() {
+ case "$GITOPS_TOOL" in
+ flux) do_teardown_flux ;;
+ argocd) do_teardown_argocd ;;
+ *) die "Unknown GitOps tool: ${GITOPS_TOOL}" ;;
+ esac
+}
+
+# ─────────────────────────────────────────────────────────────────────
+# Help
+# ─────────────────────────────────────────────────────────────────────
+
+show_help() {
+ cat </dev/null; then
+ missing+=("$cmd")
+ fi
+ done
+ if [[ ${#missing[@]} -gt 0 ]]; then
+ echo "# ERROR: Missing required commands: ${missing[*]}" >&2
+ echo "# Install with: apt install ${missing[*]} OR dnf install ${missing[*]}" >&2
+ exit 1
+ fi
+}
+
+validate_config() {
+ if [[ -z "$GLPI_URL" ]]; then
+ echo "# ERROR: GLPI_URL environment variable is required" >&2
+ exit 1
+ fi
+ if [[ -z "$GLPI_USER_TOKEN" ]]; then
+ echo "# ERROR: GLPI_USER_TOKEN environment variable is required" >&2
+ exit 1
+ fi
+ GLPI_URL="${GLPI_URL%/}"
+}
+
+init_session() {
+ local auth_headers=(-H "Authorization: user_token ${GLPI_USER_TOKEN}")
+ if [[ -n "$GLPI_APP_TOKEN" ]]; then
+ auth_headers+=(-H "App-Token: ${GLPI_APP_TOKEN}")
+ fi
+
+ local response
+ response=$(curl -sf --max-time "$CURL_TIMEOUT" \
+ "${auth_headers[@]}" \
+ "${GLPI_URL}/apirest.php/initSession" 2>/dev/null) || { echo ""; return 1; }
+
+ SESSION_TOKEN=$(echo "$response" | jq -r '.session_token // empty' 2>/dev/null)
+
+ if [[ -z "$SESSION_TOKEN" ]]; then
+ return 1
+ fi
+
+ return 0
+}
+
+kill_session() {
+ if [[ -n "$SESSION_TOKEN" ]]; then
+ local headers=(-H "Session-Token: ${SESSION_TOKEN}")
+ if [[ -n "$GLPI_APP_TOKEN" ]]; then
+ headers+=(-H "App-Token: ${GLPI_APP_TOKEN}")
+ fi
+ curl -sf --max-time "$CURL_TIMEOUT" \
+ "${headers[@]}" \
+ "${GLPI_URL}/apirest.php/killSession" &>/dev/null || true
+ SESSION_TOKEN=""
+ fi
+}
+
+api_get() {
+ local endpoint="$1"
+ local headers=(-H "Session-Token: ${SESSION_TOKEN}" -H "Content-Type: application/json")
+ if [[ -n "$GLPI_APP_TOKEN" ]]; then
+ headers+=(-H "App-Token: ${GLPI_APP_TOKEN}")
+ fi
+
+ curl -sf --max-time "$CURL_TIMEOUT" \
+ "${headers[@]}" \
+ "${GLPI_URL}/apirest.php/${endpoint}" 2>/dev/null || echo ""
+}
+
+api_get_count() {
+ local endpoint="$1"
+ local range_header
+ range_header=$(curl -sI --max-time "$CURL_TIMEOUT" \
+ -H "Session-Token: ${SESSION_TOKEN}" \
+ -H "Content-Type: application/json" \
+ ${GLPI_APP_TOKEN:+-H "App-Token: ${GLPI_APP_TOKEN}"} \
+ "${GLPI_URL}/apirest.php/${endpoint}?range=0-0" 2>/dev/null | grep -i '^Content-Range:' | tr -d '\r')
+
+ if [[ -n "$range_header" ]]; then
+ echo "$range_header" | sed 's|.*/||'
+ else
+ echo "0"
+ fi
+}
+
+sanitize_label() {
+ local value="$1"
+ echo "$value" | sed 's/[^a-zA-Z0-9_ \/.-]/_/g' | sed 's/"/\\"/g'
+}
+
+add_metric() {
+ local name="$1"
+ local type="$2"
+ local help="$3"
+ local value="$4"
+ local labels="${5:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="# HELP ${name} ${help}
+# TYPE ${name} ${type}
+${name} ${value}
+"
+ fi
+}
+
+add_metric_value() {
+ local name="$1"
+ local value="$2"
+ local labels="${3:-}"
+
+ if [[ -n "$labels" ]]; then
+ OUTPUT+="${name}{${labels}} ${value}
+"
+ else
+ OUTPUT+="${name} ${value}
+"
+ fi
+}
+
+# --- Collectors ---
+
+collect_tickets() {
+ # Total tickets (open = not closed)
+ local total
+ total=$(api_get_count "Ticket")
+ add_metric "glpi_tickets_total" "gauge" "Total number of tickets" "${total:-0}"
+
+ # Tickets by status
+ # GLPI status codes: 1=New, 2=Assigned, 3=Planned, 4=Waiting, 5=Solved, 6=Closed
+ local status_names=("new" "assigned" "planned" "waiting" "solved" "closed")
+ local status_codes=(1 2 3 4 5 6)
+
+ for i in "${!status_codes[@]}"; do
+ local code="${status_codes[$i]}"
+ local name="${status_names[$i]}"
+ local count
+ count=$(api_get_count "Ticket?searchText[status]=${code}")
+ add_metric "glpi_tickets_${name}" "gauge" "Tickets in ${name} status" "${count:-0}"
+ done
+
+ # Tickets by urgency
+ # GLPI urgency: 1=Very low, 2=Low, 3=Medium, 4=High, 5=Very high
+ OUTPUT+="# HELP glpi_tickets_by_urgency Number of tickets by urgency level
+# TYPE glpi_tickets_by_urgency gauge
+"
+ local urgency_names=("very_low" "low" "medium" "high" "very_high")
+ local urgency_codes=(1 2 3 4 5)
+
+ for i in "${!urgency_codes[@]}"; do
+ local code="${urgency_codes[$i]}"
+ local uname="${urgency_names[$i]}"
+ local count
+ count=$(api_get_count "Ticket?searchText[urgency]=${code}")
+ add_metric_value "glpi_tickets_by_urgency" "${count:-0}" "urgency=\"${uname}\""
+ done
+
+ # Tickets by category (top categories)
+ local cat_json
+ cat_json=$(api_get "ITILCategory?range=0-49")
+
+ if [[ -n "$cat_json" ]]; then
+ local is_array
+ is_array=$(echo "$cat_json" | jq -r 'if type == "array" then "yes" else "no" end' 2>/dev/null)
+
+ if [[ "$is_array" == "yes" ]]; then
+ local cat_count
+ cat_count=$(echo "$cat_json" | jq 'length' 2>/dev/null)
+
+ if [[ "$cat_count" -gt 0 ]]; then
+ OUTPUT+="# HELP glpi_tickets_by_category Number of tickets per category
+# TYPE glpi_tickets_by_category gauge
+"
+ local j
+ for ((j = 0; j < cat_count && j < 30; j++)); do
+ local cat_name cat_id
+ cat_name=$(echo "$cat_json" | jq -r ".[$j].completename // .[$j].name // empty" 2>/dev/null)
+ cat_id=$(echo "$cat_json" | jq -r ".[$j].id // empty" 2>/dev/null)
+
+ if [[ -n "$cat_name" && -n "$cat_id" ]]; then
+ local ticket_count
+ ticket_count=$(api_get_count "Ticket?searchText[itilcategories_id]=${cat_id}")
+ local safe_name
+ safe_name=$(sanitize_label "$cat_name")
+ add_metric_value "glpi_tickets_by_category" "${ticket_count:-0}" "category=\"${safe_name}\""
+ fi
+ done
+ fi
+ fi
+ fi
+}
+
+collect_assets() {
+ # Computers
+ local computers
+ computers=$(api_get_count "Computer")
+ add_metric "glpi_computers_total" "gauge" "Total number of computers" "${computers:-0}"
+
+ # Monitors
+ local monitors
+ monitors=$(api_get_count "Monitor")
+ add_metric "glpi_monitors_total" "gauge" "Total number of monitors" "${monitors:-0}"
+
+ # Network devices
+ local netdevices
+ netdevices=$(api_get_count "NetworkEquipment")
+ add_metric "glpi_network_devices_total" "gauge" "Total number of network devices" "${netdevices:-0}"
+
+ # Phones
+ local phones
+ phones=$(api_get_count "Phone")
+ add_metric "glpi_phones_total" "gauge" "Total number of phones" "${phones:-0}"
+
+ # Printers
+ local printers
+ printers=$(api_get_count "Printer")
+ add_metric "glpi_printers_total" "gauge" "Total number of printers" "${printers:-0}"
+
+ # Software
+ local software
+ software=$(api_get_count "Software")
+ add_metric "glpi_software_total" "gauge" "Total number of software entries" "${software:-0}"
+}
+
+collect_organization() {
+ # Users
+ local users
+ users=$(api_get_count "User")
+ add_metric "glpi_users_total" "gauge" "Total number of users" "${users:-0}"
+
+ # Groups
+ local groups
+ groups=$(api_get_count "Group")
+ add_metric "glpi_groups_total" "gauge" "Total number of groups" "${groups:-0}"
+
+ # Entities
+ local entities
+ entities=$(api_get_count "Entity")
+ add_metric "glpi_entities_total" "gauge" "Total number of entities" "${entities:-0}"
+
+ # Locations
+ local locations
+ locations=$(api_get_count "Location")
+ add_metric "glpi_locations_total" "gauge" "Total number of locations" "${locations:-0}"
+}
+
+# --- Output ---
+
+write_output() {
+ if [[ "$TEXTFILE_MODE" == true ]]; then
+ local output_file="${TEXTFILE_DIR}/glpi.prom"
+ local temp_file="${output_file}.$$"
+
+ mkdir -p "$TEXTFILE_DIR"
+ echo "$OUTPUT" > "$temp_file"
+ mv "$temp_file" "$output_file"
+ echo "# Wrote metrics to ${output_file}" >&2
+ else
+ echo "$OUTPUT"
+ fi
+}
+
+serve_http() {
+ if ! command -v nc &>/dev/null && ! command -v ncat &>/dev/null; then
+ echo "# ERROR: nc (netcat) or ncat required for HTTP mode" >&2
+ exit 1
+ fi
+
+ echo "# GLPI exporter listening on port ${HTTP_PORT}" >&2
+ echo "# Metrics endpoint: http://localhost:${HTTP_PORT}/metrics" >&2
+
+ local nc_cmd="nc"
+ if command -v ncat &>/dev/null; then
+ nc_cmd="ncat"
+ fi
+
+ while true; do
+ OUTPUT=""
+ START_TIME=$(date +%s%N)
+
+ add_metric "glpi_exporter_info" "gauge" "Exporter version information" "1" "version=\"${VERSION}\""
+
+ if init_session; then
+ add_metric "glpi_up" "gauge" "GLPI API reachability (1=up, 0=down)" "1"
+ collect_tickets
+ collect_assets
+ collect_organization
+ kill_session
+ else
+ add_metric "glpi_up" "gauge" "GLPI API reachability (1=up, 0=down)" "0"
+ fi
+
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
+ add_metric "glpi_exporter_duration_seconds" "gauge" "Time to generate all metrics" "$duration"
+ add_metric "glpi_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run" "$(date +%s)"
+
+ local content_length=${#OUTPUT}
+ local response="HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\nContent-Length: ${content_length}\r\nConnection: close\r\n\r\n${OUTPUT}"
+
+ echo -e "$response" | $nc_cmd -l -p "$HTTP_PORT" -q 1 2>/dev/null || \
+ echo -e "$response" | $nc_cmd -l "$HTTP_PORT" -c 2>/dev/null || \
+ echo -e "$response" | $nc_cmd -l -p "$HTTP_PORT" 2>/dev/null || true
+ done
+}
+
+install_cron() {
+ if [[ $EUID -ne 0 ]]; then
+ echo "# ERROR: --install requires root" >&2
+ exit 1
+ fi
+
+ local script_path
+ script_path=$(readlink -f "$0")
+
+ cat > /etc/cron.d/glpi-exporter </dev/null
+EOF
+
+ chmod 644 /etc/cron.d/glpi-exporter
+ echo "# Installed cron job: /etc/cron.d/glpi-exporter" >&2
+ echo "# Metrics will be written to: ${TEXTFILE_DIR}/glpi.prom" >&2
+}
+
+# --- Main ---
+
+main() {
+ for arg in "$@"; do
+ case "$arg" in
+ --textfile) TEXTFILE_MODE=true ;;
+ --http) HTTP_MODE=true ;;
+ -p|--port) shift; HTTP_PORT="${1:-$HTTP_PORT}" ;;
+ --install)
+ check_dependencies
+ validate_config
+ install_cron
+ exit 0
+ ;;
+ --help|-h) usage ;;
+ *) ;;
+ esac
+ done
+
+ check_dependencies
+ validate_config
+
+ if [[ "$HTTP_MODE" == true ]]; then
+ serve_http
+ exit 0
+ fi
+
+ START_TIME=$(date +%s%N)
+
+ add_metric "glpi_exporter_info" "gauge" "Exporter version information" "1" "version=\"${VERSION}\""
+
+ if init_session; then
+ add_metric "glpi_up" "gauge" "GLPI API reachability (1=up, 0=down)" "1"
+ collect_tickets
+ collect_assets
+ collect_organization
+ kill_session
+ else
+ add_metric "glpi_up" "gauge" "GLPI API reachability (1=up, 0=down)" "0"
+ fi
+
+ local end_time duration
+ end_time=$(date +%s%N)
+ duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
+ add_metric "glpi_exporter_duration_seconds" "gauge" "Time to generate all metrics" "$duration"
+ add_metric "glpi_exporter_last_run_timestamp" "gauge" "Unix timestamp of last successful run" "$(date +%s)"
+
+ write_output
+}
+
+main "$@"
diff --git a/gpu-exporter.sh b/gpu-exporter.sh
new file mode 100755
index 0000000..411e4a0
--- /dev/null
+++ b/gpu-exporter.sh
@@ -0,0 +1,440 @@
+#!/bin/bash
+################################################################################
+# Script Name: gpu-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for NVIDIA GPU metrics — temperature,
+# utilization, VRAM usage, power draw, fan speed, clock speeds,
+# performance state, and per-process GPU memory via nvidia-smi
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+# - NVIDIA GPU with drivers installed
+# - nvidia-smi available in PATH
+# - netcat (nc) for HTTP mode
+#
+# Usage:
+# ./gpu-exporter.sh # stdout
+# ./gpu-exporter.sh --http -p 9195 # HTTP server
+# ./gpu-exporter.sh --textfile # node_exporter textfile
+#
+# Metrics Exported:
+# - gpu_info{gpu,name,driver_version,cuda_version} - GPU info
+# - gpu_count - Number of GPUs detected
+# - gpu_temperature_celsius{gpu} - Temperature
+# - gpu_utilization_percent{gpu} - GPU utilization
+# - gpu_memory_utilization_percent{gpu} - Memory utilization
+# - gpu_memory_used_bytes{gpu} - VRAM used
+# - gpu_memory_total_bytes{gpu} - Total VRAM
+# - gpu_memory_free_bytes{gpu} - Free VRAM
+# - gpu_power_draw_watts{gpu} - Power draw
+# - gpu_power_limit_watts{gpu} - Power limit
+# - gpu_fan_speed_percent{gpu} - Fan speed
+# - gpu_clock_speed_mhz{gpu} - GPU clock
+# - gpu_memory_clock_speed_mhz{gpu} - Memory clock
+# - gpu_pstate{gpu} - Performance state
+# - gpu_process_memory_bytes{gpu,pid,process_name} - Per-process memory
+# - gpu_exporter_duration_seconds - Script execution time
+# - gpu_exporter_last_run_timestamp - Last run timestamp
+#
+# Configuration:
+# Default HTTP port: 9195
+# Textfile directory: /var/lib/node_exporter
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9195
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+show_usage() {
+ cat <&2; exit 1 ;;
+ esac
+ done
+}
+
+# Escape special characters in Prometheus label values
+# Args: $1 - string to escape
+# Returns: escaped string safe for Prometheus labels
+prom_escape() {
+ local val="$1"
+ val="${val//\\/\\\\}"
+ val="${val//\"/\\\"}"
+ val="${val//$'\n'/}"
+ echo "$val"
+}
+
+# ============================================================================
+# METRIC GENERATION
+# ============================================================================
+
+# Generate all Prometheus metrics
+# Returns: Prometheus text format metrics on stdout
+generate_metrics() {
+ local script_start
+ script_start=$(date +%s)
+
+ # Check nvidia-smi exists
+ if ! command -v nvidia-smi >/dev/null 2>&1; then
+ cat </dev/null | head -1)
+ gpu_count=${gpu_count:-0}
+
+ # Strip whitespace
+ gpu_count=$(echo "$gpu_count" | tr -d '[:space:]')
+
+ if [ "$gpu_count" -eq 0 ] 2>/dev/null; then
+ cat </dev/null | head -1)
+ driver_version=$(echo "$driver_version" | tr -d '[:space:]')
+ driver_version=${driver_version:-"unknown"}
+
+ cuda_version=$(nvidia-smi --query-gpu=cuda_version --format=csv,noheader 2>/dev/null | head -1)
+ cuda_version=$(echo "$cuda_version" | tr -d '[:space:]')
+
+ # Fallback: parse from nvidia-smi header if query fails
+ if [ -z "$cuda_version" ] || [ "$cuda_version" = "[N/A]" ]; then
+ cuda_version=$(nvidia-smi 2>/dev/null | grep -oP 'CUDA Version:\s*\K[0-9.]+' || echo "unknown")
+ fi
+
+ cat </dev/null)
+
+ if [ -n "$info_lines" ]; then
+ while IFS= read -r info_line; do
+ [ -z "$info_line" ] && continue
+ local g_idx g_name
+ g_idx=$(echo "$info_line" | cut -d',' -f1 | tr -d '[:space:]')
+ g_name=$(echo "$info_line" | cut -d',' -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ echo "gpu_info{gpu=\"$g_idx\",name=\"$(prom_escape "$g_name")\",driver_version=\"$(prom_escape "$driver_version")\",cuda_version=\"$(prom_escape "$cuda_version")\"} 1"
+ done <<< "$info_lines"
+ fi
+
+ echo ""
+
+ # ========================================================================
+ # OUTPUT PER-GPU METRICS (with HELP/TYPE headers)
+ # ========================================================================
+
+ # Helper: emit a metric block for all GPUs
+ # Args: $1=metric_name, $2=help_text, $3=query_field
+ emit_gpu_metric() {
+ local metric="$1" help="$2" query="$3"
+ echo "# HELP $metric $help"
+ echo "# TYPE $metric gauge"
+ local lines
+ lines=$(nvidia-smi --query-gpu=index,"$query" --format=csv,noheader,nounits 2>/dev/null)
+ while IFS=', ' read -r g_idx g_val; do
+ g_idx=$(echo "$g_idx" | tr -d '[:space:]')
+ g_val=$(echo "$g_val" | tr -d '[:space:]')
+ [[ "$g_val" == "[N/A]" ]] && g_val=0
+ echo "${metric}{gpu=\"$g_idx\"} $g_val"
+ done <<< "$lines"
+ echo ""
+ }
+
+ # Helper: emit a memory metric (MiB → bytes) for all GPUs
+ # Args: $1=metric_name, $2=help_text, $3=query_field
+ emit_gpu_mem_metric() {
+ local metric="$1" help="$2" query="$3"
+ echo "# HELP $metric $help"
+ echo "# TYPE $metric gauge"
+ local lines
+ lines=$(nvidia-smi --query-gpu=index,"$query" --format=csv,noheader,nounits 2>/dev/null)
+ while IFS=', ' read -r g_idx g_val; do
+ g_idx=$(echo "$g_idx" | tr -d '[:space:]')
+ g_val=$(echo "$g_val" | tr -d '[:space:]')
+ [[ "$g_val" == "[N/A]" ]] && g_val=0
+ local bytes
+ bytes=$(awk "BEGIN { printf \"%.0f\", $g_val * 1048576 }")
+ echo "${metric}{gpu=\"$g_idx\"} $bytes"
+ done <<< "$lines"
+ echo ""
+ }
+
+ emit_gpu_metric "gpu_temperature_celsius" "GPU temperature in degrees Celsius" "temperature.gpu"
+ emit_gpu_metric "gpu_utilization_percent" "GPU core utilization percentage" "utilization.gpu"
+ emit_gpu_metric "gpu_memory_utilization_percent" "GPU memory utilization percentage" "utilization.memory"
+ emit_gpu_mem_metric "gpu_memory_used_bytes" "GPU memory used in bytes" "memory.used"
+ emit_gpu_mem_metric "gpu_memory_total_bytes" "GPU total memory in bytes" "memory.total"
+ emit_gpu_mem_metric "gpu_memory_free_bytes" "GPU free memory in bytes" "memory.free"
+ emit_gpu_metric "gpu_power_draw_watts" "GPU power draw in watts" "power.draw"
+ emit_gpu_metric "gpu_power_limit_watts" "GPU power limit in watts" "power.limit"
+ emit_gpu_metric "gpu_fan_speed_percent" "GPU fan speed percentage" "fan.speed"
+ emit_gpu_metric "gpu_clock_speed_mhz" "GPU graphics clock speed in MHz" "clocks.current.graphics"
+ emit_gpu_metric "gpu_memory_clock_speed_mhz" "GPU memory clock speed in MHz" "clocks.current.memory"
+
+ # Performance state needs special handling (P0 → 0, P8 → 8, etc.)
+ echo "# HELP gpu_pstate GPU performance state (0=max, 12=min)"
+ echo "# TYPE gpu_pstate gauge"
+ local pstate_lines
+ pstate_lines=$(nvidia-smi --query-gpu=index,pstate --format=csv,noheader,nounits 2>/dev/null)
+ while IFS=', ' read -r g_idx g_pstate; do
+ g_idx=$(echo "$g_idx" | tr -d '[:space:]')
+ g_pstate=$(echo "$g_pstate" | tr -d '[:space:]')
+ local pnum=0
+ if [[ "$g_pstate" =~ ^P([0-9]+)$ ]]; then
+ pnum="${BASH_REMATCH[1]}"
+ fi
+ echo "gpu_pstate{gpu=\"$g_idx\"} $pnum"
+ done <<< "$pstate_lines"
+
+ echo ""
+
+ # ========================================================================
+ # PER-PROCESS GPU MEMORY
+ # ========================================================================
+
+ # Build UUID-to-index mapping
+ declare -A uuid_to_index
+ local uuid_lines
+ uuid_lines=$(nvidia-smi --query-gpu=index,uuid --format=csv,noheader 2>/dev/null)
+
+ if [ -n "$uuid_lines" ]; then
+ while IFS=', ' read -r g_idx g_uuid; do
+ g_idx=$(echo "$g_idx" | tr -d '[:space:]')
+ g_uuid=$(echo "$g_uuid" | tr -d '[:space:]')
+ uuid_to_index["$g_uuid"]="$g_idx"
+ done <<< "$uuid_lines"
+ fi
+
+ cat </dev/null)
+
+ if [ -n "$process_lines" ]; then
+ while IFS= read -r proc_line; do
+ [ -z "$proc_line" ] && continue
+
+ # Parse: uuid, pid, process_name, used_memory_mib
+ local proc_uuid proc_pid proc_name proc_mem_mib
+ proc_uuid=$(echo "$proc_line" | cut -d',' -f1 | tr -d '[:space:]')
+ proc_pid=$(echo "$proc_line" | cut -d',' -f2 | tr -d '[:space:]')
+ proc_name=$(echo "$proc_line" | cut -d',' -f3 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ proc_mem_mib=$(echo "$proc_line" | rev | cut -d',' -f1 | rev | tr -d '[:space:]')
+
+ # Resolve UUID to GPU index
+ local proc_gpu_idx="${uuid_to_index[$proc_uuid]:-0}"
+
+ # Handle [N/A] memory
+ if [ "$proc_mem_mib" = "[N/A]" ]; then
+ proc_mem_mib=0
+ fi
+
+ # Convert MiB to bytes
+ local proc_mem_bytes
+ proc_mem_bytes=$(awk "BEGIN { printf \"%.0f\", $proc_mem_mib * 1048576 }")
+
+ # Extract short process name from full path
+ local short_name
+ short_name=$(basename "$proc_name" 2>/dev/null || echo "$proc_name")
+
+ echo "gpu_process_memory_bytes{gpu=\"$proc_gpu_idx\",pid=\"$proc_pid\",process_name=\"$(prom_escape "$short_name")\"} $proc_mem_bytes"
+ done <<< "$process_lines"
+ fi
+
+ echo ""
+
+ # ========================================================================
+ # EXPORTER RUNTIME
+ # ========================================================================
+
+ local script_end script_duration
+ script_end=$(date +%s)
+ script_duration=$((script_end - script_start))
+
+ cat <&2
+
+ if ! command -v nc >/dev/null 2>&1; then
+ echo "ERROR: netcat (nc) required for HTTP mode" >&2
+ exit 1
+ fi
+
+ # Infinite loop accepting HTTP requests
+ while true; do
+ {
+ read -r request
+ # Check if request is for /metrics endpoint
+ if [[ "$request" =~ ^GET\ /metrics ]]; then
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+ generate_metrics
+ else # Serve HTML landing page for other requests
+ echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+ cat <
+
+GPU Exporter v1.0
+
+GPU Prometheus Exporter v1.0
+Metrics
+NVIDIA GPU metrics via nvidia-smi.
+
+
+EOF
+ fi
+ } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+ done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+# Main entry point - routes to appropriate output mode
+main() {
+ parse_args "$@"
+
+ if [ "$HTTP_MODE" = true ]; then
+ # Run HTTP server (blocks until killed)
+ run_http_server
+ elif [ -n "$OUTPUT_FILE" ]; then
+ # Textfile collector mode: write atomically using temp file
+ local output_dir
+ output_dir="$(dirname "$OUTPUT_FILE")"
+ mkdir -p "$output_dir"
+
+ # Create temp file in SAME directory for atomic rename (same filesystem)
+ local temp_file
+ temp_file=$(mktemp "${output_dir}/.gpu_metrics.XXXXXX")
+
+ # Generate metrics to temp file
+ if ! generate_metrics > "$temp_file" 2>/dev/null; then
+ rm -f "$temp_file"
+ echo "ERROR: Failed to generate metrics" >&2
+ exit 1
+ fi
+
+ # Validate: file must exist, have content
+ local file_lines
+ file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+ if [ "$file_lines" -lt 10 ]; then
+ rm -f "$temp_file"
+ echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+ exit 1
+ fi
+
+ # Set permissions before move
+ chmod 644 "$temp_file"
+
+ # Atomic rename — no gap where file is missing
+ mv -f "$temp_file" "$OUTPUT_FILE"
+
+ echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+ else
+ # Default: output to stdout
+ generate_metrics
+ fi
+}
+
+# Execute main function with all script arguments
+main "$@"
diff --git a/grafana-backup.sh b/grafana-backup.sh
new file mode 100644
index 0000000..f2ebdc7
--- /dev/null
+++ b/grafana-backup.sh
@@ -0,0 +1,396 @@
+#!/bin/bash
+
+################################################
+#### Grafana Backup & Restore Script ####
+#### Backup dashboards, datasources, alert ####
+#### rules, and folders via the HTTP API ####
+#### ####
+#### Author: Phil Connor ####
+#### Contact: contact@mylinux.work ####
+#### Version: 1.0.0.20260309 ####
+################################################
+
+set -o pipefail
+
+SCRIPT_NAME=$(basename "$0")
+readonly SCRIPT_NAME
+
+# Default configuration
+readonly DEFAULT_BACKUP_DIR="/var/backups/grafana"
+readonly DEFAULT_RETENTION_COUNT=7
+readonly DEFAULT_CURL_TIMEOUT=30
+
+# Configuration variables (can be overridden by environment)
+GRAFANA_URL=${GRAFANA_URL:-}
+GRAFANA_TOKEN=${GRAFANA_TOKEN:-}
+BACKUP_DIR=${BACKUP_DIR:-$DEFAULT_BACKUP_DIR}
+RETENTION_COUNT=${RETENTION_COUNT:-$DEFAULT_RETENTION_COUNT}
+
+# Runtime
+RUN_MODE="backup"
+RESTORE_DIR=""
+
+handle_error() {
+ local exit_code=$1
+ local line_number=$2
+ echo "Error: $SCRIPT_NAME failed at line $line_number with exit code $exit_code" >&2
+ exit "$exit_code"
+}
+
+trap 'handle_error $? $LINENO' ERR
+
+show_help() {
+ cat << EOF
+Usage: $SCRIPT_NAME [OPTIONS]
+
+Backup and restore Grafana dashboards, datasources, alert rules, and folders
+via the HTTP API. Creates timestamped backup directories with automatic retention.
+
+OPTIONS:
+ --backup Run a full backup (default)
+ --restore DIR Restore from the specified backup directory
+ --list List available backups
+ --help, -h Show this help message
+
+ENVIRONMENT VARIABLES:
+ GRAFANA_URL Grafana base URL (required, e.g. http://localhost:3000)
+ GRAFANA_TOKEN Grafana API token with Admin permissions (required)
+ BACKUP_DIR Root backup directory (default: $DEFAULT_BACKUP_DIR)
+ RETENTION_COUNT Number of backups to retain (default: $DEFAULT_RETENTION_COUNT)
+
+EXAMPLES:
+ GRAFANA_URL=http://localhost:3000 GRAFANA_TOKEN=glsa_xxxx $SCRIPT_NAME --backup
+ GRAFANA_URL=http://localhost:3000 GRAFANA_TOKEN=glsa_xxxx $SCRIPT_NAME --restore /var/backups/grafana/20260309-143022
+ GRAFANA_URL=http://localhost:3000 GRAFANA_TOKEN=glsa_xxxx $SCRIPT_NAME --list
+
+EOF
+ exit 0
+}
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --backup) RUN_MODE="backup"; shift ;;
+ --restore) RUN_MODE="restore"; RESTORE_DIR="$2"; shift 2 ;;
+ --list) RUN_MODE="list"; shift ;;
+ --help|-h) show_help ;;
+ *) echo "Unknown option: $1" >&2; show_help ;;
+ esac
+done
+
+validate_config() {
+ if [[ -z "$GRAFANA_URL" ]]; then
+ echo "Error: GRAFANA_URL is required" >&2
+ exit 1
+ fi
+
+ if [[ -z "$GRAFANA_TOKEN" ]]; then
+ echo "Error: GRAFANA_TOKEN is required" >&2
+ exit 1
+ fi
+
+ # Strip trailing slash
+ GRAFANA_URL="${GRAFANA_URL%/}"
+
+ # Check dependencies
+ for cmd in curl jq; do
+ if ! command -v "$cmd" &>/dev/null; then
+ echo "Error: $cmd is required but not installed" >&2
+ exit 1
+ fi
+ done
+}
+
+grafana_api() {
+ local method="$1"
+ local endpoint="$2"
+ local data="${3:-}"
+ local url="${GRAFANA_URL}${endpoint}"
+
+ local args=(
+ -sf
+ --max-time "$DEFAULT_CURL_TIMEOUT"
+ -H "Authorization: Bearer ${GRAFANA_TOKEN}"
+ -H "Content-Type: application/json"
+ -H "Accept: application/json"
+ -X "$method"
+ )
+
+ if [[ -n "$data" ]]; then
+ args+=(-d "$data")
+ fi
+
+ curl "${args[@]}" "$url"
+}
+
+backup_dashboards() {
+ local dest="$1/dashboards"
+ mkdir -p "$dest"
+
+ echo "Backing up dashboards..."
+
+ local search_result
+ search_result=$(grafana_api GET "/api/search?type=dash-db&limit=5000") || {
+ echo " Error: Failed to search dashboards" >&2
+ return 1
+ }
+
+ local count
+ count=$(echo "$search_result" | jq 'length')
+ echo " Found $count dashboards"
+
+ echo "$search_result" | jq -r '.[].uid' | while IFS= read -r uid; do
+ local dashboard
+ dashboard=$(grafana_api GET "/api/dashboards/uid/$uid") || {
+ echo " Warning: Failed to export dashboard $uid" >&2
+ continue
+ }
+
+ local title
+ title=$(echo "$dashboard" | jq -r '.dashboard.title // "unknown"' | tr '/ ' '__')
+ echo "$dashboard" | jq '.' > "$dest/${uid}_${title}.json"
+ done
+
+ echo " Dashboards saved to $dest"
+}
+
+backup_datasources() {
+ local dest="$1/datasources"
+ mkdir -p "$dest"
+
+ echo "Backing up datasources..."
+
+ local result
+ result=$(grafana_api GET "/api/datasources") || {
+ echo " Error: Failed to fetch datasources" >&2
+ return 1
+ }
+
+ local count
+ count=$(echo "$result" | jq 'length')
+ echo " Found $count datasources"
+
+ echo "$result" | jq -c '.[]' | while IFS= read -r ds; do
+ local id name
+ id=$(echo "$ds" | jq -r '.id')
+ name=$(echo "$ds" | jq -r '.name' | tr '/ ' '__')
+ echo "$ds" | jq '.' > "$dest/${id}_${name}.json"
+ done
+
+ echo " Datasources saved to $dest"
+}
+
+backup_alert_rules() {
+ local dest="$1/alert_rules"
+ mkdir -p "$dest"
+
+ echo "Backing up alert rules..."
+
+ local result
+ result=$(grafana_api GET "/api/v1/provisioning/alert-rules") || {
+ echo " Error: Failed to fetch alert rules" >&2
+ return 1
+ }
+
+ local count
+ count=$(echo "$result" | jq 'length')
+ echo " Found $count alert rules"
+
+ echo "$result" | jq -c '.[]' | while IFS= read -r rule; do
+ local uid title
+ uid=$(echo "$rule" | jq -r '.uid // .id // "unknown"')
+ title=$(echo "$rule" | jq -r '.title // "unknown"' | tr '/ ' '__')
+ echo "$rule" | jq '.' > "$dest/${uid}_${title}.json"
+ done
+
+ echo " Alert rules saved to $dest"
+}
+
+backup_folders() {
+ local dest="$1/folders"
+ mkdir -p "$dest"
+
+ echo "Backing up folders..."
+
+ local result
+ result=$(grafana_api GET "/api/folders?limit=1000") || {
+ echo " Error: Failed to fetch folders" >&2
+ return 1
+ }
+
+ local count
+ count=$(echo "$result" | jq 'length')
+ echo " Found $count folders"
+
+ echo "$result" | jq -c '.[]' | while IFS= read -r folder; do
+ local uid title
+ uid=$(echo "$folder" | jq -r '.uid')
+ title=$(echo "$folder" | jq -r '.title' | tr '/ ' '__')
+ echo "$folder" | jq '.' > "$dest/${uid}_${title}.json"
+ done
+
+ echo " Folders saved to $dest"
+}
+
+restore_dashboards() {
+ local src="$1/dashboards"
+
+ if [[ ! -d "$src" ]]; then
+ echo " No dashboards directory found, skipping" >&2
+ return 0
+ fi
+
+ echo "Restoring dashboards..."
+
+ local count=0
+ for file in "$src"/*.json; do
+ [[ -f "$file" ]] || continue
+
+ local payload
+ payload=$(jq '{dashboard: .dashboard, overwrite: true, folderId: (.meta.folderId // 0)}' "$file") || {
+ echo " Warning: Failed to parse $file" >&2
+ continue
+ }
+
+ if grafana_api POST "/api/dashboards/db" "$payload" >/dev/null; then
+ count=$((count + 1))
+ else
+ local title
+ title=$(basename "$file" .json)
+ echo " Warning: Failed to restore dashboard $title" >&2
+ fi
+ done
+
+ echo " Restored $count dashboards"
+}
+
+restore_datasources() {
+ local src="$1/datasources"
+
+ if [[ ! -d "$src" ]]; then
+ echo " No datasources directory found, skipping" >&2
+ return 0
+ fi
+
+ echo "Restoring datasources..."
+
+ local count=0
+ for file in "$src"/*.json; do
+ [[ -f "$file" ]] || continue
+
+ local payload
+ payload=$(jq 'del(.id, .uid, .readOnly)' "$file") || {
+ echo " Warning: Failed to parse $file" >&2
+ continue
+ }
+
+ if grafana_api POST "/api/datasources" "$payload" >/dev/null; then
+ count=$((count + 1))
+ else
+ local name
+ name=$(basename "$file" .json)
+ echo " Warning: Failed to restore datasource $name" >&2
+ fi
+ done
+
+ echo " Restored $count datasources"
+}
+
+prune_backups() {
+ echo "Pruning old backups (retaining $RETENTION_COUNT)..."
+
+ local backup_count
+ backup_count=$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l)
+
+ if [[ $backup_count -le $RETENTION_COUNT ]]; then
+ echo " No pruning needed ($backup_count backups present)"
+ return 0
+ fi
+
+ local remove_count=$((backup_count - RETENTION_COUNT))
+ find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d | sort | head -n "$remove_count" | while IFS= read -r dir; do
+ echo " Removing $(basename "$dir")"
+ rm -rf "$dir"
+ done
+
+ echo " Pruned $remove_count old backups"
+}
+
+do_backup() {
+ local timestamp
+ timestamp=$(date +%Y%m%d-%H%M%S)
+ local dest="$BACKUP_DIR/$timestamp"
+
+ mkdir -p "$dest"
+ echo "Starting backup to $dest"
+
+ backup_dashboards "$dest"
+ backup_datasources "$dest"
+ backup_alert_rules "$dest"
+ backup_folders "$dest"
+
+ prune_backups
+
+ echo "Backup complete: $dest"
+}
+
+do_restore() {
+ if [[ -z "$RESTORE_DIR" ]]; then
+ echo "Error: --restore requires a directory argument" >&2
+ exit 1
+ fi
+
+ if [[ ! -d "$RESTORE_DIR" ]]; then
+ echo "Error: Restore directory does not exist: $RESTORE_DIR" >&2
+ exit 1
+ fi
+
+ echo "Restoring from $RESTORE_DIR"
+
+ restore_dashboards "$RESTORE_DIR"
+ restore_datasources "$RESTORE_DIR"
+
+ echo "Restore complete"
+}
+
+do_list() {
+ if [[ ! -d "$BACKUP_DIR" ]]; then
+ echo "No backups found (directory does not exist: $BACKUP_DIR)"
+ return 0
+ fi
+
+ local count=0
+ for dir in "$BACKUP_DIR"/*/; do
+ [[ -d "$dir" ]] || continue
+ count=$((count + 1))
+
+ local name
+ name=$(basename "$dir")
+ local dashboards datasources alerts folders
+ dashboards=$(find "$dir/dashboards" -name '*.json' 2>/dev/null | wc -l)
+ datasources=$(find "$dir/datasources" -name '*.json' 2>/dev/null | wc -l)
+ alerts=$(find "$dir/alert_rules" -name '*.json' 2>/dev/null | wc -l)
+ folders=$(find "$dir/folders" -name '*.json' 2>/dev/null | wc -l)
+
+ printf " %s dashboards:%-4s datasources:%-4s alerts:%-4s folders:%-4s\n" \
+ "$name" "$dashboards" "$datasources" "$alerts" "$folders"
+ done
+
+ if [[ $count -eq 0 ]]; then
+ echo "No backups found in $BACKUP_DIR"
+ else
+ echo "$count backup(s) in $BACKUP_DIR"
+ fi
+}
+
+main() {
+ validate_config
+
+ case "$RUN_MODE" in
+ backup) do_backup ;;
+ restore) do_restore ;;
+ list) do_list ;;
+ esac
+}
+
+main
diff --git a/graylog-api-backup.sh b/graylog-api-backup.sh
new file mode 100755
index 0000000..0078abf
--- /dev/null
+++ b/graylog-api-backup.sh
@@ -0,0 +1,256 @@
+#!/usr/bin/env bash
+#
+# Graylog API Backup
+#
+# Backs up Graylog configuration objects via the REST API.
+# Exports inputs, streams, pipelines, dashboards, alerts,
+# index sets, users, roles, sidecar configs, content packs,
+# and lookup tables to individual JSON files in a dated directory.
+#
+# Usage:
+# GRAYLOG_URL="http://graylog.example.com:9000/api" GRAYLOG_TOKEN="abc123" ./graylog-api-backup.sh
+# GRAYLOG_URL="http://graylog.example.com:9000/api" GRAYLOG_TOKEN="abc123" ./graylog-api-backup.sh --dry-run
+# GRAYLOG_URL="http://graylog.example.com:9000/api" GRAYLOG_TOKEN="abc123" ./graylog-api-backup.sh --install
+#
+# Parameters:
+# --dry-run Show what would be backed up without writing files
+# --install Create cron job for daily backup at 3am
+# --help Show usage
+#
+# Environment:
+# GRAYLOG_URL Graylog API base URL (required, e.g. http://localhost:9000/api)
+# GRAYLOG_TOKEN API token (required)
+# BACKUP_DIR Base backup directory (default: /backup/graylog-api)
+# RETENTION_DAYS Delete backups older than this many days (default: 30)
+# CURL_TIMEOUT API request timeout in seconds (default: 10)
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+# Version: 1.01
+
+set -euo pipefail
+
+# --- Configuration ---
+readonly VERSION="1.0"
+readonly SCRIPT_NAME="$(basename "$0")"
+GRAYLOG_URL="${GRAYLOG_URL:-}"
+GRAYLOG_TOKEN="${GRAYLOG_TOKEN:-}"
+BACKUP_DIR="${BACKUP_DIR:-/backup/graylog-api}"
+RETENTION_DAYS="${RETENTION_DAYS:-30}"
+CURL_TIMEOUT="${CURL_TIMEOUT:-10}"
+DRY_RUN=false
+
+# Backup endpoints: "api_path output_filename"
+readonly ENDPOINTS=(
+ "system/inputs inputs.json"
+ "streams streams.json"
+ "system/pipelines/pipeline pipelines.json"
+ "system/pipelines/rule pipeline-rules.json"
+ "system/pipelines/connections pipeline-connections.json"
+ "views dashboards.json"
+ "alerts/definitions alert-definitions.json"
+ "alerts/notifications alert-notifications.json"
+ "system/indices/index_sets index-sets.json"
+ "users users.json"
+ "roles roles.json"
+ "sidecar/configurations sidecar-configs.json"
+ "system/content_packs content-packs.json"
+ "system/lookup/tables lookup-tables.json"
+ "system/lookup/adapters lookup-adapters.json"
+ "system/lookup/caches lookup-caches.json"
+)
+
+# --- Functions ---
+
+usage() {
+ cat </dev/null; then
+ missing+=("$cmd")
+ fi
+ done
+ if [[ ${#missing[@]} -gt 0 ]]; then
+ echo "ERROR: Missing required commands: ${missing[*]}" >&2
+ echo "Install with: apt install ${missing[*]} OR dnf install ${missing[*]}" >&2
+ exit 1
+ fi
+}
+
+validate_config() {
+ if [[ -z "$GRAYLOG_URL" ]]; then
+ echo "ERROR: GRAYLOG_URL environment variable is required" >&2
+ exit 1
+ fi
+ if [[ -z "$GRAYLOG_TOKEN" ]]; then
+ echo "ERROR: GRAYLOG_TOKEN environment variable is required" >&2
+ exit 1
+ fi
+ # Strip trailing slash
+ GRAYLOG_URL="${GRAYLOG_URL%/}"
+}
+
+api_get() {
+ local endpoint="$1"
+ curl -sf --max-time "$CURL_TIMEOUT" \
+ -u "${GRAYLOG_TOKEN}:token" \
+ -H "Accept: application/json" \
+ "${GRAYLOG_URL}/${endpoint}" 2>/dev/null || echo ""
+}
+
+backup_endpoint() {
+ local endpoint="$1"
+ local filename="$2"
+ local output_dir="$3"
+ local response
+
+ if [[ "$DRY_RUN" == true ]]; then
+ printf " %-35s → %s (dry-run)\n" "$endpoint" "$filename"
+ return 0
+ fi
+
+ response=$(api_get "$endpoint")
+
+ if [[ -z "$response" ]]; then
+ printf " %-35s → %-30s FAIL\n" "$endpoint" "$filename"
+ return 1
+ fi
+
+ echo "$response" | jq '.' > "${output_dir}/${filename}"
+ local size
+ size=$(du -h "${output_dir}/${filename}" | cut -f1)
+ printf " %-35s → %-30s OK %s\n" "$endpoint" "$filename" "$size"
+ return 0
+}
+
+cleanup_old_backups() {
+ if [[ ! -d "$BACKUP_DIR" ]]; then
+ return
+ fi
+
+ local removed=0
+ while IFS= read -r dir; do
+ rm -rf "$dir"
+ ((removed++)) || true
+ done < <(find "$BACKUP_DIR" -maxdepth 1 -mindepth 1 -type d -mtime +"$RETENTION_DAYS" 2>/dev/null)
+
+ if [[ $removed -gt 0 ]]; then
+ echo "Retention: removed $removed backup(s) older than ${RETENTION_DAYS} days"
+ fi
+}
+
+install_cron() {
+ if [[ $EUID -ne 0 ]]; then
+ echo "ERROR: --install requires root" >&2
+ exit 1
+ fi
+
+ local script_path
+ script_path=$(readlink -f "$0")
+
+ cat > /etc/cron.d/graylog-api-backup </dev/null
+EOF
+
+ chmod 644 /etc/cron.d/graylog-api-backup
+ echo "Installed cron job: /etc/cron.d/graylog-api-backup"
+ echo "Backups will be written to: ${BACKUP_DIR}/