Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.

incus-metrics-exporter.sh

@@ -0,0 +1,567 @@
+#!/bin/bash
+################################################################################
+# Script Name: incus-metrics-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for Incus container/VM metrics not covered
+#              by the built-in /1.0/metrics endpoint — storage pool usage,
+#              snapshot counts and age, instance inventory, and image cache.
+#              Designed for node_exporter textfile collector so all Incus data
+#              flows through a single Prometheus scrape target.
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+#   - Incus installed and running
+#   - User in incus/lxd group (or root)
+#   - jq for JSON parsing
+#   - netcat (nc) for HTTP mode
+#
+# Usage:
+#   ./incus-metrics-exporter.sh                     # stdout
+#   ./incus-metrics-exporter.sh --textfile          # node_exporter textfile
+#   ./incus-metrics-exporter.sh --http -p 9199      # HTTP server
+#
+################################################################################
+
+# ============================================================================
+# CONFIGURATION VARIABLES
+# ============================================================================
+
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9199
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+prom_escape() {
+    local s="$1"
+    s=${s//\\/\\\\}
+    s=${s//\"/\\\"}
+    s=${s//$'\n'/\\n}
+    printf '%s\n' "$s"
+}
+
+show_usage() {
+    cat <<EOF
+Usage: $0 [OPTIONS]
+
+Export Incus supplemental metrics as Prometheus metrics (v1.0).
+
+MODES:
+    --textfile      Write to node_exporter textfile collector
+    --http          Run HTTP server on port $HTTP_PORT
+
+OPTIONS:
+    -p, --port          HTTP port (default: 9199)
+    -o, --output        Output file path
+
+EXAMPLES:
+    $0 --textfile                              # Write to textfile collector
+    $0 --http --port 9199                      # Run HTTP server
+    $0 -o /tmp/incus_metrics.prom              # Write to custom file
+
+EOF
+    exit 0
+}
+
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -h|--help) show_usage ;;
+            --textfile) OUTPUT_FILE="$TEXTFILE_DIR/incus_metrics_exporter.prom"; shift ;;
+            --http) HTTP_MODE=true; shift ;;
+            -p|--port) HTTP_PORT="$2"; shift 2 ;;
+            -o|--output) OUTPUT_FILE="$2"; shift 2 ;;
+            *) echo "Unknown option: $1" >&2; exit 1 ;;
+        esac
+    done
+}
+
+# ============================================================================
+# METRICS GENERATION
+# ============================================================================
+
+generate_metrics() {
+    local script_start
+    script_start=$(date +%s)
+
+    # ========================================================================
+    # Prerequisite Check
+    # ========================================================================
+
+    if ! command -v jq >/dev/null 2>&1; then
+        echo "# ERROR: jq is required but not found" >&2
+        cat <<EOF
+# HELP incus_exporter_up Exporter status (1=up, 0=down)
+# TYPE incus_exporter_up gauge
+incus_exporter_up 0
+
+# HELP incus_exporter_info Exporter version information
+# TYPE incus_exporter_info gauge
+incus_exporter_info{version="1.0"} 1
+
+EOF
+        return
+    fi
+
+    if ! command -v incus >/dev/null 2>&1; then
+        echo "# ERROR: incus command not found" >&2
+        cat <<EOF
+# HELP incus_exporter_up Exporter status (1=up, 0=down)
+# TYPE incus_exporter_up gauge
+incus_exporter_up 0
+
+# HELP incus_exporter_info Exporter version information
+# TYPE incus_exporter_info gauge
+incus_exporter_info{version="1.0"} 1
+
+EOF
+        return
+    fi
+
+    cat <<EOF
+# HELP incus_exporter_up Exporter status (1=up, 0=down)
+# TYPE incus_exporter_up gauge
+incus_exporter_up 1
+
+# HELP incus_exporter_info Exporter version information
+# TYPE incus_exporter_info gauge
+incus_exporter_info{version="1.0"} 1
+
+EOF
+
+    # ========================================================================
+    # Storage Pool Metrics
+    # ========================================================================
+    local pool_total_lines=""
+    local pool_used_lines=""
+    local pool_ratio_lines=""
+
+    local pools_json
+    pools_json=$(incus query /1.0/storage-pools 2>/dev/null)
+
+    if [ -n "$pools_json" ] && [ "$pools_json" != "null" ]; then
+        local pool_urls
+        pool_urls=$(echo "$pools_json" | jq -r '.[]' 2>/dev/null)
+
+        while IFS= read -r pool_url; do
+            [ -z "$pool_url" ] && continue
+
+            local pool_name
+            pool_name=$(basename "$pool_url")
+            local esc_pool
+            esc_pool=$(prom_escape "$pool_name")
+
+            local pool_config
+            pool_config=$(incus query "/1.0/storage-pools/$pool_name" 2>/dev/null)
+            local driver
+            driver=$(echo "$pool_config" | jq -r '.driver // "unknown"' 2>/dev/null)
+            local esc_driver
+            esc_driver=$(prom_escape "$driver")
+
+            local resources_json
+            resources_json=$(incus query "/1.0/storage-pools/$pool_name/resources" 2>/dev/null)
+
+            local total_space=""
+            local used_space=""
+
+            if [ -n "$resources_json" ] && [ "$resources_json" != "null" ]; then
+                total_space=$(echo "$resources_json" | jq -r '.space.total // empty' 2>/dev/null)
+                used_space=$(echo "$resources_json" | jq -r '.space.used // empty' 2>/dev/null)
+            fi
+
+            # Fallback: parse incus storage info output
+            if [ -z "$total_space" ] || [ -z "$used_space" ]; then
+                local storage_info
+                storage_info=$(incus storage info "$pool_name" 2>/dev/null)
+
+                if [ -n "$storage_info" ]; then
+                    total_space=$(echo "$storage_info" | awk '/Total space:/ {
+                        val = $3; unit = $4
+                        if (unit == "GiB") val = val * 1073741824
+                        else if (unit == "TiB") val = val * 1099511627776
+                        else if (unit == "MiB") val = val * 1048576
+                        else if (unit == "KiB") val = val * 1024
+                        printf "%.0f", val
+                    }')
+                    used_space=$(echo "$storage_info" | awk '/Space used:/ {
+                        val = $3; unit = $4
+                        if (unit == "GiB") val = val * 1073741824
+                        else if (unit == "TiB") val = val * 1099511627776
+                        else if (unit == "MiB") val = val * 1048576
+                        else if (unit == "KiB") val = val * 1024
+                        printf "%.0f", val
+                    }')
+                fi
+            fi
+
+            if [ -n "$total_space" ] && [ -n "$used_space" ]; then
+                local labels="pool=\"$esc_pool\",driver=\"$esc_driver\""
+                pool_total_lines="${pool_total_lines}incus_storage_pool_total_bytes{${labels}} $total_space
+"
+                pool_used_lines="${pool_used_lines}incus_storage_pool_used_bytes{${labels}} $used_space
+"
+                local ratio
+                ratio=$(awk "BEGIN { if ($total_space > 0) printf \"%.6f\", $used_space / $total_space; else print \"0\" }")
+                pool_ratio_lines="${pool_ratio_lines}incus_storage_pool_usage_ratio{${labels}} $ratio
+"
+            else
+                echo "# WARNING: could not read storage pool '$pool_name' resources" >&2
+            fi
+        done <<< "$pool_urls"
+    fi
+
+    if [ -n "$pool_total_lines" ]; then
+        echo "# HELP incus_storage_pool_total_bytes Total storage pool capacity in bytes"
+        echo "# TYPE incus_storage_pool_total_bytes gauge"
+        printf '%s' "$pool_total_lines"
+        echo ""
+    fi
+
+    if [ -n "$pool_used_lines" ]; then
+        echo "# HELP incus_storage_pool_used_bytes Used storage pool space in bytes"
+        echo "# TYPE incus_storage_pool_used_bytes gauge"
+        printf '%s' "$pool_used_lines"
+        echo ""
+    fi
+
+    if [ -n "$pool_ratio_lines" ]; then
+        echo "# HELP incus_storage_pool_usage_ratio Storage pool used/total ratio"
+        echo "# TYPE incus_storage_pool_usage_ratio gauge"
+        printf '%s' "$pool_ratio_lines"
+        echo ""
+    fi
+
+    # ========================================================================
+    # Instance Inventory Metrics
+    # ========================================================================
+    local instances_json
+    instances_json=$(incus list --format json 2>/dev/null)
+
+    local instance_total_lines=""
+    local instance_info_lines=""
+    local snapshot_total_lines=""
+    local snapshot_oldest_lines=""
+    local snapshot_newest_lines=""
+
+    if [ -n "$instances_json" ] && [ "$instances_json" != "null" ] && [ "$instances_json" != "[]" ]; then
+        # Count instances by type and status
+        local counts
+        counts=$(echo "$instances_json" | jq -r '
+            group_by(.type, .status) |
+            .[] |
+            {type: .[0].type, status: .[0].status, count: length} |
+            "\(.type) \(.status) \(.count)"
+        ' 2>/dev/null)
+
+        while IFS= read -r count_line; do
+            [ -z "$count_line" ] && continue
+            local inst_type inst_status inst_count
+            inst_type=$(echo "$count_line" | awk '{print $1}')
+            inst_status=$(echo "$count_line" | awk '{print $2}')
+            inst_count=$(echo "$count_line" | awk '{print $3}')
+            local esc_type esc_status
+            esc_type=$(prom_escape "$inst_type")
+            esc_status=$(prom_escape "$inst_status")
+            instance_total_lines="${instance_total_lines}incus_instances_total{type=\"$esc_type\",status=\"$esc_status\"} $inst_count
+"
+        done <<< "$counts"
+
+        # Per-instance info gauge and snapshot metrics
+        local instance_data
+        instance_data=$(echo "$instances_json" | jq -r '
+            .[] |
+            "\(.name)\t\(.type)\t\(.status)\t\(.project // "default")"
+        ' 2>/dev/null)
+
+        local now
+        now=$(date +%s)
+
+        while IFS=$'\t' read -r inst_name inst_type inst_status inst_project; do
+            [ -z "$inst_name" ] && continue
+
+            local esc_name esc_type esc_status esc_project
+            esc_name=$(prom_escape "$inst_name")
+            esc_type=$(prom_escape "$inst_type")
+            esc_status=$(prom_escape "$inst_status")
+            esc_project=$(prom_escape "$inst_project")
+
+            instance_info_lines="${instance_info_lines}incus_instance_info{name=\"$esc_name\",type=\"$esc_type\",status=\"$esc_status\",project=\"$esc_project\"} 1
+"
+
+            # Snapshot metrics
+            local snap_json
+            snap_json=$(incus query "/1.0/instances/$inst_name/snapshots" --project "$inst_project" 2>/dev/null)
+
+            if [ -n "$snap_json" ] && [ "$snap_json" != "null" ] && [ "$snap_json" != "[]" ]; then
+                local snap_count
+                snap_count=$(echo "$snap_json" | jq 'length' 2>/dev/null)
+                snap_count=${snap_count:-0}
+
+                local snap_labels="name=\"$esc_name\",project=\"$esc_project\""
+                snapshot_total_lines="${snapshot_total_lines}incus_instance_snapshots_total{${snap_labels}} $snap_count
+"
+
+                if [ "$snap_count" -gt 0 ] 2>/dev/null; then
+                    # Fetch details for each snapshot to get created_at timestamps
+                    local snap_timestamps=""
+                    local snap_urls
+                    snap_urls=$(echo "$snap_json" | jq -r '.[]' 2>/dev/null)
+
+                    while IFS= read -r snap_url; do
+                        [ -z "$snap_url" ] && continue
+                        local snap_detail
+                        snap_detail=$(incus query "$snap_url" --project "$inst_project" 2>/dev/null)
+                        if [ -n "$snap_detail" ]; then
+                            local created_at
+                            created_at=$(echo "$snap_detail" | jq -r '.created_at // empty' 2>/dev/null)
+                            if [ -n "$created_at" ]; then
+                                local snap_epoch
+                                snap_epoch=$(date -d "$created_at" +%s 2>/dev/null)
+                                if [ -n "$snap_epoch" ]; then
+                                    snap_timestamps="${snap_timestamps}${snap_epoch}
+"
+                                fi
+                            fi
+                        fi
+                    done <<< "$snap_urls"
+
+                    if [ -n "$snap_timestamps" ]; then
+                        local oldest newest
+                        oldest=$(echo "$snap_timestamps" | sort -n | head -1)
+                        newest=$(echo "$snap_timestamps" | sort -rn | head -1)
+
+                        if [ -n "$oldest" ]; then
+                            local oldest_age
+                            oldest_age=$((now - oldest))
+                            snapshot_oldest_lines="${snapshot_oldest_lines}incus_instance_snapshot_oldest_age_seconds{${snap_labels}} $oldest_age
+"
+                        fi
+
+                        if [ -n "$newest" ]; then
+                            local newest_age
+                            newest_age=$((now - newest))
+                            snapshot_newest_lines="${snapshot_newest_lines}incus_instance_snapshot_newest_age_seconds{${snap_labels}} $newest_age
+"
+                        fi
+                    fi
+                fi
+            else
+                local snap_labels="name=\"$esc_name\",project=\"$esc_project\""
+                snapshot_total_lines="${snapshot_total_lines}incus_instance_snapshots_total{${snap_labels}} 0
+"
+            fi
+        done <<< "$instance_data"
+    fi
+
+    if [ -n "$instance_total_lines" ]; then
+        echo "# HELP incus_instances_total Number of instances by type and status"
+        echo "# TYPE incus_instances_total gauge"
+        printf '%s' "$instance_total_lines"
+        echo ""
+    fi
+
+    if [ -n "$instance_info_lines" ]; then
+        echo "# HELP incus_instance_info Instance information (always 1)"
+        echo "# TYPE incus_instance_info gauge"
+        printf '%s' "$instance_info_lines"
+        echo ""
+    fi
+
+    if [ -n "$snapshot_total_lines" ]; then
+        echo "# HELP incus_instance_snapshots_total Number of snapshots per instance"
+        echo "# TYPE incus_instance_snapshots_total gauge"
+        printf '%s' "$snapshot_total_lines"
+        echo ""
+    fi
+
+    if [ -n "$snapshot_oldest_lines" ]; then
+        echo "# HELP incus_instance_snapshot_oldest_age_seconds Age of oldest snapshot in seconds"
+        echo "# TYPE incus_instance_snapshot_oldest_age_seconds gauge"
+        printf '%s' "$snapshot_oldest_lines"
+        echo ""
+    fi
+
+    if [ -n "$snapshot_newest_lines" ]; then
+        echo "# HELP incus_instance_snapshot_newest_age_seconds Age of newest snapshot in seconds"
+        echo "# TYPE incus_instance_snapshot_newest_age_seconds gauge"
+        printf '%s' "$snapshot_newest_lines"
+        echo ""
+    fi
+
+    # ========================================================================
+    # Image Cache Metrics
+    # ========================================================================
+    local images_json
+    images_json=$(incus image list --format json 2>/dev/null)
+
+    local images_total=0
+    local image_size_lines=""
+
+    if [ -n "$images_json" ] && [ "$images_json" != "null" ] && [ "$images_json" != "[]" ]; then
+        images_total=$(echo "$images_json" | jq 'length' 2>/dev/null)
+        images_total=${images_total:-0}
+
+        local image_data
+        image_data=$(echo "$images_json" | jq -r '
+            .[] |
+            "\(.fingerprint[0:12])\t\(((.aliases // []) | if length > 0 then .[0].name else "" end))\t\(.architecture)\t\(.size)"
+        ' 2>/dev/null)
+
+        while IFS=$'\t' read -r img_fp img_alias img_arch img_size; do
+            [ -z "$img_fp" ] && continue
+            local esc_fp esc_alias esc_arch
+            esc_fp=$(prom_escape "$img_fp")
+            esc_alias=$(prom_escape "$img_alias")
+            esc_arch=$(prom_escape "$img_arch")
+            image_size_lines="${image_size_lines}incus_image_size_bytes{fingerprint=\"$esc_fp\",alias=\"$esc_alias\",arch=\"$esc_arch\"} $img_size
+"
+        done <<< "$image_data"
+    fi
+
+    echo "# HELP incus_images_total Total number of cached images"
+    echo "# TYPE incus_images_total gauge"
+    echo "incus_images_total $images_total"
+    echo ""
+
+    if [ -n "$image_size_lines" ]; then
+        echo "# HELP incus_image_size_bytes Image size in bytes"
+        echo "# TYPE incus_image_size_bytes gauge"
+        printf '%s' "$image_size_lines"
+        echo ""
+    fi
+
+    # ========================================================================
+    # Built-in Incus Metrics (/1.0/metrics)
+    # ========================================================================
+    # Pull the native metrics (CPU, memory, network, disk I/O per instance)
+    # so everything flows through node_exporter — no second scrape target.
+    local builtin_metrics
+    builtin_metrics=$(incus query /1.0/metrics 2>/dev/null)
+
+    if [ -n "$builtin_metrics" ]; then
+        echo "# ── Built-in Incus instance metrics (/1.0/metrics) ──"
+        echo "$builtin_metrics"
+        echo ""
+    else
+        echo "# WARNING: could not read /1.0/metrics (check incus daemon status)" >&2
+    fi
+
+    # ========================================================================
+    # Exporter Runtime
+    # ========================================================================
+    local script_end script_duration
+    script_end=$(date +%s)
+    script_duration=$((script_end - script_start))
+
+    cat <<EOF
+# HELP incus_exporter_duration_seconds Time to generate all metrics
+# TYPE incus_exporter_duration_seconds gauge
+incus_exporter_duration_seconds $script_duration
+
+# HELP incus_exporter_last_run_timestamp Unix timestamp of last successful run
+# TYPE incus_exporter_last_run_timestamp gauge
+incus_exporter_last_run_timestamp $script_end
+EOF
+
+    echo ""
+}
+
+# ============================================================================
+# HTTP SERVER MODE
+# ============================================================================
+
+run_http_server() {
+    echo "Starting Incus metrics exporter on port $HTTP_PORT..." >&2
+
+    if ! command -v nc >/dev/null 2>&1; then
+        echo "ERROR: netcat (nc) required for HTTP mode" >&2
+        exit 1
+    fi
+
+    trap 'echo "Shutting down Incus metrics exporter..." >&2; exit 0' INT TERM
+
+    while true; do
+        {
+            read -r request
+            local body
+            if [[ "$request" =~ ^GET\ /metrics ]]; then
+                body=$(generate_metrics)
+                printf "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s" "${#body}" "$body"
+            else
+                body=$(cat <<'HTMLEOF'
+<!DOCTYPE html>
+<html>
+<head><title>Incus Metrics Exporter v1.0</title></head>
+<body>
+<h1>Incus Metrics Exporter v1.0</h1>
+<p><a href="/metrics">Metrics</a></p>
+<h2>Sections</h2>
+<ul>
+<li>Storage pool usage (total, used, ratio)</li>
+<li>Instance inventory (count by type/status, info per instance)</li>
+<li>Snapshot counts and age per instance</li>
+<li>Image cache (count and size per image)</li>
+</ul>
+</body>
+</html>
+HTMLEOF
+)
+                printf "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s" "${#body}" "$body"
+            fi
+        } | if nc -h 2>&1 | grep -q 'GNU\|traditional'; then
+            nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+        else
+            nc -l "$HTTP_PORT" 2>/dev/null
+        fi
+    done
+}
+
+# ============================================================================
+# MAIN EXECUTION
+# ============================================================================
+
+main() {
+    parse_args "$@"
+
+    if [ "$HTTP_MODE" = true ]; then
+        run_http_server
+    elif [ -n "$OUTPUT_FILE" ]; then
+        local output_dir
+        output_dir="$(dirname "$OUTPUT_FILE")"
+        mkdir -p "$output_dir"
+
+        local temp_file
+        temp_file=$(mktemp "${output_dir}/.incus_metrics.XXXXXX")
+
+        if ! generate_metrics > "$temp_file" 2>/dev/null; then
+            rm -f "$temp_file"
+            echo "ERROR: Failed to generate metrics" >&2
+            exit 1
+        fi
+
+        local file_lines
+        file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
+
+        if [ "$file_lines" -lt 3 ]; then
+            rm -f "$temp_file"
+            echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
+            exit 1
+        fi
+
+        chmod 644 "$temp_file"
+        mv -f "$temp_file" "$OUTPUT_FILE"
+
+        echo "Metrics written to $OUTPUT_FILE ($file_lines lines)" >&2
+    else
+        generate_metrics
+    fi
+}
+
+main "$@"