Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.

syncthing-exporter.sh

@@ -0,0 +1,448 @@
+#!/bin/bash
+################################################################################
+# Script Name: syncthing-exporter.sh
+# Version: 1.0
+# Description: Prometheus exporter for Syncthing — scrapes the built-in
+#              /metrics endpoint and writes to a textfile collector .prom file,
+#              adding connection status and completion metrics from the REST API
+#
+# Author: Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Prerequisites:
+#   - Syncthing running with GUI/API enabled
+#   - curl and jq for API access
+#   - netcat (nc) for HTTP mode
+#
+# Usage:
+#   ./syncthing-exporter.sh --textfile
+#   ./syncthing-exporter.sh --http -p 9599
+#   ./syncthing-exporter.sh -o /tmp/syncthing.prom
+#
+# Configuration:
+#   Set SYNCTHING_API_KEY via environment variable, --api-key flag,
+#   or the script reads it from ~/.local/state/syncthing/config.xml
+#
+# Metrics exported (from /metrics):
+#   syncthing_model_folder_summary        - File/byte counts per folder
+#   syncthing_model_folder_state          - Current folder state
+#   syncthing_model_folder_pull_seconds   - Time in pull iterations
+#   syncthing_model_folder_scan_seconds   - Time in scan iterations
+#   syncthing_protocol_recv_bytes_total   - Bytes received per device
+#   syncthing_protocol_sent_bytes_total   - Bytes sent per device
+#   syncthing_fs_operation_bytes_total    - Filesystem bytes per operation
+#   syncthing_fs_operations_total         - Filesystem operation count
+#   syncthing_scanner_hashed_bytes_total  - Data hashed per folder
+#   syncthing_events_total                - Events created/forwarded/dropped
+#
+# Extra metrics (from REST API):
+#   syncthing_device_connected            - Per-device connection status (1/0)
+#   syncthing_device_paused               - Per-device paused status (1/0)
+#   syncthing_device_last_seen_timestamp  - Last seen Unix timestamp
+#   syncthing_connected_devices           - Total connected device count
+#   syncthing_folder_completion_pct       - Per-device folder completion %
+#
+################################################################################
+
+# ==============================================================================
+# CONFIGURATION VARIABLES
+# ==============================================================================
+
+SYNCTHING_URL="${SYNCTHING_URL:-http://127.0.0.1:8384}"
+SYNCTHING_API_KEY="${SYNCTHING_API_KEY:-}"
+TEXTFILE_DIR="/var/lib/node_exporter"
+OUTPUT_FILE=""
+HTTP_MODE=false
+HTTP_PORT=9599
+
+show_usage() {
+    cat <<EOF
+Usage: $0 [OPTIONS]
+
+Export Syncthing statistics as Prometheus metrics.
+
+MODES:
+    --textfile      Write to node_exporter textfile collector
+    --http          Run HTTP server on port $HTTP_PORT
+
+OPTIONS:
+    -u, --url       Syncthing URL (default: $SYNCTHING_URL)
+    -k, --api-key   Syncthing API key (or set SYNCTHING_API_KEY)
+    -p, --port      HTTP port (default: $HTTP_PORT)
+    -o, --output    Output file path
+
+EXAMPLES:
+    $0 --textfile
+    $0 --textfile --api-key abc123
+    $0 --http --port 9599
+    SYNCTHING_API_KEY=abc123 $0 --textfile
+
+EOF
+    exit 0
+}
+
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -h|--help) show_usage ;;
+            --textfile) OUTPUT_FILE="$TEXTFILE_DIR/syncthing.prom"; shift ;;
+            --http) HTTP_MODE=true; shift ;;
+            -u|--url) SYNCTHING_URL="$2"; shift 2 ;;
+            -k|--api-key) SYNCTHING_API_KEY="$2"; shift 2 ;;
+            -p|--port) HTTP_PORT="$2"; shift 2 ;;
+            -o|--output) OUTPUT_FILE="$2"; shift 2 ;;
+            *) echo "Unknown option: $1" >&2; exit 1 ;;
+        esac
+    done
+}
+
+# ==============================================================================
+# HELPER FUNCTIONS
+# ==============================================================================
+
+detect_api_key() {
+    if [ -n "$SYNCTHING_API_KEY" ]; then
+        return 0
+    fi
+
+    local config_paths=(
+        "$HOME/.local/state/syncthing/config.xml"
+        "$HOME/.config/syncthing/config.xml"
+        "/var/lib/syncthing/.config/syncthing/config.xml"
+        "/root/.local/state/syncthing/config.xml"
+        "/root/.config/syncthing/config.xml"
+    )
+
+    for cfg in "${config_paths[@]}"; do
+        if [ -f "$cfg" ]; then
+            SYNCTHING_API_KEY=$(grep -oP '<apikey>\K[^<]+' "$cfg" 2>/dev/null)
+            if [ -n "$SYNCTHING_API_KEY" ]; then
+                return 0
+            fi
+        fi
+    done
+
+    echo "ERROR: No API key found. Set SYNCTHING_API_KEY or use --api-key" >&2
+    return 1
+}
+
+api_call() {
+    local endpoint="$1"
+    curl -sf --max-time 10 \
+        -H "X-API-Key: $SYNCTHING_API_KEY" \
+        "${SYNCTHING_URL}${endpoint}"
+}
+
+prom_escape() {
+    local val="$1"
+    val="${val//\\/\\\\}"
+    val="${val//\"/\\\"}"
+    val="${val//$'\n'/\\n}"
+    echo "$val"
+}
+
+iso_to_unix() {
+    local ts="$1"
+    if [ -z "$ts" ] || [ "$ts" = "null" ] || [ "$ts" = "0001-01-01T00:00:00Z" ]; then
+        echo "0"
+        return
+    fi
+    local unix_ts
+    unix_ts=$(date -d "$ts" +%s 2>/dev/null)
+    echo "${unix_ts:-0}"
+}
+
+# ==============================================================================
+# METRIC GENERATION
+# ==============================================================================
+
+generate_metrics() {
+    local script_start
+    script_start=$(date +%s%N)
+
+    if ! command -v curl >/dev/null 2>&1; then
+        echo "ERROR: curl not found" >&2
+        return 1
+    fi
+    if ! command -v jq >/dev/null 2>&1; then
+        echo "ERROR: jq not found" >&2
+        return 1
+    fi
+
+    if ! detect_api_key; then
+        echo "# HELP syncthing_up Syncthing reachable (1/0)"
+        echo "# TYPE syncthing_up gauge"
+        echo "syncthing_up 0"
+        return
+    fi
+
+    # ------------------------------------------------------------------
+    # Scrape built-in /metrics endpoint
+    # ------------------------------------------------------------------
+    local builtin_metrics
+    builtin_metrics=$(curl -sf --max-time 10 \
+        -H "X-API-Key: $SYNCTHING_API_KEY" \
+        "${SYNCTHING_URL}/metrics" 2>/dev/null)
+
+    if [ -z "$builtin_metrics" ]; then
+        echo "# HELP syncthing_up Syncthing reachable (1/0)"
+        echo "# TYPE syncthing_up gauge"
+        echo "syncthing_up 0"
+        return
+    fi
+
+    echo "# HELP syncthing_up Syncthing reachable (1/0)"
+    echo "# TYPE syncthing_up gauge"
+    echo "syncthing_up 1"
+    echo ""
+
+    # Output built-in metrics (strip Go runtime metrics to keep it focused)
+    echo "$builtin_metrics" | grep -E '^(syncthing_|# (HELP|TYPE) syncthing_)' | awk '/^# HELP/ && NR>1 {print ""} {print}'
+    echo ""
+
+    # ------------------------------------------------------------------
+    # REST API: system version info
+    # ------------------------------------------------------------------
+    local version_json
+    version_json=$(api_call "/rest/system/version")
+    if [ -n "$version_json" ]; then
+        local version os arch
+        version=$(echo "$version_json" | jq -r '.version // ""')
+        os=$(echo "$version_json" | jq -r '.os // ""')
+        arch=$(echo "$version_json" | jq -r '.arch // ""')
+
+        echo "# HELP syncthing_version_info Syncthing version information"
+        echo "# TYPE syncthing_version_info gauge"
+        echo "syncthing_version_info{version=\"$(prom_escape "$version")\",os=\"$(prom_escape "$os")\",arch=\"$(prom_escape "$arch")\"} 1"
+        echo ""
+    fi
+
+    # ------------------------------------------------------------------
+    # REST API: system status (uptime, RAM)
+    # ------------------------------------------------------------------
+    local status_json
+    status_json=$(api_call "/rest/system/status")
+    if [ -n "$status_json" ]; then
+        local uptime alloc sys goroutines my_id
+        uptime=$(echo "$status_json" | jq -r '.uptime // 0')
+        alloc=$(echo "$status_json" | jq -r '.alloc // 0')
+        sys=$(echo "$status_json" | jq -r '.sys // 0')
+        goroutines=$(echo "$status_json" | jq -r '.goroutines // 0')
+        my_id=$(echo "$status_json" | jq -r '.myID // ""')
+
+        echo "# HELP syncthing_system_uptime_seconds Syncthing process uptime in seconds"
+        echo "# TYPE syncthing_system_uptime_seconds gauge"
+        echo "syncthing_system_uptime_seconds $uptime"
+        echo ""
+
+        echo "# HELP syncthing_system_alloc_bytes Memory allocated by Syncthing"
+        echo "# TYPE syncthing_system_alloc_bytes gauge"
+        echo "syncthing_system_alloc_bytes $alloc"
+        echo ""
+
+        echo "# HELP syncthing_system_sys_bytes Total system memory obtained"
+        echo "# TYPE syncthing_system_sys_bytes gauge"
+        echo "syncthing_system_sys_bytes $sys"
+        echo ""
+
+        echo "# HELP syncthing_system_goroutines Number of active goroutines"
+        echo "# TYPE syncthing_system_goroutines gauge"
+        echo "syncthing_system_goroutines $goroutines"
+        echo ""
+    fi
+
+    # ------------------------------------------------------------------
+    # REST API: device connections
+    # ------------------------------------------------------------------
+    local connections_json
+    connections_json=$(api_call "/rest/system/connections")
+    if [ -n "$connections_json" ]; then
+        local connected_count=0
+        local connected_lines=""
+        local paused_lines=""
+        local conn_type_lines=""
+
+        local device_data
+        device_data=$(echo "$connections_json" | jq -r '
+            .connections // {} | to_entries[] |
+            select(.key != "total") |
+            [
+                .key,
+                (.value.connected // false | tostring),
+                (.value.paused // false | tostring),
+                (.value.type // ""),
+                (.value.address // ""),
+                (.value.clientVersion // "")
+            ] | @tsv
+        ')
+
+        while IFS=$'\t' read -r device_id connected paused conn_type address client_version; do
+            [ -z "$device_id" ] && continue
+            local short_id="${device_id:0:7}"
+
+            local conn_val=0
+            if [ "$connected" = "true" ]; then
+                conn_val=1
+                connected_count=$((connected_count + 1))
+            fi
+            connected_lines+="syncthing_device_connected{device_id=\"$(prom_escape "$short_id")\",address=\"$(prom_escape "$address")\",client_version=\"$(prom_escape "$client_version")\"} $conn_val"$'\n'
+
+            local paused_val=0
+            if [ "$paused" = "true" ]; then
+                paused_val=1
+            fi
+            paused_lines+="syncthing_device_paused{device_id=\"$(prom_escape "$short_id")\"} $paused_val"$'\n'
+
+            if [ -n "$conn_type" ]; then
+                conn_type_lines+="syncthing_device_connection_type{device_id=\"$(prom_escape "$short_id")\",type=\"$(prom_escape "$conn_type")\"} 1"$'\n'
+            fi
+        done <<< "$device_data"
+
+        echo "# HELP syncthing_device_connected Per-device connection status (1=connected, 0=disconnected)"
+        echo "# TYPE syncthing_device_connected gauge"
+        printf '%s' "$connected_lines"
+        echo ""
+        echo "# HELP syncthing_device_paused Per-device paused status (1=paused, 0=active)"
+        echo "# TYPE syncthing_device_paused gauge"
+        printf '%s' "$paused_lines"
+        echo ""
+        echo "# HELP syncthing_device_connection_type Per-device connection type"
+        echo "# TYPE syncthing_device_connection_type gauge"
+        printf '%s' "$conn_type_lines"
+        echo ""
+        echo "# HELP syncthing_connected_devices Total number of connected devices"
+        echo "# TYPE syncthing_connected_devices gauge"
+        echo "syncthing_connected_devices $connected_count"
+        echo ""
+    fi
+
+    # ------------------------------------------------------------------
+    # REST API: device stats (last seen)
+    # ------------------------------------------------------------------
+    local stats_json
+    stats_json=$(api_call "/rest/stats/device")
+    if [ -n "$stats_json" ]; then
+        echo "# HELP syncthing_device_last_seen_timestamp Unix timestamp when device was last seen"
+        echo "# TYPE syncthing_device_last_seen_timestamp gauge"
+
+        local stats_data
+        stats_data=$(echo "$stats_json" | jq -r '
+            to_entries[] |
+            [.key, (.value.lastSeen // "")] | @tsv
+        ')
+
+        while IFS=$'\t' read -r device_id last_seen; do
+            [ -z "$device_id" ] && continue
+            local short_id="${device_id:0:7}"
+            local last_seen_unix
+            last_seen_unix=$(iso_to_unix "$last_seen")
+            echo "syncthing_device_last_seen_timestamp{device_id=\"$(prom_escape "$short_id")\"} $last_seen_unix"
+        done <<< "$stats_data"
+        echo ""
+    fi
+
+    # ------------------------------------------------------------------
+    # REST API: folder completion per device
+    # ------------------------------------------------------------------
+    local config_json
+    config_json=$(api_call "/rest/config")
+    if [ -n "$config_json" ]; then
+        local folder_ids device_ids
+        folder_ids=$(echo "$config_json" | jq -r '.folders[]?.id // empty')
+        device_ids=$(echo "$config_json" | jq -r '.devices[]?.deviceID // empty')
+
+        if [ -n "$folder_ids" ] && [ -n "$device_ids" ]; then
+            echo "# HELP syncthing_folder_completion_pct Folder sync completion percentage per device"
+            echo "# TYPE syncthing_folder_completion_pct gauge"
+
+            while IFS= read -r folder_id; do
+                [ -z "$folder_id" ] && continue
+                while IFS= read -r device_id; do
+                    [ -z "$device_id" ] && continue
+                    local completion_json
+                    completion_json=$(api_call "/rest/db/completion?folder=$(printf '%s' "$folder_id" | jq -sRr @uri)&device=$(printf '%s' "$device_id" | jq -sRr @uri)")
+                    if [ -n "$completion_json" ]; then
+                        local pct
+                        pct=$(echo "$completion_json" | jq -r '.completion // 0')
+                        local short_id="${device_id:0:7}"
+                        echo "syncthing_folder_completion_pct{folder=\"$(prom_escape "$folder_id")\",device_id=\"$(prom_escape "$short_id")\"} $pct"
+                    fi
+                done <<< "$device_ids"
+            done <<< "$folder_ids"
+            echo ""
+        fi
+    fi
+
+    # ------------------------------------------------------------------
+    # Exporter metadata
+    # ------------------------------------------------------------------
+    local script_end script_duration
+    script_end=$(date +%s)
+    local script_start_s=$((${script_start} / 1000000000))
+    script_duration=$((script_end - script_start_s))
+
+    cat <<EOF
+# HELP syncthing_exporter_duration_seconds Time to generate all metrics
+# TYPE syncthing_exporter_duration_seconds gauge
+syncthing_exporter_duration_seconds $script_duration
+
+# HELP syncthing_exporter_last_run_timestamp Unix timestamp of last successful run
+# TYPE syncthing_exporter_last_run_timestamp gauge
+syncthing_exporter_last_run_timestamp $script_end
+EOF
+}
+
+# ==============================================================================
+# HTTP SERVER
+# ==============================================================================
+
+run_http_server() {
+    echo "Starting Syncthing exporter on port $HTTP_PORT..." >&2
+    if ! command -v nc >/dev/null 2>&1; then
+        echo "ERROR: netcat (nc) required for HTTP mode" >&2
+        exit 1
+    fi
+    while true; do
+        {
+            read -r request
+            if [[ "$request" =~ ^GET\ /metrics ]]; then
+                echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\n\r"
+                generate_metrics
+            else
+                echo -e "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r"
+                echo "<html><head><title>Syncthing Exporter</title></head><body><h1>Syncthing Prometheus Exporter</h1><p><a href=\"/metrics\">Metrics</a></p></body></html>"
+            fi
+        } | nc -l -p "$HTTP_PORT" -q 1 2>/dev/null
+    done
+}
+
+# ==============================================================================
+# MAIN
+# ==============================================================================
+
+main() {
+    parse_args "$@"
+    if [ "$HTTP_MODE" = true ]; then
+        run_http_server
+    elif [ -n "$OUTPUT_FILE" ]; then
+        local output_dir
+        output_dir="$(dirname "$OUTPUT_FILE")"
+        mkdir -p "$output_dir"
+        local temp_file
+        temp_file=$(mktemp "${output_dir}/.syncthing_metrics.XXXXXX")
+        if ! generate_metrics > "$temp_file" 2>/dev/null; then
+            rm -f "$temp_file"
+            echo "ERROR: Failed to generate metrics" >&2
+            exit 1
+        fi
+        chmod 644 "$temp_file"
+        mv -f "$temp_file" "$OUTPUT_FILE"
+        echo "Metrics written to $OUTPUT_FILE" >&2
+    else
+        generate_metrics
+    fi
+}
+
+main "$@"