Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
2026-05-25 03:31:08 +02:00
parent dbd6bf0324
commit a1a17e81a1
332 changed files with 174509 additions and 1106 deletions
@@ -0,0 +1,320 @@
+#!/usr/bin/env bash
+
+#########################################################################################
+#### lighthouse-audit.sh — Automated Lighthouse audits with Prometheus integration   ####
+#### Runs headless Chrome via Lighthouse CLI, extracts scores with jq, pushes        ####
+#### metrics to Pushgateway — saves HTML reports and color-codes output               ####
+####                                                                                 ####
+#### Author: Phil Connor                                                             ####
+#### Contact: contact@mylinux.work                                                   ####
+#### License: MIT                                                                    ####
+#### Version 1.00                                                                    ####
+####                                                                                 ####
+#### Usage:                                                                          ####
+####   ./lighthouse-audit.sh https://mylinux.work                                    ####
+####   ./lighthouse-audit.sh --file urls.txt --pushgateway http://localhost:9091      ####
+####                                                                                 ####
+#### See --help for all options.                                                     ####
+#########################################################################################
+
+set -euo pipefail
+
+# ── Defaults ──────────────────────────────────────────────────────────
+URLS_FILE=""
+OUTPUT_DIR="${OUTPUT_DIR:-/var/lib/lighthouse/reports}"
+PUSHGATEWAY_URL="${PUSHGATEWAY_URL:-}"
+CHROME_FLAGS="--headless --no-sandbox --disable-dev-shm-usage --disable-gpu"
+JOB_NAME="${JOB_NAME:-lighthouse}"
+LIGHTHOUSE_TIMEOUT="${LIGHTHOUSE_TIMEOUT:-60}"
+RUNS="${RUNS:-1}"
+VERBOSE="${VERBOSE:-false}"
+COLOR="${COLOR:-auto}"
+
+# ── State ─────────────────────────────────────────────────────────────
+SCRIPT_NAME="$(basename "$0")"
+readonly SCRIPT_NAME
+TEMP_DIR=""
+
+# ── Colors ────────────────────────────────────────────────────────────
+setup_colors() {
+    if [[ "$COLOR" == "never" ]]; then
+        RED="" GREEN="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+        return
+    fi
+    if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
+        RED='\033[0;31m'
+        GREEN='\033[0;32m'
+        YELLOW='\033[0;33m'
+        CYAN='\033[0;36m'
+        BOLD='\033[1m'
+        DIM='\033[2m'
+        RESET='\033[0m'
+    else
+        RED="" GREEN="" YELLOW="" CYAN="" BOLD="" DIM="" RESET=""
+    fi
+}
+
+# ── Logging ───────────────────────────────────────────────────────────
+log()     { echo -e "${CYAN}[INFO]${RESET} $*"; }
+warn()    { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
+err()     { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
+verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
+
+# ── Cleanup ───────────────────────────────────────────────────────────
+cleanup() {
+    if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then
+        rm -rf "$TEMP_DIR"
+    fi
+}
+trap cleanup EXIT
+
+# ── Color-coded score output ─────────────────────────────────────────
+color_score() {
+    local score=$1
+    if (( score >= 90 )); then
+        echo -e "${GREEN}${score}${RESET}"
+    elif (( score >= 50 )); then
+        echo -e "${YELLOW}${score}${RESET}"
+    else
+        echo -e "${RED}${score}${RESET}"
+    fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+#                       AUDIT
+# ══════════════════════════════════════════════════════════════════════
+
+run_audit() {
+    local url="$1"
+    local date_stamp
+    date_stamp=$(date +%Y-%m-%d_%H%M%S)
+    local slug
+    slug=$(echo "$url" | sed 's|https\?://||;s|/|_|g;s|[^a-zA-Z0-9_.-]||g')
+
+    log "Auditing: ${url}"
+
+    local perf_total=0 a11y_total=0 bp_total=0 seo_total=0
+    local lcp_total=0 cls_total=0 tbt_total=0
+    local best_json=""
+    local successful_runs=0
+
+    for ((run=1; run<=RUNS; run++)); do
+        [[ "$RUNS" -gt 1 ]] && log "  Run ${run}/${RUNS}"
+
+        local json_file="${TEMP_DIR}/${slug}_run${run}.json"
+
+        lighthouse "$url" \
+            --output json \
+            --output-path "$json_file" \
+            --chrome-flags="$CHROME_FLAGS" \
+            --max-wait-for-load "$((LIGHTHOUSE_TIMEOUT * 1000))" \
+            --quiet 2>/dev/null || {
+                warn "Lighthouse failed for ${url} (run ${run})"
+                continue
+            }
+
+        local perf a11y bp seo lcp cls tbt
+        perf=$(jq -r '.categories.performance.score * 100 | floor' "$json_file")
+        a11y=$(jq -r '.categories.accessibility.score * 100 | floor' "$json_file")
+        bp=$(jq -r '.categories["best-practices"].score * 100 | floor' "$json_file")
+        seo=$(jq -r '.categories.seo.score * 100 | floor' "$json_file")
+        lcp=$(jq -r '.audits["largest-contentful-paint"].numericValue / 1000' "$json_file")
+        cls=$(jq -r '.audits["cumulative-layout-shift"].numericValue' "$json_file")
+        tbt=$(jq -r '.audits["total-blocking-time"].numericValue' "$json_file")
+
+        perf_total=$(echo "$perf_total + $perf" | bc)
+        a11y_total=$(echo "$a11y_total + $a11y" | bc)
+        bp_total=$(echo "$bp_total + $bp" | bc)
+        seo_total=$(echo "$seo_total + $seo" | bc)
+        lcp_total=$(echo "$lcp_total + $lcp" | bc)
+        cls_total=$(echo "$cls_total + $cls" | bc)
+        tbt_total=$(echo "$tbt_total + $tbt" | bc)
+
+        successful_runs=$((successful_runs + 1))
+        best_json="$json_file"
+    done
+
+    if [[ -z "$best_json" ]]; then
+        warn "All runs failed for ${url}"
+        return 1
+    fi
+
+    # Average scores
+    local perf_avg a11y_avg bp_avg seo_avg lcp_avg cls_avg tbt_avg
+    perf_avg=$(echo "$perf_total / $successful_runs" | bc)
+    a11y_avg=$(echo "$a11y_total / $successful_runs" | bc)
+    bp_avg=$(echo "$bp_total / $successful_runs" | bc)
+    seo_avg=$(echo "$seo_total / $successful_runs" | bc)
+    lcp_avg=$(echo "scale=2; $lcp_total / $successful_runs" | bc)
+    cls_avg=$(echo "scale=3; $cls_total / $successful_runs" | bc)
+    tbt_avg=$(echo "scale=0; $tbt_total / $successful_runs" | bc)
+
+    # Generate HTML report from last run's JSON
+    local report_file="${OUTPUT_DIR}/${slug}_${date_stamp}.html"
+    lighthouse "$url" \
+        --output html \
+        --output-path "$report_file" \
+        --chrome-flags="$CHROME_FLAGS" \
+        --max-wait-for-load "$((LIGHTHOUSE_TIMEOUT * 1000))" \
+        --quiet 2>/dev/null || true
+
+    # Print results
+    log "Results for ${url}:"
+    printf "  ${BOLD}Performance:${RESET}    %s\n" "$(color_score "$perf_avg")"
+    printf "  ${BOLD}Accessibility:${RESET}  %s\n" "$(color_score "$a11y_avg")"
+    printf "  ${BOLD}Best Practices:${RESET} %s\n" "$(color_score "$bp_avg")"
+    printf "  ${BOLD}SEO:${RESET}            %s\n" "$(color_score "$seo_avg")"
+    printf "  LCP: %ss  CLS: %s  TBT: %sms\n" "$lcp_avg" "$cls_avg" "$tbt_avg"
+    [[ -f "$report_file" ]] && log "Report: ${report_file}"
+
+    # Push to Prometheus Pushgateway
+    if [[ -n "$PUSHGATEWAY_URL" ]]; then
+        local encoded_url
+        encoded_url=$(echo "$url" | sed 's|/|%2F|g;s|:|%3A|g')
+
+        cat <<METRICS | curl -s --data-binary @- "${PUSHGATEWAY_URL}/metrics/job/${JOB_NAME}/url/${encoded_url}"
+# HELP lighthouse_performance_score Lighthouse performance score
+# TYPE lighthouse_performance_score gauge
+lighthouse_performance_score{url="${url}"} ${perf_avg}
+# HELP lighthouse_accessibility_score Lighthouse accessibility score
+# TYPE lighthouse_accessibility_score gauge
+lighthouse_accessibility_score{url="${url}"} ${a11y_avg}
+# HELP lighthouse_best_practices_score Lighthouse best practices score
+# TYPE lighthouse_best_practices_score gauge
+lighthouse_best_practices_score{url="${url}"} ${bp_avg}
+# HELP lighthouse_seo_score Lighthouse SEO score
+# TYPE lighthouse_seo_score gauge
+lighthouse_seo_score{url="${url}"} ${seo_avg}
+# HELP lighthouse_lcp_seconds Largest Contentful Paint in seconds
+# TYPE lighthouse_lcp_seconds gauge
+lighthouse_lcp_seconds{url="${url}"} ${lcp_avg}
+# HELP lighthouse_cls_score Cumulative Layout Shift score
+# TYPE lighthouse_cls_score gauge
+lighthouse_cls_score{url="${url}"} ${cls_avg}
+# HELP lighthouse_tbt_milliseconds Total Blocking Time in milliseconds
+# TYPE lighthouse_tbt_milliseconds gauge
+lighthouse_tbt_milliseconds{url="${url}"} ${tbt_avg}
+METRICS
+        log "Pushed metrics to ${PUSHGATEWAY_URL}"
+    fi
+}
+
+# ══════════════════════════════════════════════════════════════════════
+#                       USAGE
+# ══════════════════════════════════════════════════════════════════════
+
+usage() {
+    cat <<EOF
+${SCRIPT_NAME} — Automated Lighthouse audits with Prometheus integration
+
+USAGE:
+  ${SCRIPT_NAME} [OPTIONS] [URL...]
+  ${SCRIPT_NAME} --file urls.txt [OPTIONS]
+  echo "https://example.com" | ${SCRIPT_NAME}
+
+OPTIONS:
+  --file FILE           Read URLs from file (one per line)
+  --output-dir DIR      Directory for HTML reports (default: ${OUTPUT_DIR})
+  --pushgateway URL     Pushgateway URL (e.g., http://localhost:9091)
+  --job NAME            Pushgateway job name (default: ${JOB_NAME})
+  --timeout SECONDS     Lighthouse page load timeout (default: ${LIGHTHOUSE_TIMEOUT})
+  --runs N              Number of runs to average (default: ${RUNS})
+  --verbose             Enable debug output
+  --no-color            Disable colored output
+  --help                Show this help
+
+ENVIRONMENT VARIABLES:
+  PUSHGATEWAY_URL       Pushgateway URL
+  LIGHTHOUSE_TIMEOUT    Page load timeout in seconds (default: 60)
+  OUTPUT_DIR            Report output directory
+  JOB_NAME              Pushgateway job name (default: lighthouse)
+  RUNS                  Number of runs to average (default: 1)
+EOF
+}
+
+# ══════════════════════════════════════════════════════════════════════
+#                       ARGUMENT PARSING
+# ══════════════════════════════════════════════════════════════════════
+
+parse_args() {
+    POSITIONAL_URLS=()
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --file)          URLS_FILE="$2"; shift 2 ;;
+            --output-dir)    OUTPUT_DIR="$2"; shift 2 ;;
+            --pushgateway)   PUSHGATEWAY_URL="$2"; shift 2 ;;
+            --job)           JOB_NAME="$2"; shift 2 ;;
+            --timeout)       LIGHTHOUSE_TIMEOUT="$2"; shift 2 ;;
+            --runs)          RUNS="$2"; shift 2 ;;
+            --verbose)       VERBOSE="true"; shift ;;
+            --no-color)      COLOR="never"; shift ;;
+            --help|-h)       setup_colors; usage; exit 0 ;;
+            --)              shift; break ;;
+            -*)
+                err "Unknown option: $1"
+                echo "Run ${SCRIPT_NAME} --help for usage" >&2
+                exit 1 ;;
+            *)               POSITIONAL_URLS+=("$1"); shift ;;
+        esac
+    done
+}
+
+# ══════════════════════════════════════════════════════════════════════
+#                       MAIN
+# ══════════════════════════════════════════════════════════════════════
+
+main() {
+    parse_args "$@"
+    setup_colors
+
+    # Collect URLs
+    URLS=()
+
+    for u in "${POSITIONAL_URLS[@]+"${POSITIONAL_URLS[@]}"}"; do
+        URLS+=("$u")
+    done
+
+    if [[ -n "$URLS_FILE" ]]; then
+        if [[ ! -f "$URLS_FILE" ]]; then
+            err "URLs file not found: ${URLS_FILE}"
+            exit 1
+        fi
+        while IFS= read -r line; do
+            line=$(echo "$line" | xargs)
+            [[ -z "$line" || "$line" == \#* ]] && continue
+            URLS+=("$line")
+        done < "$URLS_FILE"
+    fi
+
+    if [[ ! -t 0 ]]; then
+        while IFS= read -r line; do
+            line=$(echo "$line" | xargs)
+            [[ -z "$line" || "$line" == \#* ]] && continue
+            URLS+=("$line")
+        done
+    fi
+
+    if [[ ${#URLS[@]} -eq 0 ]]; then
+        err "No URLs provided (see --help)"
+        exit 1
+    fi
+
+    # Validate dependencies
+    command -v lighthouse >/dev/null 2>&1 || { err "lighthouse not found — npm install -g lighthouse"; exit 1; }
+    command -v jq >/dev/null 2>&1 || { err "jq not found — apt install jq"; exit 1; }
+
+    mkdir -p "$OUTPUT_DIR"
+    TEMP_DIR=$(mktemp -d)
+
+    log "Lighthouse Audit — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    log "URLs: ${#URLS[@]} | Runs: ${RUNS} | Timeout: ${LIGHTHOUSE_TIMEOUT}s"
+
+    for url in "${URLS[@]}"; do
+        run_audit "$url" || true
+        [[ ${#URLS[@]} -gt 1 ]] && sleep 5
+    done
+
+    log "Done."
+}
+
+main "$@"