Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
2026-05-25 03:31:08 +02:00
parent dbd6bf0324
commit a1a17e81a1
332 changed files with 174509 additions and 1106 deletions
@@ -0,0 +1,456 @@
+#!/bin/bash
+################################################################################
+# Script Name: add-fail2ban-head-crawler.sh
+# Version: 1.0
+# Description: Adds a Fail2ban jail to block HEAD-only crawlers — bots that
+#              systematically send HEAD requests with no referer to probe or
+#              index your site while spoofing real browser user agents.
+#
+# Author:  Phil Connor
+# Contact: contact@mylinux.work
+# Website: https://mylinux.work
+# License: MIT
+#
+# Usage:
+#   sudo ./add-fail2ban-head-crawler.sh
+#   sudo ./add-fail2ban-head-crawler.sh --logpath /var/log/nginx/access.log
+#   sudo ./add-fail2ban-head-crawler.sh --maxretry 10
+#   sudo ./add-fail2ban-head-crawler.sh --dry-run
+#
+################################################################################
+
+set -euo pipefail
+
+# ============================================================================
+# DEFAULTS
+# ============================================================================
+
+readonly VERSION="1.0"
+readonly SCRIPT_NAME="${0##*/}"
+
+LOGPATH="auto"
+BANTIME="86400"
+MAXRETRY="5"
+FINDTIME="300"
+DRY_RUN=false
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+log_info()  { echo -e "${GREEN}[INFO]${NC} $*"; }
+log_warn()  { echo -e "${YELLOW}[WARN]${NC} $*"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
+log_step()  { echo -e "${CYAN}[STEP]${NC} $*"; }
+
+show_usage() {
+    cat <<EOF
+Usage: sudo $SCRIPT_NAME [OPTIONS]
+
+Adds a HEAD-crawler blocking jail to an existing Fail2ban installation.
+Catches bots that systematically send HEAD requests with no referer —
+a common pattern for scrapers and SEO tools that spoof real browser
+user agents to avoid detection.
+
+OPTIONS:
+    --logpath PATH      Access log path (default: auto-detect)
+    --bantime SECS      Ban duration in seconds (default: 86400 / 24 hours)
+    --maxretry NUM      HEAD requests before ban (default: 5)
+    --findtime SECS     Window for counting requests (default: 300 / 5 min)
+    --dry-run           Show what would be done without making changes
+    --remove            Remove the filter and jail
+    -h, --help          Show this help message
+
+WHAT IT CATCHES:
+    - HEAD requests with no referer ("-")
+    - Bots rotating through AWS/cloud IPs with spoofed user agents
+    - SEO crawlers, uptime probes, and content scrapers
+    - Any automated tool doing HEAD-only reconnaissance
+
+WHAT IT IGNORES:
+    - Normal GET/POST traffic (even with no referer)
+    - HEAD requests with a valid referer (browser prefetch)
+    - Single HEAD requests (below maxretry threshold)
+
+EXAMPLES:
+    # Install with defaults (ban after 5 HEAD requests in 5 min)
+    sudo $SCRIPT_NAME
+
+    # Stricter: ban after 3 requests in 2 minutes
+    sudo $SCRIPT_NAME --maxretry 3 --findtime 120
+
+    # More lenient: ban after 15 requests in 10 minutes
+    sudo $SCRIPT_NAME --maxretry 15 --findtime 600
+
+    # Preview without changes
+    sudo $SCRIPT_NAME --dry-run
+
+    # Remove the jail
+    sudo $SCRIPT_NAME --remove
+
+EOF
+    exit 0
+}
+
+# ============================================================================
+# ARGUMENT PARSING
+# ============================================================================
+
+REMOVE=false
+
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --logpath)   LOGPATH="$2"; shift 2 ;;
+            --bantime)   BANTIME="$2"; shift 2 ;;
+            --maxretry)  MAXRETRY="$2"; shift 2 ;;
+            --findtime)  FINDTIME="$2"; shift 2 ;;
+            --dry-run)   DRY_RUN=true; shift ;;
+            --remove)    REMOVE=true; shift ;;
+            -h|--help)   show_usage ;;
+            *)           log_error "Unknown option: $1"; show_usage ;;
+        esac
+    done
+}
+
+# ============================================================================
+# CHECKS
+# ============================================================================
+
+check_root() {
+    if [[ $EUID -ne 0 ]]; then
+        log_error "This script must be run as root (sudo)"
+        exit 1
+    fi
+}
+
+check_fail2ban() {
+    if ! command -v fail2ban-client &>/dev/null; then
+        log_error "Fail2ban is not installed"
+        log_error "Install it first: https://mylinux.work/guides/fail2ban-setup/"
+        exit 1
+    fi
+
+    if ! systemctl is-active --quiet fail2ban; then
+        log_error "Fail2ban is not running"
+        exit 1
+    fi
+
+    log_info "Fail2ban is installed and running"
+}
+
+detect_logpath() {
+    if [[ "$LOGPATH" != "auto" ]]; then
+        # shellcheck disable=SC2086
+        local matches=( $LOGPATH )
+        if [[ ${#matches[@]} -eq 0 || ! -f "${matches[0]}" ]]; then
+            log_error "Log file not found: $LOGPATH"
+            exit 1
+        fi
+        log_info "Using specified log path: $LOGPATH"
+        return
+    fi
+
+    log_step "Auto-detecting web server access log..."
+
+    # HestiaCP — apache domains
+    local hestia_apache=( /var/log/apache2/domains/*.log )
+    if [[ -f "${hestia_apache[0]:-}" ]]; then
+        LOGPATH="/var/log/apache2/domains/*.log"
+        log_info "Detected HestiaCP apache: $LOGPATH"
+        return
+    fi
+
+    # HestiaCP — nginx domains
+    local hestia_nginx=( /var/log/nginx/domains/*.log )
+    if [[ -f "${hestia_nginx[0]:-}" ]]; then
+        LOGPATH="/var/log/nginx/domains/*.log"
+        log_info "Detected HestiaCP nginx: $LOGPATH"
+        return
+    fi
+
+    # Nginx (standard)
+    if [[ -f /var/log/nginx/access.log ]]; then
+        LOGPATH="/var/log/nginx/access.log"
+        log_info "Detected nginx: $LOGPATH"
+        return
+    fi
+
+    # Apache (Debian/Ubuntu)
+    if [[ -f /var/log/apache2/access.log ]]; then
+        LOGPATH="/var/log/apache2/access.log"
+        log_info "Detected apache2: $LOGPATH"
+        return
+    fi
+
+    # Apache (RHEL/Rocky)
+    if [[ -f /var/log/httpd/access_log ]]; then
+        LOGPATH="/var/log/httpd/access_log"
+        log_info "Detected httpd: $LOGPATH"
+        return
+    fi
+
+    log_error "Could not auto-detect access log. Use --logpath to specify."
+    exit 1
+}
+
+# ============================================================================
+# REMOVE
+# ============================================================================
+
+do_remove() {
+    local filter_file="/etc/fail2ban/filter.d/head-crawler.conf"
+    local jail_file="/etc/fail2ban/jail.d/head-crawler.conf"
+
+    log_step "Removing HEAD crawler jail..."
+
+    if $DRY_RUN; then
+        log_info "[DRY RUN] Would remove $filter_file"
+        log_info "[DRY RUN] Would remove $jail_file"
+        log_info "[DRY RUN] Would reload fail2ban"
+        return
+    fi
+
+    if [[ -f "$jail_file" ]]; then
+        rm -f "$jail_file"
+        log_info "Removed: $jail_file"
+    else
+        log_warn "Jail config not found: $jail_file"
+    fi
+
+    if [[ -f "$filter_file" ]]; then
+        rm -f "$filter_file"
+        log_info "Removed: $filter_file"
+    else
+        log_warn "Filter not found: $filter_file"
+    fi
+
+    fail2ban-client reload
+    sleep 2
+    log_info "Fail2ban reloaded — head-crawler jail removed"
+    exit 0
+}
+
+# ============================================================================
+# INSTALL FILTER
+# ============================================================================
+
+install_filter() {
+    local filter_file="/etc/fail2ban/filter.d/head-crawler.conf"
+
+    log_step "Installing filter: $filter_file"
+
+    if $DRY_RUN; then
+        log_info "[DRY RUN] Would create $filter_file"
+        echo ""
+        generate_filter
+        echo ""
+        return
+    fi
+
+    if [[ -f "$filter_file" ]]; then
+        log_warn "Filter already exists — backing up to ${filter_file}.bak"
+        cp "$filter_file" "${filter_file}.bak"
+    fi
+
+    generate_filter > "$filter_file"
+    log_info "Filter installed: $filter_file"
+}
+
+generate_filter() {
+    cat <<'EOF'
+# Fail2ban filter to block HEAD-only crawlers
+# https://mylinux.work
+#
+# Catches bots that send HEAD requests with no referer. These are typically
+# scrapers, SEO tools, or reconnaissance bots that spoof real browser user
+# agents and rotate through cloud IPs to avoid detection.
+#
+# The filter matches:
+#   - HTTP HEAD method
+#   - No referer (logged as "-")
+#   - Any user agent (spoofed or otherwise)
+#
+# Combined with a low maxretry (default: 5 in 5 min), this catches
+# systematic crawlers while ignoring occasional legitimate HEAD requests
+# (browser prefetch, monitoring probes).
+
+[Definition]
+
+# HEAD request with no referer — combined log format
+# Format: IP - - [date] "HEAD /path HTTP/x.x" status size "-" "user agent"
+failregex = ^<HOST> \S+ \S+ \[.*\] "HEAD \S+ \S+" \d+ \d+ "-" ".*"
+
+ignoreregex =
+
+# Author: Phil Connor — https://mylinux.work
+EOF
+}
+
+# ============================================================================
+# INSTALL JAIL
+# ============================================================================
+
+install_jail() {
+    local jail_file="/etc/fail2ban/jail.d/head-crawler.conf"
+
+    log_step "Installing jail: $jail_file"
+
+    if $DRY_RUN; then
+        log_info "[DRY RUN] Would create $jail_file"
+        echo ""
+        generate_jail
+        echo ""
+        return
+    fi
+
+    if [[ -f "$jail_file" ]]; then
+        log_warn "Jail config already exists — backing up to ${jail_file}.bak"
+        cp "$jail_file" "${jail_file}.bak"
+    fi
+
+    generate_jail > "$jail_file"
+    log_info "Jail config installed: $jail_file"
+}
+
+generate_jail() {
+    cat <<EOF
+# Fail2ban jail to block HEAD-only crawlers
+# https://mylinux.work
+#
+# Bans IPs that send multiple HEAD requests with no referer.
+# Default: 5 HEAD requests in 5 minutes = 24 hour ban.
+#
+# This catches scrapers and SEO bots that spoof real browser
+# user agents and rotate through cloud provider IPs.
+
+[head-crawler]
+enabled  = true
+port     = http,https
+filter   = head-crawler
+logpath  = $LOGPATH
+maxretry = $MAXRETRY
+findtime = $FINDTIME
+bantime  = $BANTIME
+
+# Author: Phil Connor — https://mylinux.work
+EOF
+}
+
+# ============================================================================
+# RELOAD AND VERIFY
+# ============================================================================
+
+reload_fail2ban() {
+    log_step "Reloading Fail2ban..."
+
+    if $DRY_RUN; then
+        log_info "[DRY RUN] Would reload fail2ban"
+        return
+    fi
+
+    if ! fail2ban-client --test 2>/dev/null; then
+        log_warn "Config test not available — reloading directly"
+    fi
+
+    fail2ban-client reload
+    sleep 2
+
+    if systemctl is-active --quiet fail2ban; then
+        log_info "Fail2ban reloaded successfully"
+    else
+        log_error "Fail2ban failed to restart — check: journalctl -u fail2ban"
+        exit 1
+    fi
+}
+
+verify_jail() {
+    log_step "Verifying head-crawler jail..."
+
+    if $DRY_RUN; then
+        log_info "[DRY RUN] Would verify jail status"
+        return
+    fi
+
+    echo ""
+    if fail2ban-client status head-crawler 2>/dev/null; then
+        echo ""
+        log_info "HEAD crawler jail is active and monitoring $LOGPATH"
+    else
+        log_error "Jail 'head-crawler' is not running — check: fail2ban-client status"
+        log_error "Debug with: fail2ban-regex $LOGPATH /etc/fail2ban/filter.d/head-crawler.conf"
+        exit 1
+    fi
+}
+
+test_against_logs() {
+    if $DRY_RUN; then
+        # shellcheck disable=SC2086
+        local matches=( $LOGPATH )
+        if [[ -f "${matches[0]}" ]]; then
+            log_step "Testing filter against existing logs..."
+            echo ""
+            fail2ban-regex "${matches[0]}" /dev/stdin <<'FILTER' 2>&1 | tail -5
+[Definition]
+failregex = ^<HOST> \S+ \S+ \[.*\] "HEAD \S+ \S+" \d+ \d+ "-" ".*"
+ignoreregex =
+FILTER
+            echo ""
+        fi
+    fi
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+main() {
+    parse_args "$@"
+
+    echo ""
+    echo "============================================"
+    echo "  Fail2ban HEAD Crawler Blocker v${VERSION}"
+    echo "  https://mylinux.work"
+    echo "============================================"
+    echo ""
+
+    check_root
+    check_fail2ban
+
+    if $REMOVE; then
+        do_remove
+    fi
+
+    detect_logpath
+    test_against_logs
+    install_filter
+    install_jail
+    reload_fail2ban
+    verify_jail
+
+    echo ""
+    echo "============================================"
+    echo "  Setup Complete"
+    echo "============================================"
+    echo ""
+    echo "  Jail:      head-crawler"
+    echo "  Log:       $LOGPATH"
+    echo "  Ban time:  ${BANTIME}s ($(( BANTIME / 3600 ))h)"
+    echo "  Max retry: $MAXRETRY (HEAD requests before ban)"
+    echo "  Find time: ${FINDTIME}s ($(( FINDTIME / 60 ))m window)"
+    echo ""
+    echo "  Useful commands:"
+    echo "    fail2ban-client status head-crawler"
+    echo "    fail2ban-client set head-crawler unbanip <IP>"
+    echo "    fail2ban-regex $LOGPATH /etc/fail2ban/filter.d/head-crawler.conf"
+    echo ""
+}
+
+main "$@"