chiefgeek
a1a17e81a1
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
457 lines
13 KiB
Bash
Executable File
457 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
################################################################################
|
|
# Script Name: add-fail2ban-head-crawler.sh
|
|
# Version: 1.0
|
|
# Description: Adds a Fail2ban jail to block HEAD-only crawlers — bots that
|
|
# systematically send HEAD requests with no referer to probe or
|
|
# index your site while spoofing real browser user agents.
|
|
#
|
|
# Author: Phil Connor
|
|
# Contact: contact@mylinux.work
|
|
# Website: https://mylinux.work
|
|
# License: MIT
|
|
#
|
|
# Usage:
|
|
# sudo ./add-fail2ban-head-crawler.sh
|
|
# sudo ./add-fail2ban-head-crawler.sh --logpath /var/log/nginx/access.log
|
|
# sudo ./add-fail2ban-head-crawler.sh --maxretry 10
|
|
# sudo ./add-fail2ban-head-crawler.sh --dry-run
|
|
#
|
|
################################################################################
|
|
|
|
set -euo pipefail
|
|
|
|
# ============================================================================
|
|
# DEFAULTS
|
|
# ============================================================================
|
|
|
|
readonly VERSION="1.0"
|
|
readonly SCRIPT_NAME="${0##*/}"
|
|
|
|
LOGPATH="auto"
|
|
BANTIME="86400"
|
|
MAXRETRY="5"
|
|
FINDTIME="300"
|
|
DRY_RUN=false
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
# ============================================================================
|
|
# HELPER FUNCTIONS
|
|
# ============================================================================
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
|
|
log_step() { echo -e "${CYAN}[STEP]${NC} $*"; }
|
|
|
|
show_usage() {
|
|
cat <<EOF
|
|
Usage: sudo $SCRIPT_NAME [OPTIONS]
|
|
|
|
Adds a HEAD-crawler blocking jail to an existing Fail2ban installation.
|
|
Catches bots that systematically send HEAD requests with no referer —
|
|
a common pattern for scrapers and SEO tools that spoof real browser
|
|
user agents to avoid detection.
|
|
|
|
OPTIONS:
|
|
--logpath PATH Access log path (default: auto-detect)
|
|
--bantime SECS Ban duration in seconds (default: 86400 / 24 hours)
|
|
--maxretry NUM HEAD requests before ban (default: 5)
|
|
--findtime SECS Window for counting requests (default: 300 / 5 min)
|
|
--dry-run Show what would be done without making changes
|
|
--remove Remove the filter and jail
|
|
-h, --help Show this help message
|
|
|
|
WHAT IT CATCHES:
|
|
- HEAD requests with no referer ("-")
|
|
- Bots rotating through AWS/cloud IPs with spoofed user agents
|
|
- SEO crawlers, uptime probes, and content scrapers
|
|
- Any automated tool doing HEAD-only reconnaissance
|
|
|
|
WHAT IT IGNORES:
|
|
- Normal GET/POST traffic (even with no referer)
|
|
- HEAD requests with a valid referer (browser prefetch)
|
|
- Single HEAD requests (below maxretry threshold)
|
|
|
|
EXAMPLES:
|
|
# Install with defaults (ban after 5 HEAD requests in 5 min)
|
|
sudo $SCRIPT_NAME
|
|
|
|
# Stricter: ban after 3 requests in 2 minutes
|
|
sudo $SCRIPT_NAME --maxretry 3 --findtime 120
|
|
|
|
# More lenient: ban after 15 requests in 10 minutes
|
|
sudo $SCRIPT_NAME --maxretry 15 --findtime 600
|
|
|
|
# Preview without changes
|
|
sudo $SCRIPT_NAME --dry-run
|
|
|
|
# Remove the jail
|
|
sudo $SCRIPT_NAME --remove
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ============================================================================
|
|
# ARGUMENT PARSING
|
|
# ============================================================================
|
|
|
|
REMOVE=false
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--logpath) LOGPATH="$2"; shift 2 ;;
|
|
--bantime) BANTIME="$2"; shift 2 ;;
|
|
--maxretry) MAXRETRY="$2"; shift 2 ;;
|
|
--findtime) FINDTIME="$2"; shift 2 ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
--remove) REMOVE=true; shift ;;
|
|
-h|--help) show_usage ;;
|
|
*) log_error "Unknown option: $1"; show_usage ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# ============================================================================
|
|
# CHECKS
|
|
# ============================================================================
|
|
|
|
check_root() {
|
|
if [[ $EUID -ne 0 ]]; then
|
|
log_error "This script must be run as root (sudo)"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
check_fail2ban() {
|
|
if ! command -v fail2ban-client &>/dev/null; then
|
|
log_error "Fail2ban is not installed"
|
|
log_error "Install it first: https://mylinux.work/guides/fail2ban-setup/"
|
|
exit 1
|
|
fi
|
|
|
|
if ! systemctl is-active --quiet fail2ban; then
|
|
log_error "Fail2ban is not running"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "Fail2ban is installed and running"
|
|
}
|
|
|
|
detect_logpath() {
|
|
if [[ "$LOGPATH" != "auto" ]]; then
|
|
# shellcheck disable=SC2086
|
|
local matches=( $LOGPATH )
|
|
if [[ ${#matches[@]} -eq 0 || ! -f "${matches[0]}" ]]; then
|
|
log_error "Log file not found: $LOGPATH"
|
|
exit 1
|
|
fi
|
|
log_info "Using specified log path: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
log_step "Auto-detecting web server access log..."
|
|
|
|
# HestiaCP — apache domains
|
|
local hestia_apache=( /var/log/apache2/domains/*.log )
|
|
if [[ -f "${hestia_apache[0]:-}" ]]; then
|
|
LOGPATH="/var/log/apache2/domains/*.log"
|
|
log_info "Detected HestiaCP apache: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
# HestiaCP — nginx domains
|
|
local hestia_nginx=( /var/log/nginx/domains/*.log )
|
|
if [[ -f "${hestia_nginx[0]:-}" ]]; then
|
|
LOGPATH="/var/log/nginx/domains/*.log"
|
|
log_info "Detected HestiaCP nginx: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
# Nginx (standard)
|
|
if [[ -f /var/log/nginx/access.log ]]; then
|
|
LOGPATH="/var/log/nginx/access.log"
|
|
log_info "Detected nginx: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
# Apache (Debian/Ubuntu)
|
|
if [[ -f /var/log/apache2/access.log ]]; then
|
|
LOGPATH="/var/log/apache2/access.log"
|
|
log_info "Detected apache2: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
# Apache (RHEL/Rocky)
|
|
if [[ -f /var/log/httpd/access_log ]]; then
|
|
LOGPATH="/var/log/httpd/access_log"
|
|
log_info "Detected httpd: $LOGPATH"
|
|
return
|
|
fi
|
|
|
|
log_error "Could not auto-detect access log. Use --logpath to specify."
|
|
exit 1
|
|
}
|
|
|
|
# ============================================================================
|
|
# REMOVE
|
|
# ============================================================================
|
|
|
|
do_remove() {
|
|
local filter_file="/etc/fail2ban/filter.d/head-crawler.conf"
|
|
local jail_file="/etc/fail2ban/jail.d/head-crawler.conf"
|
|
|
|
log_step "Removing HEAD crawler jail..."
|
|
|
|
if $DRY_RUN; then
|
|
log_info "[DRY RUN] Would remove $filter_file"
|
|
log_info "[DRY RUN] Would remove $jail_file"
|
|
log_info "[DRY RUN] Would reload fail2ban"
|
|
return
|
|
fi
|
|
|
|
if [[ -f "$jail_file" ]]; then
|
|
rm -f "$jail_file"
|
|
log_info "Removed: $jail_file"
|
|
else
|
|
log_warn "Jail config not found: $jail_file"
|
|
fi
|
|
|
|
if [[ -f "$filter_file" ]]; then
|
|
rm -f "$filter_file"
|
|
log_info "Removed: $filter_file"
|
|
else
|
|
log_warn "Filter not found: $filter_file"
|
|
fi
|
|
|
|
fail2ban-client reload
|
|
sleep 2
|
|
log_info "Fail2ban reloaded — head-crawler jail removed"
|
|
exit 0
|
|
}
|
|
|
|
# ============================================================================
|
|
# INSTALL FILTER
|
|
# ============================================================================
|
|
|
|
install_filter() {
|
|
local filter_file="/etc/fail2ban/filter.d/head-crawler.conf"
|
|
|
|
log_step "Installing filter: $filter_file"
|
|
|
|
if $DRY_RUN; then
|
|
log_info "[DRY RUN] Would create $filter_file"
|
|
echo ""
|
|
generate_filter
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
if [[ -f "$filter_file" ]]; then
|
|
log_warn "Filter already exists — backing up to ${filter_file}.bak"
|
|
cp "$filter_file" "${filter_file}.bak"
|
|
fi
|
|
|
|
generate_filter > "$filter_file"
|
|
log_info "Filter installed: $filter_file"
|
|
}
|
|
|
|
generate_filter() {
|
|
cat <<'EOF'
|
|
# Fail2ban filter to block HEAD-only crawlers
|
|
# https://mylinux.work
|
|
#
|
|
# Catches bots that send HEAD requests with no referer. These are typically
|
|
# scrapers, SEO tools, or reconnaissance bots that spoof real browser user
|
|
# agents and rotate through cloud IPs to avoid detection.
|
|
#
|
|
# The filter matches:
|
|
# - HTTP HEAD method
|
|
# - No referer (logged as "-")
|
|
# - Any user agent (spoofed or otherwise)
|
|
#
|
|
# Combined with a low maxretry (default: 5 in 5 min), this catches
|
|
# systematic crawlers while ignoring occasional legitimate HEAD requests
|
|
# (browser prefetch, monitoring probes).
|
|
|
|
[Definition]
|
|
|
|
# HEAD request with no referer — combined log format
|
|
# Format: IP - - [date] "HEAD /path HTTP/x.x" status size "-" "user agent"
|
|
failregex = ^<HOST> \S+ \S+ \[.*\] "HEAD \S+ \S+" \d+ \d+ "-" ".*"
|
|
|
|
ignoreregex =
|
|
|
|
# Author: Phil Connor — https://mylinux.work
|
|
EOF
|
|
}
|
|
|
|
# ============================================================================
|
|
# INSTALL JAIL
|
|
# ============================================================================
|
|
|
|
install_jail() {
|
|
local jail_file="/etc/fail2ban/jail.d/head-crawler.conf"
|
|
|
|
log_step "Installing jail: $jail_file"
|
|
|
|
if $DRY_RUN; then
|
|
log_info "[DRY RUN] Would create $jail_file"
|
|
echo ""
|
|
generate_jail
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
if [[ -f "$jail_file" ]]; then
|
|
log_warn "Jail config already exists — backing up to ${jail_file}.bak"
|
|
cp "$jail_file" "${jail_file}.bak"
|
|
fi
|
|
|
|
generate_jail > "$jail_file"
|
|
log_info "Jail config installed: $jail_file"
|
|
}
|
|
|
|
generate_jail() {
|
|
cat <<EOF
|
|
# Fail2ban jail to block HEAD-only crawlers
|
|
# https://mylinux.work
|
|
#
|
|
# Bans IPs that send multiple HEAD requests with no referer.
|
|
# Default: 5 HEAD requests in 5 minutes = 24 hour ban.
|
|
#
|
|
# This catches scrapers and SEO bots that spoof real browser
|
|
# user agents and rotate through cloud provider IPs.
|
|
|
|
[head-crawler]
|
|
enabled = true
|
|
port = http,https
|
|
filter = head-crawler
|
|
logpath = $LOGPATH
|
|
maxretry = $MAXRETRY
|
|
findtime = $FINDTIME
|
|
bantime = $BANTIME
|
|
|
|
# Author: Phil Connor — https://mylinux.work
|
|
EOF
|
|
}
|
|
|
|
# ============================================================================
|
|
# RELOAD AND VERIFY
|
|
# ============================================================================
|
|
|
|
reload_fail2ban() {
|
|
log_step "Reloading Fail2ban..."
|
|
|
|
if $DRY_RUN; then
|
|
log_info "[DRY RUN] Would reload fail2ban"
|
|
return
|
|
fi
|
|
|
|
if ! fail2ban-client --test 2>/dev/null; then
|
|
log_warn "Config test not available — reloading directly"
|
|
fi
|
|
|
|
fail2ban-client reload
|
|
sleep 2
|
|
|
|
if systemctl is-active --quiet fail2ban; then
|
|
log_info "Fail2ban reloaded successfully"
|
|
else
|
|
log_error "Fail2ban failed to restart — check: journalctl -u fail2ban"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
verify_jail() {
|
|
log_step "Verifying head-crawler jail..."
|
|
|
|
if $DRY_RUN; then
|
|
log_info "[DRY RUN] Would verify jail status"
|
|
return
|
|
fi
|
|
|
|
echo ""
|
|
if fail2ban-client status head-crawler 2>/dev/null; then
|
|
echo ""
|
|
log_info "HEAD crawler jail is active and monitoring $LOGPATH"
|
|
else
|
|
log_error "Jail 'head-crawler' is not running — check: fail2ban-client status"
|
|
log_error "Debug with: fail2ban-regex $LOGPATH /etc/fail2ban/filter.d/head-crawler.conf"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
test_against_logs() {
|
|
if $DRY_RUN; then
|
|
# shellcheck disable=SC2086
|
|
local matches=( $LOGPATH )
|
|
if [[ -f "${matches[0]}" ]]; then
|
|
log_step "Testing filter against existing logs..."
|
|
echo ""
|
|
fail2ban-regex "${matches[0]}" /dev/stdin <<'FILTER' 2>&1 | tail -5
|
|
[Definition]
|
|
failregex = ^<HOST> \S+ \S+ \[.*\] "HEAD \S+ \S+" \d+ \d+ "-" ".*"
|
|
ignoreregex =
|
|
FILTER
|
|
echo ""
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# MAIN
|
|
# ============================================================================
|
|
|
|
main() {
|
|
parse_args "$@"
|
|
|
|
echo ""
|
|
echo "============================================"
|
|
echo " Fail2ban HEAD Crawler Blocker v${VERSION}"
|
|
echo " https://mylinux.work"
|
|
echo "============================================"
|
|
echo ""
|
|
|
|
check_root
|
|
check_fail2ban
|
|
|
|
if $REMOVE; then
|
|
do_remove
|
|
fi
|
|
|
|
detect_logpath
|
|
test_against_logs
|
|
install_filter
|
|
install_jail
|
|
reload_fail2ban
|
|
verify_jail
|
|
|
|
echo ""
|
|
echo "============================================"
|
|
echo " Setup Complete"
|
|
echo "============================================"
|
|
echo ""
|
|
echo " Jail: head-crawler"
|
|
echo " Log: $LOGPATH"
|
|
echo " Ban time: ${BANTIME}s ($(( BANTIME / 3600 ))h)"
|
|
echo " Max retry: $MAXRETRY (HEAD requests before ban)"
|
|
echo " Find time: ${FINDTIME}s ($(( FINDTIME / 60 ))m window)"
|
|
echo ""
|
|
echo " Useful commands:"
|
|
echo " fail2ban-client status head-crawler"
|
|
echo " fail2ban-client set head-crawler unbanip <IP>"
|
|
echo " fail2ban-regex $LOGPATH /etc/fail2ban/filter.d/head-crawler.conf"
|
|
echo ""
|
|
}
|
|
|
|
main "$@"
|