Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
@@ -0,0 +1,796 @@
|
||||
#!/bin/bash
|
||||
################################################################################
|
||||
# Script Name: hestia-js-challenge.sh
|
||||
# Version: 3.1
|
||||
# Description: Adds a lightweight JavaScript cookie challenge to nginx on
|
||||
# HestiaCP / VestaCP / myVesta servers. Creates an nginx map,
|
||||
# custom templates with JS challenge rules, and a challenge HTML
|
||||
# page. Bots that don't execute JavaScript are silently dropped.
|
||||
# Headless Chrome bots from suspect GeoIP regions with no external
|
||||
# referrer are tarpitted (served at 50 bytes/sec).
|
||||
# Detects and stacks on existing templates (e.g., geoip, botblock).
|
||||
# Works alongside hestia-bot-block.sh — run that first.
|
||||
#
|
||||
# Author: Phil Connor
|
||||
# Contact: contact@mylinux.work
|
||||
# Website: https://mylinux.work
|
||||
# License: MIT
|
||||
#
|
||||
# Prerequisites:
|
||||
# - HestiaCP, VestaCP, or myVesta installed
|
||||
# - nginx installed and running
|
||||
# - Root access
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./hestia-js-challenge.sh
|
||||
# sudo ./hestia-js-challenge.sh --dry-run
|
||||
# sudo ./hestia-js-challenge.sh --base-template default-botblock
|
||||
# sudo ./hestia-js-challenge.sh --base-template geoip-botblock --apply-all admin
|
||||
# sudo ./hestia-js-challenge.sh --remove
|
||||
#
|
||||
# How it works:
|
||||
# 1. Whitelisted bot UAs (Googlebot, Bingbot, etc.) bypass the check entirely
|
||||
# 2. All other visitors must have a cookie with a randomized name and token
|
||||
# 3. First-time visitors get a brief redirect to a challenge page that sets
|
||||
# the cookie via JS and bounces them back — takes < 100ms
|
||||
# 4. Bots that don't run JS never get the cookie and get 444'd
|
||||
# 5. Cookie name and token are randomized per installation — re-running the
|
||||
# script rotates them, immediately invalidating old pre-set cookies
|
||||
#
|
||||
# Changelog:
|
||||
# 3.1 — 2026-05-21: Challenge endpoint rate limiting. Headless Chrome bot farms
|
||||
# (China Unicom Shanghai, China Telecom) were passing the JS challenge on
|
||||
# every request by spawning fresh browser instances without persistent
|
||||
# cookies. They hit /_bc on every page load (~35s intervals) while real
|
||||
# users hit it once and keep the cookie for 24h. Added limit_req_zone on
|
||||
# the challenge endpoint: 3 requests allowed (burst), then 1/min sustained.
|
||||
# Excess requests get 444. Added --challenge-burst and --challenge-rate
|
||||
# options for tuning.
|
||||
# 3.0 — 2026-05-20: Referrer tracking through challenge redirect. Original
|
||||
# HTTP Referer is passed as &ref= param in the 302 redirect. Challenge
|
||||
# JS stores it in a _bc_ref cookie. Tarpit map: visitors from suspect
|
||||
# GeoIP countries (CN by default) with no external referrer are served
|
||||
# at 50 bytes/sec via limit_rate, draining headless Chrome resources.
|
||||
# Requires ngx_http_geoip2_module for GeoIP-based tarpitting.
|
||||
# Added --tarpit-countries and --tarpit-rate options.
|
||||
# 2.0 — 2026-05-19: Randomized cookie name and token per installation.
|
||||
# Cookie name is now a random 2-character suffix (e.g. _v7, _xq).
|
||||
# Cookie value is now a 32-char hex token instead of static "verified".
|
||||
# Values persist in /etc/nginx/js-challenge.env for --update-html-only.
|
||||
# Re-running rotates credentials and invalidates old bot bypass cookies.
|
||||
# Added no-cache headers on challenge page to prevent stale HTML after
|
||||
# rotation. Fixed challenge page Secure flag to be conditional on HTTPS.
|
||||
# 1.1 — 2026-05-13: Added --update-html-only to regenerate challenge page
|
||||
# without touching templates. Added Secure flag to cookie. Added
|
||||
# loop breaker — shows error message instead of infinite redirect
|
||||
# when browser blocks the cookie (Firefox strict mode).
|
||||
# 1.0 — 2026-05-11: Initial release
|
||||
#
|
||||
################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --- Configuration ---
|
||||
TEMPLATE_NAME="default-jschallenge"
|
||||
BASE_TEMPLATE="default"
|
||||
CONF_DIR="/etc/nginx/conf.d"
|
||||
PANEL_TPL_DIR=""
|
||||
PANEL_NAME=""
|
||||
CHALLENGE_MAP="${CONF_DIR}/js-challenge.conf"
|
||||
CHALLENGE_DIR="/var/www/js-challenge"
|
||||
CHALLENGE_HTML="${CHALLENGE_DIR}/challenge.html"
|
||||
STATE_FILE="/etc/nginx/js-challenge.env"
|
||||
CHALLENGE_PATH="/_bc"
|
||||
APPLY_USER=""
|
||||
APPLY_DOMAIN=""
|
||||
APPLY_ALL=false
|
||||
DRY_RUN=false
|
||||
REMOVE=false
|
||||
UPDATE_HTML_ONLY=false
|
||||
COOKIE_MAX_AGE=86400 # 24 hours
|
||||
TARPIT_COUNTRIES="${TARPIT_COUNTRIES:-CN}" # GeoIP country codes to tarpit (space-separated)
|
||||
TARPIT_RATE="${TARPIT_RATE:-50}" # bytes/sec for tarpitted responses
|
||||
CHALLENGE_RATE="${CHALLENGE_RATE:-1}" # sustained challenge requests per minute per IP
|
||||
CHALLENGE_BURST="${CHALLENGE_BURST:-3}" # initial burst of challenge requests allowed
|
||||
TIMESTAMP=$(date +%s)
|
||||
|
||||
# --- Colors ---
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
CYAN='\033[0;36m'
|
||||
BOLD='\033[1m'
|
||||
NC='\033[0m'
|
||||
|
||||
info() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
step() { echo -e "${CYAN}[STEP]${NC} $*"; }
|
||||
|
||||
detect_panel() {
|
||||
if [[ -d "/usr/local/hestia/data/templates/web/nginx" ]]; then
|
||||
PANEL_TPL_DIR="/usr/local/hestia/data/templates/web/nginx"
|
||||
PANEL_NAME="HestiaCP"
|
||||
elif [[ -d "/usr/local/vesta/data/templates/web/nginx" ]]; then
|
||||
PANEL_TPL_DIR="/usr/local/vesta/data/templates/web/nginx"
|
||||
PANEL_NAME="VestaCP/myVesta"
|
||||
else
|
||||
echo -e "${RED}Error: Neither HestiaCP nor VestaCP/myVesta found${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
info "Detected ${PANEL_NAME} (${PANEL_TPL_DIR})"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: sudo $(basename "$0") [OPTIONS]
|
||||
|
||||
Adds a JavaScript cookie challenge to nginx on HestiaCP / VestaCP / myVesta.
|
||||
Visitors that don't execute JavaScript (headless scrapers, curl-based bots) are
|
||||
silently dropped. Legitimate search engine crawlers are whitelisted by user agent.
|
||||
|
||||
This is designed to work alongside hestia-bot-block.sh — run that first to block
|
||||
known bad bots, then use this to catch the ones spoofing real browser user agents.
|
||||
|
||||
The cookie name and token are randomized per installation. Re-running the script
|
||||
rotates them, immediately invalidating any bot-cached bypass cookies.
|
||||
|
||||
Options:
|
||||
--template-name NAME Custom template name (default: default-jschallenge)
|
||||
--base-template NAME Template to build on (default: default)
|
||||
Use this to stack on existing templates (e.g., default-botblock,
|
||||
geoip-botblock)
|
||||
--apply USER DOMAIN Apply template to a specific domain after creation
|
||||
--apply-all USER Apply template to all domains for a user
|
||||
--cookie-ttl SECONDS Cookie lifetime in seconds (default: 86400 / 24h)
|
||||
--tarpit-countries CC Space-separated GeoIP country codes to tarpit (default: CN)
|
||||
--tarpit-rate BYTES Response rate in bytes/sec for tarpitted visitors (default: 50)
|
||||
--challenge-rate N Sustained challenge requests per minute per IP (default: 2)
|
||||
--challenge-burst N Initial burst of challenge requests allowed (default: 3)
|
||||
--dry-run Show what would be done without making changes
|
||||
--update-html-only Regenerate the challenge HTML page only — no template changes
|
||||
(reuses existing cookie name and token from state file)
|
||||
--remove Remove challenge config, HTML, and templates
|
||||
-h, --help Show this help
|
||||
|
||||
Examples:
|
||||
sudo $(basename "$0")
|
||||
sudo $(basename "$0") --update-html-only
|
||||
sudo $(basename "$0") --dry-run
|
||||
sudo $(basename "$0") --base-template default-botblock
|
||||
sudo $(basename "$0") --base-template geoip-botblock --template-name geoip-botblock-jsc
|
||||
sudo $(basename "$0") --apply admin example.com
|
||||
sudo $(basename "$0") --apply-all admin
|
||||
sudo $(basename "$0") --remove
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
# --- Argument parsing ---
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--template-name) TEMPLATE_NAME="$2"; shift 2 ;;
|
||||
--base-template) BASE_TEMPLATE="$2"; shift 2 ;;
|
||||
--apply) APPLY_USER="$2"; APPLY_DOMAIN="$3"; shift 3 ;;
|
||||
--apply-all) APPLY_USER="$2"; APPLY_ALL=true; shift 2 ;;
|
||||
--cookie-ttl) COOKIE_MAX_AGE="$2"; shift 2 ;;
|
||||
--tarpit-countries) TARPIT_COUNTRIES="$2"; shift 2 ;;
|
||||
--tarpit-rate) TARPIT_RATE="$2"; shift 2 ;;
|
||||
--challenge-rate) CHALLENGE_RATE="$2"; shift 2 ;;
|
||||
--challenge-burst) CHALLENGE_BURST="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--update-html-only) UPDATE_HTML_ONLY=true; shift ;;
|
||||
--remove) REMOVE=true; shift ;;
|
||||
-h|--help) usage ;;
|
||||
*) echo "Unknown option: $1"; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- Root check ---
|
||||
if [[ $EUID -ne 0 && "$DRY_RUN" != "true" ]]; then
|
||||
echo -e "${RED}[ERROR] Must run as root (or use --dry-run)${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Generate or load cookie credentials
|
||||
# =====================================================
|
||||
|
||||
generate_credentials() {
|
||||
COOKIE_NAME="_$(openssl rand -hex 1)"
|
||||
COOKIE_VALUE="$(openssl rand -hex 16)"
|
||||
COOKIE_VAR="\$cookie_${COOKIE_NAME}"
|
||||
}
|
||||
|
||||
save_credentials() {
|
||||
if [[ "$DRY_RUN" != "true" ]]; then
|
||||
cat > "$STATE_FILE" <<ENVEOF
|
||||
# JS challenge credentials — generated $(date -Iseconds)
|
||||
# Re-run the script to rotate these values
|
||||
COOKIE_NAME='${COOKIE_NAME}'
|
||||
COOKIE_VALUE='${COOKIE_VALUE}'
|
||||
CHALLENGE_PATH='${CHALLENGE_PATH}'
|
||||
ENVEOF
|
||||
chmod 600 "$STATE_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
load_credentials() {
|
||||
if [[ -f "$STATE_FILE" ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$STATE_FILE"
|
||||
COOKIE_VAR="\$cookie_${COOKIE_NAME}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "$UPDATE_HTML_ONLY" == "true" ]]; then
|
||||
if load_credentials; then
|
||||
info "Loaded existing credentials from ${STATE_FILE}"
|
||||
info " Cookie name: ${COOKIE_NAME} Token: ${COOKIE_VALUE:0:8}..."
|
||||
else
|
||||
echo -e "${RED}[ERROR] No state file found at ${STATE_FILE}${NC}" >&2
|
||||
echo " Run without --update-html-only first to generate credentials." >&2
|
||||
exit 1
|
||||
fi
|
||||
elif [[ "$REMOVE" != "true" ]]; then
|
||||
generate_credentials
|
||||
info "Generated new credentials — cookie: ${COOKIE_NAME} token: ${COOKIE_VALUE:0:8}..."
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Remove mode
|
||||
# =====================================================
|
||||
if [[ "$REMOVE" == "true" ]]; then
|
||||
step "Removing JS challenge configuration"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would remove: ${CHALLENGE_MAP}"
|
||||
echo " Would remove: ${CHALLENGE_DIR}"
|
||||
echo " Would remove: ${STATE_FILE}"
|
||||
echo " Would run: nginx -t && systemctl reload nginx"
|
||||
else
|
||||
[[ -f "$CHALLENGE_MAP" ]] && rm -f "$CHALLENGE_MAP" && info "Removed: ${CHALLENGE_MAP}"
|
||||
[[ -d "$CHALLENGE_DIR" ]] && rm -rf "$CHALLENGE_DIR" && info "Removed: ${CHALLENGE_DIR}"
|
||||
[[ -f "$STATE_FILE" ]] && rm -f "$STATE_FILE" && info "Removed: ${STATE_FILE}"
|
||||
|
||||
if nginx -t 2>&1; then
|
||||
systemctl reload nginx
|
||||
info "nginx reloaded"
|
||||
else
|
||||
echo -e "${RED}[ERROR] nginx config test failed after removal${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}JS challenge removed.${NC}"
|
||||
echo ""
|
||||
echo " Note: You should also remove or switch away from the JS challenge"
|
||||
echo " templates on your domains:"
|
||||
echo " v-change-web-domain-proxy-tpl <user> <domain> <previous-template>"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Panel detection ---
|
||||
detect_panel
|
||||
|
||||
if ! command -v nginx &>/dev/null; then
|
||||
echo -e "${RED}Error: nginx not found${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Step 1: Create the challenge HTML page
|
||||
# =====================================================
|
||||
step "Creating challenge page at ${CHALLENGE_HTML}"
|
||||
|
||||
CHALLENGE_CONTENT='<!DOCTYPE html>
|
||||
<html>
|
||||
<head><meta charset="utf-8"><title>Verifying</title></head>
|
||||
<body>
|
||||
<noscript><p>JavaScript is required to access this site.</p></noscript>
|
||||
<p id="msg" style="display:none;font-family:sans-serif;text-align:center;margin-top:2em;">
|
||||
Cookies must be enabled to access this site.</p>
|
||||
<script>
|
||||
(function(){
|
||||
var p = new URLSearchParams(window.location.search);
|
||||
var r = p.get("r") || "/";
|
||||
if (r.charAt(0) !== "/") r = "/";
|
||||
var cn = "'"${COOKIE_NAME}"'";
|
||||
var cv = "'"${COOKIE_VALUE}"'";
|
||||
var secure = (location.protocol === "https:") ? ";Secure" : "";
|
||||
document.cookie = cn + "=" + cv + ";path=/;max-age='"${COOKIE_MAX_AGE}"';SameSite=Lax" + secure;
|
||||
var origRef = p.get("ref") || "direct";
|
||||
if (origRef !== "direct") {
|
||||
try { var rh = new URL(origRef).hostname; if (rh === location.hostname) origRef = "direct"; } catch(e) {}
|
||||
}
|
||||
document.cookie = "_bc_ref=" + encodeURIComponent(origRef) + ";path=/;max-age=120;SameSite=Lax" + secure;
|
||||
if (document.cookie.split(/;\s*/).every(function(c){ return c.indexOf(cn + "=") !== 0; })) {
|
||||
document.getElementById("msg").style.display = "block";
|
||||
return;
|
||||
}
|
||||
window.location.replace(r);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>'
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would create: ${CHALLENGE_DIR}/"
|
||||
echo " Would create: ${CHALLENGE_HTML}"
|
||||
else
|
||||
mkdir -p "$CHALLENGE_DIR"
|
||||
echo "$CHALLENGE_CONTENT" > "$CHALLENGE_HTML"
|
||||
info "Challenge page created: ${CHALLENGE_HTML}"
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# If --update-html-only, stop here
|
||||
# =====================================================
|
||||
if [[ "$UPDATE_HTML_ONLY" == "true" ]]; then
|
||||
echo ""
|
||||
echo -e "${BOLD}Done.${NC} Challenge HTML updated — templates unchanged."
|
||||
echo " HTML: ${CHALLENGE_HTML}"
|
||||
echo " Cookie: ${COOKIE_NAME}=${COOKIE_VALUE:0:8}..."
|
||||
echo ""
|
||||
echo " No nginx reload needed — the HTML is served as a static file."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Save credentials for future --update-html-only runs
|
||||
save_credentials
|
||||
|
||||
# =====================================================
|
||||
# Step 2: Create nginx map config
|
||||
# =====================================================
|
||||
step "Creating JS challenge map at ${CHALLENGE_MAP}"
|
||||
|
||||
# Build the cookie variable name for nginx (e.g. _v7 → $cookie__v7)
|
||||
NGINX_COOKIE_VAR="\$cookie_${COOKIE_NAME}"
|
||||
|
||||
# Check if a geoip2 block already loads an mmdb anywhere in nginx config.
|
||||
# If so, $geoip2_data_country_code should already be defined — don't duplicate.
|
||||
# Exclude our own file and backup files from the search.
|
||||
GEOIP2_BLOCK=""
|
||||
if ! grep -r 'geoip2.*\.mmdb' /etc/nginx/ \
|
||||
--include='*.conf' --exclude='js-challenge.conf' --exclude='*.bak.*' \
|
||||
-q 2>/dev/null; then
|
||||
GEOIP2_BLOCK='
|
||||
# ── GeoIP2: country lookup for tarpit decisions ──────────────────────
|
||||
# Uses the City database (superset of Country). Adjust path if needed.
|
||||
geoip2 /usr/share/GeoIP/GeoLite2-City.mmdb {
|
||||
$geoip2_country_code country iso_code;
|
||||
}
|
||||
'
|
||||
step "No existing geoip2 country_code config found — adding to map config"
|
||||
fi
|
||||
|
||||
# Collect server_name values from nginx configs to build same-site referer map
|
||||
local REFERER_ENTRIES=""
|
||||
local _jsc_domain_seen=()
|
||||
for _conf in /etc/nginx/conf.d/*.conf /etc/nginx/sites-enabled/*; do
|
||||
[[ -f "$_conf" ]] || continue
|
||||
while read -r _sn; do
|
||||
for _d in $_sn; do
|
||||
[[ "$_d" == "server_name" || "$_d" == ";" || "$_d" == "_" || "$_d" =~ ^[0-9] ]] && continue
|
||||
_d="${_d%;}"
|
||||
[[ " ${_jsc_domain_seen[*]:-} " == *" $_d "* ]] && continue
|
||||
_jsc_domain_seen+=("$_d")
|
||||
local _d_escaped="${_d//./\\.}"
|
||||
REFERER_ENTRIES+=" ~^1:https?://${_d_escaped} 1;\n"
|
||||
done
|
||||
done < <(grep -oP '^\s*server_name\s+\K[^;]+;?' "$_conf" 2>/dev/null)
|
||||
done
|
||||
|
||||
if [[ -z "$REFERER_ENTRIES" ]]; then
|
||||
warn "No server_name values found — same-site image bypass will not work"
|
||||
warn "Images behind the challenge may cause redirect loops for browsers"
|
||||
fi
|
||||
|
||||
MAP_CONTENT='# JS cookie challenge — allowed bots and cookie check
|
||||
# Generated by hestia-js-challenge.sh — https://mylinux.work
|
||||
# Cookie: '"${COOKIE_NAME}"' Token: '"${COOKIE_VALUE:0:8}"'...
|
||||
# Generated: '"$(date -Iseconds)"'
|
||||
|
||||
# ── Rate limit: challenge endpoint ───────────────────────────────────
|
||||
# Real users hit the challenge once and keep the cookie. Headless bot farms
|
||||
# spawn fresh browsers per request, hitting the challenge every time.
|
||||
# Rate: '"${CHALLENGE_RATE}"'r/m with burst of '"${CHALLENGE_BURST}"' — excess gets 444.
|
||||
limit_req_zone $binary_remote_addr zone=jschallenge:10m rate='"${CHALLENGE_RATE}"'r/m;
|
||||
|
||||
# Bots that legitimately identify themselves and should bypass the JS check
|
||||
map $http_user_agent $is_allowed_bot {
|
||||
default 0;
|
||||
|
||||
# Search engines
|
||||
~*Googlebot 1;
|
||||
~*bingbot 1;
|
||||
~*Slurp 1;
|
||||
~*DuckDuckBot 1;
|
||||
~*DuckAssistBot 1;
|
||||
~*Baiduspider 1;
|
||||
~*YandexBot 1;
|
||||
~*YandexFavicons 1;
|
||||
~*Applebot 1;
|
||||
~*Qwantbot 1;
|
||||
~*Qwantify 1;
|
||||
~*Bravebot 1;
|
||||
~*kagi-fetcher 1;
|
||||
~*Kagibot 1;
|
||||
~*Yahoo! 1;
|
||||
~*Yeti 1;
|
||||
|
||||
# Social media / link previews
|
||||
~*facebookexternalhit 1;
|
||||
~*Facebot 1;
|
||||
~*Twitterbot 1;
|
||||
~*LinkedInBot 1;
|
||||
~*Slackbot 1;
|
||||
~*Slack-ImgProxy 1;
|
||||
~*Discordbot 1;
|
||||
~*TelegramBot 1;
|
||||
~*WhatsApp 1;
|
||||
~*redditbot 1;
|
||||
~*ArenaUnfurlBot 1;
|
||||
|
||||
# Feed readers
|
||||
~*Feedly 1;
|
||||
~*Miniflux 1;
|
||||
~*FreshRSS 1;
|
||||
~*NewsBlur 1;
|
||||
~*Tiny\ Tiny\ RSS 1;
|
||||
~*Inoreader 1;
|
||||
~*NetNewsWire 1;
|
||||
|
||||
# Monitoring / uptime
|
||||
~*UptimeRobot 1;
|
||||
~*Pingdom 1;
|
||||
~*StatusCake 1;
|
||||
~*Blackbox-Exporter 1;
|
||||
|
||||
# AI answer bots (user-facing, not training crawlers)
|
||||
~*OAI-SearchBot 1;
|
||||
~*ChatGPT-User 1;
|
||||
~*Claude-Web 1;
|
||||
~*Claude-User 1;
|
||||
~*MistralAI-User 1;
|
||||
|
||||
# Archive / research
|
||||
~*archive\.org_bot 1;
|
||||
|
||||
# Apple Safari prefetch
|
||||
~*safarifetcherd 1;
|
||||
|
||||
# Link checkers / validators
|
||||
~*W3C_Validator 1;
|
||||
~*W3C-checklink 1;
|
||||
~*LinkChecker 1;
|
||||
~*link-check 1;
|
||||
|
||||
# Decentralized search
|
||||
~*yacybot 1;
|
||||
|
||||
# Add your own allowed bots below
|
||||
}
|
||||
|
||||
# Validate the challenge cookie — exact token match
|
||||
map '"${NGINX_COOKIE_VAR}"' $js_cookie_valid {
|
||||
default 0;
|
||||
"'"${COOKIE_VALUE}"'" 1;
|
||||
}
|
||||
|
||||
# Detect requests to the challenge page and download paths (prevent redirect loops)
|
||||
map $uri $is_challenge_uri {
|
||||
default 0;
|
||||
"'"${CHALLENGE_PATH}"'" 1;
|
||||
~^/downloads/ 1;
|
||||
~*\.(css|js|woff2?)$ 1;
|
||||
~*favicon 1;
|
||||
~*apple-touch-icon 1;
|
||||
}
|
||||
|
||||
# Detect image sub-resource requests with same-site referer (browser <img> loads)
|
||||
# These bypass the challenge because: (a) images cannot execute JS challenges,
|
||||
# and (b) the same-site referer proves the browser loaded a page from this domain.
|
||||
# Direct image requests from scrapers (no referer or external referer) still get challenged.
|
||||
map $uri $is_image_request {
|
||||
default 0;
|
||||
~*\.(png|jpe?g|gif|svg|webp|ico|avif)$ 1;
|
||||
}
|
||||
map "$is_image_request:$http_referer" $is_samesite_image {
|
||||
default 0;
|
||||
'"${REFERER_ENTRIES}"'}
|
||||
|
||||
# Combined check: need challenge if not allowed bot, no valid cookie, and not the challenge page
|
||||
map "$is_allowed_bot:$js_cookie_valid:$is_challenge_uri:$is_samesite_image" $needs_js_challenge {
|
||||
default 1;
|
||||
"1:0:0:0" 0;
|
||||
"1:0:0:1" 0;
|
||||
"1:0:1:0" 0;
|
||||
"1:0:1:1" 0;
|
||||
"1:1:0:0" 0;
|
||||
"1:1:0:1" 0;
|
||||
"1:1:1:0" 0;
|
||||
"1:1:1:1" 0;
|
||||
"0:1:0:0" 0;
|
||||
"0:1:0:1" 0;
|
||||
"0:1:1:0" 0;
|
||||
"0:1:1:1" 0;
|
||||
"0:0:1:0" 0;
|
||||
"0:0:1:1" 0;
|
||||
"0:0:0:1" 0;
|
||||
}
|
||||
'"${GEOIP2_BLOCK}"'
|
||||
# ── Tarpit: headless Chrome bots from suspect regions ─────────────────
|
||||
# Visitors from tarpit countries with no external referrer (passed through
|
||||
# the challenge redirect as the _bc_ref cookie) are served at a crawl.
|
||||
# This drains headless Chrome resources (~200-500 MB RAM per instance)
|
||||
# without giving the bot a clear "blocked" signal to adapt to.
|
||||
#
|
||||
# The _bc_ref cookie is set by the challenge page JS from the &ref= param.
|
||||
# It contains the original HTTP Referer before the 302 redirect destroyed it.
|
||||
# "direct" = no external referrer (typed URL or bot). Cookie expires in 120s.
|
||||
|
||||
# Check if visitor is from a tarpit country (requires geoip2 module)
|
||||
map $geoip2_country_code $is_tarpit_country {
|
||||
default 0;
|
||||
'"$(for cc in $TARPIT_COUNTRIES; do echo " \"${cc}\" 1;"; done)"'
|
||||
}
|
||||
|
||||
# Tarpit only if: tarpit country + no external referrer + passed JS challenge
|
||||
map "$is_tarpit_country:$cookie__bc_ref" $tarpit_client {
|
||||
default 0;
|
||||
"1:direct" 1;
|
||||
"1:" 1;
|
||||
}
|
||||
|
||||
# Serve the challenge page
|
||||
server {
|
||||
listen 127.0.0.1:18444;
|
||||
server_name _;
|
||||
root /var/www/js-challenge;
|
||||
|
||||
location / {
|
||||
add_header Cache-Control "no-store, no-cache, must-revalidate" always;
|
||||
add_header Pragma "no-cache" always;
|
||||
try_files /challenge.html =404;
|
||||
}
|
||||
}'
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would create: ${CHALLENGE_MAP}"
|
||||
else
|
||||
if [[ -f "$CHALLENGE_MAP" ]]; then
|
||||
cp "$CHALLENGE_MAP" "${CHALLENGE_MAP}.bak.${TIMESTAMP}"
|
||||
warn "Existing config backed up"
|
||||
fi
|
||||
echo "$MAP_CONTENT" > "$CHALLENGE_MAP"
|
||||
info "Map config created: ${CHALLENGE_MAP}"
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Step 3: Create custom Hestia templates
|
||||
# =====================================================
|
||||
JS_CHALLENGE_DIRECTIVE='
|
||||
# JS cookie challenge — added by hestia-js-challenge.sh
|
||||
location = '"${CHALLENGE_PATH}"' {
|
||||
limit_req zone=jschallenge burst='"${CHALLENGE_BURST}"' nodelay;
|
||||
limit_req_status 444;
|
||||
proxy_pass http://127.0.0.1:18444/;
|
||||
}
|
||||
|
||||
# Redirect non-JS visitors to challenge page (pass original referrer)
|
||||
if ($needs_js_challenge) {
|
||||
return 302 '"${CHALLENGE_PATH}"'?r=$request_uri&ref=$http_referer;
|
||||
}
|
||||
|
||||
# Tarpit headless Chrome bots from suspect GeoIP regions
|
||||
if ($tarpit_client) {
|
||||
set $limit_rate '"${TARPIT_RATE}"';
|
||||
}'
|
||||
|
||||
create_template() {
|
||||
local src="$1" dst="$2" label="$3"
|
||||
|
||||
if [[ ! -f "$src" ]]; then
|
||||
warn "Source template not found: ${src} — skipping ${label}"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would create: ${dst} (from ${src})"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -f "$dst" ]]; then
|
||||
cp "$dst" "${dst}.bak.${TIMESTAMP}"
|
||||
warn "Existing ${label} template backed up"
|
||||
fi
|
||||
|
||||
# Check if the source already has JS challenge (avoid double-injection)
|
||||
if grep -q 'needs_js_challenge' "$src"; then
|
||||
cp "$src" "$dst"
|
||||
info "Created ${label} template: ${dst} (JS challenge already present in base)"
|
||||
else
|
||||
# Insert JS challenge directive before the first 'location' line
|
||||
awk -v block="$JS_CHALLENGE_DIRECTIVE" '
|
||||
!inserted && /^[[:space:]]*location[[:space:]]/ {
|
||||
print block
|
||||
print ""
|
||||
inserted = 1
|
||||
}
|
||||
{ print }
|
||||
' "$src" > "$dst"
|
||||
info "Created ${label} template: ${dst}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Resolve the base template — verify it exists
|
||||
BASE_TPL="${PANEL_TPL_DIR}/${BASE_TEMPLATE}.tpl"
|
||||
if [[ ! -f "$BASE_TPL" ]]; then
|
||||
if [[ "$BASE_TEMPLATE" != "default" ]]; then
|
||||
warn "Base template '${BASE_TEMPLATE}' not found — falling back to 'default'"
|
||||
BASE_TEMPLATE="default"
|
||||
fi
|
||||
BASE_TPL="${PANEL_TPL_DIR}/${BASE_TEMPLATE}.tpl"
|
||||
if [[ ! -f "$BASE_TPL" ]]; then
|
||||
echo -e "${RED}Error: Default template not found: ${BASE_TPL}${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
step "Creating custom ${PANEL_NAME} nginx templates (${TEMPLATE_NAME}) from ${BASE_TEMPLATE}"
|
||||
|
||||
# Proxy templates
|
||||
create_template \
|
||||
"${PANEL_TPL_DIR}/${BASE_TEMPLATE}.tpl" \
|
||||
"${PANEL_TPL_DIR}/${TEMPLATE_NAME}.tpl" \
|
||||
"HTTP (.tpl)"
|
||||
|
||||
create_template \
|
||||
"${PANEL_TPL_DIR}/${BASE_TEMPLATE}.stpl" \
|
||||
"${PANEL_TPL_DIR}/${TEMPLATE_NAME}.stpl" \
|
||||
"SSL (.stpl)"
|
||||
|
||||
# php-fpm templates (if they exist for the base)
|
||||
if [[ -d "${PANEL_TPL_DIR}/php-fpm" ]]; then
|
||||
if [[ -f "${PANEL_TPL_DIR}/php-fpm/${BASE_TEMPLATE}.tpl" ]]; then
|
||||
create_template \
|
||||
"${PANEL_TPL_DIR}/php-fpm/${BASE_TEMPLATE}.tpl" \
|
||||
"${PANEL_TPL_DIR}/php-fpm/${TEMPLATE_NAME}.tpl" \
|
||||
"php-fpm HTTP (.tpl)"
|
||||
|
||||
create_template \
|
||||
"${PANEL_TPL_DIR}/php-fpm/${BASE_TEMPLATE}.stpl" \
|
||||
"${PANEL_TPL_DIR}/php-fpm/${TEMPLATE_NAME}.stpl" \
|
||||
"php-fpm SSL (.stpl)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Copy .sh hooks from the base template if they exist
|
||||
for ext in tpl stpl; do
|
||||
base_sh="${PANEL_TPL_DIR}/${BASE_TEMPLATE}.${ext}.sh"
|
||||
dst_sh="${PANEL_TPL_DIR}/${TEMPLATE_NAME}.${ext}.sh"
|
||||
if [[ -f "$base_sh" && ! -f "$dst_sh" ]]; then
|
||||
cp "$base_sh" "$dst_sh"
|
||||
info "Copied hook: ${dst_sh}"
|
||||
fi
|
||||
done
|
||||
|
||||
# =====================================================
|
||||
# Step 4: Validate nginx config
|
||||
# =====================================================
|
||||
step "Testing nginx configuration"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would run: nginx -t"
|
||||
else
|
||||
if nginx -t 2>&1; then
|
||||
info "nginx config valid"
|
||||
else
|
||||
echo -e "${RED}[ERROR] nginx config test failed${NC}" >&2
|
||||
echo " Restore backups from ${PANEL_TPL_DIR} and ${CONF_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Step 5: Apply template (optional)
|
||||
# =====================================================
|
||||
if [[ -n "$APPLY_USER" ]]; then
|
||||
if ! command -v v-change-web-domain-proxy-tpl &>/dev/null; then
|
||||
echo -e "${RED}Error: v-change-web-domain-proxy-tpl not found${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$APPLY_ALL" == "true" ]]; then
|
||||
step "Applying template to all domains for user: ${APPLY_USER}"
|
||||
domains=$(v-list-web-domains "$APPLY_USER" plain 2>/dev/null | awk '{print $1}')
|
||||
if [[ -z "$domains" ]]; then
|
||||
warn "No domains found for user: ${APPLY_USER}"
|
||||
else
|
||||
while IFS= read -r domain; do
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would apply: v-change-web-domain-proxy-tpl ${APPLY_USER} ${domain} ${TEMPLATE_NAME}"
|
||||
else
|
||||
v-change-web-domain-proxy-tpl "$APPLY_USER" "$domain" "$TEMPLATE_NAME"
|
||||
info "Applied to: ${domain}"
|
||||
fi
|
||||
done <<< "$domains"
|
||||
fi
|
||||
else
|
||||
step "Applying template to: ${APPLY_DOMAIN}"
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would apply: v-change-web-domain-proxy-tpl ${APPLY_USER} ${APPLY_DOMAIN} ${TEMPLATE_NAME}"
|
||||
else
|
||||
v-change-web-domain-proxy-tpl "$APPLY_USER" "$APPLY_DOMAIN" "$TEMPLATE_NAME"
|
||||
info "Applied to: ${APPLY_DOMAIN}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Step 6: Reload nginx
|
||||
# =====================================================
|
||||
step "Reloading nginx"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo " Would run: systemctl reload nginx"
|
||||
else
|
||||
systemctl reload nginx
|
||||
info "nginx reloaded"
|
||||
fi
|
||||
|
||||
# =====================================================
|
||||
# Summary
|
||||
# =====================================================
|
||||
echo ""
|
||||
echo -e "${BOLD}Done.${NC}"
|
||||
echo ""
|
||||
echo " Challenge map: ${CHALLENGE_MAP}"
|
||||
echo " Challenge page: ${CHALLENGE_HTML}"
|
||||
echo " State file: ${STATE_FILE}"
|
||||
echo " Base template: ${BASE_TEMPLATE}"
|
||||
echo " Template: ${TEMPLATE_NAME} (.tpl + .stpl)"
|
||||
echo " Cookie name: ${COOKIE_NAME}"
|
||||
echo " Cookie token: ${COOKIE_VALUE:0:8}... (32 hex chars)"
|
||||
echo " Cookie TTL: ${COOKIE_MAX_AGE}s"
|
||||
echo " Challenge rate: ${CHALLENGE_RATE}r/m (burst: ${CHALLENGE_BURST})"
|
||||
if [[ -n "$APPLY_USER" ]]; then
|
||||
if [[ "$APPLY_ALL" == "true" ]]; then
|
||||
echo " Applied: All domains for ${APPLY_USER}"
|
||||
else
|
||||
echo " Applied: ${APPLY_DOMAIN}"
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo " To apply to a domain:"
|
||||
echo " v-change-web-domain-proxy-tpl <user> <domain> ${TEMPLATE_NAME}"
|
||||
echo ""
|
||||
echo " To apply to all domains for a user:"
|
||||
echo " sudo $(basename "$0") --apply-all <user>"
|
||||
fi
|
||||
echo ""
|
||||
echo " To rotate credentials (invalidate bot-cached cookies):"
|
||||
echo " sudo $(basename "$0") --base-template ${BASE_TEMPLATE}"
|
||||
echo ""
|
||||
echo " Test (bot without cookie gets redirected to challenge):"
|
||||
echo " curl -o /dev/null -s -w '%{http_code}' https://yourdomain.com"
|
||||
echo " Expected: 302"
|
||||
echo ""
|
||||
echo " Test (browser completes challenge — 302 → 200):"
|
||||
echo " Open https://yourdomain.com in a browser"
|
||||
echo " Expected: brief redirect then page loads normally"
|
||||
echo ""
|
||||
echo " Test (old static bypass no longer works):"
|
||||
echo " curl -b '_bc=verified' -o /dev/null -s -w '%{http_code}' https://yourdomain.com"
|
||||
echo " Expected: 302 (not 200 — old cookie is invalid)"
|
||||
echo ""
|
||||
echo " Test (rate limit on challenge endpoint):"
|
||||
echo " for i in 1 2 3 4 5; do curl -o /dev/null -s -w \"\$i: %{http_code}\n\" https://yourdomain.com${CHALLENGE_PATH}; done"
|
||||
echo " Expected: first 3 return 200, then 444 (rate limited)"
|
||||
echo ""
|
||||
echo " Test (allowed bot bypasses challenge):"
|
||||
echo " curl -A 'Googlebot' -o /dev/null -s -w '%{http_code}' https://yourdomain.com"
|
||||
echo " Expected: 200"
|
||||
Reference in New Issue
Block a user