Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.
This commit is contained in:
2026-05-25 03:31:08 +02:00
parent dbd6bf0324
commit a1a17e81a1
332 changed files with 174509 additions and 1106 deletions
+545
View File
@@ -0,0 +1,545 @@
#!/bin/bash
################################################################################
# Script Name: ip-intel-exporter.sh
# Version: 1.1
# Description: Prometheus exporter for IP intelligence metrics from nginx logs.
# Parses access logs (GeoIP-enriched or standard), classifies
# traffic by country, ASN, cloud provider, and threat type.
#
# Author: Phil Connor
# Contact: contact@mylinux.work
# Website: https://mylinux.work
# License: MIT
#
# Prerequisites:
# - nginx access log (enriched or combined format)
# - mmdblookup (mmdb-bin) for standard log format GeoIP lookups
# - netcat (nc) for HTTP mode
#
# Usage:
# ./ip-intel-exporter.sh # stdout
# ./ip-intel-exporter.sh --textfile # node_exporter textfile
# ./ip-intel-exporter.sh --http # HTTP server on port 9199
# ./ip-intel-exporter.sh --log /path/to/access.log
#
################################################################################
set -euo pipefail
# ============================================================================
# CONFIGURATION
# ============================================================================
NGINX_LOG="/var/log/nginx/access.log"
TEXTFILE_DIR="/var/lib/node_exporter"
OUTPUT_FILE=""
HTTP_MODE=false
HTTP_PORT=9199
LOG_FORMAT="auto"
# shellcheck disable=SC2034
TOP_N=10
# shellcheck disable=SC2034
MMDB_CITY="/usr/share/GeoIP/GeoLite2-City.mmdb"
# shellcheck disable=SC2034
MMDB_ASN="/usr/share/GeoIP/GeoLite2-ASN.mmdb"
# ============================================================================
# COLORS
# ============================================================================
if [[ -t 1 ]]; then
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[0;33m'; RESET='\033[0m'
else
RED="" GREEN="" YELLOW="" RESET=""
fi
log() { echo -e "${GREEN}[OK]${RESET} $*" >&2; }
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
# Sanitize a string for use as a Prometheus label value.
# Converts \xNN escape sequences to real UTF-8, escapes backslashes and quotes.
prom_label() {
local val
val=$(printf '%b' "$1")
val="${val//\\/\\\\}"
val="${val//\"/\\\"}"
printf '%s' "$val"
}
# ============================================================================
# USAGE
# ============================================================================
show_usage() {
cat <<EOF
Usage: $(basename "$0") [OPTIONS]
Export IP intelligence metrics from nginx access logs as Prometheus metrics.
MODES:
--textfile Write to node_exporter textfile collector
--http Run HTTP server on port ${HTTP_PORT}
(default) Print metrics to stdout
OPTIONS:
--log PATH Nginx access log path (default: ${NGINX_LOG})
--format FMT Log format: enriched, standard, auto (default: auto)
-p, --port N HTTP port (default: ${HTTP_PORT})
-o, --output F Output file path
-h, --help Show this help
LOG FORMATS:
enriched GeoIP-enriched: includes country code and ASN org fields
standard Standard nginx combined format (uses mmdblookup if available)
auto Auto-detect based on log content
EXAMPLES:
$(basename "$0") --textfile
$(basename "$0") --http --port 9199
$(basename "$0") --log /var/log/nginx/access.log --format enriched
EOF
exit 0
}
# ============================================================================
# ARGUMENT PARSING
# ============================================================================
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help) show_usage ;;
--textfile) OUTPUT_FILE="${TEXTFILE_DIR}/ip_intel.prom"; shift ;;
--http) HTTP_MODE=true; shift ;;
-p|--port) HTTP_PORT="$2"; shift 2 ;;
-o|--output) OUTPUT_FILE="$2"; shift 2 ;;
--log) NGINX_LOG="$2"; shift 2 ;;
--format) LOG_FORMAT="$2"; shift 2 ;;
*) err "Unknown option: $1"; show_usage ;;
esac
done
}
# ============================================================================
# PREFLIGHT
# ============================================================================
preflight() {
if [[ ! -f "$NGINX_LOG" ]]; then
err "Nginx log not found: ${NGINX_LOG}"
exit 1
fi
if [[ ! -r "$NGINX_LOG" ]]; then
err "Cannot read: ${NGINX_LOG} (check permissions)"
exit 1
fi
# Detect log format
if [[ "$LOG_FORMAT" == "auto" ]]; then
# Enriched format ends with: COUNTRY_CODE "ASN Org"
if tail -20 "$NGINX_LOG" | grep -qP '\s[A-Z]{2}\s+"[^"]*"\s*$'; then
LOG_FORMAT="enriched"
else
LOG_FORMAT="standard"
fi
log "Auto-detected log format: ${LOG_FORMAT}"
fi
if [[ "$LOG_FORMAT" == "standard" ]]; then
if ! command -v mmdblookup &>/dev/null; then
warn "mmdblookup not found — country/ASN metrics unavailable for standard log format"
warn "Install: apt install mmdb-bin"
fi
fi
}
# ============================================================================
# METRICS COLLECTION
# ============================================================================
collect_metrics() {
local start_time
start_time=$(date +%s%N)
local metrics=""
local now
now=$(date +%s)
# --- Parse log with awk (single pass for performance) ---
local awk_output
awk_output=$(awk '
BEGIN {
FS=" "
# Known cloud provider patterns (lowercase match)
split("amazon,aws,cloudflare,google cloud,microsoft,azure,digitalocean,linode,vultr,hetzner,ovh", providers, ",")
}
{
ip = $1
# Extract status code — field after "HTTP/x.x"
status = ""
for (i = 1; i <= NF; i++) {
if ($i ~ /^[0-9]{3}$/ && $(i-1) ~ /HTTP\//) {
status = $i
break
}
}
if (status == "") {
# fallback: find first standalone 3-digit number after the request
for (i = 7; i <= NF; i++) {
if ($i ~ /^[0-9]{3}$/) {
status = $i
break
}
}
}
# Extract method
for (i = 1; i <= NF; i++) {
if ($i ~ /^"(GET|POST|HEAD|PUT|DELETE|PATCH|OPTIONS|CONNECT)$/) {
method = substr($i, 2)
break
}
}
# Extract user agent (between second and third set of quotes after status)
ua = ""
quote_count = 0
ua_start = 0
for (i = 1; i <= NF; i++) {
if ($i ~ /^"/) quote_count++
if (quote_count == 6 && ua_start == 0) {
ua_start = i
}
}
# Simpler: reconstruct the full line and parse with regex
line = $0
# Country and ASN (enriched format: last fields are CC "ASN Org")
country = ""
asn_org = ""
if (format == "enriched") {
# Match trailing: XX "Some ASN Org"
if (match(line, /([A-Z]{2}|-) "([^"]*)"[[:space:]]*$/, m)) {
country = m[1]
asn_org = m[2]
}
}
# Count total requests per IP
ip_total[ip]++
# Count by status class
if (status >= 400) {
ip_blocked[ip]++
blocked_total++
}
# Count by country
if (country != "" && country != "-") {
country_req[country]++
if (status >= 400) country_blocked[country]++
}
# Count by ASN
if (asn_org != "" && asn_org != "-") {
asn_req[asn_org]++
if (status >= 400) asn_blocked[asn_org]++
# Classify cloud provider
lower_asn = tolower(asn_org)
provider = "other"
if (lower_asn ~ /amazon|aws/) provider = "aws"
else if (lower_asn ~ /cloudflare/) provider = "cloudflare"
else if (lower_asn ~ /google/) provider = "gcp"
else if (lower_asn ~ /microsoft|azure/) provider = "azure"
else if (lower_asn ~ /digitalocean/) provider = "digitalocean"
else if (lower_asn ~ /hetzner/) provider = "hetzner"
provider_req[provider]++
if (status >= 400) provider_blocked[provider]++
}
# POST probes (POST returning 4xx/5xx)
if (method == "POST" && status >= 400) post_probes++
# Scanner detection
if (tolower(line) ~ /(nikto|sqlmap|nmap|masscan|zgrab|zmeu|morpheus)/) scanners++
# Empty user agent
if (line ~ /" "-"$/ || line ~ /" ""$/ || line ~ /" ""-"/) empty_ua++
total_requests++
}
END {
# Output delimited data for bash to parse
print "TOTAL_REQUESTS=" total_requests
print "BLOCKED_TOTAL=" blocked_total+0
print "UNIQUE_IPS=" length(ip_total)
print "UNIQUE_BLOCKED_IPS=" length(ip_blocked)
print "POST_PROBES=" post_probes+0
print "SCANNERS=" scanners+0
print "EMPTY_UA=" empty_ua+0
# Country requests
for (c in country_req) print "COUNTRY_REQ|" c "|" country_req[c]
for (c in country_blocked) print "COUNTRY_BLK|" c "|" country_blocked[c]
# ASN requests (top 20)
for (a in asn_req) print "ASN_REQ|" a "|" asn_req[a]
for (a in asn_blocked) print "ASN_BLK|" a "|" asn_blocked[a]
# Provider
for (p in provider_req) print "PROVIDER_REQ|" p "|" provider_req[p]
for (p in provider_blocked) print "PROVIDER_BLK|" p "|" provider_blocked[p]
# Top blocked IPs (by blocked count)
PROCINFO["sorted_in"] = "@val_num_desc"
n = 0
for (ip in ip_blocked) {
if (n++ >= 10) break
print "TOP_BLOCKED|" ip "|" ip_blocked[ip]
}
}
' format="$LOG_FORMAT" "$NGINX_LOG" 2>/dev/null)
# --- Parse awk output ---
local total_requests=0 unique_ips=0 unique_blocked=0
local blocked_total=0
local post_probes=0 scanners=0 empty_ua=0
declare -A country_req=() country_blk=() asn_req=() asn_blk=() provider_req=() provider_blk=()
declare -a top_blocked_ips=()
while IFS= read -r line; do
case "$line" in
TOTAL_REQUESTS=*) total_requests="${line#*=}" ;;
BLOCKED_TOTAL=*) blocked_total="${line#*=}" ;;
UNIQUE_IPS=*) unique_ips="${line#*=}" ;;
UNIQUE_BLOCKED_IPS=*) unique_blocked="${line#*=}" ;;
POST_PROBES=*) post_probes="${line#*=}" ;;
SCANNERS=*) scanners="${line#*=}" ;;
EMPTY_UA=*) empty_ua="${line#*=}" ;;
COUNTRY_REQ\|*)
IFS='|' read -r _ key val <<< "$line"
country_req["$key"]="$val" ;;
COUNTRY_BLK\|*)
IFS='|' read -r _ key val <<< "$line"
country_blk["$key"]="$val" ;;
ASN_REQ\|*)
IFS='|' read -r _ key val <<< "$line"
key=$(prom_label "$key")
asn_req["$key"]=$(( ${asn_req["$key"]:-0} + val )) ;;
ASN_BLK\|*)
IFS='|' read -r _ key val <<< "$line"
key=$(prom_label "$key")
asn_blk["$key"]=$(( ${asn_blk["$key"]:-0} + val )) ;;
PROVIDER_REQ\|*)
IFS='|' read -r _ key val <<< "$line"
provider_req["$key"]="$val" ;;
PROVIDER_BLK\|*)
IFS='|' read -r _ key val <<< "$line"
provider_blk["$key"]="$val" ;;
TOP_BLOCKED\|*)
top_blocked_ips+=("$line") ;;
esac
done <<< "$awk_output"
# --- Build Prometheus metrics ---
metrics+="# HELP ip_intel_up Exporter status (1=up)
# TYPE ip_intel_up gauge
ip_intel_up 1
# HELP ip_intel_requests_total Total requests in log
# TYPE ip_intel_requests_total gauge
ip_intel_requests_total ${total_requests:-0}
# HELP ip_intel_blocked_total Total blocked requests (4xx/5xx)
# TYPE ip_intel_blocked_total gauge
ip_intel_blocked_total ${blocked_total:-0}
# HELP ip_intel_unique_ips_total Unique IPs seen
# TYPE ip_intel_unique_ips_total gauge
ip_intel_unique_ips_total ${unique_ips:-0}
# HELP ip_intel_unique_blocked_ips_total Unique IPs returning 4xx/5xx
# TYPE ip_intel_unique_blocked_ips_total gauge
ip_intel_unique_blocked_ips_total ${unique_blocked:-0}
# HELP ip_intel_post_probe_total POST requests returning 4xx/5xx
# TYPE ip_intel_post_probe_total gauge
ip_intel_post_probe_total ${post_probes:-0}
# HELP ip_intel_scanner_total Requests from known scanner user agents
# TYPE ip_intel_scanner_total gauge
ip_intel_scanner_total ${scanners:-0}
# HELP ip_intel_empty_ua_total Requests with empty user agent
# TYPE ip_intel_empty_ua_total gauge
ip_intel_empty_ua_total ${empty_ua:-0}
"
# Country metrics
if [[ ${#country_req[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_requests_by_country Requests by country code
# TYPE ip_intel_requests_by_country gauge
"
for c in "${!country_req[@]}"; do
metrics+="ip_intel_requests_by_country{country=\"${c}\"} ${country_req[$c]}
"
done
metrics+="
"
fi
if [[ ${#country_blk[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_blocked_by_country Blocked requests by country code
# TYPE ip_intel_blocked_by_country gauge
"
for c in "${!country_blk[@]}"; do
metrics+="ip_intel_blocked_by_country{country=\"${c}\"} ${country_blk[$c]}
"
done
metrics+="
"
fi
# ASN metrics
if [[ ${#asn_req[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_requests_by_asn Requests by ASN organization
# TYPE ip_intel_requests_by_asn gauge
"
for a in "${!asn_req[@]}"; do
metrics+="ip_intel_requests_by_asn{asn_org=\"${a}\"} ${asn_req[$a]}
"
done
metrics+="
"
fi
if [[ ${#asn_blk[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_blocked_by_asn Blocked requests by ASN organization
# TYPE ip_intel_blocked_by_asn gauge
"
for a in "${!asn_blk[@]}"; do
metrics+="ip_intel_blocked_by_asn{asn_org=\"${a}\"} ${asn_blk[$a]}
"
done
metrics+="
"
fi
# Provider metrics
if [[ ${#provider_req[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_requests_by_provider Requests by cloud provider
# TYPE ip_intel_requests_by_provider gauge
"
for p in "${!provider_req[@]}"; do
metrics+="ip_intel_requests_by_provider{provider=\"${p}\"} ${provider_req[$p]}
"
done
metrics+="
"
fi
if [[ ${#provider_blk[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_blocked_by_provider Blocked requests by cloud provider
# TYPE ip_intel_blocked_by_provider gauge
"
for p in "${!provider_blk[@]}"; do
metrics+="ip_intel_blocked_by_provider{provider=\"${p}\"} ${provider_blk[$p]}
"
done
metrics+="
"
fi
# Top blocked IPs
if [[ ${#top_blocked_ips[@]} -gt 0 ]]; then
metrics+="# HELP ip_intel_top_blocked_ip_requests Top blocked IPs by blocked request count
# TYPE ip_intel_top_blocked_ip_requests gauge
"
for entry in "${top_blocked_ips[@]}"; do
IFS='|' read -r _ ip count <<< "$entry"
metrics+="ip_intel_top_blocked_ip_requests{ip=\"${ip}\"} ${count}
"
done
metrics+="
"
fi
# Exporter metadata
local end_time
end_time=$(date +%s%N)
local duration
duration=$(echo "scale=3; ($end_time - $start_time) / 1000000000" | bc 2>/dev/null || echo "0")
metrics+="# HELP ip_intel_exporter_duration_seconds Script execution time
# TYPE ip_intel_exporter_duration_seconds gauge
ip_intel_exporter_duration_seconds ${duration}
# HELP ip_intel_exporter_last_run_timestamp Last successful run (unix timestamp)
# TYPE ip_intel_exporter_last_run_timestamp gauge
ip_intel_exporter_last_run_timestamp ${now}
"
echo "$metrics"
}
# ============================================================================
# OUTPUT HANDLING
# ============================================================================
output_metrics() {
local metrics
metrics=$(collect_metrics)
if [[ -n "$OUTPUT_FILE" ]]; then
echo "$metrics" > "${OUTPUT_FILE}.tmp"
mv "${OUTPUT_FILE}.tmp" "$OUTPUT_FILE"
log "Metrics written to ${OUTPUT_FILE}"
else
echo "$metrics"
fi
}
serve_http() {
log "Starting HTTP server on port ${HTTP_PORT}"
log "Metrics endpoint: http://localhost:${HTTP_PORT}/metrics"
if ! command -v nc &>/dev/null && ! command -v ncat &>/dev/null; then
err "netcat (nc/ncat) not found — required for HTTP mode"
exit 1
fi
local nc_cmd="nc"
command -v ncat &>/dev/null && nc_cmd="ncat"
while true; do
local metrics
metrics=$(collect_metrics)
local content_length=${#metrics}
local response="HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: ${content_length}\r\nConnection: close\r\n\r\n${metrics}"
echo -e "$response" | $nc_cmd -l -p "$HTTP_PORT" -q 1 2>/dev/null || \
echo -e "$response" | $nc_cmd -l "$HTTP_PORT" 2>/dev/null || true
done
}
# ============================================================================
# MAIN
# ============================================================================
parse_args "$@"
preflight
if [[ "$HTTP_MODE" == "true" ]]; then
serve_http
else
output_metrics
fi