Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.

ufw-blocklists.sh

@@ -1,13 +1,25 @@
 #!/bin/bash
 ################################################################################
 # Script Name: ufw-blocklists.sh
-# Version: 1.0
+# Version: 1.23
 # Description: Per-feed UFW threat intelligence blocking with ipset
 # Author: Phil Connor
 # Contact: contact@mylinux.work
 # Website: https://mylinux.work
 # License: MIT
 ################################################################################
+# Changelog:
+#   1.23 - Fix ipset restore: write batch to temp file instead of pipe to avoid
+#          silent failures; log errors instead of suppressing; clean up temp
+#          ipsets on failure
+#        - Fix IPv6 parser: reject timestamps (e.g. 14:34:21) misidentified
+#          as IPv6 addresses by requiring hex letters, 3+ groups, or ::
+#   1.22 - Flush temp ipsets before loading to avoid stale entries after crashes
+#        - Always load ipsets even when feed is empty (clears stale blocks)
+#        - show-stats: cache journalctl once, count both v4 and v6 blocks,
+#          use Members: header for ipset counting
+#        - add-feed: validate that URL is provided
+################################################################################
 # Don't use 'set -e' - it causes silent failures when log file has permission issues
 
 CONFIG_DIR="/etc/ufw-threats"
@@ -156,8 +168,13 @@ check_requirements() {
     # with "Set ufw-feed-XXX doesn't exist" and block ALL traffic including DNS.
     ensure_ipsets_exist
 
+    # NOTE: Do NOT enable UFW here. UFW should only be enabled/reloaded in
+    # apply_ufw_rules() AFTER ipsets are verified. Enabling too early causes
+    # lockouts when ipsets aren't fully loaded yet.
     if [ "$enable_ufw" = true ]; then
-        ufw --force enable
+        if ! ufw status | grep -q "Status: active"; then
+            log_message "UFW is not active — it will be enabled after rules are applied"
+        fi
     fi
 
     cleanup_old_backups
@@ -213,7 +230,7 @@ create_directory_structure() {
 
 initialize_feeds_config() {
     local has_feeds
-    has_feeds=$(grep -c '^[01]|' "$FEEDS_CONFIG" 2>/dev/null || echo 0)
+    has_feeds=$(grep -c '^[01]|' "$FEEDS_CONFIG" 2>/dev/null || true)
 
     if [ -f "$FEEDS_CONFIG" ] && [ "$has_feeds" -gt 0 ]; then
         log_message "Feeds configuration already exists with $has_feeds feeds"
@@ -230,8 +247,11 @@ initialize_feeds_config() {
 #
 # ENABLED: 1 (enabled) or 0 (disabled)
 # NAME: Unique feed identifier
-# URL: Feed URL
-# TYPE: Format type (plain, cidr, commented, custom)
+# URL: Feed URL (http/https) or local file (file:///path/to/file)
+# TYPE: Format type (plain, cidr, commented)
+#   plain     - One IP/CIDR per line, no comments
+#   cidr      - IP/CIDR with optional inline comments/fields
+#   commented - Lines starting with # or ; are ignored, IPs extracted
 # DESCRIPTION: Feed description
 
 1|cinsarmy|http://cinsscore.com/list/ci-badguys.txt|plain|CINS Army Malicious IPs
@@ -300,7 +320,10 @@ setup_ipsets() {
 }
 
 setup_ipset_persistence() {
-    cat > /etc/systemd/system/ipset-persistent.service <<'EOF'
+    local script_path
+    script_path=$(readlink -f "$0")
+
+    cat > /etc/systemd/system/ipset-persistent.service <<EOF
 [Unit]
 Description=ipset persistent configuration
 Before=network-pre.target ufw.service
@@ -309,21 +332,35 @@ Wants=network-pre.target
 [Service]
 Type=oneshot
 RemainAfterExit=yes
-ExecStart=-/sbin/ipset restore -f /etc/ipset.conf
+# Restore saved ipsets, then ensure all feed ipsets exist (creates empty ones
+# for any that failed to restore). This prevents UFW from failing on boot
+# when before.rules references ipsets that don't exist.
+ExecStart=/bin/sh -c '/sbin/ipset restore -f /etc/ipset.conf 2>/dev/null || true; ${script_path} _ensure-ipsets 2>/dev/null || true'
 ExecStop=/sbin/ipset save -f /etc/ipset.conf
-StandardOutput=null
-StandardError=null
 
 [Install]
 WantedBy=multi-user.target
 EOF
 
     ipset save > /etc/ipset.conf
+    systemctl daemon-reload 2>/dev/null || true
     systemctl enable ipset-persistent.service 2>/dev/null || true
 }
 
 download_feed() {
     local url="$1" output="$2"
+
+    # Local file support: file:///path or file://path
+    if [[ "$url" == file://* ]]; then
+        local local_path="${url#file://}"
+        if [ ! -f "$local_path" ]; then
+            log_message "    Local file not found: $local_path"
+            return 1
+        fi
+        cp "$local_path" "$output" 2>/dev/null || return 1
+        return 0
+    fi
+
     local http_code
     http_code=$(curl -f -s -m 60 --connect-timeout 10 -L \
         -A "ufw-threat-feeds-per-feed/1.0" \
@@ -342,31 +379,45 @@ parse_feed() {
     : > "$output_v4"
     : > "$output_v6"
 
+    # Strip carriage returns from all feed types (common in downloaded feeds)
+    local cleaned
+    cleaned=$(mktemp)
+    tr -d '\r' < "$file" > "$cleaned"
+
+    # IPv6 filter: require either a hex letter [a-fA-F], or 3+ colon-separated
+    # groups, or ::. This excludes timestamps like 14:34:21 which only have
+    # digits and exactly two colon-separated groups.
+    local v6_filter='([a-fA-F]|:.*:.*:|::)'
+
     case "$type" in
         plain)
-            grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?$' "$file" >> "$output_v4" 2>/dev/null || true
+            grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?$' "$cleaned" >> "$output_v4" 2>/dev/null || true
             if [ "$ENABLE_IPV6" = true ]; then
-                grep -E '^[0-9a-fA-F:]+(/[0-9]+)?$' "$file" | grep ':' >> "$output_v6" 2>/dev/null || true
+                grep -E '^[0-9a-fA-F:]+(/[0-9]+)?$' "$cleaned" | grep ':' \
+                    | grep -E "$v6_filter" >> "$output_v6" 2>/dev/null || true
             fi
             ;;
         cidr)
-            grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?' "$file" \
-                | cut -d' ' -f1 | cut -d'#' -f1 | grep -v '^$' >> "$output_v4" 2>/dev/null || true
+            grep -oE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?' "$cleaned" \
+                | grep -v '^$' >> "$output_v4" 2>/dev/null || true
             if [ "$ENABLE_IPV6" = true ]; then
-                grep -E '^[0-9a-fA-F:]+(/[0-9]+)?' "$file" \
-                    | grep ':' | cut -d' ' -f1 | cut -d'#' -f1 | grep -v '^$' >> "$output_v6" 2>/dev/null || true
+                grep -oE '^[0-9a-fA-F:]+(/[0-9]+)?' "$cleaned" \
+                    | grep ':' | grep -E "$v6_filter" | grep -v '^$' >> "$output_v6" 2>/dev/null || true
             fi
             ;;
         commented)
-            grep -v -E '^[#;]|^$' "$file" \
+            grep -v -E '^[#;]|^$' "$cleaned" \
                 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?' >> "$output_v4" 2>/dev/null || true
             if [ "$ENABLE_IPV6" = true ]; then
-                grep -v -E '^[#;]|^$' "$file" \
+                grep -v -E '^[#;]|^$' "$cleaned" \
                     | grep -oE '[0-9a-fA-F:]+(/[0-9]+)?' \
-                    | grep -E '^[0-9a-fA-F]{1,4}:[0-9a-fA-F:]+' >> "$output_v6" 2>/dev/null || true
+                    | grep -E '^[0-9a-fA-F]{1,4}:[0-9a-fA-F:]+' \
+                    | grep -E "$v6_filter" >> "$output_v6" 2>/dev/null || true
             fi
             ;;
     esac
+
+    rm -f "$cleaned"
 }
 
 _clean_stale_cache() {
@@ -391,28 +442,48 @@ _clean_stale_cache() {
 
 _load_ipset_v4() {
     local name="$1" v4_file="$2"
+    local batch_file restore_err
+    batch_file=$(mktemp /tmp/ipset-v4-XXXXXX)
+
     {
         echo "create ${IPSET_PREFIX}-${name}-tmp hash:net family inet hashsize 4096 maxelem 200000"
+        echo "flush ${IPSET_PREFIX}-${name}-tmp"
         while IFS= read -r ip; do
             [ -z "$ip" ] && continue
             echo "add ${IPSET_PREFIX}-${name}-tmp $ip"
         done < "$v4_file"
         echo "swap ${IPSET_PREFIX}-${name} ${IPSET_PREFIX}-${name}-tmp"
         echo "destroy ${IPSET_PREFIX}-${name}-tmp"
-    } | ipset restore 2>/dev/null
+    } > "$batch_file"
+
+    if ! restore_err=$(ipset restore -exist < "$batch_file" 2>&1); then
+        log_message "WARNING: ipset restore failed for ${name} (v4): $restore_err"
+        ipset destroy "${IPSET_PREFIX}-${name}-tmp" 2>/dev/null || true
+    fi
+    rm -f "$batch_file"
 }
 
 _load_ipset_v6() {
     local name="$1" v6_file="$2"
+    local batch_file restore_err
+    batch_file=$(mktemp /tmp/ipset-v6-XXXXXX)
+
     {
         echo "create ${IPSET_PREFIX}-${name}-v6-tmp hash:net family inet6 hashsize 4096 maxelem 200000"
+        echo "flush ${IPSET_PREFIX}-${name}-v6-tmp"
         while IFS= read -r ip; do
             [ -z "$ip" ] && continue
             echo "add ${IPSET_PREFIX}-${name}-v6-tmp $ip"
         done < "$v6_file"
         echo "swap ${IPSET_PREFIX}-${name}-v6 ${IPSET_PREFIX}-${name}-v6-tmp"
         echo "destroy ${IPSET_PREFIX}-${name}-v6-tmp"
-    } | ipset restore 2>/dev/null
+    } > "$batch_file"
+
+    if ! restore_err=$(ipset restore -exist < "$batch_file" 2>&1); then
+        log_message "WARNING: ipset restore failed for ${name} (v6): $restore_err"
+        ipset destroy "${IPSET_PREFIX}-${name}-v6-tmp" 2>/dev/null || true
+    fi
+    rm -f "$batch_file"
 }
 
 update_feeds() {
@@ -425,7 +496,7 @@ update_feeds() {
     fi
 
     local enabled_count
-    enabled_count=$(grep -c '^1|' "$FEEDS_CONFIG" 2>/dev/null || echo 0)
+    enabled_count=$(grep -c '^1|' "$FEEDS_CONFIG" 2>/dev/null || true)
     if [ "$enabled_count" -eq 0 ]; then
         echo "ERROR: No enabled feeds found in $FEEDS_CONFIG"
         echo "Check the config file format"
@@ -465,8 +536,8 @@ update_feeds() {
             count_v6=0
             [ "$ENABLE_IPV6" = true ] && count_v6=$(wc -l < "$v6_file" 2>/dev/null || echo 0)
 
-            [ "$count_v4" -gt 0 ] && _load_ipset_v4 "$name" "$v4_file"
-            [ "$ENABLE_IPV6" = true ] && [ "$count_v6" -gt 0 ] && _load_ipset_v6 "$name" "$v6_file"
+            _load_ipset_v4 "$name" "$v4_file"
+            [ "$ENABLE_IPV6" = true ] && _load_ipset_v6 "$name" "$v6_file"
 
             log_message "  $name: $count_v4 IPv4, $count_v6 IPv6"
         else
@@ -539,7 +610,7 @@ _insert_and_validate_rules() {
     tail -n +"$((insert_line + 1))" "$template" >> "$output"
 
     local filter_count
-    filter_count=$(grep -c '^\*filter' "$output" 2>/dev/null || echo 0)
+    filter_count=$(grep -c '^\*filter' "$output" 2>/dev/null || true)
     if [ "$filter_count" -ne 1 ]; then
         log_message "ERROR: Generated rules file has $filter_count *filter blocks (expected 1)"
         return 1
@@ -608,8 +679,6 @@ apply_ufw_rules() {
         fi
     fi
 
-    ufw limit "$SSH_PORT/tcp" 2>/dev/null || ufw allow "$SSH_PORT/tcp"
-
     # CRITICAL: Ensure all ipsets exist BEFORE reloading UFW
     log_message "  Verifying ipsets exist..."
     ensure_ipsets_exist
@@ -705,24 +774,28 @@ cmd_show_stats() {
         return 1
     fi
 
+    local journal_cache
+    journal_cache=$(journalctl --since "1 hour ago" 2>/dev/null || true)
+
     local enabled name url type description
-    local v4_count v6_count blocks
+    local v4_count v6_count blocks_v4 blocks_v6 blocks
     while IFS='|' read -r enabled name url type description; do
         [[ "$enabled" =~ ^#.*$ ]] && continue
         [[ -z "$enabled" ]] && continue
         [ "$enabled" != "1" ] && continue
 
-        v4_count=$(ipset list "${IPSET_PREFIX}-${name}" 2>/dev/null | grep -c '^[0-9]' 2>/dev/null)
+        v4_count=$(ipset list "${IPSET_PREFIX}-${name}" 2>/dev/null | awk '/^Members:/{found=1; next} found' | wc -l)
         v4_count=${v4_count:-0}
 
         v6_count=0
         if [ "$ENABLE_IPV6" = true ]; then
-            v6_count=$(ipset list "${IPSET_PREFIX}-${name}-v6" 2>/dev/null | grep -c '^[0-9a-fA-F:]' 2>/dev/null)
+            v6_count=$(ipset list "${IPSET_PREFIX}-${name}-v6" 2>/dev/null | awk '/^Members:/{found=1; next} found' | wc -l)
             v6_count=${v6_count:-0}
         fi
 
-        blocks=$(journalctl --since "1 hour ago" 2>/dev/null | grep -c "\[THREAT:${name}\]" 2>/dev/null)
-        blocks=${blocks:-0}
+        blocks_v4=$(grep -c "\[THREAT:${name}\]" <<< "$journal_cache" 2>/dev/null || true)
+        blocks_v6=$(grep -c "\[THREAT-v6:${name}\]" <<< "$journal_cache" 2>/dev/null || true)
+        blocks=$(( ${blocks_v4:-0} + ${blocks_v6:-0} ))
 
         printf "%-25s %10d %10d %12d\n" "$name" "$v4_count" "$v6_count" "$blocks"
     done < "$FEEDS_CONFIG"
@@ -743,6 +816,7 @@ cmd_list_feeds() {
 
 cmd_add_feed() {
     validate_feed_name "$FEED_NAME" || exit 1
+    [ -z "$FEED_URL" ] && { echo "Usage: $0 add-feed <NAME> <URL>"; exit 1; }
     grep -q "^[01]|${FEED_NAME}|" "$FEEDS_CONFIG" 2>/dev/null && { echo "Feed exists"; exit 1; }
     echo "1|${FEED_NAME}|${FEED_URL}|plain|Custom: ${FEED_NAME}" >> "$FEEDS_CONFIG"
     log_message "Added feed: $FEED_NAME"
@@ -908,7 +982,7 @@ cmd_test_rules() {
     _build_rules_block "v4" "$v4_rules"
 
     local feed_count
-    feed_count=$(grep -c '^1|' "$FEEDS_CONFIG" 2>/dev/null || echo 0)
+    feed_count=$(grep -c '^1|' "$FEEDS_CONFIG" 2>/dev/null || true)
     echo "Generated rules for $feed_count enabled feeds"
 
     if ! _insert_and_validate_rules "$test_v4" "$v4_rules" "$v4_output"; then
@@ -920,7 +994,7 @@ cmd_test_rules() {
 
     local total_lines rule_lines
     total_lines=$(wc -l < "$v4_output")
-    rule_lines=$(grep -c "^-A " "$v4_output" 2>/dev/null || echo 0)
+    rule_lines=$(grep -c "^-A " "$v4_output" 2>/dev/null || true)
 
     echo "Generated $rule_lines iptables rules in $total_lines total lines"
     echo ""
@@ -943,6 +1017,10 @@ cmd_install() {
     initialize_feeds_config
     setup_ipsets
     update_feeds
+
+    # Ensure SSH is allowed before applying threat rules (only during install)
+    ufw limit "$SSH_PORT/tcp" 2>/dev/null || ufw allow "$SSH_PORT/tcp"
+
     apply_ufw_rules
     setup_auto_update
     create_management_commands
@@ -966,6 +1044,12 @@ cmd_install() {
 }
 
 main() {
+    # Internal command used by ipset-persistent.service at boot
+    if [ "${1:-}" = "_ensure-ipsets" ]; then
+        ensure_ipsets_exist
+        exit 0
+    fi
+
     parse_args "$@"
     case "$COMMAND" in
         install) cmd_install ;;