Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
@@ -0,0 +1,413 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# server-forensics.sh - Post-mortem forensics for crashed/locked servers
|
||||
#
|
||||
# Collects system state, logs, crash dumps, resource usage history, and
|
||||
# network info into a timestamped report for root-cause analysis.
|
||||
#
|
||||
# Author: Phil Connor
|
||||
# Contact: contact@mylinux.work
|
||||
# License: MIT
|
||||
# Version 1.00
|
||||
#
|
||||
# Usage:
|
||||
# ./server-forensics.sh # Full forensic collection
|
||||
# ./server-forensics.sh --quick # Quick summary only
|
||||
# ./server-forensics.sh --service nginx # Focus on a specific service
|
||||
# ./server-forensics.sh --since "1 hour ago" # Logs since a time
|
||||
# ./server-forensics.sh --output /tmp # Custom output directory
|
||||
###############################################################################
|
||||
|
||||
set -uo pipefail
|
||||
# NOTE: no -e — we want to keep collecting even if a command fails
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# CONFIGURATION
|
||||
#------------------------------------------------------------------------------
|
||||
REPORT_DIR="/var/log/forensics"
|
||||
SINCE="4 hours ago"
|
||||
TARGET_SERVICE=""
|
||||
QUICK_MODE=false
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() { echo -e "${GREEN}[forensics]${NC} $1"; }
|
||||
warn() { echo -e "${YELLOW}[forensics]${NC} $1"; }
|
||||
error() { echo -e "${RED}[forensics]${NC} $1" >&2; }
|
||||
header(){ echo -e "${CYAN}=== $1 ===${NC}"; }
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# PARSE ARGUMENTS
|
||||
#------------------------------------------------------------------------------
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--quick) QUICK_MODE=true ;;
|
||||
--service) shift; TARGET_SERVICE="$1" ;;
|
||||
--since) shift; SINCE="$1" ;;
|
||||
--output) shift; REPORT_DIR="$1" ;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo " --quick Quick summary (skip deep collection)"
|
||||
echo " --service <name> Focus forensics on a specific service"
|
||||
echo " --since <time> How far back to look (default: '4 hours ago')"
|
||||
echo " --output <dir> Output directory (default: /var/log/forensics)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Full forensic sweep"
|
||||
echo " $0 --quick # Quick triage"
|
||||
echo " $0 --service nginx # Focus on nginx"
|
||||
echo " $0 --since '2 hours ago' # Recent logs only"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# SETUP
|
||||
#------------------------------------------------------------------------------
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
warn "Running without root — some data may be inaccessible. Consider: sudo $0"
|
||||
fi
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
CASE_DIR="${REPORT_DIR}/${TIMESTAMP}"
|
||||
mkdir -p "$CASE_DIR"
|
||||
|
||||
REPORT="${CASE_DIR}/forensics-report.txt"
|
||||
|
||||
collect() {
|
||||
local label="$1"
|
||||
local cmd="$2"
|
||||
header "$label" | tee -a "$REPORT"
|
||||
eval "$cmd" 2>&1 | tee -a "$REPORT"
|
||||
echo "" | tee -a "$REPORT"
|
||||
}
|
||||
|
||||
log "Forensics case: ${CASE_DIR}"
|
||||
log "Looking back since: ${SINCE}"
|
||||
echo "Forensics Report — $(date)" > "$REPORT"
|
||||
echo "Hostname: $(hostname)" >> "$REPORT"
|
||||
echo "Kernel: $(uname -r)" >> "$REPORT"
|
||||
echo "=============================================" >> "$REPORT"
|
||||
echo "" >> "$REPORT"
|
||||
|
||||
###############################################################################
|
||||
# SECTION 1: SYSTEM STATE SNAPSHOT
|
||||
###############################################################################
|
||||
log "Collecting system state..."
|
||||
|
||||
collect "Uptime & Load" \
|
||||
"uptime"
|
||||
|
||||
collect "Last Reboot History" \
|
||||
"last reboot | head -20"
|
||||
|
||||
collect "System Boot Time" \
|
||||
"who -b 2>/dev/null || uptime -s 2>/dev/null"
|
||||
|
||||
collect "Kernel Messages (last 100 lines)" \
|
||||
"dmesg --time-format iso 2>/dev/null | tail -100 || dmesg | tail -100"
|
||||
|
||||
collect "Kernel Panics / Oops / OOM" \
|
||||
"dmesg | grep -iE 'panic|oops|oom|kill|segfault|bug:|call trace' | tail -50"
|
||||
|
||||
###############################################################################
|
||||
# SECTION 2: RESOURCE EXHAUSTION CHECK
|
||||
###############################################################################
|
||||
log "Checking resource exhaustion..."
|
||||
|
||||
collect "Memory Usage" \
|
||||
"free -h"
|
||||
|
||||
collect "Swap Usage Detail" \
|
||||
"swapon --show 2>/dev/null; echo '---'; cat /proc/meminfo | grep -iE 'swap|mem|commit|dirty|writeback'"
|
||||
|
||||
collect "Disk Usage" \
|
||||
"df -h"
|
||||
|
||||
collect "Inode Usage" \
|
||||
"df -i"
|
||||
|
||||
collect "Largest Files (top 20 in /var/log)" \
|
||||
"find /var/log -type f -exec du -h {} + 2>/dev/null | sort -rh | head -20"
|
||||
|
||||
collect "Open File Descriptors" \
|
||||
"cat /proc/sys/fs/file-nr"
|
||||
|
||||
if ! $QUICK_MODE; then
|
||||
collect "Top File Descriptor Consumers" \
|
||||
"for pid in /proc/[0-9]*/fd; do echo \"\$(ls \"\$pid\" 2>/dev/null | wc -l) \$pid\"; done 2>/dev/null | sort -rn | head -15"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 3: PROCESS STATE
|
||||
###############################################################################
|
||||
log "Collecting process info..."
|
||||
|
||||
collect "Top Processes by CPU" \
|
||||
"ps auxf --sort=-%cpu | head -30"
|
||||
|
||||
collect "Top Processes by Memory" \
|
||||
"ps auxf --sort=-%mem | head -30"
|
||||
|
||||
collect "Zombie Processes" \
|
||||
"ps aux | awk '\$8 ~ /Z/ {print}'"
|
||||
|
||||
collect "Processes in D-state (uninterruptible sleep)" \
|
||||
"ps aux | awk '\$8 ~ /D/ {print}'"
|
||||
|
||||
collect "Process Count by User" \
|
||||
"ps -eo user= | sort | uniq -c | sort -rn | head -15"
|
||||
|
||||
if ! $QUICK_MODE; then
|
||||
collect "Full Process Tree" \
|
||||
"pstree -palT 2>/dev/null || ps auxf"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 4: SYSTEMD / SERVICES
|
||||
###############################################################################
|
||||
log "Checking services..."
|
||||
|
||||
collect "Failed systemd Units" \
|
||||
"systemctl --failed 2>/dev/null"
|
||||
|
||||
collect "Recently Crashed Services" \
|
||||
"systemctl list-units --state=failed --no-legend 2>/dev/null | while read -r unit _rest; do
|
||||
echo \"--- \$unit ---\"
|
||||
systemctl status \"\$unit\" --no-pager -l 2>/dev/null | head -20
|
||||
echo ''
|
||||
done"
|
||||
|
||||
if [ -n "$TARGET_SERVICE" ]; then
|
||||
log "Focused forensics on: ${TARGET_SERVICE}"
|
||||
|
||||
collect "Service Status: ${TARGET_SERVICE}" \
|
||||
"systemctl status '$TARGET_SERVICE' --no-pager -l 2>/dev/null"
|
||||
|
||||
collect "Service Journal: ${TARGET_SERVICE}" \
|
||||
"journalctl -u '$TARGET_SERVICE' --since '$SINCE' --no-pager -l 2>/dev/null | tail -200"
|
||||
|
||||
collect "Service Config: ${TARGET_SERVICE}" \
|
||||
"systemctl cat '$TARGET_SERVICE' 2>/dev/null"
|
||||
|
||||
collect "Service Restarts: ${TARGET_SERVICE}" \
|
||||
"journalctl -u '$TARGET_SERVICE' --no-pager | grep -iE 'start|stop|fail|exit|restart|kill' | tail -30"
|
||||
|
||||
# Try to find coredumps for that service
|
||||
collect "Coredumps: ${TARGET_SERVICE}" \
|
||||
"coredumpctl list '$TARGET_SERVICE' 2>/dev/null | tail -10"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 5: LOGS (THE GOLD MINE)
|
||||
###############################################################################
|
||||
log "Mining logs..."
|
||||
|
||||
collect "Syslog / Messages (errors since ${SINCE})" \
|
||||
"journalctl --since '$SINCE' -p err --no-pager 2>/dev/null | tail -100 || \
|
||||
grep -iE 'error|fail|panic|kill|oom|segfault' /var/log/syslog 2>/dev/null | tail -100 || \
|
||||
grep -iE 'error|fail|panic|kill|oom|segfault' /var/log/messages 2>/dev/null | tail -100"
|
||||
|
||||
collect "Auth / Security Events" \
|
||||
"journalctl --since '$SINCE' -t sshd -t sudo --no-pager 2>/dev/null | tail -50 || \
|
||||
tail -50 /var/log/auth.log 2>/dev/null || tail -50 /var/log/secure 2>/dev/null"
|
||||
|
||||
collect "OOM Killer Events" \
|
||||
"journalctl --since '$SINCE' --no-pager 2>/dev/null | grep -i 'oom\|out of memory\|killed process' | tail -30 || \
|
||||
grep -i 'oom\|out of memory\|killed process' /var/log/syslog 2>/dev/null | tail -30"
|
||||
|
||||
if ! $QUICK_MODE; then
|
||||
collect "Kernel Ring Buffer (full)" \
|
||||
"dmesg -T 2>/dev/null | tail -200"
|
||||
|
||||
collect "Boot Log" \
|
||||
"journalctl -b --no-pager 2>/dev/null | head -100"
|
||||
|
||||
collect "Previous Boot Log (if available)" \
|
||||
"journalctl -b -1 --no-pager 2>/dev/null | tail -100 || echo 'No previous boot journal available'"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 6: NETWORK STATE
|
||||
###############################################################################
|
||||
log "Checking network..."
|
||||
|
||||
collect "Listening Ports" \
|
||||
"ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null"
|
||||
|
||||
collect "Established Connections" \
|
||||
"ss -tnp 2>/dev/null | head -50 || netstat -tnp 2>/dev/null | head -50"
|
||||
|
||||
collect "Connection Count by State" \
|
||||
"ss -tan 2>/dev/null | awk 'NR>1 {print \$1}' | sort | uniq -c | sort -rn"
|
||||
|
||||
collect "Network Interface Errors" \
|
||||
"ip -s link 2>/dev/null | grep -A2 -iE 'errors|dropped|overrun' || netstat -i 2>/dev/null"
|
||||
|
||||
if ! $QUICK_MODE; then
|
||||
collect "Firewall Rules" \
|
||||
"iptables -L -n -v 2>/dev/null | head -50; echo '--- nftables ---'; nft list ruleset 2>/dev/null | head -50"
|
||||
|
||||
collect "ARP Table" \
|
||||
"ip neigh 2>/dev/null || arp -a 2>/dev/null"
|
||||
|
||||
collect "Routing Table" \
|
||||
"ip route 2>/dev/null || route -n 2>/dev/null"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 7: CRASH DUMPS & COREDUMPS
|
||||
###############################################################################
|
||||
if ! $QUICK_MODE; then
|
||||
log "Looking for crash dumps..."
|
||||
|
||||
collect "Coredump List (coredumpctl)" \
|
||||
"coredumpctl list --since '$SINCE' 2>/dev/null || echo 'coredumpctl not available'"
|
||||
|
||||
collect "Recent Coredumps on Disk" \
|
||||
"find /var/lib/systemd/coredump /var/crash /tmp /var/tmp -name '*.core' -o -name 'core.*' -o -name 'vmcore' 2>/dev/null | head -20 || echo 'No coredumps found'"
|
||||
|
||||
collect "Crash Report Files" \
|
||||
"ls -lt /var/crash/ 2>/dev/null | head -10 || echo 'No /var/crash/ directory'"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 8: HARDWARE / TEMPERATURE / STORAGE
|
||||
###############################################################################
|
||||
if ! $QUICK_MODE; then
|
||||
log "Checking hardware health..."
|
||||
|
||||
collect "Hardware Errors (MCE)" \
|
||||
"dmesg | grep -iE 'mce|hardware error|machine check' | tail -20"
|
||||
|
||||
collect "SMART Disk Health" \
|
||||
"for disk in /dev/sd? /dev/nvme?n?; do
|
||||
[ -b \"\$disk\" ] || continue
|
||||
echo \"--- \$disk ---\"
|
||||
smartctl -H \"\$disk\" 2>/dev/null | grep -iE 'result|health|temperature|reallocated|pending|uncorrectable' || echo 'smartctl not available or no permission'
|
||||
done"
|
||||
|
||||
collect "Temperature Sensors" \
|
||||
"sensors 2>/dev/null || echo 'lm-sensors not installed'"
|
||||
|
||||
collect "RAID Status" \
|
||||
"cat /proc/mdstat 2>/dev/null || echo 'No software RAID'; \
|
||||
megacli -LDInfo -Lall -aALL 2>/dev/null; \
|
||||
arcconf getconfig 1 2>/dev/null"
|
||||
|
||||
collect "Filesystem Errors in dmesg" \
|
||||
"dmesg | grep -iE 'ext4|xfs|btrfs|filesystem|remount|read-only|i/o error' | tail -20"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 9: SECURITY TRIAGE
|
||||
###############################################################################
|
||||
if ! $QUICK_MODE; then
|
||||
log "Security quick-check..."
|
||||
|
||||
collect "Failed Login Attempts (last 24h)" \
|
||||
"journalctl --since '24 hours ago' --no-pager 2>/dev/null | grep -i 'failed password\|authentication failure' | wc -l; \
|
||||
echo 'Top offending IPs:'; \
|
||||
journalctl --since '24 hours ago' --no-pager 2>/dev/null | grep -i 'failed password' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort | uniq -c | sort -rn | head -10"
|
||||
|
||||
collect "Root Login Activity" \
|
||||
"grep 'root' /var/log/auth.log 2>/dev/null | tail -20 || \
|
||||
journalctl --since '$SINCE' --no-pager 2>/dev/null | grep -i 'root.*session' | tail -20"
|
||||
|
||||
collect "Recently Modified System Binaries" \
|
||||
"find /usr/bin /usr/sbin /usr/local/bin -mtime -1 -type f 2>/dev/null | head -20 || echo 'None found'"
|
||||
|
||||
collect "Suspicious Cron Jobs" \
|
||||
"for u in \$(cut -f1 -d: /etc/passwd); do
|
||||
crontab_out=\$(crontab -l -u \"\$u\" 2>/dev/null)
|
||||
[ -n \"\$crontab_out\" ] && echo \"--- \$u ---\" && echo \"\$crontab_out\"
|
||||
done; echo '--- /etc/cron.d ---'; ls -la /etc/cron.d/ 2>/dev/null"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SECTION 10: SAR / HISTORICAL METRICS
|
||||
###############################################################################
|
||||
if ! $QUICK_MODE && command -v sar &>/dev/null; then
|
||||
log "Pulling SAR historical data..."
|
||||
|
||||
collect "CPU History (today)" \
|
||||
"sar -u 2>/dev/null | tail -30"
|
||||
|
||||
collect "Memory History (today)" \
|
||||
"sar -r 2>/dev/null | tail -30"
|
||||
|
||||
collect "Load Average History (today)" \
|
||||
"sar -q 2>/dev/null | tail -30"
|
||||
|
||||
collect "Disk I/O History (today)" \
|
||||
"sar -d 2>/dev/null | tail -30"
|
||||
|
||||
collect "Network History (today)" \
|
||||
"sar -n DEV 2>/dev/null | tail -30"
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# SUMMARY
|
||||
###############################################################################
|
||||
log "Generating summary..."
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "============================================="
|
||||
echo "FORENSICS SUMMARY"
|
||||
echo "============================================="
|
||||
echo "Report generated: $(date)"
|
||||
echo "Hostname: $(hostname)"
|
||||
echo "Uptime: $(uptime -p 2>/dev/null || uptime)"
|
||||
echo "Kernel: $(uname -r)"
|
||||
echo ""
|
||||
|
||||
echo "🔴 CRITICAL FINDINGS:"
|
||||
|
||||
# OOM kills
|
||||
oom_count=$(dmesg 2>/dev/null | grep -ci 'oom\|out of memory\|killed process' || true)
|
||||
[ "${oom_count:-0}" -gt 0 ] 2>/dev/null && echo " - OOM Killer fired ${oom_count} time(s)"
|
||||
|
||||
# Segfaults
|
||||
seg_count=$(dmesg 2>/dev/null | grep -ci 'segfault' || true)
|
||||
[ "${seg_count:-0}" -gt 0 ] 2>/dev/null && echo " - ${seg_count} segfault(s) detected"
|
||||
|
||||
# Panics
|
||||
panic_count=$(dmesg 2>/dev/null | grep -ci 'panic' || true)
|
||||
[ "${panic_count:-0}" -gt 0 ] 2>/dev/null && echo " - ${panic_count} kernel panic(s)"
|
||||
|
||||
# Failed services
|
||||
failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
|
||||
[ "$failed_count" -gt 0 ] && echo " - ${failed_count} failed systemd unit(s)"
|
||||
|
||||
# Disk full
|
||||
full_disks=$(df -h 2>/dev/null | awk 'NR>1 && int($5) >= 90 {print $6 " (" $5 " full)"}')
|
||||
[ -n "$full_disks" ] && echo " - Disk(s) near full:" && echo "$full_disks" | sed 's/^/ /'
|
||||
|
||||
# D-state processes (I/O hung)
|
||||
dstate=$(ps aux 2>/dev/null | awk '$8 ~ /D/ {count++} END {print count+0}')
|
||||
[ "$dstate" -gt 0 ] && echo " - ${dstate} process(es) in D-state (I/O wait / hung)"
|
||||
|
||||
# Zombies
|
||||
zombies=$(ps aux 2>/dev/null | awk '$8 ~ /Z/ {count++} END {print count+0}')
|
||||
[ "$zombies" -gt 0 ] && echo " - ${zombies} zombie process(es)"
|
||||
|
||||
echo ""
|
||||
echo "📁 Full report: ${REPORT}"
|
||||
echo "📁 Case folder: ${CASE_DIR}"
|
||||
|
||||
} | tee -a "$REPORT"
|
||||
|
||||
# Save key files for offline analysis
|
||||
cp /var/log/syslog "$CASE_DIR/syslog.snapshot" 2>/dev/null
|
||||
cp /var/log/auth.log "$CASE_DIR/auth.snapshot" 2>/dev/null
|
||||
cp /var/log/kern.log "$CASE_DIR/kern.snapshot" 2>/dev/null
|
||||
journalctl --since "$SINCE" --no-pager > "$CASE_DIR/journal.snapshot" 2>/dev/null
|
||||
|
||||
log "✅ Forensics collection complete: ${CASE_DIR}"
|
||||
Reference in New Issue
Block a user