#!/bin/bash
###############################################################################
# server-forensics.sh - Post-mortem forensics for crashed/locked servers
#
# Collects system state, logs, crash dumps, resource usage history, and
# network info into a timestamped report for root-cause analysis.
#
# Author: Phil Connor
# Contact: contact@mylinux.work
# License: MIT
# Version 1.00
#
# Usage:
#   ./server-forensics.sh                  # Full forensic collection
#   ./server-forensics.sh --quick          # Quick summary only
#   ./server-forensics.sh --service nginx  # Focus on a specific service
#   ./server-forensics.sh --since "1 hour ago"  # Logs since a time
#   ./server-forensics.sh --output /tmp    # Custom output directory
###############################################################################

set -uo pipefail
# NOTE: no -e — we want to keep collecting even if a command fails

#------------------------------------------------------------------------------
# CONFIGURATION
#------------------------------------------------------------------------------
REPORT_DIR="/var/log/forensics"
SINCE="4 hours ago"
TARGET_SERVICE=""
QUICK_MODE=false

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'

log()   { echo -e "${GREEN}[forensics]${NC} $1"; }
warn()  { echo -e "${YELLOW}[forensics]${NC} $1"; }
error() { echo -e "${RED}[forensics]${NC} $1" >&2; }
header(){ echo -e "${CYAN}=== $1 ===${NC}"; }

#------------------------------------------------------------------------------
# PARSE ARGUMENTS
#------------------------------------------------------------------------------
for arg in "$@"; do
    case "$arg" in
        --quick)       QUICK_MODE=true ;;
        --service)     shift; TARGET_SERVICE="$1" ;;
        --since)       shift; SINCE="$1" ;;
        --output)      shift; REPORT_DIR="$1" ;;
        --help|-h)
            echo "Usage: $0 [OPTIONS]"
            echo ""
            echo "  --quick              Quick summary (skip deep collection)"
            echo "  --service <name>     Focus forensics on a specific service"
            echo "  --since <time>       How far back to look (default: '4 hours ago')"
            echo "  --output <dir>       Output directory (default: /var/log/forensics)"
            echo ""
            echo "Examples:"
            echo "  $0                           # Full forensic sweep"
            echo "  $0 --quick                   # Quick triage"
            echo "  $0 --service nginx           # Focus on nginx"
            echo "  $0 --since '2 hours ago'     # Recent logs only"
            exit 0
            ;;
    esac
done

#------------------------------------------------------------------------------
# SETUP
#------------------------------------------------------------------------------
if [ "$(id -u)" -ne 0 ]; then
    warn "Running without root — some data may be inaccessible. Consider: sudo $0"
fi

TIMESTAMP=$(date +%Y%m%d_%H%M%S)
CASE_DIR="${REPORT_DIR}/${TIMESTAMP}"
mkdir -p "$CASE_DIR"

REPORT="${CASE_DIR}/forensics-report.txt"

collect() {
    local label="$1"
    local cmd="$2"
    header "$label" | tee -a "$REPORT"
    eval "$cmd" 2>&1 | tee -a "$REPORT"
    echo "" | tee -a "$REPORT"
}

log "Forensics case: ${CASE_DIR}"
log "Looking back since: ${SINCE}"
echo "Forensics Report — $(date)" > "$REPORT"
echo "Hostname: $(hostname)" >> "$REPORT"
echo "Kernel: $(uname -r)" >> "$REPORT"
echo "=============================================" >> "$REPORT"
echo "" >> "$REPORT"

###############################################################################
# SECTION 1: SYSTEM STATE SNAPSHOT
###############################################################################
log "Collecting system state..."

collect "Uptime & Load" \
    "uptime"

collect "Last Reboot History" \
    "last reboot | head -20"

collect "System Boot Time" \
    "who -b 2>/dev/null || uptime -s 2>/dev/null"

collect "Kernel Messages (last 100 lines)" \
    "dmesg --time-format iso 2>/dev/null | tail -100 || dmesg | tail -100"

collect "Kernel Panics / Oops / OOM" \
    "dmesg | grep -iE 'panic|oops|oom|kill|segfault|bug:|call trace' | tail -50"

###############################################################################
# SECTION 2: RESOURCE EXHAUSTION CHECK
###############################################################################
log "Checking resource exhaustion..."

collect "Memory Usage" \
    "free -h"

collect "Swap Usage Detail" \
    "swapon --show 2>/dev/null; echo '---'; cat /proc/meminfo | grep -iE 'swap|mem|commit|dirty|writeback'"

collect "Disk Usage" \
    "df -h"

collect "Inode Usage" \
    "df -i"

collect "Largest Files (top 20 in /var/log)" \
    "find /var/log -type f -exec du -h {} + 2>/dev/null | sort -rh | head -20"

collect "Open File Descriptors" \
    "cat /proc/sys/fs/file-nr"

if ! $QUICK_MODE; then
    collect "Top File Descriptor Consumers" \
        "for pid in /proc/[0-9]*/fd; do echo \"\$(ls \"\$pid\" 2>/dev/null | wc -l) \$pid\"; done 2>/dev/null | sort -rn | head -15"
fi

###############################################################################
# SECTION 3: PROCESS STATE
###############################################################################
log "Collecting process info..."

collect "Top Processes by CPU" \
    "ps auxf --sort=-%cpu | head -30"

collect "Top Processes by Memory" \
    "ps auxf --sort=-%mem | head -30"

collect "Zombie Processes" \
    "ps aux | awk '\$8 ~ /Z/ {print}'"

collect "Processes in D-state (uninterruptible sleep)" \
    "ps aux | awk '\$8 ~ /D/ {print}'"

collect "Process Count by User" \
    "ps -eo user= | sort | uniq -c | sort -rn | head -15"

if ! $QUICK_MODE; then
    collect "Full Process Tree" \
        "pstree -palT 2>/dev/null || ps auxf"
fi

###############################################################################
# SECTION 4: SYSTEMD / SERVICES
###############################################################################
log "Checking services..."

collect "Failed systemd Units" \
    "systemctl --failed 2>/dev/null"

collect "Recently Crashed Services" \
    "systemctl list-units --state=failed --no-legend 2>/dev/null | while read -r unit _rest; do
        echo \"--- \$unit ---\"
        systemctl status \"\$unit\" --no-pager -l 2>/dev/null | head -20
        echo ''
    done"

if [ -n "$TARGET_SERVICE" ]; then
    log "Focused forensics on: ${TARGET_SERVICE}"

    collect "Service Status: ${TARGET_SERVICE}" \
        "systemctl status '$TARGET_SERVICE' --no-pager -l 2>/dev/null"

    collect "Service Journal: ${TARGET_SERVICE}" \
        "journalctl -u '$TARGET_SERVICE' --since '$SINCE' --no-pager -l 2>/dev/null | tail -200"

    collect "Service Config: ${TARGET_SERVICE}" \
        "systemctl cat '$TARGET_SERVICE' 2>/dev/null"

    collect "Service Restarts: ${TARGET_SERVICE}" \
        "journalctl -u '$TARGET_SERVICE' --no-pager | grep -iE 'start|stop|fail|exit|restart|kill' | tail -30"

    # Try to find coredumps for that service
    collect "Coredumps: ${TARGET_SERVICE}" \
        "coredumpctl list '$TARGET_SERVICE' 2>/dev/null | tail -10"
fi

###############################################################################
# SECTION 5: LOGS (THE GOLD MINE)
###############################################################################
log "Mining logs..."

collect "Syslog / Messages (errors since ${SINCE})" \
    "journalctl --since '$SINCE' -p err --no-pager 2>/dev/null | tail -100 || \
     grep -iE 'error|fail|panic|kill|oom|segfault' /var/log/syslog 2>/dev/null | tail -100 || \
     grep -iE 'error|fail|panic|kill|oom|segfault' /var/log/messages 2>/dev/null | tail -100"

collect "Auth / Security Events" \
    "journalctl --since '$SINCE' -t sshd -t sudo --no-pager 2>/dev/null | tail -50 || \
     tail -50 /var/log/auth.log 2>/dev/null || tail -50 /var/log/secure 2>/dev/null"

collect "OOM Killer Events" \
    "journalctl --since '$SINCE' --no-pager 2>/dev/null | grep -i 'oom\|out of memory\|killed process' | tail -30 || \
     grep -i 'oom\|out of memory\|killed process' /var/log/syslog 2>/dev/null | tail -30"

if ! $QUICK_MODE; then
    collect "Kernel Ring Buffer (full)" \
        "dmesg -T 2>/dev/null | tail -200"

    collect "Boot Log" \
        "journalctl -b --no-pager 2>/dev/null | head -100"

    collect "Previous Boot Log (if available)" \
        "journalctl -b -1 --no-pager 2>/dev/null | tail -100 || echo 'No previous boot journal available'"
fi

###############################################################################
# SECTION 6: NETWORK STATE
###############################################################################
log "Checking network..."

collect "Listening Ports" \
    "ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null"

collect "Established Connections" \
    "ss -tnp 2>/dev/null | head -50 || netstat -tnp 2>/dev/null | head -50"

collect "Connection Count by State" \
    "ss -tan 2>/dev/null | awk 'NR>1 {print \$1}' | sort | uniq -c | sort -rn"

collect "Network Interface Errors" \
    "ip -s link 2>/dev/null | grep -A2 -iE 'errors|dropped|overrun' || netstat -i 2>/dev/null"

if ! $QUICK_MODE; then
    collect "Firewall Rules" \
        "iptables -L -n -v 2>/dev/null | head -50; echo '--- nftables ---'; nft list ruleset 2>/dev/null | head -50"

    collect "ARP Table" \
        "ip neigh 2>/dev/null || arp -a 2>/dev/null"

    collect "Routing Table" \
        "ip route 2>/dev/null || route -n 2>/dev/null"
fi

###############################################################################
# SECTION 7: CRASH DUMPS & COREDUMPS
###############################################################################
if ! $QUICK_MODE; then
    log "Looking for crash dumps..."

    collect "Coredump List (coredumpctl)" \
        "coredumpctl list --since '$SINCE' 2>/dev/null || echo 'coredumpctl not available'"

    collect "Recent Coredumps on Disk" \
        "find /var/lib/systemd/coredump /var/crash /tmp /var/tmp -name '*.core' -o -name 'core.*' -o -name 'vmcore' 2>/dev/null | head -20 || echo 'No coredumps found'"

    collect "Crash Report Files" \
        "ls -lt /var/crash/ 2>/dev/null | head -10 || echo 'No /var/crash/ directory'"
fi

###############################################################################
# SECTION 8: HARDWARE / TEMPERATURE / STORAGE
###############################################################################
if ! $QUICK_MODE; then
    log "Checking hardware health..."

    collect "Hardware Errors (MCE)" \
        "dmesg | grep -iE 'mce|hardware error|machine check' | tail -20"

    collect "SMART Disk Health" \
        "for disk in /dev/sd? /dev/nvme?n?; do
            [ -b \"\$disk\" ] || continue
            echo \"--- \$disk ---\"
            smartctl -H \"\$disk\" 2>/dev/null | grep -iE 'result|health|temperature|reallocated|pending|uncorrectable' || echo 'smartctl not available or no permission'
        done"

    collect "Temperature Sensors" \
        "sensors 2>/dev/null || echo 'lm-sensors not installed'"

    collect "RAID Status" \
        "cat /proc/mdstat 2>/dev/null || echo 'No software RAID'; \
         megacli -LDInfo -Lall -aALL 2>/dev/null; \
         arcconf getconfig 1 2>/dev/null" 

    collect "Filesystem Errors in dmesg" \
        "dmesg | grep -iE 'ext4|xfs|btrfs|filesystem|remount|read-only|i/o error' | tail -20"
fi

###############################################################################
# SECTION 9: SECURITY TRIAGE
###############################################################################
if ! $QUICK_MODE; then
    log "Security quick-check..."

    collect "Failed Login Attempts (last 24h)" \
        "journalctl --since '24 hours ago' --no-pager 2>/dev/null | grep -i 'failed password\|authentication failure' | wc -l; \
         echo 'Top offending IPs:'; \
         journalctl --since '24 hours ago' --no-pager 2>/dev/null | grep -i 'failed password' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort | uniq -c | sort -rn | head -10"

    collect "Root Login Activity" \
        "grep 'root' /var/log/auth.log 2>/dev/null | tail -20 || \
         journalctl --since '$SINCE' --no-pager 2>/dev/null | grep -i 'root.*session' | tail -20"

    collect "Recently Modified System Binaries" \
        "find /usr/bin /usr/sbin /usr/local/bin -mtime -1 -type f 2>/dev/null | head -20 || echo 'None found'"

    collect "Suspicious Cron Jobs" \
        "for u in \$(cut -f1 -d: /etc/passwd); do
            crontab_out=\$(crontab -l -u \"\$u\" 2>/dev/null)
            [ -n \"\$crontab_out\" ] && echo \"--- \$u ---\" && echo \"\$crontab_out\"
        done; echo '--- /etc/cron.d ---'; ls -la /etc/cron.d/ 2>/dev/null"
fi

###############################################################################
# SECTION 10: SAR / HISTORICAL METRICS
###############################################################################
if ! $QUICK_MODE && command -v sar &>/dev/null; then
    log "Pulling SAR historical data..."

    collect "CPU History (today)" \
        "sar -u 2>/dev/null | tail -30"

    collect "Memory History (today)" \
        "sar -r 2>/dev/null | tail -30"

    collect "Load Average History (today)" \
        "sar -q 2>/dev/null | tail -30"

    collect "Disk I/O History (today)" \
        "sar -d 2>/dev/null | tail -30"

    collect "Network History (today)" \
        "sar -n DEV 2>/dev/null | tail -30"
fi

###############################################################################
# SUMMARY
###############################################################################
log "Generating summary..."

{
    echo ""
    echo "============================================="
    echo "FORENSICS SUMMARY"
    echo "============================================="
    echo "Report generated: $(date)"
    echo "Hostname:         $(hostname)"
    echo "Uptime:           $(uptime -p 2>/dev/null || uptime)"
    echo "Kernel:           $(uname -r)"
    echo ""

    echo "🔴 CRITICAL FINDINGS:"

    # OOM kills
    oom_count=$(dmesg 2>/dev/null | grep -ci 'oom\|out of memory\|killed process' || true)
    [ "${oom_count:-0}" -gt 0 ] 2>/dev/null && echo "  - OOM Killer fired ${oom_count} time(s)"

    # Segfaults
    seg_count=$(dmesg 2>/dev/null | grep -ci 'segfault' || true)
    [ "${seg_count:-0}" -gt 0 ] 2>/dev/null && echo "  - ${seg_count} segfault(s) detected"

    # Panics
    panic_count=$(dmesg 2>/dev/null | grep -ci 'panic' || true)
    [ "${panic_count:-0}" -gt 0 ] 2>/dev/null && echo "  - ${panic_count} kernel panic(s)"

    # Failed services
    failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
    [ "$failed_count" -gt 0 ] && echo "  - ${failed_count} failed systemd unit(s)"

    # Disk full
    full_disks=$(df -h 2>/dev/null | awk 'NR>1 && int($5) >= 90 {print $6 " (" $5 " full)"}')
    [ -n "$full_disks" ] && echo "  - Disk(s) near full:" && echo "$full_disks" | sed 's/^/      /'

    # D-state processes (I/O hung)
    dstate=$(ps aux 2>/dev/null | awk '$8 ~ /D/ {count++} END {print count+0}')
    [ "$dstate" -gt 0 ] && echo "  - ${dstate} process(es) in D-state (I/O wait / hung)"

    # Zombies
    zombies=$(ps aux 2>/dev/null | awk '$8 ~ /Z/ {count++} END {print count+0}')
    [ "$zombies" -gt 0 ] && echo "  - ${zombies} zombie process(es)"

    echo ""
    echo "📁 Full report: ${REPORT}"
    echo "📁 Case folder: ${CASE_DIR}"

} | tee -a "$REPORT"

# Save key files for offline analysis
cp /var/log/syslog "$CASE_DIR/syslog.snapshot" 2>/dev/null
cp /var/log/auth.log "$CASE_DIR/auth.snapshot" 2>/dev/null
cp /var/log/kern.log "$CASE_DIR/kern.snapshot" 2>/dev/null
journalctl --since "$SINCE" --no-pager > "$CASE_DIR/journal.snapshot" 2>/dev/null

log "✅ Forensics collection complete: ${CASE_DIR}"