chiefgeek
a1a17e81a1
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
1718 lines
61 KiB
Bash
Executable File
1718 lines
61 KiB
Bash
Executable File
#!/bin/bash
|
|
################################################################################
|
|
# Script Name: linux-log-analyzer.sh
|
|
# Version: 1.00
|
|
# Description: Analyze Linux system logs across five log types — system, auth,
|
|
# Docker, Java, GitLab, and PostgreSQL. Parses syslog, auth.log,
|
|
# journalctl, Java/log4j logs, GitLab JSON logs, and PostgreSQL
|
|
# logs to surface errors, failed logins, service failures, stack
|
|
# traces, slow queries, and crash events.
|
|
#
|
|
# Author: Phil Connor
|
|
# Contact: contact@mylinux.work
|
|
# Website: https://mylinux.work
|
|
# License: MIT
|
|
#
|
|
# Prerequisites:
|
|
# - bash, awk, grep, sort
|
|
# - journalctl (optional, for systemd-based systems)
|
|
# - Root/sudo for auth and system log types
|
|
#
|
|
# Usage:
|
|
# sudo ./linux-log-analyzer.sh --type system
|
|
# sudo ./linux-log-analyzer.sh --type auth --since 24h
|
|
# sudo ./linux-log-analyzer.sh --type docker
|
|
# ./linux-log-analyzer.sh --type java --log /opt/app/logs/application.log
|
|
# sudo ./linux-log-analyzer.sh --type gitlab
|
|
# ./linux-log-analyzer.sh --type postgresql
|
|
# sudo ./linux-log-analyzer.sh --all-types
|
|
# sudo ./linux-log-analyzer.sh --type system --json
|
|
# sudo ./linux-log-analyzer.sh --all-types --no-color > report.txt
|
|
#
|
|
# Log Types:
|
|
# system - syslog/messages: service failures, OOM, disk errors, panics
|
|
# auth - auth.log/secure: SSH brute force, sudo, account changes
|
|
# docker - Docker daemon: container crashes, restart loops, OOM
|
|
# java - log4j/logback/catalina: stack traces, OOM, GC pauses, errors
|
|
# gitlab - GitLab Omnibus: 5xx errors, slow requests, Sidekiq failures
|
|
# postgresql - PostgreSQL: deadlocks, slow queries, connection errors
|
|
################################################################################
|
|
|
|
set -uo pipefail
|
|
|
|
# ============================================================================
|
|
# DEFAULTS
|
|
# ============================================================================
|
|
|
|
VERSION="1.00"
|
|
LOG_TYPE=""
|
|
ALL_TYPES=false
|
|
SINCE=""
|
|
SINCE_EPOCH=""
|
|
TOP_N=20
|
|
JSON_MODE=false
|
|
NO_COLOR=false
|
|
OUTPUT_FILE=""
|
|
CUSTOM_LOG=""
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
YELLOW='\033[1;33m'
|
|
GREEN='\033[0;32m'
|
|
CYAN='\033[0;36m'
|
|
BOLD='\033[1m'
|
|
DIM='\033[2m'
|
|
NC='\033[0m'
|
|
|
|
# JSON accumulator
|
|
JSON_OUTPUT=""
|
|
|
|
# ============================================================================
|
|
# USAGE
|
|
# ============================================================================
|
|
|
|
show_usage() {
|
|
cat <<EOF
|
|
Usage: $0 [OPTIONS]
|
|
|
|
Analyze Linux system logs (v${VERSION}).
|
|
|
|
LOG TYPES:
|
|
--type TYPE Log type: system, auth, docker, java, gitlab, postgresql
|
|
--all-types Run all available analyzers in one pass
|
|
|
|
OPTIONS:
|
|
--since RANGE Time filter: 1h, 24h, 7d, 30d, or ISO date (default: all)
|
|
--top N Entries per section (default: ${TOP_N})
|
|
--json Output as JSON
|
|
--no-color Disable colored output
|
|
--output FILE Save report to file
|
|
--log PATH Override default log path
|
|
-h, --help Show this help
|
|
|
|
EXAMPLES:
|
|
sudo $0 --type system
|
|
sudo $0 --type auth --since 24h
|
|
sudo $0 --type docker --since 7d
|
|
$0 --type java --log /opt/tomcat/logs/catalina.out
|
|
sudo $0 --type gitlab
|
|
$0 --type postgresql --log /var/log/postgresql/postgresql-16-main.log
|
|
sudo $0 --all-types --no-color > report.txt
|
|
sudo $0 --type system --json | jq .
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ============================================================================
|
|
# HELPERS
|
|
# ============================================================================
|
|
|
|
disable_colors() {
|
|
RED="" YELLOW="" GREEN="" CYAN="" BOLD="" DIM="" NC=""
|
|
}
|
|
|
|
log_info() { echo -e "${CYAN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
check_root() {
|
|
if [[ $EUID -ne 0 ]]; then
|
|
log_error "This log type requires root. Run with: sudo $0 $*"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
section_header() {
|
|
local title="$1"
|
|
echo ""
|
|
echo -e "${BOLD}===================================================="
|
|
echo -e " ${title}"
|
|
echo -e "====================================================${NC}"
|
|
}
|
|
|
|
subsection_header() {
|
|
local title="$1"
|
|
echo ""
|
|
echo -e "${BOLD}── ${title} ─────────────────────────────────────────${NC}"
|
|
}
|
|
|
|
# Convert relative time strings to epoch
|
|
parse_since() {
|
|
local since="$1"
|
|
local now
|
|
now=$(date +%s)
|
|
|
|
case "$since" in
|
|
*h)
|
|
local hours="${since%h}"
|
|
SINCE_EPOCH=$((now - hours * 3600))
|
|
;;
|
|
*d)
|
|
local days="${since%d}"
|
|
SINCE_EPOCH=$((now - days * 86400))
|
|
;;
|
|
*)
|
|
# Try ISO date
|
|
SINCE_EPOCH=$(date -d "$since" +%s 2>/dev/null || echo "")
|
|
if [[ -z "$SINCE_EPOCH" ]]; then
|
|
log_error "Invalid --since value: $since"
|
|
exit 1
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Get journalctl --since string from SINCE
|
|
journalctl_since_arg() {
|
|
if [[ -n "$SINCE" ]]; then
|
|
case "$SINCE" in
|
|
*h) echo "--since=${SINCE%h} hours ago" ;;
|
|
*d) echo "--since=${SINCE%d} days ago" ;;
|
|
*) echo "--since=$SINCE" ;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# Filter lines by timestamp — uses a reference date to avoid per-line date calls
|
|
filter_by_time() {
|
|
if [[ -z "$SINCE_EPOCH" ]]; then
|
|
cat
|
|
return
|
|
fi
|
|
local cutoff_date
|
|
cutoff_date=$(date -d "@${SINCE_EPOCH}" "+%Y-%m-%d %H:%M:%S" 2>/dev/null)
|
|
local cutoff_syslog
|
|
cutoff_syslog=$(date -d "@${SINCE_EPOCH}" "+%b %d %H:%M:%S" 2>/dev/null)
|
|
local current_year
|
|
current_year=$(date +%Y)
|
|
|
|
awk -v cutoff_iso="$cutoff_date" -v cutoff_syslog="$cutoff_syslog" -v yr="$current_year" '
|
|
BEGIN {
|
|
# Month name to number for syslog comparison
|
|
split("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", m)
|
|
for (i = 1; i <= 12; i++) mon_num[m[i]] = sprintf("%02d", i)
|
|
}
|
|
{
|
|
# ISO: 2026-04-13 14:22:03 or 2026-04-13T14:22:03
|
|
if (match($0, /^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}/)) {
|
|
ts = substr($0, RSTART, RLENGTH)
|
|
gsub(/T/, " ", ts)
|
|
if (ts >= cutoff_iso) print
|
|
next
|
|
}
|
|
# Syslog: Apr 13 14:22:03
|
|
if (match($0, /^[A-Z][a-z]{2} [ 0-9][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}/)) {
|
|
ts = substr($0, RSTART, RLENGTH)
|
|
# Convert to sortable: YYYY-MM-DD HH:MM:SS
|
|
mon_name = substr(ts, 1, 3)
|
|
day = substr(ts, 5, 2)
|
|
gsub(/ /, "0", day)
|
|
time_part = substr(ts, 8)
|
|
sortable = yr "-" mon_num[mon_name] "-" day " " time_part
|
|
if (sortable >= cutoff_iso) print
|
|
next
|
|
}
|
|
# No timestamp — include line (continuation / multi-line)
|
|
print
|
|
}'
|
|
}
|
|
|
|
# Find first available log file from a list
|
|
find_log() {
|
|
for f in "$@"; do
|
|
if [[ -r "$f" ]]; then
|
|
echo "$f"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Read log source — file or journalctl
|
|
read_log_source() {
|
|
local log_file="$1"
|
|
local journal_unit="${2:-}"
|
|
|
|
if [[ -n "$CUSTOM_LOG" && -r "$CUSTOM_LOG" ]]; then
|
|
cat "$CUSTOM_LOG" | filter_by_time
|
|
return
|
|
fi
|
|
|
|
if [[ -n "$log_file" && -r "$log_file" ]]; then
|
|
cat "$log_file" | filter_by_time
|
|
return
|
|
fi
|
|
|
|
if [[ -n "$journal_unit" ]] && command -v journalctl &>/dev/null; then
|
|
local since_arg
|
|
since_arg=$(journalctl_since_arg)
|
|
if [[ -n "$since_arg" ]]; then
|
|
journalctl -u "$journal_unit" --no-pager -q $since_arg 2>/dev/null
|
|
else
|
|
journalctl -u "$journal_unit" --no-pager -q 2>/dev/null
|
|
fi
|
|
return
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# Emit a JSON key-value pair (accumulates into JSON_OUTPUT)
|
|
json_add() {
|
|
local key="$1"
|
|
local value="$2"
|
|
if [[ -n "$JSON_OUTPUT" ]]; then
|
|
JSON_OUTPUT="${JSON_OUTPUT},"
|
|
fi
|
|
JSON_OUTPUT="${JSON_OUTPUT}\"${key}\":${value}"
|
|
}
|
|
|
|
json_string() {
|
|
local s="$1"
|
|
s="${s//\\/\\\\}"
|
|
s="${s//\"/\\\"}"
|
|
printf '"%s"' "$s"
|
|
}
|
|
|
|
# ============================================================================
|
|
# SYSTEM LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_system() {
|
|
if ! check_root; then return 1; fi
|
|
|
|
local log_file
|
|
log_file=$(find_log /var/log/syslog /var/log/messages)
|
|
|
|
local log_data
|
|
log_data=$(read_log_source "${log_file:-}" "")
|
|
|
|
# Also pull journalctl if available and no file source
|
|
if [[ -z "$log_data" ]] && command -v journalctl &>/dev/null; then
|
|
local since_arg
|
|
since_arg=$(journalctl_since_arg)
|
|
if [[ -n "$since_arg" ]]; then
|
|
log_data=$(journalctl --no-pager -q $since_arg 2>/dev/null)
|
|
else
|
|
log_data=$(journalctl --no-pager -q --since "24 hours ago" 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
log_warn "No system log sources found (syslog, messages, or journalctl)"
|
|
return 1
|
|
fi
|
|
|
|
local total_lines
|
|
total_lines=$(echo "$log_data" | wc -l)
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "System Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}${log_file:-journalctl}${NC}"
|
|
echo -e " Lines: ${total_lines}"
|
|
fi
|
|
|
|
# Service failures
|
|
local service_failures
|
|
service_failures=$(echo "$log_data" | grep -iE "(failed|error|fatal)" | \
|
|
grep -oP '(\S+\.service)' | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Service Failures"
|
|
if [[ -n "$service_failures" ]]; then
|
|
printf " ${BOLD}%-4s %-35s %s${NC}\n" "#" "Service" "Failures"
|
|
echo "$service_failures" | awk '{printf " %-4d %-35s %d\n", NR, $2, $1}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# OOM kills
|
|
local oom_kills
|
|
oom_kills=$(echo "$log_data" | grep -i "oom-kill\|oom_kill\|out of memory\|killed process" | head -"$TOP_N")
|
|
|
|
local oom_count=0
|
|
if [[ -n "$oom_kills" ]]; then
|
|
oom_count=$(echo "$oom_kills" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "OOM Kills"
|
|
if [[ -n "$oom_kills" ]]; then
|
|
echo "$oom_kills" | while IFS= read -r line; do
|
|
local proc
|
|
proc=$(echo "$line" | grep -oP "process \d+ \(\K[^)]+|Killed process \d+ \(\K[^)]+|oom_kill.*task=\K\S+" | head -1)
|
|
local pid
|
|
pid=$(echo "$line" | grep -oP "process \K\d+|Killed process \K\d+" | head -1)
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
printf " %-22s PID %-7s %s\n" "${ts:-unknown}" "${pid:-?}" "${proc:-unknown}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Disk errors
|
|
local disk_errors
|
|
disk_errors=$(echo "$log_data" | grep -iE "(I/O error|EXT[234]-fs.*(warning|error)|XFS.*(error|corruption)|BTRFS.*error|SMART.*error|end_request.*I/O|medium error|blk_update_request.*error|Buffer I/O error|remount.*read-only)" | head -"$TOP_N")
|
|
|
|
local disk_error_count=0
|
|
if [[ -n "$disk_errors" ]]; then
|
|
disk_error_count=$(echo "$disk_errors" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Disk Errors"
|
|
if [[ -n "$disk_errors" ]]; then
|
|
echo "$disk_errors" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9]{2}:[0-9]{2}:[0-9]{2}")
|
|
local msg
|
|
msg=$(echo "$line" | sed -E 's/^.*kernel(\[[0-9.]+\])?: //' | head -c 80)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "$msg"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Failed logins (from syslog)
|
|
local failed_logins
|
|
failed_logins=$(echo "$log_data" | grep -iE "(failed password|authentication failure|failed login)" | \
|
|
grep -oP "user[ =]\K\S+|for \K\S+(?= from)" | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
local failed_login_total=0
|
|
if [[ -n "$failed_logins" ]]; then
|
|
failed_login_total=$(echo "$failed_logins" | awk '{s+=$1} END {print s+0}')
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Failed Logins"
|
|
if [[ -n "$failed_logins" ]]; then
|
|
printf " ${BOLD}%-4s %-25s %s${NC}\n" "#" "User" "Failures"
|
|
echo "$failed_logins" | awk '{printf " %-4d %-25s %d\n", NR, $2, $1}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Sudo usage
|
|
local sudo_usage
|
|
sudo_usage=$(echo "$log_data" | grep -i "sudo:" | grep "COMMAND=" | \
|
|
grep -oP "^\S+\s+\S+\s+\S+\s+\S+\s+\K\S+(?=\s)" | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
local sudo_total=0
|
|
if [[ -n "$sudo_usage" ]]; then
|
|
sudo_total=$(echo "$sudo_usage" | awk '{s+=$1} END {print s+0}')
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Sudo Usage"
|
|
if [[ -n "$sudo_usage" ]]; then
|
|
printf " ${BOLD}%-4s %-25s %s${NC}\n" "#" "User" "Commands"
|
|
echo "$sudo_usage" | awk '{printf " %-4d %-25s %d\n", NR, $2, $1}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Kernel panics / segfaults
|
|
local kernel_panics
|
|
kernel_panics=$(echo "$log_data" | grep -iE "(kernel panic|segfault|general protection fault|BUG:|Oops:)" | head -"$TOP_N")
|
|
|
|
local panic_count=0
|
|
if [[ -n "$kernel_panics" ]]; then
|
|
panic_count=$(echo "$kernel_panics" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Kernel Panics / Segfaults"
|
|
if [[ -n "$kernel_panics" ]]; then
|
|
echo "$kernel_panics" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
local msg
|
|
msg=$(echo "$line" | sed 's/^.*\(kernel\)[^:]*: //')
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:0:80}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Cron failures
|
|
local cron_failures
|
|
cron_failures=$(echo "$log_data" | grep -iE "(cron.*error|CRON.*FAILED|cron.*exit status [^0])" | head -"$TOP_N")
|
|
|
|
local cron_fail_count=0
|
|
if [[ -n "$cron_failures" ]]; then
|
|
cron_fail_count=$(echo "$cron_failures" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Cron Failures"
|
|
if [[ -n "$cron_failures" ]]; then
|
|
echo "$cron_failures" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
local msg
|
|
msg=$(echo "$line" | sed 's/^.*CRON[^:]*: //' | head -c 80)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "$msg"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Top error patterns
|
|
local error_patterns
|
|
error_patterns=$(echo "$log_data" | grep -iE "(error|fail|fatal|critical)" | \
|
|
sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[^ ]+ [^ ]+ [^:]+: //' | \
|
|
sed -E 's/^[A-Z][a-z]{2} +[0-9]{1,2} [0-9:]{8} [^ ]+ [^:]+: //' | \
|
|
sed -E ':a; s/^(time|ts|level|caller|source|component|host)=[^ ]+ //; ta' | \
|
|
sed -E 's/^\[[0-9T:.Z+ -]+ [A-Z]+ [^]]+\] //' | \
|
|
sed 's/[0-9]\{2,\}/#/g' | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top Error Patterns"
|
|
if [[ -n "$error_patterns" ]]; then
|
|
printf " ${BOLD}%-4s %-6s %s${NC}\n" "#" "Count" "Pattern"
|
|
echo "$error_patterns" | awk '{
|
|
count = $1
|
|
$1 = ""
|
|
sub(/^ +/, "")
|
|
msg = substr($0, 1, 70)
|
|
printf " %-4d %-6d %s\n", NR, count, msg
|
|
}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Service failure count
|
|
local svc_fail_count=0
|
|
if [[ -n "$service_failures" ]]; then
|
|
svc_fail_count=$(echo "$service_failures" | awk '{s+=$1} END {print s+0}')
|
|
fi
|
|
|
|
# Summary
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "Service failures:" "$svc_fail_count"
|
|
printf " %-25s %d\n" "OOM kills:" "$oom_count"
|
|
printf " %-25s %d\n" "Disk errors:" "$disk_error_count"
|
|
printf " %-25s %d\n" "Failed logins:" "$failed_login_total"
|
|
printf " %-25s %d\n" "Sudo commands:" "$sudo_total"
|
|
printf " %-25s %d\n" "Kernel panics:" "$panic_count"
|
|
printf " %-25s %d\n" "Cron failures:" "$cron_fail_count"
|
|
printf " %-25s %d\n" "Total lines parsed:" "$total_lines"
|
|
else
|
|
json_add "system" "{\"service_failures\":${svc_fail_count},\"oom_kills\":${oom_count},\"disk_errors\":${disk_error_count},\"failed_logins\":${failed_login_total},\"sudo_commands\":${sudo_total},\"kernel_panics\":${panic_count},\"cron_failures\":${cron_fail_count},\"total_lines\":${total_lines}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# AUTH LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_auth() {
|
|
if ! check_root; then return 1; fi
|
|
|
|
local log_file
|
|
log_file=$(find_log /var/log/auth.log /var/log/secure)
|
|
|
|
local log_data
|
|
log_data=$(read_log_source "${log_file:-}" "ssh")
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
log_warn "No auth log sources found (auth.log, secure, or journalctl)"
|
|
return 1
|
|
fi
|
|
|
|
local total_lines
|
|
total_lines=$(echo "$log_data" | wc -l)
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Auth Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}${log_file:-journalctl}${NC}"
|
|
echo -e " Lines: ${total_lines}"
|
|
fi
|
|
|
|
# SSH login failures by IP
|
|
local ssh_fail_ips
|
|
ssh_fail_ips=$(echo "$log_data" | grep -i "failed password" | \
|
|
grep -oP "from \K[0-9a-f.:]+(?= port)" | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "SSH Failed Logins by IP"
|
|
if [[ -n "$ssh_fail_ips" ]]; then
|
|
printf " ${BOLD}%-4s %-40s %s${NC}\n" "#" "IP Address" "Failures"
|
|
echo "$ssh_fail_ips" | awk '{printf " %-4d %-40s %d\n", NR, $2, $1}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Brute force detection (>10 failures)
|
|
local brute_force
|
|
brute_force=$(echo "$log_data" | grep -i "failed password" | \
|
|
grep -oP "from \K[0-9a-f.:]+(?= port)" | sort | uniq -c | sort -rn | awk '$1 > 10')
|
|
|
|
local brute_force_count=0
|
|
if [[ -n "$brute_force" ]]; then
|
|
brute_force_count=$(echo "$brute_force" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Brute Force Suspects (>10 failures)"
|
|
if [[ -n "$brute_force" ]]; then
|
|
printf " ${BOLD}%-40s %s${NC}\n" "IP Address" "Failures"
|
|
echo "$brute_force" | awk '{printf " %-40s %s\n", $2, $1}'
|
|
echo ""
|
|
echo -e " ${RED}${brute_force_count} IP(s) with >10 failed attempts${NC}"
|
|
else
|
|
echo -e " ${GREEN}None found.${NC}"
|
|
fi
|
|
fi
|
|
|
|
# SSH login successes by user
|
|
local ssh_success
|
|
ssh_success=$(echo "$log_data" | grep -i "accepted" | \
|
|
grep -oP "for \K\S+(?= from)" | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "SSH Successful Logins by User"
|
|
if [[ -n "$ssh_success" ]]; then
|
|
printf " ${BOLD}%-4s %-25s %s${NC}\n" "#" "User" "Logins"
|
|
echo "$ssh_success" | awk '{printf " %-4d %-25s %d\n", NR, $2, $1}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# SSH key vs password
|
|
local key_logins=0 pass_logins=0
|
|
key_logins=$(echo "$log_data" | grep -ic "accepted publickey" || true)
|
|
pass_logins=$(echo "$log_data" | grep -ic "accepted password" || true)
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Auth Method Breakdown"
|
|
printf " %-25s %d\n" "SSH key:" "$key_logins"
|
|
printf " %-25s %d\n" "Password:" "$pass_logins"
|
|
fi
|
|
|
|
# Root login attempts
|
|
local root_attempts
|
|
root_attempts=$(echo "$log_data" | grep -iE "(failed password|accepted).* for root " | wc -l || true)
|
|
local root_success
|
|
root_success=$(echo "$log_data" | grep -i "accepted.* for root " | wc -l || true)
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Root Login Attempts"
|
|
printf " %-25s %d\n" "Total attempts:" "$root_attempts"
|
|
printf " %-25s %d\n" "Successful:" "$root_success"
|
|
if [[ "$root_success" -gt 0 ]]; then
|
|
echo ""
|
|
echo "$log_data" | grep -i "accepted.* for root " | tail -5 | while IFS= read -r line; do
|
|
local ts ip
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
ip=$(echo "$line" | grep -oP "from \K[0-9a-f.:]+")
|
|
printf " %-22s from %s\n" "${ts:-unknown}" "${ip:-unknown}"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# Sudo failures
|
|
local sudo_failures
|
|
sudo_failures=$(echo "$log_data" | grep -iE "sudo.*authentication failure|sudo.*incorrect password|sudo.*3 incorrect" | head -"$TOP_N")
|
|
|
|
local sudo_fail_count=0
|
|
if [[ -n "$sudo_failures" ]]; then
|
|
sudo_fail_count=$(echo "$sudo_failures" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Sudo Failures"
|
|
if [[ -n "$sudo_failures" ]]; then
|
|
echo "$sudo_failures" | while IFS= read -r line; do
|
|
local ts user
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
user=$(echo "$line" | grep -oP "user[ =]\K\S+|;\s+\K\S+(?=\s+:)" | head -1)
|
|
printf " %-22s user: %s\n" "${ts:-unknown}" "${user:-unknown}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Account changes (useradd, usermod, groupadd, userdel, passwd)
|
|
local account_changes
|
|
account_changes=$(echo "$log_data" | grep -iE "(useradd|usermod|userdel|groupadd|groupdel|passwd|new user|new group|delete user)" | head -"$TOP_N")
|
|
|
|
local account_change_count=0
|
|
if [[ -n "$account_changes" ]]; then
|
|
account_change_count=$(echo "$account_changes" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Account Changes"
|
|
if [[ -n "$account_changes" ]]; then
|
|
echo "$account_changes" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
msg=$(echo "$line" | sed 's/^.*\(useradd\|usermod\|userdel\|groupadd\|passwd\)/\1/' | head -c 70)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "$msg"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Totals
|
|
local total_failures
|
|
total_failures=$(echo "$log_data" | grep -ic "failed password" || true)
|
|
local total_success
|
|
total_success=$(echo "$log_data" | grep -ic "accepted" || true)
|
|
local total_auth=$((total_failures + total_success))
|
|
local success_rate=0
|
|
if [[ "$total_auth" -gt 0 ]]; then
|
|
success_rate=$(awk "BEGIN {printf \"%.1f\", ($total_success / $total_auth) * 100}")
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "Total auth events:" "$total_auth"
|
|
printf " %-25s %d\n" "Successful logins:" "$total_success"
|
|
printf " %-25s %d\n" "Failed logins:" "$total_failures"
|
|
printf " %-25s %s%%\n" "Success rate:" "$success_rate"
|
|
printf " %-25s %d\n" "Brute force IPs:" "$brute_force_count"
|
|
printf " %-25s %d\n" "Sudo failures:" "$sudo_fail_count"
|
|
printf " %-25s %d\n" "Account changes:" "$account_change_count"
|
|
else
|
|
json_add "auth" "{\"total_auth_events\":${total_auth},\"successful_logins\":${total_success},\"failed_logins\":${total_failures},\"success_rate\":${success_rate},\"brute_force_ips\":${brute_force_count},\"sudo_failures\":${sudo_fail_count},\"account_changes\":${account_change_count}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# DOCKER LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_docker() {
|
|
if ! command -v docker &>/dev/null && ! [[ -r /var/log/docker.log ]]; then
|
|
log_warn "Docker not found and no /var/log/docker.log — skipping"
|
|
return 1
|
|
fi
|
|
|
|
local log_data=""
|
|
|
|
# Try journalctl first (most common on systemd systems)
|
|
if command -v journalctl &>/dev/null; then
|
|
local since_arg
|
|
since_arg=$(journalctl_since_arg)
|
|
if [[ -n "$since_arg" ]]; then
|
|
log_data=$(journalctl -u docker.service --no-pager -q $since_arg 2>/dev/null)
|
|
else
|
|
log_data=$(journalctl -u docker.service --no-pager -q 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
# Fallback to docker.log
|
|
if [[ -z "$log_data" && -r /var/log/docker.log ]]; then
|
|
log_data=$(cat /var/log/docker.log | filter_by_time)
|
|
fi
|
|
|
|
# Also pull docker events if docker is available
|
|
local docker_ps_data=""
|
|
if command -v docker &>/dev/null; then
|
|
docker_ps_data=$(docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.CreatedAt}}' 2>/dev/null || true)
|
|
fi
|
|
|
|
if [[ -z "$log_data" && -z "$docker_ps_data" ]]; then
|
|
log_warn "No Docker log data available"
|
|
return 1
|
|
fi
|
|
|
|
local total_lines=0
|
|
if [[ -n "$log_data" ]]; then
|
|
total_lines=$(echo "$log_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Docker Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}journalctl/docker.log${NC}"
|
|
echo -e " Lines: ${total_lines}"
|
|
fi
|
|
|
|
# Container start/stop/restart events
|
|
local container_events
|
|
container_events=$(echo "$log_data" | grep -oP "container \K(start|stop|restart|kill|die|create|destroy)\S*" 2>/dev/null | \
|
|
sort | uniq -c | sort -rn)
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Container Events"
|
|
if [[ -n "$container_events" ]]; then
|
|
printf " ${BOLD}%-20s %s${NC}\n" "Event" "Count"
|
|
echo "$container_events" | awk '{printf " %-20s %d\n", $2, $1}'
|
|
else
|
|
echo " None found in daemon logs."
|
|
fi
|
|
fi
|
|
|
|
# Container restart loops (from docker ps)
|
|
local restart_loops=""
|
|
if [[ -n "$docker_ps_data" ]]; then
|
|
restart_loops=$(echo "$docker_ps_data" | grep -i "restarting" || true)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Containers in Restart Loop"
|
|
if [[ -n "$restart_loops" ]]; then
|
|
echo "$restart_loops" | while IFS=$'\t' read -r name status created; do
|
|
printf " %-30s %s\n" "$name" "$status"
|
|
done
|
|
else
|
|
echo -e " ${GREEN}None found.${NC}"
|
|
fi
|
|
fi
|
|
|
|
# OOM kills in containers
|
|
local docker_oom
|
|
docker_oom=$(echo "$log_data" | grep -iE "(oom|out of memory)" | head -"$TOP_N")
|
|
|
|
local docker_oom_count=0
|
|
if [[ -n "$docker_oom" ]]; then
|
|
docker_oom_count=$(echo "$docker_oom" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Container OOM Kills"
|
|
if [[ -n "$docker_oom" ]]; then
|
|
echo "$docker_oom" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
local msg
|
|
msg=$(echo "$line" | sed 's/^.*dockerd[^:]*: //' | head -c 80)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "$msg"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Docker daemon errors
|
|
local daemon_errors
|
|
daemon_errors=$(echo "$log_data" | grep -iE "(error|fatal|panic)" | \
|
|
grep -v "level=info" | grep -v "level=warning" | head -"$TOP_N")
|
|
|
|
local daemon_error_count=0
|
|
if [[ -n "$daemon_errors" ]]; then
|
|
daemon_error_count=$(echo "$daemon_errors" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Daemon Errors"
|
|
if [[ -n "$daemon_errors" ]]; then
|
|
echo "$daemon_errors" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
msg=$(echo "$line" | grep -oP 'msg="\K[^"]+|msg=\K\S+' | head -1)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:-$(echo "$line" | tail -c 80)}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Health check failures
|
|
local health_failures
|
|
health_failures=$(echo "$log_data" | grep -iE "(health.*unhealthy|health check|health_status)" | head -"$TOP_N")
|
|
|
|
local health_fail_count=0
|
|
if [[ -n "$health_failures" ]]; then
|
|
health_fail_count=$(echo "$health_failures" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Health Check Failures"
|
|
if [[ -n "$health_failures" ]]; then
|
|
echo "$health_failures" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+|^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9:]+")
|
|
msg=$(echo "$line" | sed 's/^.*dockerd[^:]*: //' | head -c 80)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "$msg"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Docker warnings
|
|
local daemon_warnings
|
|
daemon_warnings=$(echo "$log_data" | grep -i "level=warning\|WARN" | \
|
|
grep -oP 'msg="\K[^"]+|msg=\K\S+' | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top Warning Patterns"
|
|
if [[ -n "$daemon_warnings" ]]; then
|
|
printf " ${BOLD}%-6s %s${NC}\n" "Count" "Warning"
|
|
echo "$daemon_warnings" | awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, substr($0, 1, 70)
|
|
}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Exited containers (from docker ps -a)
|
|
local exited_containers=""
|
|
if [[ -n "$docker_ps_data" ]]; then
|
|
exited_containers=$(echo "$docker_ps_data" | grep -i "exited" | head -"$TOP_N" || true)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Exited Containers"
|
|
if [[ -n "$exited_containers" ]]; then
|
|
printf " ${BOLD}%-30s %s${NC}\n" "Container" "Status"
|
|
echo "$exited_containers" | while IFS=$'\t' read -r name status created; do
|
|
printf " %-30s %s\n" "$name" "$status"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
local restart_count=0
|
|
if [[ -n "$restart_loops" ]]; then
|
|
restart_count=$(echo "$restart_loops" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "Daemon errors:" "$daemon_error_count"
|
|
printf " %-25s %d\n" "OOM kills:" "$docker_oom_count"
|
|
printf " %-25s %d\n" "Health check failures:" "$health_fail_count"
|
|
printf " %-25s %d\n" "Restart loops:" "$restart_count"
|
|
printf " %-25s %d\n" "Log lines parsed:" "$total_lines"
|
|
else
|
|
json_add "docker" "{\"daemon_errors\":${daemon_error_count},\"oom_kills\":${docker_oom_count},\"health_check_failures\":${health_fail_count},\"restart_loops\":${restart_count},\"total_lines\":${total_lines}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# JAVA LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_java() {
|
|
local log_file=""
|
|
|
|
if [[ -n "$CUSTOM_LOG" ]]; then
|
|
log_file="$CUSTOM_LOG"
|
|
else
|
|
# Search common Java/Tomcat log locations
|
|
log_file=$(find_log \
|
|
/opt/tomcat/logs/catalina.out \
|
|
/var/log/tomcat*/catalina.out \
|
|
/opt/tomcat*/logs/catalina.out \
|
|
/var/log/wildfly/server.log \
|
|
/opt/wildfly/standalone/log/server.log \
|
|
/var/log/jetty/jetty.log \
|
|
/opt/sonarqube/logs/sonar.log \
|
|
/opt/nexus/sonatype-work/nexus3/log/nexus.log \
|
|
/opt/jenkins/log/jenkins.log \
|
|
/var/log/jenkins/jenkins.log)
|
|
fi
|
|
|
|
if [[ -z "$log_file" || ! -r "$log_file" ]]; then
|
|
log_warn "No Java log sources found — use --log to specify the path"
|
|
return 1
|
|
fi
|
|
|
|
local log_data
|
|
log_data=$(cat "$log_file" | filter_by_time)
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
log_warn "No log data in ${log_file} for the specified time range"
|
|
return 1
|
|
fi
|
|
|
|
local total_lines
|
|
total_lines=$(echo "$log_data" | wc -l)
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Java Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}${log_file}${NC}"
|
|
echo -e " Lines: ${total_lines}"
|
|
fi
|
|
|
|
# ERROR / WARN / FATAL counts
|
|
local error_count warn_count fatal_count
|
|
error_count=$(echo "$log_data" | grep -cE "\bERROR\b" || true)
|
|
warn_count=$(echo "$log_data" | grep -cE "\bWARN\b" || true)
|
|
fatal_count=$(echo "$log_data" | grep -cE "\bFATAL\b" || true)
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Log Level Breakdown"
|
|
printf " %-15s %d\n" "FATAL:" "$fatal_count"
|
|
printf " %-15s %d\n" "ERROR:" "$error_count"
|
|
printf " %-15s %d\n" "WARN:" "$warn_count"
|
|
fi
|
|
|
|
# Stack traces (lines starting with "at " or "Caused by:")
|
|
local stack_traces
|
|
stack_traces=$(echo "$log_data" | grep -cE "^\s+at |^Caused by:" || true)
|
|
local exception_types
|
|
exception_types=$(echo "$log_data" | grep -oP "^\S+Exception|^\S+Error|Caused by: \K\S+Exception|\S+Exception(?=:)" | \
|
|
sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Exceptions / Stack Traces"
|
|
printf " %-25s %d\n" "Stack trace lines:" "$stack_traces"
|
|
echo ""
|
|
if [[ -n "$exception_types" ]]; then
|
|
printf " ${BOLD}%-4s %-6s %s${NC}\n" "#" "Count" "Exception"
|
|
echo "$exception_types" | awk '{printf " %-4d %-6d %s\n", NR, $1, $2}' | head -"$TOP_N"
|
|
else
|
|
echo " No exceptions found."
|
|
fi
|
|
fi
|
|
|
|
# OOM errors
|
|
local oom_errors
|
|
oom_errors=$(echo "$log_data" | grep -iE "(OutOfMemoryError|java.lang.OutOfMemoryError|GC overhead limit exceeded|Java heap space|Metaspace)" | head -"$TOP_N")
|
|
|
|
local oom_count=0
|
|
if [[ -n "$oom_errors" ]]; then
|
|
oom_count=$(echo "$oom_errors" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "OutOfMemoryErrors"
|
|
if [[ -n "$oom_errors" ]]; then
|
|
echo "$oom_errors" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9]{2}:[0-9]{2}:[0-9]{2}|^[A-Z][a-z]{2} [0-9]{2}, [0-9]{4} [0-9:]+")
|
|
local msg
|
|
msg=$(echo "$line" | grep -oP "OutOfMemoryError.*|GC overhead limit exceeded|Java heap space|Metaspace" | head -c 70)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:-OOM}"
|
|
done
|
|
echo ""
|
|
echo -e " ${RED}${oom_count} OOM event(s) — check JVM heap settings${NC}"
|
|
else
|
|
echo -e " ${GREEN}None found.${NC}"
|
|
fi
|
|
fi
|
|
|
|
# GC pauses (from GC log lines embedded in app logs or GC-related messages)
|
|
local gc_events
|
|
gc_events=$(echo "$log_data" | grep -iE "(GC pause|Full GC|GC\(|Allocation Failure|G1 Evacuation Pause|CMS-concurrent|to-space exhausted)" | head -"$TOP_N")
|
|
|
|
local gc_count=0
|
|
if [[ -n "$gc_events" ]]; then
|
|
gc_count=$(echo "$gc_events" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "GC Events"
|
|
if [[ -n "$gc_events" ]]; then
|
|
# Show Full GC vs minor GC breakdown
|
|
local full_gc_count minor_gc_count
|
|
full_gc_count=$(echo "$gc_events" | grep -c "Full GC" || true)
|
|
minor_gc_count=$((gc_count - full_gc_count))
|
|
printf " %-25s %d\n" "Full GC:" "$full_gc_count"
|
|
printf " %-25s %d\n" "Minor GC:" "$minor_gc_count"
|
|
printf " %-25s %d\n" "Total:" "$gc_count"
|
|
else
|
|
echo " None found in application log (check dedicated GC log if separate)."
|
|
fi
|
|
fi
|
|
|
|
# Thread deadlocks
|
|
local deadlocks
|
|
deadlocks=$(echo "$log_data" | grep -iE "(deadlock|DEADLOCK|Found one Java-level deadlock)" | head -"$TOP_N")
|
|
|
|
local deadlock_count=0
|
|
if [[ -n "$deadlocks" ]]; then
|
|
deadlock_count=$(echo "$deadlocks" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Thread Deadlocks"
|
|
if [[ -n "$deadlocks" ]]; then
|
|
echo "$deadlocks" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9]{2}:[0-9]{2}:[0-9]{2}|^[A-Z][a-z]{2} [0-9]{2}, [0-9]{4} [0-9:]+")
|
|
printf " %s\n" "${ts:-unknown} — deadlock detected"
|
|
done
|
|
echo ""
|
|
echo -e " ${RED}${deadlock_count} deadlock(s) found${NC}"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Connection/network errors
|
|
local conn_errors
|
|
conn_errors=$(echo "$log_data" | grep -iE "(ConnectException|SocketTimeoutException|ConnectionRefused|Connection reset|NoRouteToHostException|UnknownHostException)" | \
|
|
grep -oP "\S*(Exception|Error)\S*" | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Connection / Network Errors"
|
|
if [[ -n "$conn_errors" ]]; then
|
|
printf " ${BOLD}%-6s %s${NC}\n" "Count" "Exception"
|
|
echo "$conn_errors" | awk '{printf " %-6d %s\n", $1, $2}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Top ERROR messages (deduped)
|
|
local error_patterns
|
|
error_patterns=$(echo "$log_data" | grep -E "\bERROR\b" | \
|
|
sed 's/^[0-9T:.+Z -]*//' | sed 's/\[.*\] //' | \
|
|
sed 's/[0-9]\{3,\}/#/g' | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top ERROR Patterns"
|
|
if [[ -n "$error_patterns" ]]; then
|
|
printf " ${BOLD}%-6s %s${NC}\n" "Count" "Pattern"
|
|
echo "$error_patterns" | awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, substr($0, 1, 65)
|
|
}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Top WARN messages (deduped)
|
|
local warn_patterns
|
|
warn_patterns=$(echo "$log_data" | grep -E "\bWARN\b" | \
|
|
sed 's/^[0-9T:.+Z -]*//' | sed 's/\[.*\] //' | \
|
|
sed 's/[0-9]\{3,\}/#/g' | sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top WARN Patterns"
|
|
if [[ -n "$warn_patterns" ]]; then
|
|
printf " ${BOLD}%-6s %s${NC}\n" "Count" "Pattern"
|
|
echo "$warn_patterns" | awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, substr($0, 1, 65)
|
|
}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Unique exception count
|
|
local unique_exceptions=0
|
|
if [[ -n "$exception_types" ]]; then
|
|
unique_exceptions=$(echo "$exception_types" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "FATAL entries:" "$fatal_count"
|
|
printf " %-25s %d\n" "ERROR entries:" "$error_count"
|
|
printf " %-25s %d\n" "WARN entries:" "$warn_count"
|
|
printf " %-25s %d\n" "Stack trace lines:" "$stack_traces"
|
|
printf " %-25s %d\n" "Unique exceptions:" "$unique_exceptions"
|
|
printf " %-25s %d\n" "OOM errors:" "$oom_count"
|
|
printf " %-25s %d\n" "GC events:" "$gc_count"
|
|
printf " %-25s %d\n" "Deadlocks:" "$deadlock_count"
|
|
printf " %-25s %d\n" "Total lines parsed:" "$total_lines"
|
|
else
|
|
json_add "java" "{\"fatal\":${fatal_count},\"error\":${error_count},\"warn\":${warn_count},\"stack_trace_lines\":${stack_traces},\"unique_exceptions\":${unique_exceptions},\"oom_errors\":${oom_count},\"gc_events\":${gc_count},\"deadlocks\":${deadlock_count},\"total_lines\":${total_lines}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# GITLAB LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_gitlab() {
|
|
local gitlab_log_dir="/var/log/gitlab"
|
|
|
|
if [[ -n "$CUSTOM_LOG" ]]; then
|
|
gitlab_log_dir="$CUSTOM_LOG"
|
|
fi
|
|
|
|
if [[ ! -d "$gitlab_log_dir" ]]; then
|
|
log_warn "GitLab log directory not found: ${gitlab_log_dir}"
|
|
return 1
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "GitLab Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}${gitlab_log_dir}${NC}"
|
|
fi
|
|
|
|
# Production JSON log (Rails)
|
|
local prod_log="${gitlab_log_dir}/gitlab-rails/production_json.log"
|
|
local total_requests=0 error_5xx=0 slow_requests=0 auth_failures=0
|
|
|
|
if [[ -r "$prod_log" ]]; then
|
|
local prod_data
|
|
prod_data=$(cat "$prod_log" | filter_by_time)
|
|
total_requests=$(echo "$prod_data" | wc -l)
|
|
|
|
# 5xx errors
|
|
local fivexx_data
|
|
fivexx_data=$(echo "$prod_data" | awk -F'"status":' '{
|
|
if (NF > 1) {
|
|
split($2, a, /[,}]/)
|
|
status = a[1] + 0
|
|
if (status >= 500) print
|
|
}
|
|
}')
|
|
|
|
error_5xx=0
|
|
if [[ -n "$fivexx_data" ]]; then
|
|
error_5xx=$(echo "$fivexx_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "HTTP 5xx Errors (Rails)"
|
|
if [[ "$error_5xx" -gt 0 ]]; then
|
|
# Top paths with 5xx
|
|
echo "$fivexx_data" | awk -F'"path":"' '{
|
|
if (NF > 1) {
|
|
split($2, a, "\"")
|
|
print a[1]
|
|
}
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{printf " %-6d %s\n", $1, $2}'
|
|
else
|
|
echo -e " ${GREEN}None found.${NC}"
|
|
fi
|
|
fi
|
|
|
|
# Slow requests (>5s)
|
|
local slow_data
|
|
slow_data=$(echo "$prod_data" | awk -F'"duration_s":' '{
|
|
if (NF > 1) {
|
|
split($2, a, /[,}]/)
|
|
dur = a[1] + 0
|
|
if (dur > 5.0) print
|
|
}
|
|
}')
|
|
|
|
slow_requests=0
|
|
if [[ -n "$slow_data" ]]; then
|
|
slow_requests=$(echo "$slow_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Slow Requests (>5s)"
|
|
if [[ "$slow_requests" -gt 0 ]]; then
|
|
echo "$slow_data" | awk -F'"' '{
|
|
path = ""; dur = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "path") path = $(i+2)
|
|
if ($i == "duration_s") {
|
|
split($(i+1), d, /[:,}]/)
|
|
dur = d[2]
|
|
}
|
|
}
|
|
if (path != "") printf " %.2fs %s\n", dur+0, path
|
|
}' | sort -rn | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Auth failures
|
|
local auth_fail_data
|
|
auth_fail_data=$(echo "$prod_data" | grep -i '"status":401\|"status":403' || true)
|
|
auth_failures=0
|
|
if [[ -n "$auth_fail_data" ]]; then
|
|
auth_failures=$(echo "$auth_fail_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Authentication Failures (401/403)"
|
|
if [[ "$auth_failures" -gt 0 ]]; then
|
|
echo "$auth_fail_data" | awk -F'"' '{
|
|
path = ""; remote_ip = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "path") path = $(i+2)
|
|
if ($i == "remote_ip") remote_ip = $(i+2)
|
|
}
|
|
if (remote_ip != "") printf " %-20s %s\n", remote_ip, path
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{printf " %-6d %-20s %s\n", $1, $2, $3}'
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Top endpoints by request count
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top Endpoints by Request Count"
|
|
echo "$prod_data" | awk -F'"path":"' '{
|
|
if (NF > 1) {
|
|
split($2, a, "\"")
|
|
print a[1]
|
|
}
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{printf " %-6d %s\n", $1, $2}'
|
|
fi
|
|
else
|
|
if ! $JSON_MODE; then
|
|
log_warn "GitLab production log not found: ${prod_log}"
|
|
fi
|
|
fi
|
|
|
|
# Sidekiq failures
|
|
local sidekiq_log="${gitlab_log_dir}/sidekiq/current"
|
|
local sidekiq_failures=0
|
|
|
|
if [[ -r "$sidekiq_log" ]]; then
|
|
local sidekiq_data
|
|
sidekiq_data=$(cat "$sidekiq_log" | filter_by_time)
|
|
|
|
local sidekiq_fail_data
|
|
sidekiq_fail_data=$(echo "$sidekiq_data" | grep -i '"job_status":"fail"\|"severity":"ERROR"' || true)
|
|
|
|
if [[ -n "$sidekiq_fail_data" ]]; then
|
|
sidekiq_failures=$(echo "$sidekiq_fail_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Sidekiq Failed Jobs"
|
|
if [[ "$sidekiq_failures" -gt 0 ]]; then
|
|
echo "$sidekiq_fail_data" | awk -F'"' '{
|
|
job_class = ""; error = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "class") job_class = $(i+2)
|
|
if ($i == "error_class") error = $(i+2)
|
|
}
|
|
if (job_class != "") printf " %s — %s\n", job_class, error
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, $0
|
|
}'
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Gitaly errors
|
|
local gitaly_log="${gitlab_log_dir}/gitaly/current"
|
|
local gitaly_errors=0
|
|
|
|
if [[ -r "$gitaly_log" ]]; then
|
|
local gitaly_data
|
|
gitaly_data=$(cat "$gitaly_log" | filter_by_time)
|
|
|
|
local gitaly_error_data
|
|
gitaly_error_data=$(echo "$gitaly_data" | grep -iE '"level":"error"|"level":"fatal"' || true)
|
|
|
|
if [[ -n "$gitaly_error_data" ]]; then
|
|
gitaly_errors=$(echo "$gitaly_error_data" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Gitaly Errors"
|
|
if [[ "$gitaly_errors" -gt 0 ]]; then
|
|
echo "$gitaly_error_data" | awk -F'"' '{
|
|
msg = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "msg" || $i == "error") {
|
|
msg = $(i+2)
|
|
break
|
|
}
|
|
}
|
|
if (msg != "") print msg
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, substr($0, 1, 70)
|
|
}'
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Error rate
|
|
local error_rate="0.0"
|
|
if [[ "$total_requests" -gt 0 ]]; then
|
|
error_rate=$(awk "BEGIN {printf \"%.2f\", ($error_5xx / $total_requests) * 100}")
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "Total requests:" "$total_requests"
|
|
printf " %-25s %d (%.2f%%)\n" "5xx errors:" "$error_5xx" "$error_rate"
|
|
printf " %-25s %d\n" "Slow requests (>5s):" "$slow_requests"
|
|
printf " %-25s %d\n" "Auth failures:" "$auth_failures"
|
|
printf " %-25s %d\n" "Sidekiq failures:" "$sidekiq_failures"
|
|
printf " %-25s %d\n" "Gitaly errors:" "$gitaly_errors"
|
|
else
|
|
json_add "gitlab" "{\"total_requests\":${total_requests},\"5xx_errors\":${error_5xx},\"error_rate\":${error_rate},\"slow_requests\":${slow_requests},\"auth_failures\":${auth_failures},\"sidekiq_failures\":${sidekiq_failures},\"gitaly_errors\":${gitaly_errors}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# POSTGRESQL LOG ANALYZER
|
|
# ============================================================================
|
|
|
|
analyze_postgresql() {
|
|
local log_file=""
|
|
|
|
if [[ -n "$CUSTOM_LOG" ]]; then
|
|
log_file="$CUSTOM_LOG"
|
|
else
|
|
# Search common PostgreSQL log locations
|
|
log_file=$(find_log \
|
|
/var/log/postgresql/postgresql-*-main.log \
|
|
/var/log/postgresql/postgresql.log \
|
|
/var/lib/pgsql/data/log/postgresql-*.log \
|
|
/var/lib/pgsql/data/pg_log/postgresql-*.log \
|
|
/var/log/pgsql/postgresql.log)
|
|
fi
|
|
|
|
local log_data=""
|
|
|
|
if [[ -n "$log_file" && -r "$log_file" ]]; then
|
|
log_data=$(cat "$log_file" | filter_by_time)
|
|
elif command -v journalctl &>/dev/null; then
|
|
local since_arg
|
|
since_arg=$(journalctl_since_arg)
|
|
if [[ -n "$since_arg" ]]; then
|
|
log_data=$(journalctl -u postgresql* --no-pager -q $since_arg 2>/dev/null)
|
|
else
|
|
log_data=$(journalctl -u postgresql* --no-pager -q 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
log_warn "No PostgreSQL log sources found"
|
|
return 1
|
|
fi
|
|
|
|
local total_lines
|
|
total_lines=$(echo "$log_data" | wc -l)
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "PostgreSQL Log Analysis"
|
|
echo ""
|
|
echo -e " Source: ${DIM}${log_file:-journalctl}${NC}"
|
|
echo -e " Lines: ${total_lines}"
|
|
fi
|
|
|
|
# FATAL/ERROR/PANIC counts
|
|
local fatal_count error_count panic_count
|
|
fatal_count=$(echo "$log_data" | grep -c " FATAL: " || true)
|
|
error_count=$(echo "$log_data" | grep -c " ERROR: " || true)
|
|
panic_count=$(echo "$log_data" | grep -c " PANIC: " || true)
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Severity Breakdown"
|
|
printf " %-15s %d\n" "PANIC:" "$panic_count"
|
|
printf " %-15s %d\n" "FATAL:" "$fatal_count"
|
|
printf " %-15s %d\n" "ERROR:" "$error_count"
|
|
fi
|
|
|
|
# Connection errors
|
|
local conn_errors
|
|
conn_errors=$(echo "$log_data" | grep -iE "(too many connections|connection refused|remaining connection slots|sorry, too many clients)" | head -"$TOP_N")
|
|
|
|
local conn_error_count=0
|
|
if [[ -n "$conn_errors" ]]; then
|
|
conn_error_count=$(echo "$conn_errors" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Connection Errors"
|
|
if [[ -n "$conn_errors" ]]; then
|
|
echo "$conn_errors" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+|^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+")
|
|
msg=$(echo "$line" | grep -oP "(FATAL|ERROR): \K.*" | head -c 70)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:-$line}"
|
|
done
|
|
else
|
|
echo -e " ${GREEN}None found.${NC}"
|
|
fi
|
|
fi
|
|
|
|
# Deadlocks
|
|
local deadlocks
|
|
deadlocks=$(echo "$log_data" | grep -i "deadlock detected" | head -"$TOP_N")
|
|
|
|
local deadlock_count=0
|
|
if [[ -n "$deadlocks" ]]; then
|
|
deadlock_count=$(echo "$deadlocks" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Deadlocks"
|
|
if [[ -n "$deadlocks" ]]; then
|
|
echo "$deadlocks" | while IFS= read -r line; do
|
|
local ts
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+|^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+")
|
|
printf " %s\n" "${ts:-unknown}"
|
|
done
|
|
echo ""
|
|
echo " Total: ${deadlock_count}"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Slow queries (log_min_duration_statement)
|
|
local slow_queries
|
|
slow_queries=$(echo "$log_data" | grep -i "duration:" | grep -v "LOG: connection\|LOG: disconnection" | head -"$TOP_N")
|
|
|
|
local slow_query_count=0
|
|
if [[ -n "$slow_queries" ]]; then
|
|
slow_query_count=$(echo "$slow_queries" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Slow Queries"
|
|
if [[ "$slow_query_count" -gt 0 ]]; then
|
|
echo "$slow_queries" | while IFS= read -r line; do
|
|
local ts dur stmt
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+|^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+")
|
|
dur=$(echo "$line" | grep -oP "duration: \K[0-9.]+ ms")
|
|
stmt=$(echo "$line" | grep -oP "statement: \K.*" | head -c 60)
|
|
printf " %-22s %-12s %s\n" "${ts:-unknown}" "${dur:-?}" "${stmt:-...}"
|
|
done
|
|
else
|
|
echo " None found (requires log_min_duration_statement in postgresql.conf)."
|
|
fi
|
|
fi
|
|
|
|
# Checkpoint warnings
|
|
local checkpoint_warns
|
|
checkpoint_warns=$(echo "$log_data" | grep -iE "(checkpoint.*too frequent|checkpoints are occurring too frequently|checkpoint.*taking)" | head -"$TOP_N")
|
|
|
|
local checkpoint_count=0
|
|
if [[ -n "$checkpoint_warns" ]]; then
|
|
checkpoint_count=$(echo "$checkpoint_warns" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Checkpoint Warnings"
|
|
if [[ -n "$checkpoint_warns" ]]; then
|
|
echo "$checkpoint_warns" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+|^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+")
|
|
msg=$(echo "$line" | grep -oP "LOG: \K.*|WARNING: \K.*" | head -c 70)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:-checkpoint warning}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# WAL/replication errors
|
|
local wal_errors
|
|
wal_errors=$(echo "$log_data" | grep -iE "(WAL.*error|replication.*error|could not receive|streaming replication|recovery target)" | head -"$TOP_N")
|
|
|
|
local wal_error_count=0
|
|
if [[ -n "$wal_errors" ]]; then
|
|
wal_error_count=$(echo "$wal_errors" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "WAL / Replication Errors"
|
|
if [[ -n "$wal_errors" ]]; then
|
|
echo "$wal_errors" | while IFS= read -r line; do
|
|
local ts msg
|
|
ts=$(echo "$line" | grep -oP "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:]+|^[A-Z][a-z]{2} [ 0-9]{2} [0-9:]+")
|
|
msg=$(echo "$line" | grep -oP "(FATAL|ERROR|LOG): \K.*" | head -c 70)
|
|
printf " %-22s %s\n" "${ts:-unknown}" "${msg:-$line}"
|
|
done
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Auth failures
|
|
local pg_auth_failures
|
|
pg_auth_failures=$(echo "$log_data" | grep -iE "(password authentication failed|no pg_hba.conf entry|authentication failed)" | head -"$TOP_N")
|
|
|
|
local pg_auth_fail_count=0
|
|
if [[ -n "$pg_auth_failures" ]]; then
|
|
pg_auth_fail_count=$(echo "$pg_auth_failures" | wc -l)
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Authentication Failures"
|
|
if [[ -n "$pg_auth_failures" ]]; then
|
|
echo "$pg_auth_failures" | awk -F'"' '{
|
|
# Try to extract user and database
|
|
line = $0
|
|
}
|
|
{
|
|
match($0, /user "([^"]+)"/, u)
|
|
match($0, /database "([^"]+)"/, d)
|
|
user = (u[1] != "") ? u[1] : "?"
|
|
db = (d[1] != "") ? d[1] : "?"
|
|
printf " user=%-15s db=%-15s\n", user, db
|
|
}' | sort | uniq -c | sort -rn | head -"$TOP_N" | \
|
|
awk '{printf " %-6d %s %s\n", $1, $2, $3}'
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
# Top error patterns
|
|
local pg_error_patterns
|
|
pg_error_patterns=$(echo "$log_data" | grep -E " (ERROR|FATAL): " | \
|
|
sed 's/^[^:]*: //' | sed 's/[0-9]\{2,\}/#/g' | \
|
|
sort | uniq -c | sort -rn | head -"$TOP_N")
|
|
|
|
if ! $JSON_MODE; then
|
|
subsection_header "Top Error Patterns"
|
|
if [[ -n "$pg_error_patterns" ]]; then
|
|
printf " ${BOLD}%-6s %s${NC}\n" "Count" "Pattern"
|
|
echo "$pg_error_patterns" | awk '{
|
|
count = $1; $1 = ""
|
|
sub(/^ +/, "")
|
|
printf " %-6d %s\n", count, substr($0, 1, 65)
|
|
}' | head -"$TOP_N"
|
|
else
|
|
echo " None found."
|
|
fi
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
section_header "Summary"
|
|
echo ""
|
|
printf " %-25s %d\n" "PANIC entries:" "$panic_count"
|
|
printf " %-25s %d\n" "FATAL entries:" "$fatal_count"
|
|
printf " %-25s %d\n" "ERROR entries:" "$error_count"
|
|
printf " %-25s %d\n" "Connection errors:" "$conn_error_count"
|
|
printf " %-25s %d\n" "Deadlocks:" "$deadlock_count"
|
|
printf " %-25s %d\n" "Slow queries:" "$slow_query_count"
|
|
printf " %-25s %d\n" "Checkpoint warnings:" "$checkpoint_count"
|
|
printf " %-25s %d\n" "WAL/replication errors:" "$wal_error_count"
|
|
printf " %-25s %d\n" "Auth failures:" "$pg_auth_fail_count"
|
|
printf " %-25s %d\n" "Total lines parsed:" "$total_lines"
|
|
else
|
|
json_add "postgresql" "{\"panic\":${panic_count},\"fatal\":${fatal_count},\"error\":${error_count},\"connection_errors\":${conn_error_count},\"deadlocks\":${deadlock_count},\"slow_queries\":${slow_query_count},\"checkpoint_warnings\":${checkpoint_count},\"wal_errors\":${wal_error_count},\"auth_failures\":${pg_auth_fail_count},\"total_lines\":${total_lines}}"
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# ARGUMENT PARSING
|
|
# ============================================================================
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help) show_usage ;;
|
|
--type) LOG_TYPE="$2"; shift 2 ;;
|
|
--all-types) ALL_TYPES=true; shift ;;
|
|
--since) SINCE="$2"; parse_since "$2"; shift 2 ;;
|
|
--top) TOP_N="$2"; shift 2 ;;
|
|
--json) JSON_MODE=true; shift ;;
|
|
--no-color) NO_COLOR=true; shift ;;
|
|
--output) OUTPUT_FILE="$2"; shift 2 ;;
|
|
--log) CUSTOM_LOG="$2"; shift 2 ;;
|
|
*) log_error "Unknown option: $1"; show_usage ;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$LOG_TYPE" ]] && ! $ALL_TYPES; then
|
|
log_error "Specify --type or --all-types"
|
|
echo ""
|
|
show_usage
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# MAIN
|
|
# ============================================================================
|
|
|
|
main() {
|
|
parse_args "$@"
|
|
|
|
# Disable colors if requested or not a terminal
|
|
if $NO_COLOR || [[ ! -t 1 ]]; then
|
|
disable_colors
|
|
fi
|
|
|
|
# Redirect output to file if specified
|
|
if [[ -n "$OUTPUT_FILE" ]]; then
|
|
disable_colors
|
|
exec > >(tee "$OUTPUT_FILE") 2>&1
|
|
fi
|
|
|
|
if ! $JSON_MODE; then
|
|
echo -e "${BOLD}Linux Log Analyzer v${VERSION}${NC}"
|
|
if [[ -n "$SINCE" ]]; then
|
|
echo -e "Since: ${SINCE}"
|
|
fi
|
|
fi
|
|
|
|
if $ALL_TYPES; then
|
|
# Run all available analyzers
|
|
local ran_any=false
|
|
|
|
for type in system auth docker java gitlab postgresql; do
|
|
if ! $JSON_MODE; then
|
|
echo ""
|
|
echo -e "${DIM}────────────────────────────────────────────────────${NC}"
|
|
echo -e "${BOLD} Analyzing: ${type}${NC}"
|
|
echo -e "${DIM}────────────────────────────────────────────────────${NC}"
|
|
fi
|
|
|
|
case "$type" in
|
|
system) analyze_system && ran_any=true || log_warn "system: skipped" ;;
|
|
auth) analyze_auth && ran_any=true || log_warn "auth: skipped" ;;
|
|
docker) analyze_docker && ran_any=true || log_warn "docker: skipped" ;;
|
|
java) analyze_java && ran_any=true || log_warn "java: skipped" ;;
|
|
gitlab) analyze_gitlab && ran_any=true || log_warn "gitlab: skipped" ;;
|
|
postgresql) analyze_postgresql && ran_any=true || log_warn "postgresql: skipped" ;;
|
|
esac
|
|
done
|
|
|
|
if ! $ran_any; then
|
|
log_error "No log sources found for any type"
|
|
exit 1
|
|
fi
|
|
else
|
|
case "$LOG_TYPE" in
|
|
system) analyze_system ;;
|
|
auth) analyze_auth ;;
|
|
docker) analyze_docker ;;
|
|
java) analyze_java ;;
|
|
gitlab) analyze_gitlab ;;
|
|
postgresql) analyze_postgresql ;;
|
|
*) log_error "Unknown log type: $LOG_TYPE"; show_usage ;;
|
|
esac
|
|
fi
|
|
|
|
if $JSON_MODE; then
|
|
echo "{${JSON_OUTPUT}}"
|
|
fi
|
|
}
|
|
|
|
main "$@"
|