chiefgeek
a1a17e81a1
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
356 lines
18 KiB
Bash
Executable File
356 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
#########################################################################################
|
|
#### log-disk-analyzer.sh — Analyze log directory disk usage and report problems ####
|
|
#### Reports largest files, growth rates, unrotated logs, broken symlinks, ####
|
|
#### empty files, and subdirectory breakdown ####
|
|
#### ####
|
|
#### Author: Phil Connor ####
|
|
#### Contact: contact@mylinux.work ####
|
|
#### License: MIT ####
|
|
#### Version 1.00 ####
|
|
#### ####
|
|
#### Usage: ####
|
|
#### ./log-disk-analyzer.sh ####
|
|
#### ./log-disk-analyzer.sh --path /opt/app/logs --top 10 ####
|
|
#### ./log-disk-analyzer.sh --json ####
|
|
#### ####
|
|
#### See --help for all options. ####
|
|
#########################################################################################
|
|
|
|
set -euo pipefail
|
|
|
|
# ── Defaults ──────────────────────────────────────────────────────────
|
|
LOG_PATH="${LOG_PATH:-/var/log}"
|
|
TOP_COUNT="${TOP_COUNT:-20}"
|
|
COLOR="${COLOR:-auto}"
|
|
TEXTFILE_DIR="/var/lib/node_exporter"
|
|
PROM_FILE=""
|
|
JSON_OUTPUT="false"
|
|
VERBOSE="false"
|
|
|
|
# ── State ─────────────────────────────────────────────────────────────
|
|
SCRIPT_NAME="$(basename "$0")"
|
|
readonly SCRIPT_NAME
|
|
TOTAL_SIZE=0 TOTAL_FILES=0 EMPTY_COUNT=0 BROKEN_COUNT=0 UNROTATED_COUNT=0
|
|
FILE_LIST=() SIZE_LIST=() RECOMMENDATIONS=()
|
|
|
|
# ── Colors ────────────────────────────────────────────────────────────
|
|
setup_colors() {
|
|
if [[ "$COLOR" == "never" ]]; then
|
|
RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""; return
|
|
fi
|
|
if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
|
|
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
|
|
BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m'
|
|
else
|
|
RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""
|
|
fi
|
|
}
|
|
|
|
# ── Logging ───────────────────────────────────────────────────────────
|
|
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
|
|
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────
|
|
human_size() {
|
|
local b="$1"
|
|
if [[ "$b" -ge 1073741824 ]]; then printf "%.1f GB" "$(echo "scale=1; $b/1073741824" | bc)"
|
|
elif [[ "$b" -ge 1048576 ]]; then printf "%.1f MB" "$(echo "scale=1; $b/1048576" | bc)"
|
|
elif [[ "$b" -ge 1024 ]]; then printf "%.1f KB" "$(echo "scale=1; $b/1024" | bc)"
|
|
else printf "%d B" "$b"; fi
|
|
}
|
|
|
|
file_age_days() {
|
|
local mtime now age
|
|
mtime=$(stat -c %Y "$1" 2>/dev/null) || return 1
|
|
now=$(date +%s); age=$(( (now - mtime) / 86400 ))
|
|
[[ "$age" -lt 1 ]] && age=1
|
|
echo "$age"
|
|
}
|
|
|
|
separator() { printf " %s\n" "$(printf '%.0s─' {1..62})"; }
|
|
|
|
json_escape() { local s="$1"; s="${s//\\/\\\\}"; s="${s//\"/\\\"}"; printf '%s' "$s"; }
|
|
|
|
# ── Collect all files ─────────────────────────────────────────────────
|
|
collect_files() {
|
|
while IFS= read -r line; do
|
|
local size="${line%% *}" file="${line#* }"
|
|
[[ -z "$size" || -z "$file" ]] && continue
|
|
FILE_LIST+=("$file"); SIZE_LIST+=("$size")
|
|
TOTAL_SIZE=$((TOTAL_SIZE + size)); TOTAL_FILES=$((TOTAL_FILES + 1))
|
|
done < <(find "$LOG_PATH" -type f -printf '%s\t%p\n' 2>/dev/null | sort -t$'\t' -k1 -rn)
|
|
}
|
|
|
|
# ── Top files by size ─────────────────────────────────────────────────
|
|
print_top_files() {
|
|
local limit="$TOP_COUNT"
|
|
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
|
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
|
echo ""; echo -e " ${BOLD}TOP ${limit} FILES BY SIZE${RESET}"; separator
|
|
for (( i = 0; i < limit; i++ )); do
|
|
local size_h color=""
|
|
size_h="$(human_size "${SIZE_LIST[$i]}")"
|
|
[[ "${SIZE_LIST[$i]}" -ge 104857600 ]] && color="$RED"
|
|
[[ "${SIZE_LIST[$i]}" -lt 104857600 && "${SIZE_LIST[$i]}" -ge 52428800 ]] && color="$YELLOW"
|
|
printf " %b%-10s%b %s\n" "$color" "$size_h" "$RESET" "${FILE_LIST[$i]}"
|
|
done
|
|
}
|
|
|
|
# ── Growth rates ──────────────────────────────────────────────────────
|
|
print_growth_rates() {
|
|
local limit="$TOP_COUNT"
|
|
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
|
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
|
local -a rates=() rate_files=() rate_ages=()
|
|
for (( i = 0; i < ${#FILE_LIST[@]}; i++ )); do
|
|
local age; age=$(file_age_days "${FILE_LIST[$i]}") || continue
|
|
rates+=("$(( SIZE_LIST[i] / age ))"); rate_files+=("${FILE_LIST[$i]}"); rate_ages+=("$age")
|
|
done
|
|
[[ ${#rates[@]} -eq 0 ]] && return
|
|
local -a sorted_idx=()
|
|
mapfile -t sorted_idx < <(for i in "${!rates[@]}"; do echo "$i ${rates[$i]}"; done | sort -k2 -rn | head -n "$limit" | awk '{print $1}')
|
|
echo ""; echo -e " ${BOLD}GROWTH RATE (SIZE / AGE)${RESET}"; separator
|
|
for idx in "${sorted_idx[@]}"; do
|
|
printf " %-16s %s (%d days old)\n" "$(human_size "${rates[$idx]}")/day" "${rate_files[$idx]}" "${rate_ages[$idx]}"
|
|
done
|
|
}
|
|
|
|
# ── Unrotated logs ────────────────────────────────────────────────────
|
|
print_unrotated() {
|
|
echo ""; echo -e " ${BOLD}UNROTATED LOGS (>100 MB, >7 days old)${RESET}"; separator
|
|
local found=0
|
|
while IFS= read -r file; do
|
|
[[ -z "$file" ]] && continue
|
|
local size age; size=$(stat -c %s "$file" 2>/dev/null) || continue
|
|
[[ "$size" -lt 104857600 ]] && continue
|
|
age=$(file_age_days "$file") || continue; [[ "$age" -lt 7 ]] && continue
|
|
found=1; UNROTATED_COUNT=$((UNROTATED_COUNT + 1))
|
|
local size_h; size_h="$(human_size "$size")"
|
|
printf " %b%-10s%b %3d days %s\n" "$RED" "$size_h" "$RESET" "$age" "$file"
|
|
RECOMMENDATIONS+=("Rotate $(basename "$file") — ${size_h} and ${age} days old")
|
|
done < <(find "$LOG_PATH" -type f -name '*.log' 2>/dev/null)
|
|
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
|
}
|
|
|
|
# ── Empty files ───────────────────────────────────────────────────────
|
|
print_empty_files() {
|
|
echo ""; echo -e " ${BOLD}EMPTY FILES${RESET}"; separator
|
|
local found=0
|
|
while IFS= read -r file; do
|
|
[[ -z "$file" ]] && continue; found=1; EMPTY_COUNT=$((EMPTY_COUNT + 1))
|
|
echo " $file"
|
|
done < <(find "$LOG_PATH" -type f -empty 2>/dev/null)
|
|
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
|
if [[ "$EMPTY_COUNT" -gt 0 ]]; then
|
|
RECOMMENDATIONS+=("Remove ${EMPTY_COUNT} empty log files to reclaim inodes")
|
|
fi
|
|
}
|
|
|
|
# ── Broken symlinks ──────────────────────────────────────────────────
|
|
print_broken_symlinks() {
|
|
echo ""; echo -e " ${BOLD}BROKEN SYMLINKS${RESET}"; separator
|
|
local found=0
|
|
while IFS= read -r link; do
|
|
[[ -z "$link" ]] && continue; found=1; BROKEN_COUNT=$((BROKEN_COUNT + 1))
|
|
local target; target=$(readlink "$link" 2>/dev/null || echo "unknown")
|
|
printf " %b%s%b -> %s\n" "$YELLOW" "$link" "$RESET" "$target"
|
|
done < <(find "$LOG_PATH" -xtype l 2>/dev/null)
|
|
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
|
if [[ "$BROKEN_COUNT" -gt 0 ]]; then
|
|
RECOMMENDATIONS+=("Fix ${BROKEN_COUNT} broken symlinks")
|
|
fi
|
|
}
|
|
|
|
# ── Subdirectory breakdown ────────────────────────────────────────────
|
|
print_subdir_breakdown() {
|
|
echo ""; echo -e " ${BOLD}DISK USAGE BY SUBDIRECTORY${RESET}"; separator
|
|
du -d 1 "$LOG_PATH" 2>/dev/null | sort -rn | while IFS=$'\t' read -r kb path; do
|
|
[[ "$path" == "$LOG_PATH" ]] && continue
|
|
printf " %-10s %s\n" "$(human_size $((kb * 1024)))" "$path"
|
|
done || true
|
|
}
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────
|
|
print_summary() {
|
|
echo ""; echo -e " ${BOLD}Summary${RESET}"
|
|
printf " %-20s %s\n" "Total size:" "$(human_size "$TOTAL_SIZE")"
|
|
printf " %-20s %d\n" "Total files:" "$TOTAL_FILES"
|
|
printf " %-20s %d\n" "Empty files:" "$EMPTY_COUNT"
|
|
printf " %-20s %d\n" "Broken symlinks:" "$BROKEN_COUNT"
|
|
printf " %-20s %d\n" "Unrotated logs:" "$UNROTATED_COUNT"
|
|
if [[ ${#RECOMMENDATIONS[@]} -gt 0 ]]; then
|
|
echo ""; echo -e " ${BOLD}Recommendations:${RESET}"
|
|
for rec in "${RECOMMENDATIONS[@]}"; do echo -e " ${YELLOW}•${RESET} $rec"; done
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
# ── JSON output ───────────────────────────────────────────────────────
|
|
print_json() {
|
|
local limit="$TOP_COUNT" i first
|
|
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
|
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
|
printf '{"path":"%s","top_files":[' "$(json_escape "$LOG_PATH")"
|
|
for (( i = 0; i < limit; i++ )); do
|
|
[[ "$i" -gt 0 ]] && printf ','
|
|
printf '{"file":"%s","size":%s}' "$(json_escape "${FILE_LIST[$i]}")" "${SIZE_LIST[$i]}"
|
|
done
|
|
printf '],"growth_rates":['
|
|
first=1
|
|
for (( i = 0; i < limit && i < ${#FILE_LIST[@]}; i++ )); do
|
|
local age; age=$(file_age_days "${FILE_LIST[$i]}") || continue
|
|
[[ "$first" -eq 0 ]] && printf ','; first=0
|
|
printf '{"file":"%s","bytes_per_day":%d,"age_days":%d}' \
|
|
"$(json_escape "${FILE_LIST[$i]}")" "$(( SIZE_LIST[i] / age ))" "$age"
|
|
done
|
|
printf '],"unrotated_logs":['
|
|
first=1
|
|
while IFS= read -r file; do
|
|
[[ -z "$file" ]] && continue
|
|
local size age; size=$(stat -c %s "$file" 2>/dev/null) || continue
|
|
[[ "$size" -lt 104857600 ]] && continue
|
|
age=$(file_age_days "$file") || continue; [[ "$age" -lt 7 ]] && continue
|
|
[[ "$first" -eq 0 ]] && printf ','; first=0
|
|
printf '{"file":"%s","size":%d,"age_days":%d}' "$(json_escape "$file")" "$size" "$age"
|
|
done < <(find "$LOG_PATH" -type f -name '*.log' 2>/dev/null)
|
|
printf '],"empty_files":['
|
|
first=1
|
|
while IFS= read -r file; do
|
|
[[ -z "$file" ]] && continue; [[ "$first" -eq 0 ]] && printf ','; first=0
|
|
printf '"%s"' "$(json_escape "$file")"
|
|
done < <(find "$LOG_PATH" -type f -empty 2>/dev/null)
|
|
printf '],"broken_symlinks":['
|
|
first=1
|
|
while IFS= read -r link; do
|
|
[[ -z "$link" ]] && continue; [[ "$first" -eq 0 ]] && printf ','; first=0
|
|
printf '{"link":"%s","target":"%s"}' "$(json_escape "$link")" "$(json_escape "$(readlink "$link" 2>/dev/null || echo unknown)")"
|
|
done < <(find "$LOG_PATH" -xtype l 2>/dev/null)
|
|
printf '],"summary":{"total_size":%d,"total_files":%d,"empty_files":%d,"broken_symlinks":%d,"unrotated_logs":%d}}\n' \
|
|
"$TOTAL_SIZE" "$TOTAL_FILES" "$EMPTY_COUNT" "$BROKEN_COUNT" "$UNROTATED_COUNT"
|
|
}
|
|
|
|
# ── Prometheus output ─────────────────────────────────────────────────
|
|
write_prometheus() {
|
|
local file="$1"
|
|
local output_dir
|
|
output_dir="$(dirname "$file")"
|
|
mkdir -p "$output_dir"
|
|
local tmp
|
|
tmp=$(mktemp "${output_dir}/.log_disk.XXXXXX")
|
|
{
|
|
echo "# HELP log_disk_total_bytes Total size of log directory in bytes"
|
|
echo "# TYPE log_disk_total_bytes gauge"
|
|
printf 'log_disk_total_bytes{path="%s"} %d\n' "$LOG_PATH" "$TOTAL_SIZE"
|
|
echo "# HELP log_disk_total_files Total number of files in log directory"
|
|
echo "# TYPE log_disk_total_files gauge"
|
|
printf 'log_disk_total_files{path="%s"} %d\n' "$LOG_PATH" "$TOTAL_FILES"
|
|
echo "# HELP log_disk_empty_files Number of empty files in log directory"
|
|
echo "# TYPE log_disk_empty_files gauge"
|
|
printf 'log_disk_empty_files{path="%s"} %d\n' "$LOG_PATH" "$EMPTY_COUNT"
|
|
echo "# HELP log_disk_broken_symlinks Number of broken symlinks in log directory"
|
|
echo "# TYPE log_disk_broken_symlinks gauge"
|
|
printf 'log_disk_broken_symlinks{path="%s"} %d\n' "$LOG_PATH" "$BROKEN_COUNT"
|
|
echo "# HELP log_disk_unrotated_logs Number of unrotated log files"
|
|
echo "# TYPE log_disk_unrotated_logs gauge"
|
|
printf 'log_disk_unrotated_logs{path="%s"} %d\n' "$LOG_PATH" "$UNROTATED_COUNT"
|
|
} > "$tmp"
|
|
chmod 644 "$tmp"
|
|
mv -f "$tmp" "$file"
|
|
verbose "Metrics written to ${file}"
|
|
}
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# USAGE
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
${SCRIPT_NAME} — Analyze log directory disk usage and report problems
|
|
|
|
USAGE:
|
|
${SCRIPT_NAME} [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--path DIR Directory to scan (default: ${LOG_PATH})
|
|
--top N Show top N largest files (default: ${TOP_COUNT})
|
|
--json Output results in JSON format
|
|
--no-color Disable colored output
|
|
--textfile Write metrics to node_exporter textfile collector
|
|
-o, --output PATH Write metrics to custom file path
|
|
--verbose Show all files, not just top N
|
|
--help Show this help
|
|
|
|
ENVIRONMENT VARIABLES:
|
|
LOG_PATH Directory to scan (default: /var/log)
|
|
TOP_COUNT Number of top files to show (default: 20)
|
|
COLOR Color mode: auto, always, never (default: auto)
|
|
|
|
EXAMPLES:
|
|
sudo ./log-disk-analyzer.sh
|
|
./log-disk-analyzer.sh --path /opt/app/logs
|
|
./log-disk-analyzer.sh --json
|
|
./log-disk-analyzer.sh --textfile
|
|
./log-disk-analyzer.sh -o /tmp/log_disk.prom
|
|
EOF
|
|
}
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# ARGUMENT PARSING
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--path) LOG_PATH="$2"; shift 2 ;;
|
|
--top) TOP_COUNT="$2"; shift 2 ;;
|
|
--json) JSON_OUTPUT="true"; shift ;;
|
|
--verbose) VERBOSE="true"; shift ;;
|
|
--no-color) COLOR="never"; shift ;;
|
|
--textfile) PROM_FILE="$TEXTFILE_DIR/log_disk.prom"; shift ;;
|
|
-o|--output) PROM_FILE="$2"; shift 2 ;;
|
|
--help|-h) setup_colors; usage; exit 0 ;;
|
|
-*) err "Unknown option: $1"; echo "Run ${SCRIPT_NAME} --help for usage" >&2; exit 1 ;;
|
|
*) err "Unexpected argument: $1"; echo "Run ${SCRIPT_NAME} --help for usage" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# MAIN
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
main() {
|
|
parse_args "$@"
|
|
setup_colors
|
|
|
|
if [[ ! -d "$LOG_PATH" ]]; then
|
|
err "Directory not found: $LOG_PATH"
|
|
exit 1
|
|
fi
|
|
|
|
collect_files
|
|
|
|
if [[ "$JSON_OUTPUT" == "true" ]]; then
|
|
print_json
|
|
else
|
|
echo ""
|
|
echo -e "${BOLD}Log Disk Analyzer${RESET}"
|
|
echo -e "${DIM}Path: ${LOG_PATH}${RESET}"
|
|
print_top_files
|
|
print_growth_rates
|
|
print_unrotated
|
|
print_empty_files
|
|
print_broken_symlinks
|
|
print_subdir_breakdown
|
|
print_summary
|
|
fi
|
|
|
|
if [[ -n "$PROM_FILE" ]]; then
|
|
write_prometheus "$PROM_FILE"
|
|
fi
|
|
}
|
|
|
|
main "$@"
|