Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
Executable
+355
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#########################################################################################
|
||||
#### log-disk-analyzer.sh — Analyze log directory disk usage and report problems ####
|
||||
#### Reports largest files, growth rates, unrotated logs, broken symlinks, ####
|
||||
#### empty files, and subdirectory breakdown ####
|
||||
#### ####
|
||||
#### Author: Phil Connor ####
|
||||
#### Contact: contact@mylinux.work ####
|
||||
#### License: MIT ####
|
||||
#### Version 1.00 ####
|
||||
#### ####
|
||||
#### Usage: ####
|
||||
#### ./log-disk-analyzer.sh ####
|
||||
#### ./log-disk-analyzer.sh --path /opt/app/logs --top 10 ####
|
||||
#### ./log-disk-analyzer.sh --json ####
|
||||
#### ####
|
||||
#### See --help for all options. ####
|
||||
#########################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
LOG_PATH="${LOG_PATH:-/var/log}"
|
||||
TOP_COUNT="${TOP_COUNT:-20}"
|
||||
COLOR="${COLOR:-auto}"
|
||||
TEXTFILE_DIR="/var/lib/node_exporter"
|
||||
PROM_FILE=""
|
||||
JSON_OUTPUT="false"
|
||||
VERBOSE="false"
|
||||
|
||||
# ── State ─────────────────────────────────────────────────────────────
|
||||
SCRIPT_NAME="$(basename "$0")"
|
||||
readonly SCRIPT_NAME
|
||||
TOTAL_SIZE=0 TOTAL_FILES=0 EMPTY_COUNT=0 BROKEN_COUNT=0 UNROTATED_COUNT=0
|
||||
FILE_LIST=() SIZE_LIST=() RECOMMENDATIONS=()
|
||||
|
||||
# ── Colors ────────────────────────────────────────────────────────────
|
||||
setup_colors() {
|
||||
if [[ "$COLOR" == "never" ]]; then
|
||||
RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""; return
|
||||
fi
|
||||
if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
|
||||
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m'
|
||||
BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m'
|
||||
else
|
||||
RED="" GREEN="" YELLOW="" BOLD="" DIM="" RESET=""
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────
|
||||
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
|
||||
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────
|
||||
human_size() {
|
||||
local b="$1"
|
||||
if [[ "$b" -ge 1073741824 ]]; then printf "%.1f GB" "$(echo "scale=1; $b/1073741824" | bc)"
|
||||
elif [[ "$b" -ge 1048576 ]]; then printf "%.1f MB" "$(echo "scale=1; $b/1048576" | bc)"
|
||||
elif [[ "$b" -ge 1024 ]]; then printf "%.1f KB" "$(echo "scale=1; $b/1024" | bc)"
|
||||
else printf "%d B" "$b"; fi
|
||||
}
|
||||
|
||||
file_age_days() {
|
||||
local mtime now age
|
||||
mtime=$(stat -c %Y "$1" 2>/dev/null) || return 1
|
||||
now=$(date +%s); age=$(( (now - mtime) / 86400 ))
|
||||
[[ "$age" -lt 1 ]] && age=1
|
||||
echo "$age"
|
||||
}
|
||||
|
||||
separator() { printf " %s\n" "$(printf '%.0s─' {1..62})"; }
|
||||
|
||||
json_escape() { local s="$1"; s="${s//\\/\\\\}"; s="${s//\"/\\\"}"; printf '%s' "$s"; }
|
||||
|
||||
# ── Collect all files ─────────────────────────────────────────────────
|
||||
collect_files() {
|
||||
while IFS= read -r line; do
|
||||
local size="${line%% *}" file="${line#* }"
|
||||
[[ -z "$size" || -z "$file" ]] && continue
|
||||
FILE_LIST+=("$file"); SIZE_LIST+=("$size")
|
||||
TOTAL_SIZE=$((TOTAL_SIZE + size)); TOTAL_FILES=$((TOTAL_FILES + 1))
|
||||
done < <(find "$LOG_PATH" -type f -printf '%s\t%p\n' 2>/dev/null | sort -t$'\t' -k1 -rn)
|
||||
}
|
||||
|
||||
# ── Top files by size ─────────────────────────────────────────────────
|
||||
print_top_files() {
|
||||
local limit="$TOP_COUNT"
|
||||
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
||||
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
||||
echo ""; echo -e " ${BOLD}TOP ${limit} FILES BY SIZE${RESET}"; separator
|
||||
for (( i = 0; i < limit; i++ )); do
|
||||
local size_h color=""
|
||||
size_h="$(human_size "${SIZE_LIST[$i]}")"
|
||||
[[ "${SIZE_LIST[$i]}" -ge 104857600 ]] && color="$RED"
|
||||
[[ "${SIZE_LIST[$i]}" -lt 104857600 && "${SIZE_LIST[$i]}" -ge 52428800 ]] && color="$YELLOW"
|
||||
printf " %b%-10s%b %s\n" "$color" "$size_h" "$RESET" "${FILE_LIST[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
# ── Growth rates ──────────────────────────────────────────────────────
|
||||
print_growth_rates() {
|
||||
local limit="$TOP_COUNT"
|
||||
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
||||
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
||||
local -a rates=() rate_files=() rate_ages=()
|
||||
for (( i = 0; i < ${#FILE_LIST[@]}; i++ )); do
|
||||
local age; age=$(file_age_days "${FILE_LIST[$i]}") || continue
|
||||
rates+=("$(( SIZE_LIST[i] / age ))"); rate_files+=("${FILE_LIST[$i]}"); rate_ages+=("$age")
|
||||
done
|
||||
[[ ${#rates[@]} -eq 0 ]] && return
|
||||
local -a sorted_idx=()
|
||||
mapfile -t sorted_idx < <(for i in "${!rates[@]}"; do echo "$i ${rates[$i]}"; done | sort -k2 -rn | head -n "$limit" | awk '{print $1}')
|
||||
echo ""; echo -e " ${BOLD}GROWTH RATE (SIZE / AGE)${RESET}"; separator
|
||||
for idx in "${sorted_idx[@]}"; do
|
||||
printf " %-16s %s (%d days old)\n" "$(human_size "${rates[$idx]}")/day" "${rate_files[$idx]}" "${rate_ages[$idx]}"
|
||||
done
|
||||
}
|
||||
|
||||
# ── Unrotated logs ────────────────────────────────────────────────────
|
||||
print_unrotated() {
|
||||
echo ""; echo -e " ${BOLD}UNROTATED LOGS (>100 MB, >7 days old)${RESET}"; separator
|
||||
local found=0
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
local size age; size=$(stat -c %s "$file" 2>/dev/null) || continue
|
||||
[[ "$size" -lt 104857600 ]] && continue
|
||||
age=$(file_age_days "$file") || continue; [[ "$age" -lt 7 ]] && continue
|
||||
found=1; UNROTATED_COUNT=$((UNROTATED_COUNT + 1))
|
||||
local size_h; size_h="$(human_size "$size")"
|
||||
printf " %b%-10s%b %3d days %s\n" "$RED" "$size_h" "$RESET" "$age" "$file"
|
||||
RECOMMENDATIONS+=("Rotate $(basename "$file") — ${size_h} and ${age} days old")
|
||||
done < <(find "$LOG_PATH" -type f -name '*.log' 2>/dev/null)
|
||||
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
||||
}
|
||||
|
||||
# ── Empty files ───────────────────────────────────────────────────────
|
||||
print_empty_files() {
|
||||
echo ""; echo -e " ${BOLD}EMPTY FILES${RESET}"; separator
|
||||
local found=0
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue; found=1; EMPTY_COUNT=$((EMPTY_COUNT + 1))
|
||||
echo " $file"
|
||||
done < <(find "$LOG_PATH" -type f -empty 2>/dev/null)
|
||||
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
||||
if [[ "$EMPTY_COUNT" -gt 0 ]]; then
|
||||
RECOMMENDATIONS+=("Remove ${EMPTY_COUNT} empty log files to reclaim inodes")
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Broken symlinks ──────────────────────────────────────────────────
|
||||
print_broken_symlinks() {
|
||||
echo ""; echo -e " ${BOLD}BROKEN SYMLINKS${RESET}"; separator
|
||||
local found=0
|
||||
while IFS= read -r link; do
|
||||
[[ -z "$link" ]] && continue; found=1; BROKEN_COUNT=$((BROKEN_COUNT + 1))
|
||||
local target; target=$(readlink "$link" 2>/dev/null || echo "unknown")
|
||||
printf " %b%s%b -> %s\n" "$YELLOW" "$link" "$RESET" "$target"
|
||||
done < <(find "$LOG_PATH" -xtype l 2>/dev/null)
|
||||
[[ "$found" -eq 0 ]] && echo -e " ${GREEN}None found${RESET}"
|
||||
if [[ "$BROKEN_COUNT" -gt 0 ]]; then
|
||||
RECOMMENDATIONS+=("Fix ${BROKEN_COUNT} broken symlinks")
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Subdirectory breakdown ────────────────────────────────────────────
|
||||
print_subdir_breakdown() {
|
||||
echo ""; echo -e " ${BOLD}DISK USAGE BY SUBDIRECTORY${RESET}"; separator
|
||||
du -d 1 "$LOG_PATH" 2>/dev/null | sort -rn | while IFS=$'\t' read -r kb path; do
|
||||
[[ "$path" == "$LOG_PATH" ]] && continue
|
||||
printf " %-10s %s\n" "$(human_size $((kb * 1024)))" "$path"
|
||||
done || true
|
||||
}
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────
|
||||
print_summary() {
|
||||
echo ""; echo -e " ${BOLD}Summary${RESET}"
|
||||
printf " %-20s %s\n" "Total size:" "$(human_size "$TOTAL_SIZE")"
|
||||
printf " %-20s %d\n" "Total files:" "$TOTAL_FILES"
|
||||
printf " %-20s %d\n" "Empty files:" "$EMPTY_COUNT"
|
||||
printf " %-20s %d\n" "Broken symlinks:" "$BROKEN_COUNT"
|
||||
printf " %-20s %d\n" "Unrotated logs:" "$UNROTATED_COUNT"
|
||||
if [[ ${#RECOMMENDATIONS[@]} -gt 0 ]]; then
|
||||
echo ""; echo -e " ${BOLD}Recommendations:${RESET}"
|
||||
for rec in "${RECOMMENDATIONS[@]}"; do echo -e " ${YELLOW}•${RESET} $rec"; done
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ── JSON output ───────────────────────────────────────────────────────
|
||||
print_json() {
|
||||
local limit="$TOP_COUNT" i first
|
||||
[[ "$VERBOSE" == "true" ]] && limit="${#FILE_LIST[@]}"
|
||||
[[ "$limit" -gt "${#FILE_LIST[@]}" ]] && limit="${#FILE_LIST[@]}"
|
||||
printf '{"path":"%s","top_files":[' "$(json_escape "$LOG_PATH")"
|
||||
for (( i = 0; i < limit; i++ )); do
|
||||
[[ "$i" -gt 0 ]] && printf ','
|
||||
printf '{"file":"%s","size":%s}' "$(json_escape "${FILE_LIST[$i]}")" "${SIZE_LIST[$i]}"
|
||||
done
|
||||
printf '],"growth_rates":['
|
||||
first=1
|
||||
for (( i = 0; i < limit && i < ${#FILE_LIST[@]}; i++ )); do
|
||||
local age; age=$(file_age_days "${FILE_LIST[$i]}") || continue
|
||||
[[ "$first" -eq 0 ]] && printf ','; first=0
|
||||
printf '{"file":"%s","bytes_per_day":%d,"age_days":%d}' \
|
||||
"$(json_escape "${FILE_LIST[$i]}")" "$(( SIZE_LIST[i] / age ))" "$age"
|
||||
done
|
||||
printf '],"unrotated_logs":['
|
||||
first=1
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
local size age; size=$(stat -c %s "$file" 2>/dev/null) || continue
|
||||
[[ "$size" -lt 104857600 ]] && continue
|
||||
age=$(file_age_days "$file") || continue; [[ "$age" -lt 7 ]] && continue
|
||||
[[ "$first" -eq 0 ]] && printf ','; first=0
|
||||
printf '{"file":"%s","size":%d,"age_days":%d}' "$(json_escape "$file")" "$size" "$age"
|
||||
done < <(find "$LOG_PATH" -type f -name '*.log' 2>/dev/null)
|
||||
printf '],"empty_files":['
|
||||
first=1
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue; [[ "$first" -eq 0 ]] && printf ','; first=0
|
||||
printf '"%s"' "$(json_escape "$file")"
|
||||
done < <(find "$LOG_PATH" -type f -empty 2>/dev/null)
|
||||
printf '],"broken_symlinks":['
|
||||
first=1
|
||||
while IFS= read -r link; do
|
||||
[[ -z "$link" ]] && continue; [[ "$first" -eq 0 ]] && printf ','; first=0
|
||||
printf '{"link":"%s","target":"%s"}' "$(json_escape "$link")" "$(json_escape "$(readlink "$link" 2>/dev/null || echo unknown)")"
|
||||
done < <(find "$LOG_PATH" -xtype l 2>/dev/null)
|
||||
printf '],"summary":{"total_size":%d,"total_files":%d,"empty_files":%d,"broken_symlinks":%d,"unrotated_logs":%d}}\n' \
|
||||
"$TOTAL_SIZE" "$TOTAL_FILES" "$EMPTY_COUNT" "$BROKEN_COUNT" "$UNROTATED_COUNT"
|
||||
}
|
||||
|
||||
# ── Prometheus output ─────────────────────────────────────────────────
|
||||
write_prometheus() {
|
||||
local file="$1"
|
||||
local output_dir
|
||||
output_dir="$(dirname "$file")"
|
||||
mkdir -p "$output_dir"
|
||||
local tmp
|
||||
tmp=$(mktemp "${output_dir}/.log_disk.XXXXXX")
|
||||
{
|
||||
echo "# HELP log_disk_total_bytes Total size of log directory in bytes"
|
||||
echo "# TYPE log_disk_total_bytes gauge"
|
||||
printf 'log_disk_total_bytes{path="%s"} %d\n' "$LOG_PATH" "$TOTAL_SIZE"
|
||||
echo "# HELP log_disk_total_files Total number of files in log directory"
|
||||
echo "# TYPE log_disk_total_files gauge"
|
||||
printf 'log_disk_total_files{path="%s"} %d\n' "$LOG_PATH" "$TOTAL_FILES"
|
||||
echo "# HELP log_disk_empty_files Number of empty files in log directory"
|
||||
echo "# TYPE log_disk_empty_files gauge"
|
||||
printf 'log_disk_empty_files{path="%s"} %d\n' "$LOG_PATH" "$EMPTY_COUNT"
|
||||
echo "# HELP log_disk_broken_symlinks Number of broken symlinks in log directory"
|
||||
echo "# TYPE log_disk_broken_symlinks gauge"
|
||||
printf 'log_disk_broken_symlinks{path="%s"} %d\n' "$LOG_PATH" "$BROKEN_COUNT"
|
||||
echo "# HELP log_disk_unrotated_logs Number of unrotated log files"
|
||||
echo "# TYPE log_disk_unrotated_logs gauge"
|
||||
printf 'log_disk_unrotated_logs{path="%s"} %d\n' "$LOG_PATH" "$UNROTATED_COUNT"
|
||||
} > "$tmp"
|
||||
chmod 644 "$tmp"
|
||||
mv -f "$tmp" "$file"
|
||||
verbose "Metrics written to ${file}"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# USAGE
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
${SCRIPT_NAME} — Analyze log directory disk usage and report problems
|
||||
|
||||
USAGE:
|
||||
${SCRIPT_NAME} [OPTIONS]
|
||||
|
||||
OPTIONS:
|
||||
--path DIR Directory to scan (default: ${LOG_PATH})
|
||||
--top N Show top N largest files (default: ${TOP_COUNT})
|
||||
--json Output results in JSON format
|
||||
--no-color Disable colored output
|
||||
--textfile Write metrics to node_exporter textfile collector
|
||||
-o, --output PATH Write metrics to custom file path
|
||||
--verbose Show all files, not just top N
|
||||
--help Show this help
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
LOG_PATH Directory to scan (default: /var/log)
|
||||
TOP_COUNT Number of top files to show (default: 20)
|
||||
COLOR Color mode: auto, always, never (default: auto)
|
||||
|
||||
EXAMPLES:
|
||||
sudo ./log-disk-analyzer.sh
|
||||
./log-disk-analyzer.sh --path /opt/app/logs
|
||||
./log-disk-analyzer.sh --json
|
||||
./log-disk-analyzer.sh --textfile
|
||||
./log-disk-analyzer.sh -o /tmp/log_disk.prom
|
||||
EOF
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# ARGUMENT PARSING
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--path) LOG_PATH="$2"; shift 2 ;;
|
||||
--top) TOP_COUNT="$2"; shift 2 ;;
|
||||
--json) JSON_OUTPUT="true"; shift ;;
|
||||
--verbose) VERBOSE="true"; shift ;;
|
||||
--no-color) COLOR="never"; shift ;;
|
||||
--textfile) PROM_FILE="$TEXTFILE_DIR/log_disk.prom"; shift ;;
|
||||
-o|--output) PROM_FILE="$2"; shift 2 ;;
|
||||
--help|-h) setup_colors; usage; exit 0 ;;
|
||||
-*) err "Unknown option: $1"; echo "Run ${SCRIPT_NAME} --help for usage" >&2; exit 1 ;;
|
||||
*) err "Unexpected argument: $1"; echo "Run ${SCRIPT_NAME} --help for usage" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MAIN
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
setup_colors
|
||||
|
||||
if [[ ! -d "$LOG_PATH" ]]; then
|
||||
err "Directory not found: $LOG_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
collect_files
|
||||
|
||||
if [[ "$JSON_OUTPUT" == "true" ]]; then
|
||||
print_json
|
||||
else
|
||||
echo ""
|
||||
echo -e "${BOLD}Log Disk Analyzer${RESET}"
|
||||
echo -e "${DIM}Path: ${LOG_PATH}${RESET}"
|
||||
print_top_files
|
||||
print_growth_rates
|
||||
print_unrotated
|
||||
print_empty_files
|
||||
print_broken_symlinks
|
||||
print_subdir_breakdown
|
||||
print_summary
|
||||
fi
|
||||
|
||||
if [[ -n "$PROM_FILE" ]]; then
|
||||
write_prometheus "$PROM_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user