chiefgeek
a1a17e81a1
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
386 lines
13 KiB
Bash
Executable File
386 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
#########################################################################################
|
|
#### logrotate-check-exporter.sh — Logrotate health metrics for Prometheus ####
|
|
#### Tracks rotation timestamps, log file sizes/ages, and stale log detection ####
|
|
#### Requires: bash 4+, coreutils ####
|
|
#### ####
|
|
#### Author: Phil Connor ####
|
|
#### Contact: contact@mylinux.work ####
|
|
#### License: MIT ####
|
|
#### Version: 1.2 ####
|
|
#### ####
|
|
#### Usage: ####
|
|
#### ./logrotate-check-exporter.sh # stdout ####
|
|
#### ./logrotate-check-exporter.sh --textfile # node_exporter textfile ####
|
|
#### ./logrotate-check-exporter.sh --daemon # continuous collection ####
|
|
#### ####
|
|
#### See --help for all options. ####
|
|
#########################################################################################
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_NAME=$(basename "$0")
|
|
readonly SCRIPT_NAME
|
|
|
|
# Default configuration
|
|
TEXTFILE_DIR="/var/lib/node_exporter"
|
|
OUTPUT_FILE=""
|
|
LOG_DIR="${LOG_DIR:-/var/log}"
|
|
WATCH_PATHS=""
|
|
readonly DEFAULT_STALE_THRESHOLD=172800
|
|
readonly DEFAULT_COLLECTION_INTERVAL=60
|
|
|
|
# Configuration variables (can be overridden by environment)
|
|
STALE_THRESHOLD=${STALE_THRESHOLD:-$DEFAULT_STALE_THRESHOLD}
|
|
COLLECTION_INTERVAL=${COLLECTION_INTERVAL:-$DEFAULT_COLLECTION_INTERVAL}
|
|
DEBUG=${DEBUG:-}
|
|
|
|
# Runtime flags
|
|
RUN_MODE="once"
|
|
|
|
debug_echo() {
|
|
if [[ -n "$DEBUG" ]]; then
|
|
echo "[DEBUG] $*" >&2
|
|
fi
|
|
}
|
|
|
|
show_help() {
|
|
cat << EOF
|
|
Usage: $SCRIPT_NAME [OPTIONS]
|
|
|
|
Logrotate health metrics for Prometheus (v1.2).
|
|
|
|
Parses the logrotate status file to track rotation timestamps, monitors log file
|
|
sizes and ages, and detects stale logs that haven't been rotated within a
|
|
configurable threshold.
|
|
|
|
By default, discovers all log files in $LOG_DIR (excluding compressed archives).
|
|
Use --watch to monitor specific files instead.
|
|
|
|
MODES:
|
|
--textfile Write to node_exporter textfile collector
|
|
--daemon Run continuously at COLLECTION_INTERVAL
|
|
(default) Output metrics to stdout
|
|
|
|
OPTIONS:
|
|
--watch PATHS Comma-separated log file paths to monitor instead of auto-discovery
|
|
--log-dir DIR Base directory for auto-discovery (default: /var/log)
|
|
-o, --output Output file path
|
|
-h, --help Show this help message
|
|
|
|
ENVIRONMENT VARIABLES:
|
|
LOG_DIR Base directory for log file discovery (default: /var/log)
|
|
STALE_THRESHOLD Seconds before a file is considered stale (default: $DEFAULT_STALE_THRESHOLD = 48h)
|
|
COLLECTION_INTERVAL Seconds between collections in daemon mode (default: $DEFAULT_COLLECTION_INTERVAL)
|
|
DEBUG Enable debug output
|
|
|
|
EXAMPLES:
|
|
$SCRIPT_NAME # Auto-discover all logs in /var/log
|
|
$SCRIPT_NAME --textfile # Write to textfile collector
|
|
$SCRIPT_NAME --watch /var/log/syslog,/var/log/auth.log # Monitor specific files
|
|
$SCRIPT_NAME --log-dir /opt/app/logs # Scan a custom log directory
|
|
$SCRIPT_NAME --daemon # Continuous collection
|
|
|
|
METRICS:
|
|
- logrotate_last_run_timestamp Unix timestamp of last logrotate run
|
|
- logrotate_status Whether logrotate has run recently (1/0)
|
|
- logrotate_files_total Files tracked in logrotate status
|
|
- logrotate_stale_files_total Files not rotated within threshold
|
|
- log_file_size_bytes Size of monitored log files
|
|
- log_file_age_seconds Age of monitored log files
|
|
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--textfile) OUTPUT_FILE="$TEXTFILE_DIR/logrotate_check.prom"; shift ;;
|
|
--daemon) RUN_MODE="daemon"; shift ;;
|
|
--watch) WATCH_PATHS="${2:?--watch requires paths}"; shift 2 ;;
|
|
--log-dir) LOG_DIR="${2:?--log-dir requires a path}"; shift 2 ;;
|
|
-o|--output) OUTPUT_FILE="${2:?--output requires a path}"; shift 2 ;;
|
|
--help|-h) show_help ;;
|
|
*) echo "Unknown option: $1" >&2; show_help ;;
|
|
esac
|
|
done
|
|
|
|
# Locate the logrotate status file
|
|
find_status_file() {
|
|
if [[ -f /var/lib/logrotate/status ]]; then
|
|
echo "/var/lib/logrotate/status"
|
|
elif [[ -f /var/lib/logrotate.status ]]; then
|
|
echo "/var/lib/logrotate.status"
|
|
else
|
|
debug_echo "No logrotate status file found"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# Validate output directory exists when writing to file
|
|
validate_output() {
|
|
if [[ -n "$OUTPUT_FILE" ]]; then
|
|
local output_dir
|
|
output_dir="$(dirname "$OUTPUT_FILE")"
|
|
if [[ ! -d "$output_dir" ]]; then
|
|
echo "Error: Output directory not found: $output_dir" >&2
|
|
echo "Create it: sudo mkdir -p $output_dir" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Collect logrotate status metrics
|
|
collect_logrotate_status() {
|
|
local status_file
|
|
status_file=$(find_status_file)
|
|
|
|
if [[ -z "$status_file" ]]; then
|
|
cat <<EOF
|
|
# HELP logrotate_last_run_timestamp Unix timestamp of last logrotate run.
|
|
# TYPE logrotate_last_run_timestamp gauge
|
|
logrotate_last_run_timestamp 0
|
|
|
|
# HELP logrotate_status Whether logrotate has run recently (1=recent, 0=stale).
|
|
# TYPE logrotate_status gauge
|
|
logrotate_status 0
|
|
|
|
# HELP logrotate_files_total Total number of files tracked in logrotate status.
|
|
# TYPE logrotate_files_total gauge
|
|
logrotate_files_total 0
|
|
|
|
# HELP logrotate_stale_files_total Number of files not rotated within threshold.
|
|
# TYPE logrotate_stale_files_total gauge
|
|
logrotate_stale_files_total 0
|
|
|
|
EOF
|
|
return
|
|
fi
|
|
|
|
debug_echo "Using status file: $status_file"
|
|
|
|
# Last run timestamp from status file mtime
|
|
local last_run_ts
|
|
last_run_ts=$(stat -c %Y "$status_file" 2>/dev/null || echo 0)
|
|
|
|
# Determine if logrotate is stale
|
|
local now
|
|
now=$(date +%s)
|
|
local age=$(( now - last_run_ts ))
|
|
local status=1
|
|
if [[ $age -gt $STALE_THRESHOLD ]]; then
|
|
status=0
|
|
fi
|
|
|
|
# Count total tracked files and stale files in status
|
|
local files_total=0
|
|
local stale_files=0
|
|
|
|
while IFS= read -r line; do
|
|
# Skip header line and empty lines
|
|
[[ "$line" =~ ^\".*\" ]] || continue
|
|
|
|
files_total=$(( files_total + 1 ))
|
|
|
|
# Extract the date portion — format: "filename" date
|
|
local date_str
|
|
date_str=$(echo "$line" | sed 's/^"[^"]*"[[:space:]]*//')
|
|
|
|
if [[ -n "$date_str" ]]; then
|
|
local file_ts
|
|
file_ts=$(date -d "$date_str" +%s 2>/dev/null || echo 0)
|
|
if [[ $file_ts -gt 0 ]]; then
|
|
local file_age=$(( now - file_ts ))
|
|
if [[ $file_age -gt $STALE_THRESHOLD ]]; then
|
|
stale_files=$(( stale_files + 1 ))
|
|
fi
|
|
fi
|
|
fi
|
|
done < "$status_file"
|
|
|
|
debug_echo "Status file: $files_total files tracked, $stale_files stale"
|
|
|
|
cat <<EOF
|
|
# HELP logrotate_last_run_timestamp Unix timestamp of last logrotate run.
|
|
# TYPE logrotate_last_run_timestamp gauge
|
|
logrotate_last_run_timestamp $last_run_ts
|
|
|
|
# HELP logrotate_status Whether logrotate has run recently (1=recent, 0=stale).
|
|
# TYPE logrotate_status gauge
|
|
logrotate_status $status
|
|
|
|
# HELP logrotate_files_total Total number of files tracked in logrotate status.
|
|
# TYPE logrotate_files_total gauge
|
|
logrotate_files_total $files_total
|
|
|
|
# HELP logrotate_stale_files_total Number of files not rotated within threshold.
|
|
# TYPE logrotate_stale_files_total gauge
|
|
logrotate_stale_files_total $stale_files
|
|
|
|
EOF
|
|
}
|
|
|
|
# Discover log files in LOG_DIR (excludes compressed archives)
|
|
discover_log_files() {
|
|
find "$LOG_DIR" -maxdepth 2 -type f \
|
|
-not -name '*.gz' -not -name '*.xz' -not -name '*.bz2' \
|
|
-not -name '*.zst' -not -name '*.zip' \
|
|
-not -name '*.prom' -not -name '*.json' \
|
|
-not -name 'wtmp' -not -name 'btmp' -not -name 'lastlog' \
|
|
2>/dev/null | sort
|
|
}
|
|
|
|
# Collect log file size and age metrics
|
|
collect_log_files() {
|
|
local now
|
|
now=$(date +%s)
|
|
|
|
local size_lines=""
|
|
local age_lines=""
|
|
local has_entries=0
|
|
|
|
if [[ -n "$WATCH_PATHS" ]]; then
|
|
# Explicit watch list
|
|
IFS=',' read -ra paths <<< "$WATCH_PATHS"
|
|
for path in "${paths[@]}"; do
|
|
path=$(echo "$path" | xargs)
|
|
[[ -f "$path" ]] || continue
|
|
|
|
has_entries=1
|
|
|
|
local size
|
|
size=$(stat -c %s "$path" 2>/dev/null || echo 0)
|
|
local mtime
|
|
mtime=$(stat -c %Y "$path" 2>/dev/null || echo 0)
|
|
local age=$(( now - mtime ))
|
|
|
|
size_lines+="log_file_size_bytes{path=\"${path}\"} ${size}\n"
|
|
age_lines+="log_file_age_seconds{path=\"${path}\"} ${age}\n"
|
|
debug_echo "Log file: $path size=$size age=${age}s"
|
|
done
|
|
else
|
|
# Auto-discover log files
|
|
debug_echo "Auto-discovering log files in $LOG_DIR"
|
|
while IFS= read -r path; do
|
|
[[ -f "$path" ]] || continue
|
|
|
|
has_entries=1
|
|
|
|
local size
|
|
size=$(stat -c %s "$path" 2>/dev/null || echo 0)
|
|
local mtime
|
|
mtime=$(stat -c %Y "$path" 2>/dev/null || echo 0)
|
|
local age=$(( now - mtime ))
|
|
|
|
size_lines+="log_file_size_bytes{path=\"${path}\"} ${size}\n"
|
|
age_lines+="log_file_age_seconds{path=\"${path}\"} ${age}\n"
|
|
debug_echo "Log file: $path size=$size age=${age}s"
|
|
done < <(discover_log_files)
|
|
fi
|
|
|
|
if [[ $has_entries -eq 1 ]]; then
|
|
echo "# HELP log_file_size_bytes Size of monitored log file in bytes."
|
|
echo "# TYPE log_file_size_bytes gauge"
|
|
echo -e "$size_lines"
|
|
|
|
echo "# HELP log_file_age_seconds Seconds since last modification of monitored log file."
|
|
echo "# TYPE log_file_age_seconds gauge"
|
|
echo -e "$age_lines"
|
|
fi
|
|
}
|
|
|
|
# Collect exporter metadata
|
|
collect_metadata() {
|
|
local duration="$1"
|
|
local success="$2"
|
|
|
|
cat <<EOF
|
|
# HELP logrotate_exporter_duration_seconds Time taken for the collection run.
|
|
# TYPE logrotate_exporter_duration_seconds gauge
|
|
logrotate_exporter_duration_seconds $duration
|
|
|
|
# HELP logrotate_exporter_last_run_timestamp Unix timestamp of last exporter run.
|
|
# TYPE logrotate_exporter_last_run_timestamp gauge
|
|
logrotate_exporter_last_run_timestamp $(date +%s)
|
|
|
|
# HELP logrotate_exporter_success Whether the last collection completed successfully.
|
|
# TYPE logrotate_exporter_success gauge
|
|
logrotate_exporter_success $success
|
|
|
|
EOF
|
|
}
|
|
|
|
# Generate all metrics to stdout
|
|
generate_metrics() {
|
|
local start_time end_time duration
|
|
start_time=$(date +%s%N)
|
|
local success=1
|
|
|
|
{
|
|
collect_logrotate_status
|
|
collect_log_files
|
|
} || success=0
|
|
|
|
end_time=$(date +%s%N)
|
|
duration=$(awk "BEGIN {printf \"%.3f\", ($end_time - $start_time) / 1000000000}")
|
|
|
|
collect_metadata "$duration" "$success"
|
|
}
|
|
|
|
# Write metrics to file atomically
|
|
write_metrics() {
|
|
local output_dir
|
|
output_dir="$(dirname "$OUTPUT_FILE")"
|
|
|
|
local temp_file
|
|
temp_file=$(mktemp "${output_dir}/.logrotate_check.XXXXXX")
|
|
|
|
if ! generate_metrics > "$temp_file" 2>/dev/null; then
|
|
rm -f "$temp_file"
|
|
echo "ERROR: Failed to generate metrics" >&2
|
|
exit 1
|
|
fi
|
|
|
|
local file_lines
|
|
file_lines=$(wc -l < "$temp_file" 2>/dev/null || echo 0)
|
|
|
|
if [[ "$file_lines" -lt 5 ]]; then
|
|
rm -f "$temp_file"
|
|
echo "ERROR: Metrics file too small ($file_lines lines), keeping previous" >&2
|
|
exit 1
|
|
fi
|
|
|
|
chmod 644 "$temp_file"
|
|
mv -f "$temp_file" "$OUTPUT_FILE"
|
|
|
|
debug_echo "Metrics written to $OUTPUT_FILE ($file_lines lines)"
|
|
}
|
|
|
|
# Main
|
|
main() {
|
|
validate_output
|
|
|
|
case "$RUN_MODE" in
|
|
once)
|
|
if [[ -n "$OUTPUT_FILE" ]]; then
|
|
write_metrics
|
|
else
|
|
generate_metrics
|
|
fi
|
|
;;
|
|
daemon)
|
|
[[ -z "$OUTPUT_FILE" ]] && OUTPUT_FILE="$TEXTFILE_DIR/logrotate_check.prom"
|
|
validate_output
|
|
echo "$SCRIPT_NAME running in daemon mode (interval: ${COLLECTION_INTERVAL}s)" >&2
|
|
while true; do
|
|
write_metrics
|
|
sleep "$COLLECTION_INTERVAL"
|
|
done
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main
|