chiefgeek
a1a17e81a1
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
1309 lines
44 KiB
Bash
1309 lines
44 KiB
Bash
#!/bin/bash
|
|
#############################################################
|
|
#### Apache Metrics Exporter for Prometheus ####
|
|
#### Comprehensive Apache monitoring via mod_status, ####
|
|
#### logs, SSL, process, and config metrics ####
|
|
#### ####
|
|
#### Author: Phil Connor ####
|
|
#### Contact: contact@mylinux.work ####
|
|
#### License: MIT ####
|
|
#### Version: 1.01 ####
|
|
#### ####
|
|
#### Usage: ./apache-metrics-exporter.sh [OPTIONS] ####
|
|
#############################################################
|
|
#
|
|
# Metrics collected:
|
|
# - mod_status: accesses, bytes, req/sec, busy/idle workers, scoreboard
|
|
# - Process: worker count, memory usage, CPU usage, open files
|
|
# - Access logs: requests by status code, response times, bytes transferred
|
|
# - SSL: certificate expiry days for configured domains
|
|
# - Config: MPM type, MaxRequestWorkers, KeepAliveTimeout
|
|
# - Upstream: proxy/balancer status (if configured)
|
|
#
|
|
# Requirements:
|
|
# - Apache with mod_status enabled (ExtendedStatus On)
|
|
# - socat (for HTTP server)
|
|
# - curl (for server-status fetching)
|
|
#
|
|
set -euo pipefail
|
|
|
|
#########################
|
|
### Auto-detect Apache ###
|
|
#########################
|
|
|
|
APACHE_BIN=""
|
|
APACHECTL=""
|
|
APACHE_PROC=""
|
|
|
|
detect_apache_flavor() {
|
|
if command -v apache2 &>/dev/null; then
|
|
APACHE_BIN="apache2"
|
|
APACHECTL="apache2ctl"
|
|
APACHE_PROC="apache2"
|
|
elif command -v httpd &>/dev/null; then
|
|
APACHE_BIN="httpd"
|
|
APACHECTL="httpd"
|
|
APACHE_PROC="httpd"
|
|
else
|
|
APACHE_BIN=""
|
|
APACHECTL=""
|
|
APACHE_PROC=""
|
|
fi
|
|
}
|
|
|
|
detect_apache_flavor
|
|
|
|
#########################
|
|
### Configuration ###
|
|
#########################
|
|
|
|
LISTEN_PORT="${APACHE_EXPORTER_PORT:-9117}"
|
|
STATUS_URL="${APACHE_STATUS_URL:-http://127.0.0.1/server-status?auto}"
|
|
SSL_CHECK_DOMAINS="${SSL_CHECK_DOMAINS:-}" # Comma-separated list of domains to check SSL
|
|
SCRAPE_INTERVAL="${SCRAPE_INTERVAL:-15}"
|
|
|
|
# Auto-detect paths based on distro
|
|
if [[ -d /etc/apache2 ]]; then
|
|
ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/apache2/access.log}"
|
|
ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/apache2/error.log}"
|
|
APACHE_CONF="${APACHE_CONF:-/etc/apache2/apache2.conf}"
|
|
SITES_DIR="${APACHE_SITES_DIR:-/etc/apache2/sites-enabled}"
|
|
CONF_D_DIR="${APACHE_CONF_D:-/etc/apache2/conf-enabled}"
|
|
elif [[ -d /etc/httpd ]]; then
|
|
ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/httpd/access_log}"
|
|
ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/httpd/error_log}"
|
|
APACHE_CONF="${APACHE_CONF:-/etc/httpd/conf/httpd.conf}"
|
|
SITES_DIR="${APACHE_SITES_DIR:-/etc/httpd/conf.d}"
|
|
CONF_D_DIR="${APACHE_CONF_D:-/etc/httpd/conf.d}"
|
|
else
|
|
ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/apache2/access.log}"
|
|
ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/apache2/error.log}"
|
|
APACHE_CONF="${APACHE_CONF:-/etc/apache2/apache2.conf}"
|
|
SITES_DIR="${APACHE_SITES_DIR:-/etc/apache2/sites-enabled}"
|
|
CONF_D_DIR="${APACHE_CONF_D:-/etc/apache2/conf-enabled}"
|
|
fi
|
|
|
|
# Log parsing settings
|
|
LOG_TAIL_LINES="${LOG_TAIL_LINES:-10000}" # Number of lines to parse from access log
|
|
LOG_PARSE_INTERVAL="${LOG_PARSE_INTERVAL:-60}" # How often to parse logs (seconds)
|
|
|
|
# State files for log metrics
|
|
STATE_DIR="/tmp/apache-metrics"
|
|
LAST_LOG_PARSE=0
|
|
|
|
# Output mode
|
|
TEXTFILE_DIR="/var/lib/node_exporter"
|
|
OUTPUT_FILE=""
|
|
HTTP_MODE=false
|
|
|
|
#########################
|
|
### Logging ###
|
|
#########################
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
|
|
}
|
|
|
|
#########################
|
|
### Parse Arguments ###
|
|
#########################
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--textfile)
|
|
OUTPUT_FILE="$TEXTFILE_DIR/apache.prom"
|
|
shift
|
|
;;
|
|
--http)
|
|
HTTP_MODE=true
|
|
shift
|
|
;;
|
|
--output|-o)
|
|
OUTPUT_FILE="$2"
|
|
shift 2
|
|
;;
|
|
--port)
|
|
LISTEN_PORT="$2"
|
|
shift 2
|
|
;;
|
|
--status-url)
|
|
STATUS_URL="$2"
|
|
shift 2
|
|
;;
|
|
--access-log)
|
|
ACCESS_LOG="$2"
|
|
shift 2
|
|
;;
|
|
--error-log)
|
|
ERROR_LOG="$2"
|
|
shift 2
|
|
;;
|
|
--apache-conf)
|
|
APACHE_CONF="$2"
|
|
shift 2
|
|
;;
|
|
--ssl-domains)
|
|
SSL_CHECK_DOMAINS="$2"
|
|
shift 2
|
|
;;
|
|
--help)
|
|
cat <<EOF
|
|
Apache Metrics Exporter for Prometheus
|
|
|
|
Usage: $0 [OPTIONS]
|
|
|
|
MODES:
|
|
--textfile Write to node_exporter textfile collector
|
|
--http Run HTTP server on port $LISTEN_PORT (default mode with no other output)
|
|
(no flag) Output to stdout
|
|
|
|
OPTIONS:
|
|
--port PORT HTTP port (default: 9117)
|
|
-o, --output PATH Output file path
|
|
--status-url URL Apache server-status URL (default: http://127.0.0.1/server-status?auto)
|
|
--access-log PATH Path to access log (default: $ACCESS_LOG)
|
|
--error-log PATH Path to error log (default: $ERROR_LOG)
|
|
--apache-conf PATH Path to Apache config (default: $APACHE_CONF)
|
|
--ssl-domains LIST Comma-separated domains to check SSL expiry
|
|
--help Show this help
|
|
|
|
EXAMPLES:
|
|
$0 --textfile # Write to textfile collector
|
|
$0 --http --port 9117 # Run HTTP server
|
|
$0 -o /tmp/apache.prom # Write to custom file
|
|
$0 # Output to stdout
|
|
|
|
EOF
|
|
exit 0
|
|
;;
|
|
*)
|
|
log "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
#########################
|
|
### Setup ###
|
|
#########################
|
|
|
|
detect_package_manager() {
|
|
if command -v apt-get &>/dev/null; then
|
|
echo "apt"
|
|
elif command -v dnf &>/dev/null; then
|
|
echo "dnf"
|
|
elif command -v yum &>/dev/null; then
|
|
echo "yum"
|
|
elif command -v zypper &>/dev/null; then
|
|
echo "zypper"
|
|
elif command -v pacman &>/dev/null; then
|
|
echo "pacman"
|
|
elif command -v apk &>/dev/null; then
|
|
echo "apk"
|
|
else
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
install_package() {
|
|
local pkg="$1"
|
|
local pkgmgr
|
|
pkgmgr=$(detect_package_manager)
|
|
|
|
log "Installing $pkg..."
|
|
|
|
case "$pkgmgr" in
|
|
apt)
|
|
apt-get update -qq && apt-get install -y -qq "$pkg"
|
|
;;
|
|
dnf)
|
|
dnf install -y -q "$pkg"
|
|
;;
|
|
yum)
|
|
yum install -y -q "$pkg"
|
|
;;
|
|
zypper)
|
|
zypper install -y -q "$pkg"
|
|
;;
|
|
pacman)
|
|
pacman -S --noconfirm "$pkg"
|
|
;;
|
|
apk)
|
|
apk add --quiet "$pkg"
|
|
;;
|
|
*)
|
|
log "ERROR: Unknown package manager. Please install $pkg manually."
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
setup() {
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
# Check for required tools and install if missing
|
|
if ! command -v socat &>/dev/null; then
|
|
log "socat not found, attempting to install..."
|
|
if [[ $EUID -eq 0 ]]; then
|
|
if ! install_package socat; then
|
|
log "ERROR: Failed to install socat"
|
|
exit 1
|
|
fi
|
|
log "socat installed successfully"
|
|
else
|
|
log "ERROR: socat is required. Run as root to auto-install, or install manually:"
|
|
log " Debian/Ubuntu: apt install socat"
|
|
log " RHEL/CentOS: yum install socat"
|
|
log " Fedora: dnf install socat"
|
|
log " Alpine: apk add socat"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if ! command -v curl &>/dev/null; then
|
|
log "curl not found, attempting to install..."
|
|
if [[ $EUID -eq 0 ]]; then
|
|
if ! install_package curl; then
|
|
log "ERROR: Failed to install curl"
|
|
exit 1
|
|
fi
|
|
log "curl installed successfully"
|
|
else
|
|
log "ERROR: curl is required. Run as root to auto-install, or install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Check if Apache is running
|
|
if [[ -n "$APACHE_PROC" ]]; then
|
|
if ! pgrep -x "$APACHE_PROC" &>/dev/null && ! pidof "$APACHE_PROC" &>/dev/null; then
|
|
log "WARNING: $APACHE_PROC process not found - process metrics will show apache_process_running=0"
|
|
fi
|
|
else
|
|
log "WARNING: Apache binary not found (neither apache2 nor httpd)"
|
|
fi
|
|
|
|
# Check if server-status is accessible
|
|
check_server_status
|
|
}
|
|
|
|
check_server_status() {
|
|
log "Checking server-status at $STATUS_URL..."
|
|
|
|
local response http_code
|
|
response=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$STATUS_URL" 2>/dev/null)
|
|
|
|
if [[ "$response" == "200" ]]; then
|
|
# Verify it's actually mod_status output
|
|
local content
|
|
content=$(curl -s --max-time 5 "$STATUS_URL" 2>/dev/null)
|
|
if echo "$content" | grep -q "Total Accesses"; then
|
|
log "✓ mod_status is working correctly"
|
|
return 0
|
|
else
|
|
log "WARNING: $STATUS_URL returned 200 but doesn't look like mod_status output"
|
|
log " Expected 'Total Accesses' in response (ensure ExtendedStatus On)"
|
|
show_server_status_help
|
|
return 1
|
|
fi
|
|
elif [[ "$response" == "000" ]]; then
|
|
log "WARNING: Cannot connect to $STATUS_URL (connection refused/timeout)"
|
|
log " server-status metrics will show apache_up=0"
|
|
show_server_status_help
|
|
return 1
|
|
elif [[ "$response" == "403" ]]; then
|
|
log "WARNING: Access denied to $STATUS_URL (HTTP 403)"
|
|
log " Check 'Require' directives in server-status location block"
|
|
show_server_status_help
|
|
return 1
|
|
elif [[ "$response" == "404" ]]; then
|
|
log "WARNING: server-status endpoint not found at $STATUS_URL (HTTP 404)"
|
|
log " mod_status may not be enabled"
|
|
show_server_status_help
|
|
return 1
|
|
else
|
|
log "WARNING: Unexpected response from $STATUS_URL (HTTP $response)"
|
|
show_server_status_help
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
show_server_status_help() {
|
|
log ""
|
|
log "To enable mod_status, configure Apache as follows:"
|
|
log ""
|
|
log " Debian/Ubuntu:"
|
|
log " sudo a2enmod status"
|
|
log " # Edit /etc/apache2/mods-enabled/status.conf:"
|
|
log " ExtendedStatus On"
|
|
log " <Location \"/server-status\">"
|
|
log " SetHandler server-status"
|
|
log " Require local"
|
|
log " </Location>"
|
|
log ""
|
|
log " RHEL/CentOS/Rocky:"
|
|
log " # Add to /etc/httpd/conf.d/status.conf:"
|
|
log " ExtendedStatus On"
|
|
log " <Location \"/server-status\">"
|
|
log " SetHandler server-status"
|
|
log " Require local"
|
|
log " </Location>"
|
|
log ""
|
|
log "Then reload: apachectl configtest && systemctl reload apache2 (or httpd)"
|
|
log ""
|
|
log "Or specify a different URL with: --status-url <url>"
|
|
log ""
|
|
}
|
|
|
|
#########################
|
|
### Server Status Metrics ###
|
|
#########################
|
|
|
|
collect_server_status() {
|
|
local status_output
|
|
|
|
echo "# HELP apache_up Whether Apache mod_status is reachable"
|
|
echo "# TYPE apache_up gauge"
|
|
|
|
if ! status_output=$(curl -s --max-time 5 "$STATUS_URL" 2>/dev/null); then
|
|
echo "apache_up 0"
|
|
return
|
|
fi
|
|
|
|
# Verify we got valid mod_status output
|
|
if ! echo "$status_output" | grep -q "Total Accesses"; then
|
|
echo "apache_up 0"
|
|
return
|
|
fi
|
|
|
|
echo "apache_up 1"
|
|
|
|
# Parse mod_status ?auto output
|
|
# Format:
|
|
# Total Accesses: 12345
|
|
# Total kBytes: 67890
|
|
# CPULoad: .0123456
|
|
# Uptime: 86400
|
|
# ReqPerSec: .142857
|
|
# BytesPerSec: 804.571
|
|
# BytesPerReq: 5632
|
|
# BusyWorkers: 3
|
|
# IdleWorkers: 7
|
|
# Scoreboard: __W_K....._R..
|
|
|
|
local total_accesses total_kbytes cpu_load uptime req_per_sec bytes_per_sec bytes_per_req
|
|
local busy_workers idle_workers scoreboard
|
|
|
|
total_accesses=$(echo "$status_output" | grep '^Total Accesses:' | awk '{print $3}') || total_accesses=0
|
|
total_kbytes=$(echo "$status_output" | grep '^Total kBytes:' | awk '{print $3}') || total_kbytes=0
|
|
cpu_load=$(echo "$status_output" | grep '^CPULoad:' | awk '{print $2}') || cpu_load=0
|
|
uptime=$(echo "$status_output" | grep '^Uptime:' | awk '{print $2}') || uptime=0
|
|
req_per_sec=$(echo "$status_output" | grep '^ReqPerSec:' | awk '{print $2}') || req_per_sec=0
|
|
bytes_per_sec=$(echo "$status_output" | grep '^BytesPerSec:' | awk '{print $2}') || bytes_per_sec=0
|
|
bytes_per_req=$(echo "$status_output" | grep '^BytesPerReq:' | awk '{print $2}') || bytes_per_req=0
|
|
busy_workers=$(echo "$status_output" | grep '^BusyWorkers:' | awk '{print $2}') || busy_workers=0
|
|
idle_workers=$(echo "$status_output" | grep '^IdleWorkers:' | awk '{print $2}') || idle_workers=0
|
|
scoreboard=$(echo "$status_output" | grep '^Scoreboard:' | awk '{print $2}') || scoreboard=""
|
|
|
|
# Convert kBytes to bytes
|
|
local total_bytes
|
|
total_bytes=$(echo "$total_kbytes * 1024" | bc 2>/dev/null || echo "$((total_kbytes * 1024))")
|
|
|
|
cat <<EOF
|
|
# HELP apache_uptime_seconds Server uptime in seconds
|
|
# TYPE apache_uptime_seconds counter
|
|
apache_uptime_seconds $uptime
|
|
|
|
# HELP apache_accesses_total Total number of requests served
|
|
# TYPE apache_accesses_total counter
|
|
apache_accesses_total $total_accesses
|
|
|
|
# HELP apache_sent_bytes_total Total bytes transferred
|
|
# TYPE apache_sent_bytes_total counter
|
|
apache_sent_bytes_total $total_bytes
|
|
|
|
# HELP apache_cpu_load Current CPU load
|
|
# TYPE apache_cpu_load gauge
|
|
apache_cpu_load $cpu_load
|
|
|
|
# HELP apache_requests_per_second Current requests per second
|
|
# TYPE apache_requests_per_second gauge
|
|
apache_requests_per_second $req_per_sec
|
|
|
|
# HELP apache_bytes_per_second Current bytes per second
|
|
# TYPE apache_bytes_per_second gauge
|
|
apache_bytes_per_second $bytes_per_sec
|
|
|
|
# HELP apache_bytes_per_request Average bytes per request
|
|
# TYPE apache_bytes_per_request gauge
|
|
apache_bytes_per_request $bytes_per_req
|
|
|
|
# HELP apache_workers_busy Workers currently serving requests
|
|
# TYPE apache_workers_busy gauge
|
|
apache_workers_busy $busy_workers
|
|
|
|
# HELP apache_workers_idle Workers waiting for connections
|
|
# TYPE apache_workers_idle gauge
|
|
apache_workers_idle $idle_workers
|
|
EOF
|
|
|
|
# Parse scoreboard
|
|
if [[ -n "$scoreboard" ]]; then
|
|
local sb_waiting=0 sb_starting=0 sb_reading=0 sb_sending=0 sb_keepalive=0
|
|
local sb_dns=0 sb_closing=0 sb_logging=0 sb_graceful=0 sb_idle_cleanup=0 sb_open=0
|
|
|
|
local i char
|
|
for (( i=0; i<${#scoreboard}; i++ )); do
|
|
char="${scoreboard:$i:1}"
|
|
case "$char" in
|
|
'_') sb_waiting=$((sb_waiting + 1)) ;;
|
|
'S') sb_starting=$((sb_starting + 1)) ;;
|
|
'R') sb_reading=$((sb_reading + 1)) ;;
|
|
'W') sb_sending=$((sb_sending + 1)) ;;
|
|
'K') sb_keepalive=$((sb_keepalive + 1)) ;;
|
|
'D') sb_dns=$((sb_dns + 1)) ;;
|
|
'C') sb_closing=$((sb_closing + 1)) ;;
|
|
'L') sb_logging=$((sb_logging + 1)) ;;
|
|
'G') sb_graceful=$((sb_graceful + 1)) ;;
|
|
'I') sb_idle_cleanup=$((sb_idle_cleanup + 1)) ;;
|
|
'.') sb_open=$((sb_open + 1)) ;;
|
|
esac
|
|
done
|
|
|
|
cat <<EOF
|
|
|
|
# HELP apache_scoreboard_waiting Workers in waiting for connection state
|
|
# TYPE apache_scoreboard_waiting gauge
|
|
apache_scoreboard_waiting $sb_waiting
|
|
|
|
# HELP apache_scoreboard_starting Workers in starting up state
|
|
# TYPE apache_scoreboard_starting gauge
|
|
apache_scoreboard_starting $sb_starting
|
|
|
|
# HELP apache_scoreboard_reading Workers reading request
|
|
# TYPE apache_scoreboard_reading gauge
|
|
apache_scoreboard_reading $sb_reading
|
|
|
|
# HELP apache_scoreboard_sending Workers sending reply
|
|
# TYPE apache_scoreboard_sending gauge
|
|
apache_scoreboard_sending $sb_sending
|
|
|
|
# HELP apache_scoreboard_keepalive Workers in keepalive state
|
|
# TYPE apache_scoreboard_keepalive gauge
|
|
apache_scoreboard_keepalive $sb_keepalive
|
|
|
|
# HELP apache_scoreboard_dns_lookup Workers doing DNS lookup
|
|
# TYPE apache_scoreboard_dns_lookup gauge
|
|
apache_scoreboard_dns_lookup $sb_dns
|
|
|
|
# HELP apache_scoreboard_closing Workers closing connection
|
|
# TYPE apache_scoreboard_closing gauge
|
|
apache_scoreboard_closing $sb_closing
|
|
|
|
# HELP apache_scoreboard_logging Workers logging
|
|
# TYPE apache_scoreboard_logging gauge
|
|
apache_scoreboard_logging $sb_logging
|
|
|
|
# HELP apache_scoreboard_graceful_finishing Workers gracefully finishing
|
|
# TYPE apache_scoreboard_graceful_finishing gauge
|
|
apache_scoreboard_graceful_finishing $sb_graceful
|
|
|
|
# HELP apache_scoreboard_idle_cleanup Workers in idle cleanup state
|
|
# TYPE apache_scoreboard_idle_cleanup gauge
|
|
apache_scoreboard_idle_cleanup $sb_idle_cleanup
|
|
|
|
# HELP apache_scoreboard_open_slot Open slots with no current process
|
|
# TYPE apache_scoreboard_open_slot gauge
|
|
apache_scoreboard_open_slot $sb_open
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### Process Metrics ###
|
|
#########################
|
|
|
|
collect_process_metrics() {
|
|
local apache_master_pid apache_pids worker_count total_memory total_cpu total_fds
|
|
local total_threads
|
|
|
|
if [[ -z "$APACHE_PROC" ]]; then
|
|
echo "# HELP apache_process_running Whether Apache process is running"
|
|
echo "# TYPE apache_process_running gauge"
|
|
echo "apache_process_running 0"
|
|
return
|
|
fi
|
|
|
|
# Find Apache master process
|
|
apache_master_pid=$(pgrep -x "$APACHE_PROC" -o 2>/dev/null || pidof "$APACHE_PROC" 2>/dev/null | awk '{print $1}' || echo "")
|
|
|
|
if [[ -z "$apache_master_pid" ]]; then
|
|
echo "# HELP apache_process_running Whether Apache process is running"
|
|
echo "# TYPE apache_process_running gauge"
|
|
echo "apache_process_running 0"
|
|
return
|
|
fi
|
|
|
|
echo "# HELP apache_process_running Whether Apache process is running"
|
|
echo "# TYPE apache_process_running gauge"
|
|
echo "apache_process_running 1"
|
|
|
|
# Get all Apache PIDs
|
|
apache_pids=$(pgrep -x "$APACHE_PROC" 2>/dev/null || pidof "$APACHE_PROC" 2>/dev/null || echo "")
|
|
|
|
# Count workers (total processes minus master)
|
|
worker_count=$(echo "$apache_pids" | wc -w)
|
|
if [[ $worker_count -gt 0 ]]; then
|
|
worker_count=$((worker_count - 1)) # Subtract master
|
|
fi
|
|
|
|
echo "# HELP apache_workers_count Number of Apache worker processes"
|
|
echo "# TYPE apache_workers_count gauge"
|
|
echo "apache_workers_count $worker_count"
|
|
|
|
# Calculate total memory usage (RSS in bytes)
|
|
total_memory=0
|
|
total_cpu=0
|
|
total_fds=0
|
|
total_threads=0
|
|
|
|
for pid in $apache_pids; do
|
|
if [[ -d "/proc/$pid" ]]; then
|
|
# Memory (RSS in KB from /proc/pid/status, convert to bytes)
|
|
local rss
|
|
rss=$(grep -m1 'VmRSS:' "/proc/$pid/status" 2>/dev/null | awk '{print $2}' || echo "0")
|
|
total_memory=$((total_memory + rss * 1024))
|
|
|
|
# CPU time (from /proc/pid/stat - utime + stime in jiffies)
|
|
local stat_line utime stime
|
|
if stat_line=$(cat "/proc/$pid/stat" 2>/dev/null); then
|
|
utime=$(echo "$stat_line" | awk '{print $14}')
|
|
stime=$(echo "$stat_line" | awk '{print $15}')
|
|
total_cpu=$((total_cpu + utime + stime))
|
|
fi
|
|
|
|
# Open file descriptors
|
|
local fds
|
|
fds=$(ls -1 "/proc/$pid/fd" 2>/dev/null | wc -l || echo "0")
|
|
total_fds=$((total_fds + fds))
|
|
|
|
# Threads
|
|
local threads
|
|
threads=$(grep -c '^Threads:' "/proc/$pid/status" 2>/dev/null || true)
|
|
if [[ "$threads" -eq 0 ]]; then
|
|
threads=$(grep 'Threads:' "/proc/$pid/status" 2>/dev/null | awk '{print $2}' || echo "1")
|
|
fi
|
|
total_threads=$((total_threads + threads))
|
|
fi
|
|
done
|
|
|
|
# Convert CPU jiffies to seconds (assuming 100 Hz)
|
|
local cpu_seconds
|
|
cpu_seconds=$(echo "scale=2; $total_cpu / 100" | bc 2>/dev/null || echo "$total_cpu")
|
|
|
|
cat <<EOF
|
|
# HELP apache_process_memory_bytes Total memory used by Apache processes (RSS)
|
|
# TYPE apache_process_memory_bytes gauge
|
|
apache_process_memory_bytes $total_memory
|
|
|
|
# HELP apache_process_cpu_seconds_total Total CPU time consumed by Apache processes
|
|
# TYPE apache_process_cpu_seconds_total counter
|
|
apache_process_cpu_seconds_total $cpu_seconds
|
|
|
|
# HELP apache_process_open_fds Total number of open file descriptors
|
|
# TYPE apache_process_open_fds gauge
|
|
apache_process_open_fds $total_fds
|
|
|
|
# HELP apache_process_threads_total Total number of threads
|
|
# TYPE apache_process_threads_total gauge
|
|
apache_process_threads_total $total_threads
|
|
|
|
# HELP apache_process_start_time_seconds Start time of Apache master process
|
|
# TYPE apache_process_start_time_seconds gauge
|
|
EOF
|
|
|
|
# Get start time of master process
|
|
if [[ -f "/proc/$apache_master_pid/stat" ]]; then
|
|
local starttime start_seconds
|
|
starttime=$(awk '{print $22}' "/proc/$apache_master_pid/stat" 2>/dev/null || echo "0")
|
|
# starttime is in jiffies since boot
|
|
start_seconds=$(awk "BEGIN {printf \"%.0f\", $(cat /proc/uptime | awk '{print $1}') - ($starttime / 100)}")
|
|
local now_epoch
|
|
now_epoch=$(date +%s)
|
|
local process_start=$((now_epoch - start_seconds))
|
|
echo "apache_process_start_time_seconds $process_start"
|
|
else
|
|
echo "apache_process_start_time_seconds 0"
|
|
fi
|
|
|
|
# Get max open files limit
|
|
if [[ -f "/proc/$apache_master_pid/limits" ]]; then
|
|
local max_fds
|
|
max_fds=$(grep 'Max open files' "/proc/$apache_master_pid/limits" 2>/dev/null | awk '{print $4}' || echo "0")
|
|
echo ""
|
|
echo "# HELP apache_process_max_fds Maximum number of open file descriptors"
|
|
echo "# TYPE apache_process_max_fds gauge"
|
|
echo "apache_process_max_fds $max_fds"
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### Config Metrics ###
|
|
#########################
|
|
|
|
collect_config_metrics() {
|
|
if [[ ! -f "$APACHE_CONF" ]]; then
|
|
echo "# Apache config not found at $APACHE_CONF"
|
|
return
|
|
fi
|
|
|
|
local max_request_workers keepalive_timeout keepalive_enabled
|
|
local mpm_type
|
|
|
|
# Parse MaxRequestWorkers (or MaxClients for older Apache)
|
|
max_request_workers=$(grep -rihE '^\s*MaxRequestWorkers' "$APACHE_CONF" "$CONF_D_DIR" "$SITES_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "")
|
|
if [[ -z "$max_request_workers" ]]; then
|
|
max_request_workers=$(grep -rihE '^\s*MaxClients' "$APACHE_CONF" "$CONF_D_DIR" "$SITES_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
|
|
fi
|
|
max_request_workers="${max_request_workers:-0}"
|
|
|
|
# Parse KeepAliveTimeout
|
|
keepalive_timeout=$(grep -rihE '^\s*KeepAliveTimeout' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
|
|
keepalive_timeout="${keepalive_timeout:-0}"
|
|
|
|
# Check KeepAlive on/off
|
|
keepalive_enabled=$(grep -rihE '^\s*KeepAlive\s' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print tolower($2)}' || echo "on")
|
|
if [[ "$keepalive_enabled" == "on" ]]; then
|
|
keepalive_enabled=1
|
|
else
|
|
keepalive_enabled=0
|
|
fi
|
|
|
|
# Detect MPM type
|
|
mpm_type="unknown"
|
|
if [[ -n "$APACHECTL" ]]; then
|
|
local modules_list
|
|
modules_list=$($APACHECTL -M 2>/dev/null || echo "")
|
|
if echo "$modules_list" | grep -q 'mpm_event_module'; then
|
|
mpm_type="event"
|
|
elif echo "$modules_list" | grep -q 'mpm_worker_module'; then
|
|
mpm_type="worker"
|
|
elif echo "$modules_list" | grep -q 'mpm_prefork_module'; then
|
|
mpm_type="prefork"
|
|
fi
|
|
fi
|
|
|
|
cat <<EOF
|
|
# HELP apache_config_max_request_workers MaxRequestWorkers setting
|
|
# TYPE apache_config_max_request_workers gauge
|
|
apache_config_max_request_workers $max_request_workers
|
|
|
|
# HELP apache_config_keep_alive_timeout KeepAliveTimeout setting in seconds
|
|
# TYPE apache_config_keep_alive_timeout gauge
|
|
apache_config_keep_alive_timeout $keepalive_timeout
|
|
|
|
# HELP apache_config_keep_alive Whether KeepAlive is enabled
|
|
# TYPE apache_config_keep_alive gauge
|
|
apache_config_keep_alive $keepalive_enabled
|
|
|
|
# HELP apache_config_mpm_type MPM type in use (label)
|
|
# TYPE apache_config_mpm_type gauge
|
|
apache_config_mpm_type{mpm="$mpm_type"} 1
|
|
EOF
|
|
|
|
# Count virtual hosts
|
|
local vhost_count=0
|
|
if [[ -d "$SITES_DIR" ]]; then
|
|
vhost_count=$(find "$SITES_DIR" -type f -o -type l 2>/dev/null | wc -l)
|
|
elif [[ -d "$CONF_D_DIR" ]]; then
|
|
vhost_count=$(find "$CONF_D_DIR" -name "*.conf" -type f 2>/dev/null | wc -l)
|
|
fi
|
|
|
|
echo ""
|
|
echo "# HELP apache_config_vhosts_total Number of configured virtual hosts"
|
|
echo "# TYPE apache_config_vhosts_total gauge"
|
|
echo "apache_config_vhosts_total $vhost_count"
|
|
|
|
# Parse ServerLimit if available
|
|
local server_limit
|
|
server_limit=$(grep -rihE '^\s*ServerLimit' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
|
|
if [[ "$server_limit" != "0" ]] && [[ -n "$server_limit" ]]; then
|
|
echo ""
|
|
echo "# HELP apache_config_server_limit ServerLimit setting"
|
|
echo "# TYPE apache_config_server_limit gauge"
|
|
echo "apache_config_server_limit $server_limit"
|
|
fi
|
|
|
|
# Parse Timeout
|
|
local timeout_val
|
|
timeout_val=$(grep -rihE '^\s*Timeout\s' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
|
|
if [[ "$timeout_val" != "0" ]] && [[ -n "$timeout_val" ]]; then
|
|
echo ""
|
|
echo "# HELP apache_config_timeout Timeout setting in seconds"
|
|
echo "# TYPE apache_config_timeout gauge"
|
|
echo "apache_config_timeout $timeout_val"
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### Access Log Metrics ###
|
|
#########################
|
|
|
|
collect_access_log_metrics() {
|
|
if [[ ! -f "$ACCESS_LOG" ]] || [[ ! -r "$ACCESS_LOG" ]]; then
|
|
echo "# Access log not readable at $ACCESS_LOG"
|
|
return
|
|
fi
|
|
|
|
local now
|
|
now=$(date +%s)
|
|
|
|
# Only parse logs every LOG_PARSE_INTERVAL seconds
|
|
if [[ -f "$STATE_DIR/last_parse" ]]; then
|
|
LAST_LOG_PARSE=$(cat "$STATE_DIR/last_parse")
|
|
fi
|
|
|
|
if [[ $((now - LAST_LOG_PARSE)) -lt $LOG_PARSE_INTERVAL ]] && [[ -f "$STATE_DIR/log_metrics" ]]; then
|
|
cat "$STATE_DIR/log_metrics"
|
|
return
|
|
fi
|
|
|
|
echo "$now" > "$STATE_DIR/last_parse"
|
|
|
|
# Parse access log for status codes and other metrics
|
|
# Assuming combined log format: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
|
|
|
|
local log_data
|
|
log_data=$(tail -n "$LOG_TAIL_LINES" "$ACCESS_LOG" 2>/dev/null || echo "")
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
echo "# No log data available"
|
|
return
|
|
fi
|
|
|
|
local metrics_output=""
|
|
|
|
# Count by status code
|
|
local status_counts
|
|
status_counts=$(echo "$log_data" | awk '{print $9}' | { grep -E '^[0-9]{3}$' || true; } | sort | uniq -c | sort -rn)
|
|
|
|
metrics_output+="# HELP apache_http_requests_by_status_total HTTP requests by status code (from last $LOG_TAIL_LINES log lines)
|
|
# TYPE apache_http_requests_by_status_total gauge
|
|
"
|
|
|
|
# Initialize counters for status code groups
|
|
local count_1xx=0 count_2xx=0 count_3xx=0 count_4xx=0 count_5xx=0
|
|
|
|
while read -r count status; do
|
|
if [[ -n "$status" ]] && [[ -n "$count" ]]; then
|
|
metrics_output+="apache_http_requests_by_status_total{status=\"$status\"} $count
|
|
"
|
|
# Aggregate by category
|
|
case "${status:0:1}" in
|
|
1) count_1xx=$((count_1xx + count)) ;;
|
|
2) count_2xx=$((count_2xx + count)) ;;
|
|
3) count_3xx=$((count_3xx + count)) ;;
|
|
4) count_4xx=$((count_4xx + count)) ;;
|
|
5) count_5xx=$((count_5xx + count)) ;;
|
|
esac
|
|
fi
|
|
done <<< "$status_counts"
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_requests_by_status_class_total HTTP requests by status class
|
|
# TYPE apache_http_requests_by_status_class_total gauge
|
|
apache_http_requests_by_status_class_total{class=\"1xx\"} $count_1xx
|
|
apache_http_requests_by_status_class_total{class=\"2xx\"} $count_2xx
|
|
apache_http_requests_by_status_class_total{class=\"3xx\"} $count_3xx
|
|
apache_http_requests_by_status_class_total{class=\"4xx\"} $count_4xx
|
|
apache_http_requests_by_status_class_total{class=\"5xx\"} $count_5xx
|
|
"
|
|
|
|
# Calculate total bytes sent
|
|
local total_bytes
|
|
total_bytes=$(echo "$log_data" | awk '{sum += $10} END {print sum+0}')
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_response_bytes_total Total bytes sent in responses (from last $LOG_TAIL_LINES log lines)
|
|
# TYPE apache_http_response_bytes_total gauge
|
|
apache_http_response_bytes_total $total_bytes
|
|
"
|
|
|
|
# Count requests by method
|
|
local method_counts
|
|
method_counts=$(echo "$log_data" | awk -F'"' '{print $2}' | awk '{print $1}' | { grep -E '^(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)$' || true; } | sort | uniq -c)
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_requests_by_method_total HTTP requests by method (from last $LOG_TAIL_LINES log lines)
|
|
# TYPE apache_http_requests_by_method_total gauge
|
|
"
|
|
|
|
while read -r count method; do
|
|
if [[ -n "$method" ]] && [[ -n "$count" ]]; then
|
|
metrics_output+="apache_http_requests_by_method_total{method=\"$method\"} $count
|
|
"
|
|
fi
|
|
done <<< "$method_counts"
|
|
|
|
# Count unique IPs
|
|
local unique_ips
|
|
unique_ips=$(echo "$log_data" | awk '{print $1}' | sort -u | wc -l)
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_unique_clients Unique client IPs (from last $LOG_TAIL_LINES log lines)
|
|
# TYPE apache_http_unique_clients gauge
|
|
apache_http_unique_clients $unique_ips
|
|
"
|
|
|
|
# Top URIs (for potential abuse detection)
|
|
local top_uris
|
|
top_uris=$(echo "$log_data" | awk -F'"' '{print $2}' | awk '{print $2}' | { grep -v '^-$' || true; } | sort | uniq -c | sort -rn | head -5)
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_top_uri_requests_total Top requested URIs (from last $LOG_TAIL_LINES log lines)
|
|
# TYPE apache_http_top_uri_requests_total gauge
|
|
"
|
|
|
|
local rank=1
|
|
while read -r count uri; do
|
|
if [[ -n "$uri" ]] && [[ -n "$count" ]]; then
|
|
# Truncate URI and escape quotes
|
|
uri="${uri:0:100}"
|
|
uri="${uri//\"/\\\"}"
|
|
metrics_output+="apache_http_top_uri_requests_total{uri=\"$uri\",rank=\"$rank\"} $count
|
|
"
|
|
rank=$((rank + 1))
|
|
fi
|
|
done <<< "$top_uris"
|
|
|
|
# Count requests in time windows
|
|
local recent_requests
|
|
recent_requests=$(echo "$log_data" | wc -l)
|
|
|
|
metrics_output+="
|
|
# HELP apache_http_requests_in_sample Total requests in sample window
|
|
# TYPE apache_http_requests_in_sample gauge
|
|
apache_http_requests_in_sample $recent_requests
|
|
"
|
|
|
|
# Save metrics for caching
|
|
echo "$metrics_output" > "$STATE_DIR/log_metrics"
|
|
echo "$metrics_output"
|
|
}
|
|
|
|
#########################
|
|
### Error Log Metrics ###
|
|
#########################
|
|
|
|
collect_error_log_metrics() {
|
|
if [[ ! -f "$ERROR_LOG" ]] || [[ ! -r "$ERROR_LOG" ]]; then
|
|
echo "# Error log not readable at $ERROR_LOG"
|
|
return
|
|
fi
|
|
|
|
# Count errors by level from last 1000 lines
|
|
local log_data
|
|
log_data=$(tail -n 1000 "$ERROR_LOG" 2>/dev/null || echo "")
|
|
|
|
if [[ -z "$log_data" ]]; then
|
|
return
|
|
fi
|
|
|
|
local emerg_count alert_count crit_count error_count warn_count notice_count info_count
|
|
|
|
emerg_count=$(echo "$log_data" | grep -c '\[emerg\]' 2>/dev/null) || emerg_count=0
|
|
alert_count=$(echo "$log_data" | grep -c '\[alert\]' 2>/dev/null) || alert_count=0
|
|
crit_count=$(echo "$log_data" | grep -c '\[crit\]' 2>/dev/null) || crit_count=0
|
|
error_count=$(echo "$log_data" | grep -c '\[error\]' 2>/dev/null) || error_count=0
|
|
warn_count=$(echo "$log_data" | grep -c '\[warn\]' 2>/dev/null) || warn_count=0
|
|
notice_count=$(echo "$log_data" | grep -c '\[notice\]' 2>/dev/null) || notice_count=0
|
|
info_count=$(echo "$log_data" | grep -c '\[info\]' 2>/dev/null) || info_count=0
|
|
|
|
cat <<EOF
|
|
# HELP apache_error_log_messages_total Error log messages by level (from last 1000 lines)
|
|
# TYPE apache_error_log_messages_total gauge
|
|
apache_error_log_messages_total{level="emerg"} $emerg_count
|
|
apache_error_log_messages_total{level="alert"} $alert_count
|
|
apache_error_log_messages_total{level="crit"} $crit_count
|
|
apache_error_log_messages_total{level="error"} $error_count
|
|
apache_error_log_messages_total{level="warn"} $warn_count
|
|
apache_error_log_messages_total{level="notice"} $notice_count
|
|
apache_error_log_messages_total{level="info"} $info_count
|
|
EOF
|
|
|
|
# Check error log file size and age
|
|
local log_size log_mtime now log_age
|
|
log_size=$(stat -c %s "$ERROR_LOG" 2>/dev/null || echo "0")
|
|
log_mtime=$(stat -c %Y "$ERROR_LOG" 2>/dev/null || echo "0")
|
|
now=$(date +%s)
|
|
log_age=$((now - log_mtime))
|
|
|
|
cat <<EOF
|
|
|
|
# HELP apache_error_log_size_bytes Size of error log file
|
|
# TYPE apache_error_log_size_bytes gauge
|
|
apache_error_log_size_bytes $log_size
|
|
|
|
# HELP apache_error_log_last_modified_seconds Seconds since error log was last modified
|
|
# TYPE apache_error_log_last_modified_seconds gauge
|
|
apache_error_log_last_modified_seconds $log_age
|
|
EOF
|
|
}
|
|
|
|
#########################
|
|
### SSL Certificate Metrics ###
|
|
#########################
|
|
|
|
collect_ssl_metrics() {
|
|
local domains="$SSL_CHECK_DOMAINS"
|
|
|
|
# If no domains specified, try to find them from Apache config
|
|
if [[ -z "$domains" ]]; then
|
|
# Look for SSLCertificateFile directives
|
|
local cert_files
|
|
cert_files=$(grep -rh 'SSLCertificateFile\s' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | grep -v '#' | awk '{print $2}' | tr -d '"' | sort -u || echo "")
|
|
|
|
if [[ -z "$cert_files" ]]; then
|
|
echo "# No SSL certificates found in Apache config"
|
|
return
|
|
fi
|
|
|
|
echo "# HELP apache_ssl_certificate_expiry_days Days until SSL certificate expires"
|
|
echo "# TYPE apache_ssl_certificate_expiry_days gauge"
|
|
echo "# HELP apache_ssl_certificate_expiry_timestamp Unix timestamp when certificate expires"
|
|
echo "# TYPE apache_ssl_certificate_expiry_timestamp gauge"
|
|
|
|
while read -r cert_file; do
|
|
if [[ -f "$cert_file" ]]; then
|
|
local expiry_date expiry_epoch now_epoch days_left cn
|
|
|
|
expiry_date=$(openssl x509 -enddate -noout -in "$cert_file" 2>/dev/null | cut -d= -f2 || echo "")
|
|
if [[ -n "$expiry_date" ]]; then
|
|
expiry_epoch=$(date -d "$expiry_date" +%s 2>/dev/null || echo "0")
|
|
now_epoch=$(date +%s)
|
|
days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
|
|
|
|
# Get CN from certificate
|
|
cn=$(openssl x509 -subject -noout -in "$cert_file" 2>/dev/null | grep -oP 'CN\s*=\s*\K[^,/]+' || basename "$cert_file")
|
|
cn="${cn// /_}"
|
|
|
|
echo "apache_ssl_certificate_expiry_days{certificate=\"$cn\",file=\"$cert_file\"} $days_left"
|
|
echo "apache_ssl_certificate_expiry_timestamp{certificate=\"$cn\",file=\"$cert_file\"} $expiry_epoch"
|
|
fi
|
|
fi
|
|
done <<< "$cert_files"
|
|
return
|
|
fi
|
|
|
|
# Check specified domains via network
|
|
echo "# HELP apache_ssl_certificate_expiry_days Days until SSL certificate expires"
|
|
echo "# TYPE apache_ssl_certificate_expiry_days gauge"
|
|
echo "# HELP apache_ssl_certificate_expiry_timestamp Unix timestamp when certificate expires"
|
|
echo "# TYPE apache_ssl_certificate_expiry_timestamp gauge"
|
|
|
|
IFS=',' read -ra domain_array <<< "$domains"
|
|
for domain in "${domain_array[@]}"; do
|
|
domain=$(echo "$domain" | tr -d ' ')
|
|
if [[ -n "$domain" ]]; then
|
|
local expiry_date expiry_epoch now_epoch days_left
|
|
|
|
expiry_date=$(echo | openssl s_client -servername "$domain" -connect "$domain:443" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "")
|
|
|
|
if [[ -n "$expiry_date" ]]; then
|
|
expiry_epoch=$(date -d "$expiry_date" +%s 2>/dev/null || echo "0")
|
|
now_epoch=$(date +%s)
|
|
days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
|
|
|
|
echo "apache_ssl_certificate_expiry_days{domain=\"$domain\"} $days_left"
|
|
echo "apache_ssl_certificate_expiry_timestamp{domain=\"$domain\"} $expiry_epoch"
|
|
else
|
|
echo "apache_ssl_certificate_expiry_days{domain=\"$domain\"} -1"
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
#########################
|
|
### Proxy/Upstream Metrics ###
|
|
#########################
|
|
|
|
collect_upstream_metrics() {
|
|
# Check for proxy/balancer configurations
|
|
local proxy_passes
|
|
proxy_passes=$(grep -rh 'ProxyPass\s' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | grep -v '#' | grep -v 'ProxyPassReverse' | awk '{print $2}' | sort -u || echo "")
|
|
|
|
local balancers
|
|
balancers=$(grep -rhoE 'balancer://[a-zA-Z0-9_-]+' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | sort -u || echo "")
|
|
|
|
if [[ -z "$proxy_passes" ]] && [[ -z "$balancers" ]]; then
|
|
return
|
|
fi
|
|
|
|
local proxy_count=0
|
|
if [[ -n "$proxy_passes" ]]; then
|
|
proxy_count=$(echo "$proxy_passes" | wc -l)
|
|
fi
|
|
|
|
local balancer_count=0
|
|
if [[ -n "$balancers" ]]; then
|
|
balancer_count=$(echo "$balancers" | wc -l)
|
|
fi
|
|
|
|
cat <<EOF
|
|
# HELP apache_proxy_backends_configured Number of configured proxy backends
|
|
# TYPE apache_proxy_backends_configured gauge
|
|
apache_proxy_backends_configured $proxy_count
|
|
|
|
# HELP apache_balancers_configured Number of configured load balancer groups
|
|
# TYPE apache_balancers_configured gauge
|
|
apache_balancers_configured $balancer_count
|
|
EOF
|
|
|
|
# Count members per balancer
|
|
if [[ -n "$balancers" ]]; then
|
|
echo ""
|
|
echo "# HELP apache_balancer_members_total Members configured per balancer"
|
|
echo "# TYPE apache_balancer_members_total gauge"
|
|
|
|
while read -r balancer; do
|
|
if [[ -n "$balancer" ]]; then
|
|
local name
|
|
name=$(echo "$balancer" | sed 's|balancer://||')
|
|
local member_count
|
|
member_count=$(grep -A 20 "Proxy $balancer" "$SITES_DIR"/* "$CONF_D_DIR"/* "$APACHE_CONF" 2>/dev/null | grep -c 'BalancerMember' 2>/dev/null) || member_count=0
|
|
echo "apache_balancer_members_total{balancer=\"$name\"} $member_count"
|
|
fi
|
|
done <<< "$balancers"
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### Version Metrics ###
|
|
#########################
|
|
|
|
collect_version_metrics() {
|
|
local version="unknown"
|
|
|
|
if [[ -n "$APACHE_BIN" ]]; then
|
|
version=$($APACHE_BIN -v 2>&1 | grep -oP 'Apache/\K[0-9.]+' || echo "unknown")
|
|
fi
|
|
|
|
echo "# HELP apache_version_info Apache version information"
|
|
echo "# TYPE apache_version_info gauge"
|
|
echo "apache_version_info{version=\"$version\"} 1"
|
|
|
|
# Check loaded modules
|
|
if [[ -n "$APACHECTL" ]]; then
|
|
local modules_output
|
|
modules_output=$($APACHECTL -M 2>/dev/null || echo "")
|
|
|
|
local has_ssl has_proxy has_proxy_http has_proxy_balancer has_rewrite
|
|
local has_headers has_deflate has_expires has_status has_http2
|
|
|
|
has_ssl=$(echo "$modules_output" | grep -q 'ssl_module' && echo "1" || echo "0")
|
|
has_proxy=$(echo "$modules_output" | grep -q 'proxy_module' && echo "1" || echo "0")
|
|
has_proxy_http=$(echo "$modules_output" | grep -q 'proxy_http_module' && echo "1" || echo "0")
|
|
has_proxy_balancer=$(echo "$modules_output" | grep -q 'proxy_balancer_module' && echo "1" || echo "0")
|
|
has_rewrite=$(echo "$modules_output" | grep -q 'rewrite_module' && echo "1" || echo "0")
|
|
has_headers=$(echo "$modules_output" | grep -q 'headers_module' && echo "1" || echo "0")
|
|
has_deflate=$(echo "$modules_output" | grep -q 'deflate_module' && echo "1" || echo "0")
|
|
has_expires=$(echo "$modules_output" | grep -q 'expires_module' && echo "1" || echo "0")
|
|
has_status=$(echo "$modules_output" | grep -q 'status_module' && echo "1" || echo "0")
|
|
has_http2=$(echo "$modules_output" | grep -q 'http2_module' && echo "1" || echo "0")
|
|
|
|
cat <<EOF
|
|
|
|
# HELP apache_module_loaded Whether Apache module is loaded
|
|
# TYPE apache_module_loaded gauge
|
|
apache_module_loaded{module="ssl"} $has_ssl
|
|
apache_module_loaded{module="proxy"} $has_proxy
|
|
apache_module_loaded{module="proxy_http"} $has_proxy_http
|
|
apache_module_loaded{module="proxy_balancer"} $has_proxy_balancer
|
|
apache_module_loaded{module="rewrite"} $has_rewrite
|
|
apache_module_loaded{module="headers"} $has_headers
|
|
apache_module_loaded{module="deflate"} $has_deflate
|
|
apache_module_loaded{module="expires"} $has_expires
|
|
apache_module_loaded{module="status"} $has_status
|
|
apache_module_loaded{module="http2"} $has_http2
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### File Descriptor Metrics ###
|
|
#########################
|
|
|
|
collect_system_metrics() {
|
|
# System-wide limits that affect Apache
|
|
local max_files ulimit_n
|
|
max_files=$(cat /proc/sys/fs/file-max 2>/dev/null || echo "0")
|
|
ulimit_n=$(ulimit -n 2>/dev/null || echo "0")
|
|
|
|
cat <<EOF
|
|
# HELP apache_system_file_max System-wide maximum file descriptors
|
|
# TYPE apache_system_file_max gauge
|
|
apache_system_file_max $max_files
|
|
|
|
# HELP apache_system_ulimit_n Current shell ulimit for open files
|
|
# TYPE apache_system_ulimit_n gauge
|
|
apache_system_ulimit_n $ulimit_n
|
|
EOF
|
|
|
|
# Current system-wide open files
|
|
local open_files
|
|
open_files=$(cat /proc/sys/fs/file-nr 2>/dev/null | awk '{print $1}' || echo "0")
|
|
|
|
echo ""
|
|
echo "# HELP apache_system_open_files Current system-wide open files"
|
|
echo "# TYPE apache_system_open_files gauge"
|
|
echo "apache_system_open_files $open_files"
|
|
}
|
|
|
|
#########################
|
|
### Collect All Metrics ###
|
|
#########################
|
|
|
|
collect_all_metrics() {
|
|
local hostname
|
|
hostname=$(hostname -f 2>/dev/null || hostname)
|
|
|
|
cat <<EOF
|
|
# Apache Metrics Exporter
|
|
# Host: $hostname
|
|
# Collected at: $(date -Iseconds)
|
|
|
|
EOF
|
|
|
|
collect_version_metrics
|
|
echo ""
|
|
collect_server_status
|
|
echo ""
|
|
collect_process_metrics
|
|
echo ""
|
|
collect_config_metrics
|
|
echo ""
|
|
collect_access_log_metrics
|
|
echo ""
|
|
collect_error_log_metrics
|
|
echo ""
|
|
collect_ssl_metrics
|
|
echo ""
|
|
collect_upstream_metrics
|
|
echo ""
|
|
collect_system_metrics
|
|
}
|
|
|
|
#########################
|
|
### HTTP Server ###
|
|
#########################
|
|
|
|
handle_request() {
|
|
local request_line=""
|
|
local content_length=0
|
|
|
|
# Read request
|
|
while IFS= read -r line; do
|
|
line="${line%%$'\r'}"
|
|
[[ -z "$line" ]] && break
|
|
[[ -z "$request_line" ]] && request_line="$line"
|
|
done
|
|
|
|
local method path
|
|
method=$(echo "$request_line" | awk '{print $1}')
|
|
path=$(echo "$request_line" | awk '{print $2}')
|
|
|
|
case "$path" in
|
|
/metrics|/)
|
|
local metrics
|
|
metrics=$(collect_all_metrics)
|
|
local body_length=${#metrics}
|
|
|
|
cat <<EOF
|
|
HTTP/1.1 200 OK
|
|
Content-Type: text/plain; charset=utf-8
|
|
Content-Length: $body_length
|
|
Connection: close
|
|
|
|
$metrics
|
|
EOF
|
|
;;
|
|
/health|/healthz)
|
|
local health_body="OK"
|
|
cat <<EOF
|
|
HTTP/1.1 200 OK
|
|
Content-Type: text/plain
|
|
Content-Length: 2
|
|
Connection: close
|
|
|
|
OK
|
|
EOF
|
|
;;
|
|
*)
|
|
cat <<EOF
|
|
HTTP/1.1 404 Not Found
|
|
Content-Type: text/plain
|
|
Content-Length: 9
|
|
Connection: close
|
|
|
|
Not Found
|
|
EOF
|
|
;;
|
|
esac
|
|
}
|
|
|
|
start_server() {
|
|
log "Starting Apache Metrics Exporter on port $LISTEN_PORT"
|
|
log "Metrics available at http://localhost:$LISTEN_PORT/metrics"
|
|
log "Server status URL: $STATUS_URL"
|
|
|
|
while true; do
|
|
socat TCP-LISTEN:"$LISTEN_PORT",reuseaddr,fork EXEC:"$0 --handle-request" 2>/dev/null || {
|
|
log "Server error, restarting in 5 seconds..."
|
|
sleep 5
|
|
}
|
|
done
|
|
}
|
|
|
|
#########################
|
|
### Output ###
|
|
#########################
|
|
|
|
write_output() {
|
|
local metrics
|
|
metrics=$(collect_all_metrics)
|
|
|
|
if [[ -n "$OUTPUT_FILE" ]]; then
|
|
local tmp_file="${OUTPUT_FILE}.$$"
|
|
echo "$metrics" > "$tmp_file"
|
|
mv "$tmp_file" "$OUTPUT_FILE"
|
|
else
|
|
echo "$metrics"
|
|
fi
|
|
}
|
|
|
|
#########################
|
|
### Main ###
|
|
#########################
|
|
|
|
main() {
|
|
if [[ "${1:-}" == "--handle-request" ]]; then
|
|
handle_request
|
|
exit 0
|
|
fi
|
|
|
|
parse_args "$@"
|
|
setup
|
|
|
|
if [[ "$HTTP_MODE" == true ]]; then
|
|
start_server
|
|
elif [[ -n "$OUTPUT_FILE" ]]; then
|
|
write_output
|
|
else
|
|
collect_all_metrics
|
|
fi
|
|
}
|
|
|
|
main "$@"
|