#!/bin/bash
#############################################################
#### Apache Metrics Exporter for Prometheus              ####
#### Comprehensive Apache monitoring via mod_status,     ####
#### logs, SSL, process, and config metrics              ####
####                                                     ####
#### Author: Phil Connor                                 ####
#### Contact: contact@mylinux.work                       ####
#### License: MIT                                        ####
#### Version: 1.01                                       ####
####                                                     ####
#### Usage: ./apache-metrics-exporter.sh [OPTIONS]       ####
#############################################################
#
# Metrics collected:
#   - mod_status: accesses, bytes, req/sec, busy/idle workers, scoreboard
#   - Process: worker count, memory usage, CPU usage, open files
#   - Access logs: requests by status code, response times, bytes transferred
#   - SSL: certificate expiry days for configured domains
#   - Config: MPM type, MaxRequestWorkers, KeepAliveTimeout
#   - Upstream: proxy/balancer status (if configured)
#
# Requirements:
#   - Apache with mod_status enabled (ExtendedStatus On)
#   - socat (for HTTP server)
#   - curl (for server-status fetching)
#
set -euo pipefail

#########################
### Auto-detect Apache ###
#########################

APACHE_BIN=""
APACHECTL=""
APACHE_PROC=""

detect_apache_flavor() {
    if command -v apache2 &>/dev/null; then
        APACHE_BIN="apache2"
        APACHECTL="apache2ctl"
        APACHE_PROC="apache2"
    elif command -v httpd &>/dev/null; then
        APACHE_BIN="httpd"
        APACHECTL="httpd"
        APACHE_PROC="httpd"
    else
        APACHE_BIN=""
        APACHECTL=""
        APACHE_PROC=""
    fi
}

detect_apache_flavor

#########################
### Configuration ###
#########################

LISTEN_PORT="${APACHE_EXPORTER_PORT:-9117}"
STATUS_URL="${APACHE_STATUS_URL:-http://127.0.0.1/server-status?auto}"
SSL_CHECK_DOMAINS="${SSL_CHECK_DOMAINS:-}"  # Comma-separated list of domains to check SSL
SCRAPE_INTERVAL="${SCRAPE_INTERVAL:-15}"

# Auto-detect paths based on distro
if [[ -d /etc/apache2 ]]; then
    ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/apache2/access.log}"
    ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/apache2/error.log}"
    APACHE_CONF="${APACHE_CONF:-/etc/apache2/apache2.conf}"
    SITES_DIR="${APACHE_SITES_DIR:-/etc/apache2/sites-enabled}"
    CONF_D_DIR="${APACHE_CONF_D:-/etc/apache2/conf-enabled}"
elif [[ -d /etc/httpd ]]; then
    ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/httpd/access_log}"
    ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/httpd/error_log}"
    APACHE_CONF="${APACHE_CONF:-/etc/httpd/conf/httpd.conf}"
    SITES_DIR="${APACHE_SITES_DIR:-/etc/httpd/conf.d}"
    CONF_D_DIR="${APACHE_CONF_D:-/etc/httpd/conf.d}"
else
    ACCESS_LOG="${APACHE_ACCESS_LOG:-/var/log/apache2/access.log}"
    ERROR_LOG="${APACHE_ERROR_LOG:-/var/log/apache2/error.log}"
    APACHE_CONF="${APACHE_CONF:-/etc/apache2/apache2.conf}"
    SITES_DIR="${APACHE_SITES_DIR:-/etc/apache2/sites-enabled}"
    CONF_D_DIR="${APACHE_CONF_D:-/etc/apache2/conf-enabled}"
fi

# Log parsing settings
LOG_TAIL_LINES="${LOG_TAIL_LINES:-10000}"  # Number of lines to parse from access log
LOG_PARSE_INTERVAL="${LOG_PARSE_INTERVAL:-60}"  # How often to parse logs (seconds)

# State files for log metrics
STATE_DIR="/tmp/apache-metrics"
LAST_LOG_PARSE=0

# Output mode
TEXTFILE_DIR="/var/lib/node_exporter"
OUTPUT_FILE=""
HTTP_MODE=false

#########################
### Logging ###
#########################

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
}

#########################
### Parse Arguments ###
#########################

parse_args() {
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --textfile)
                OUTPUT_FILE="$TEXTFILE_DIR/apache.prom"
                shift
                ;;
            --http)
                HTTP_MODE=true
                shift
                ;;
            --output|-o)
                OUTPUT_FILE="$2"
                shift 2
                ;;
            --port)
                LISTEN_PORT="$2"
                shift 2
                ;;
            --status-url)
                STATUS_URL="$2"
                shift 2
                ;;
            --access-log)
                ACCESS_LOG="$2"
                shift 2
                ;;
            --error-log)
                ERROR_LOG="$2"
                shift 2
                ;;
            --apache-conf)
                APACHE_CONF="$2"
                shift 2
                ;;
            --ssl-domains)
                SSL_CHECK_DOMAINS="$2"
                shift 2
                ;;
            --help)
                cat <<EOF
Apache Metrics Exporter for Prometheus

Usage: $0 [OPTIONS]

MODES:
  --textfile             Write to node_exporter textfile collector
  --http                 Run HTTP server on port $LISTEN_PORT (default mode with no other output)
  (no flag)              Output to stdout

OPTIONS:
  --port PORT            HTTP port (default: 9117)
  -o, --output PATH      Output file path
  --status-url URL       Apache server-status URL (default: http://127.0.0.1/server-status?auto)
  --access-log PATH      Path to access log (default: $ACCESS_LOG)
  --error-log PATH       Path to error log (default: $ERROR_LOG)
  --apache-conf PATH     Path to Apache config (default: $APACHE_CONF)
  --ssl-domains LIST     Comma-separated domains to check SSL expiry
  --help                 Show this help

EXAMPLES:
  $0 --textfile                    # Write to textfile collector
  $0 --http --port 9117            # Run HTTP server
  $0 -o /tmp/apache.prom           # Write to custom file
  $0                               # Output to stdout

EOF
                exit 0
                ;;
            *)
                log "Unknown option: $1"
                exit 1
                ;;
        esac
    done
}

#########################
### Setup ###
#########################

detect_package_manager() {
    if command -v apt-get &>/dev/null; then
        echo "apt"
    elif command -v dnf &>/dev/null; then
        echo "dnf"
    elif command -v yum &>/dev/null; then
        echo "yum"
    elif command -v zypper &>/dev/null; then
        echo "zypper"
    elif command -v pacman &>/dev/null; then
        echo "pacman"
    elif command -v apk &>/dev/null; then
        echo "apk"
    else
        echo ""
    fi
}

install_package() {
    local pkg="$1"
    local pkgmgr
    pkgmgr=$(detect_package_manager)
    
    log "Installing $pkg..."
    
    case "$pkgmgr" in
        apt)
            apt-get update -qq && apt-get install -y -qq "$pkg"
            ;;
        dnf)
            dnf install -y -q "$pkg"
            ;;
        yum)
            yum install -y -q "$pkg"
            ;;
        zypper)
            zypper install -y -q "$pkg"
            ;;
        pacman)
            pacman -S --noconfirm "$pkg"
            ;;
        apk)
            apk add --quiet "$pkg"
            ;;
        *)
            log "ERROR: Unknown package manager. Please install $pkg manually."
            return 1
            ;;
    esac
}

setup() {
    mkdir -p "$STATE_DIR"
    
    # Check for required tools and install if missing
    if ! command -v socat &>/dev/null; then
        log "socat not found, attempting to install..."
        if [[ $EUID -eq 0 ]]; then
            if ! install_package socat; then
                log "ERROR: Failed to install socat"
                exit 1
            fi
            log "socat installed successfully"
        else
            log "ERROR: socat is required. Run as root to auto-install, or install manually:"
            log "  Debian/Ubuntu: apt install socat"
            log "  RHEL/CentOS:   yum install socat"
            log "  Fedora:        dnf install socat"
            log "  Alpine:        apk add socat"
            exit 1
        fi
    fi
    
    if ! command -v curl &>/dev/null; then
        log "curl not found, attempting to install..."
        if [[ $EUID -eq 0 ]]; then
            if ! install_package curl; then
                log "ERROR: Failed to install curl"
                exit 1
            fi
            log "curl installed successfully"
        else
            log "ERROR: curl is required. Run as root to auto-install, or install manually."
            exit 1
        fi
    fi
    
    # Check if Apache is running
    if [[ -n "$APACHE_PROC" ]]; then
        if ! pgrep -x "$APACHE_PROC" &>/dev/null && ! pidof "$APACHE_PROC" &>/dev/null; then
            log "WARNING: $APACHE_PROC process not found - process metrics will show apache_process_running=0"
        fi
    else
        log "WARNING: Apache binary not found (neither apache2 nor httpd)"
    fi
    
    # Check if server-status is accessible
    check_server_status
}

check_server_status() {
    log "Checking server-status at $STATUS_URL..."
    
    local response http_code
    response=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$STATUS_URL" 2>/dev/null)
    
    if [[ "$response" == "200" ]]; then
        # Verify it's actually mod_status output
        local content
        content=$(curl -s --max-time 5 "$STATUS_URL" 2>/dev/null)
        if echo "$content" | grep -q "Total Accesses"; then
            log "✓ mod_status is working correctly"
            return 0
        else
            log "WARNING: $STATUS_URL returned 200 but doesn't look like mod_status output"
            log "  Expected 'Total Accesses' in response (ensure ExtendedStatus On)"
            show_server_status_help
            return 1
        fi
    elif [[ "$response" == "000" ]]; then
        log "WARNING: Cannot connect to $STATUS_URL (connection refused/timeout)"
        log "  server-status metrics will show apache_up=0"
        show_server_status_help
        return 1
    elif [[ "$response" == "403" ]]; then
        log "WARNING: Access denied to $STATUS_URL (HTTP 403)"
        log "  Check 'Require' directives in server-status location block"
        show_server_status_help
        return 1
    elif [[ "$response" == "404" ]]; then
        log "WARNING: server-status endpoint not found at $STATUS_URL (HTTP 404)"
        log "  mod_status may not be enabled"
        show_server_status_help
        return 1
    else
        log "WARNING: Unexpected response from $STATUS_URL (HTTP $response)"
        show_server_status_help
        return 1
    fi
}

show_server_status_help() {
    log ""
    log "To enable mod_status, configure Apache as follows:"
    log ""
    log "  Debian/Ubuntu:"
    log "    sudo a2enmod status"
    log "    # Edit /etc/apache2/mods-enabled/status.conf:"
    log "    ExtendedStatus On"
    log "    <Location \"/server-status\">"
    log "        SetHandler server-status"
    log "        Require local"
    log "    </Location>"
    log ""
    log "  RHEL/CentOS/Rocky:"
    log "    # Add to /etc/httpd/conf.d/status.conf:"
    log "    ExtendedStatus On"
    log "    <Location \"/server-status\">"
    log "        SetHandler server-status"
    log "        Require local"
    log "    </Location>"
    log ""
    log "Then reload: apachectl configtest && systemctl reload apache2  (or httpd)"
    log ""
    log "Or specify a different URL with: --status-url <url>"
    log ""
}

#########################
### Server Status Metrics ###
#########################

collect_server_status() {
    local status_output
    
    echo "# HELP apache_up Whether Apache mod_status is reachable"
    echo "# TYPE apache_up gauge"
    
    if ! status_output=$(curl -s --max-time 5 "$STATUS_URL" 2>/dev/null); then
        echo "apache_up 0"
        return
    fi
    
    # Verify we got valid mod_status output
    if ! echo "$status_output" | grep -q "Total Accesses"; then
        echo "apache_up 0"
        return
    fi
    
    echo "apache_up 1"
    
    # Parse mod_status ?auto output
    # Format:
    # Total Accesses: 12345
    # Total kBytes: 67890
    # CPULoad: .0123456
    # Uptime: 86400
    # ReqPerSec: .142857
    # BytesPerSec: 804.571
    # BytesPerReq: 5632
    # BusyWorkers: 3
    # IdleWorkers: 7
    # Scoreboard: __W_K....._R..
    
    local total_accesses total_kbytes cpu_load uptime req_per_sec bytes_per_sec bytes_per_req
    local busy_workers idle_workers scoreboard
    
    total_accesses=$(echo "$status_output" | grep '^Total Accesses:' | awk '{print $3}') || total_accesses=0
    total_kbytes=$(echo "$status_output" | grep '^Total kBytes:' | awk '{print $3}') || total_kbytes=0
    cpu_load=$(echo "$status_output" | grep '^CPULoad:' | awk '{print $2}') || cpu_load=0
    uptime=$(echo "$status_output" | grep '^Uptime:' | awk '{print $2}') || uptime=0
    req_per_sec=$(echo "$status_output" | grep '^ReqPerSec:' | awk '{print $2}') || req_per_sec=0
    bytes_per_sec=$(echo "$status_output" | grep '^BytesPerSec:' | awk '{print $2}') || bytes_per_sec=0
    bytes_per_req=$(echo "$status_output" | grep '^BytesPerReq:' | awk '{print $2}') || bytes_per_req=0
    busy_workers=$(echo "$status_output" | grep '^BusyWorkers:' | awk '{print $2}') || busy_workers=0
    idle_workers=$(echo "$status_output" | grep '^IdleWorkers:' | awk '{print $2}') || idle_workers=0
    scoreboard=$(echo "$status_output" | grep '^Scoreboard:' | awk '{print $2}') || scoreboard=""
    
    # Convert kBytes to bytes
    local total_bytes
    total_bytes=$(echo "$total_kbytes * 1024" | bc 2>/dev/null || echo "$((total_kbytes * 1024))")
    
    cat <<EOF
# HELP apache_uptime_seconds Server uptime in seconds
# TYPE apache_uptime_seconds counter
apache_uptime_seconds $uptime

# HELP apache_accesses_total Total number of requests served
# TYPE apache_accesses_total counter
apache_accesses_total $total_accesses

# HELP apache_sent_bytes_total Total bytes transferred
# TYPE apache_sent_bytes_total counter
apache_sent_bytes_total $total_bytes

# HELP apache_cpu_load Current CPU load
# TYPE apache_cpu_load gauge
apache_cpu_load $cpu_load

# HELP apache_requests_per_second Current requests per second
# TYPE apache_requests_per_second gauge
apache_requests_per_second $req_per_sec

# HELP apache_bytes_per_second Current bytes per second
# TYPE apache_bytes_per_second gauge
apache_bytes_per_second $bytes_per_sec

# HELP apache_bytes_per_request Average bytes per request
# TYPE apache_bytes_per_request gauge
apache_bytes_per_request $bytes_per_req

# HELP apache_workers_busy Workers currently serving requests
# TYPE apache_workers_busy gauge
apache_workers_busy $busy_workers

# HELP apache_workers_idle Workers waiting for connections
# TYPE apache_workers_idle gauge
apache_workers_idle $idle_workers
EOF
    
    # Parse scoreboard
    if [[ -n "$scoreboard" ]]; then
        local sb_waiting=0 sb_starting=0 sb_reading=0 sb_sending=0 sb_keepalive=0
        local sb_dns=0 sb_closing=0 sb_logging=0 sb_graceful=0 sb_idle_cleanup=0 sb_open=0
        
        local i char
        for (( i=0; i<${#scoreboard}; i++ )); do
            char="${scoreboard:$i:1}"
            case "$char" in
                '_') sb_waiting=$((sb_waiting + 1)) ;;
                'S') sb_starting=$((sb_starting + 1)) ;;
                'R') sb_reading=$((sb_reading + 1)) ;;
                'W') sb_sending=$((sb_sending + 1)) ;;
                'K') sb_keepalive=$((sb_keepalive + 1)) ;;
                'D') sb_dns=$((sb_dns + 1)) ;;
                'C') sb_closing=$((sb_closing + 1)) ;;
                'L') sb_logging=$((sb_logging + 1)) ;;
                'G') sb_graceful=$((sb_graceful + 1)) ;;
                'I') sb_idle_cleanup=$((sb_idle_cleanup + 1)) ;;
                '.') sb_open=$((sb_open + 1)) ;;
            esac
        done
        
        cat <<EOF

# HELP apache_scoreboard_waiting Workers in waiting for connection state
# TYPE apache_scoreboard_waiting gauge
apache_scoreboard_waiting $sb_waiting

# HELP apache_scoreboard_starting Workers in starting up state
# TYPE apache_scoreboard_starting gauge
apache_scoreboard_starting $sb_starting

# HELP apache_scoreboard_reading Workers reading request
# TYPE apache_scoreboard_reading gauge
apache_scoreboard_reading $sb_reading

# HELP apache_scoreboard_sending Workers sending reply
# TYPE apache_scoreboard_sending gauge
apache_scoreboard_sending $sb_sending

# HELP apache_scoreboard_keepalive Workers in keepalive state
# TYPE apache_scoreboard_keepalive gauge
apache_scoreboard_keepalive $sb_keepalive

# HELP apache_scoreboard_dns_lookup Workers doing DNS lookup
# TYPE apache_scoreboard_dns_lookup gauge
apache_scoreboard_dns_lookup $sb_dns

# HELP apache_scoreboard_closing Workers closing connection
# TYPE apache_scoreboard_closing gauge
apache_scoreboard_closing $sb_closing

# HELP apache_scoreboard_logging Workers logging
# TYPE apache_scoreboard_logging gauge
apache_scoreboard_logging $sb_logging

# HELP apache_scoreboard_graceful_finishing Workers gracefully finishing
# TYPE apache_scoreboard_graceful_finishing gauge
apache_scoreboard_graceful_finishing $sb_graceful

# HELP apache_scoreboard_idle_cleanup Workers in idle cleanup state
# TYPE apache_scoreboard_idle_cleanup gauge
apache_scoreboard_idle_cleanup $sb_idle_cleanup

# HELP apache_scoreboard_open_slot Open slots with no current process
# TYPE apache_scoreboard_open_slot gauge
apache_scoreboard_open_slot $sb_open
EOF
    fi
}

#########################
### Process Metrics ###
#########################

collect_process_metrics() {
    local apache_master_pid apache_pids worker_count total_memory total_cpu total_fds
    local total_threads
    
    if [[ -z "$APACHE_PROC" ]]; then
        echo "# HELP apache_process_running Whether Apache process is running"
        echo "# TYPE apache_process_running gauge"
        echo "apache_process_running 0"
        return
    fi
    
    # Find Apache master process
    apache_master_pid=$(pgrep -x "$APACHE_PROC" -o 2>/dev/null || pidof "$APACHE_PROC" 2>/dev/null | awk '{print $1}' || echo "")
    
    if [[ -z "$apache_master_pid" ]]; then
        echo "# HELP apache_process_running Whether Apache process is running"
        echo "# TYPE apache_process_running gauge"
        echo "apache_process_running 0"
        return
    fi
    
    echo "# HELP apache_process_running Whether Apache process is running"
    echo "# TYPE apache_process_running gauge"
    echo "apache_process_running 1"
    
    # Get all Apache PIDs
    apache_pids=$(pgrep -x "$APACHE_PROC" 2>/dev/null || pidof "$APACHE_PROC" 2>/dev/null || echo "")
    
    # Count workers (total processes minus master)
    worker_count=$(echo "$apache_pids" | wc -w)
    if [[ $worker_count -gt 0 ]]; then
        worker_count=$((worker_count - 1))  # Subtract master
    fi
    
    echo "# HELP apache_workers_count Number of Apache worker processes"
    echo "# TYPE apache_workers_count gauge"
    echo "apache_workers_count $worker_count"
    
    # Calculate total memory usage (RSS in bytes)
    total_memory=0
    total_cpu=0
    total_fds=0
    total_threads=0
    
    for pid in $apache_pids; do
        if [[ -d "/proc/$pid" ]]; then
            # Memory (RSS in KB from /proc/pid/status, convert to bytes)
            local rss
            rss=$(grep -m1 'VmRSS:' "/proc/$pid/status" 2>/dev/null | awk '{print $2}' || echo "0")
            total_memory=$((total_memory + rss * 1024))
            
            # CPU time (from /proc/pid/stat - utime + stime in jiffies)
            local stat_line utime stime
            if stat_line=$(cat "/proc/$pid/stat" 2>/dev/null); then
                utime=$(echo "$stat_line" | awk '{print $14}')
                stime=$(echo "$stat_line" | awk '{print $15}')
                total_cpu=$((total_cpu + utime + stime))
            fi
            
            # Open file descriptors
            local fds
            fds=$(ls -1 "/proc/$pid/fd" 2>/dev/null | wc -l || echo "0")
            total_fds=$((total_fds + fds))
            
            # Threads
            local threads
            threads=$(grep -c '^Threads:' "/proc/$pid/status" 2>/dev/null || true)
            if [[ "$threads" -eq 0 ]]; then
                threads=$(grep 'Threads:' "/proc/$pid/status" 2>/dev/null | awk '{print $2}' || echo "1")
            fi
            total_threads=$((total_threads + threads))
        fi
    done
    
    # Convert CPU jiffies to seconds (assuming 100 Hz)
    local cpu_seconds
    cpu_seconds=$(echo "scale=2; $total_cpu / 100" | bc 2>/dev/null || echo "$total_cpu")
    
    cat <<EOF
# HELP apache_process_memory_bytes Total memory used by Apache processes (RSS)
# TYPE apache_process_memory_bytes gauge
apache_process_memory_bytes $total_memory

# HELP apache_process_cpu_seconds_total Total CPU time consumed by Apache processes
# TYPE apache_process_cpu_seconds_total counter
apache_process_cpu_seconds_total $cpu_seconds

# HELP apache_process_open_fds Total number of open file descriptors
# TYPE apache_process_open_fds gauge
apache_process_open_fds $total_fds

# HELP apache_process_threads_total Total number of threads
# TYPE apache_process_threads_total gauge
apache_process_threads_total $total_threads

# HELP apache_process_start_time_seconds Start time of Apache master process
# TYPE apache_process_start_time_seconds gauge
EOF
    
    # Get start time of master process
    if [[ -f "/proc/$apache_master_pid/stat" ]]; then
        local starttime start_seconds
        starttime=$(awk '{print $22}' "/proc/$apache_master_pid/stat" 2>/dev/null || echo "0")
        # starttime is in jiffies since boot
        start_seconds=$(awk "BEGIN {printf \"%.0f\", $(cat /proc/uptime | awk '{print $1}') - ($starttime / 100)}")
        local now_epoch
        now_epoch=$(date +%s)
        local process_start=$((now_epoch - start_seconds))
        echo "apache_process_start_time_seconds $process_start"
    else
        echo "apache_process_start_time_seconds 0"
    fi
    
    # Get max open files limit
    if [[ -f "/proc/$apache_master_pid/limits" ]]; then
        local max_fds
        max_fds=$(grep 'Max open files' "/proc/$apache_master_pid/limits" 2>/dev/null | awk '{print $4}' || echo "0")
        echo ""
        echo "# HELP apache_process_max_fds Maximum number of open file descriptors"
        echo "# TYPE apache_process_max_fds gauge"
        echo "apache_process_max_fds $max_fds"
    fi
}

#########################
### Config Metrics ###
#########################

collect_config_metrics() {
    if [[ ! -f "$APACHE_CONF" ]]; then
        echo "# Apache config not found at $APACHE_CONF"
        return
    fi
    
    local max_request_workers keepalive_timeout keepalive_enabled
    local mpm_type
    
    # Parse MaxRequestWorkers (or MaxClients for older Apache)
    max_request_workers=$(grep -rihE '^\s*MaxRequestWorkers' "$APACHE_CONF" "$CONF_D_DIR" "$SITES_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "")
    if [[ -z "$max_request_workers" ]]; then
        max_request_workers=$(grep -rihE '^\s*MaxClients' "$APACHE_CONF" "$CONF_D_DIR" "$SITES_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
    fi
    max_request_workers="${max_request_workers:-0}"
    
    # Parse KeepAliveTimeout
    keepalive_timeout=$(grep -rihE '^\s*KeepAliveTimeout' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
    keepalive_timeout="${keepalive_timeout:-0}"
    
    # Check KeepAlive on/off
    keepalive_enabled=$(grep -rihE '^\s*KeepAlive\s' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print tolower($2)}' || echo "on")
    if [[ "$keepalive_enabled" == "on" ]]; then
        keepalive_enabled=1
    else
        keepalive_enabled=0
    fi
    
    # Detect MPM type
    mpm_type="unknown"
    if [[ -n "$APACHECTL" ]]; then
        local modules_list
        modules_list=$($APACHECTL -M 2>/dev/null || echo "")
        if echo "$modules_list" | grep -q 'mpm_event_module'; then
            mpm_type="event"
        elif echo "$modules_list" | grep -q 'mpm_worker_module'; then
            mpm_type="worker"
        elif echo "$modules_list" | grep -q 'mpm_prefork_module'; then
            mpm_type="prefork"
        fi
    fi
    
    cat <<EOF
# HELP apache_config_max_request_workers MaxRequestWorkers setting
# TYPE apache_config_max_request_workers gauge
apache_config_max_request_workers $max_request_workers

# HELP apache_config_keep_alive_timeout KeepAliveTimeout setting in seconds
# TYPE apache_config_keep_alive_timeout gauge
apache_config_keep_alive_timeout $keepalive_timeout

# HELP apache_config_keep_alive Whether KeepAlive is enabled
# TYPE apache_config_keep_alive gauge
apache_config_keep_alive $keepalive_enabled

# HELP apache_config_mpm_type MPM type in use (label)
# TYPE apache_config_mpm_type gauge
apache_config_mpm_type{mpm="$mpm_type"} 1
EOF

    # Count virtual hosts
    local vhost_count=0
    if [[ -d "$SITES_DIR" ]]; then
        vhost_count=$(find "$SITES_DIR" -type f -o -type l 2>/dev/null | wc -l)
    elif [[ -d "$CONF_D_DIR" ]]; then
        vhost_count=$(find "$CONF_D_DIR" -name "*.conf" -type f 2>/dev/null | wc -l)
    fi
    
    echo ""
    echo "# HELP apache_config_vhosts_total Number of configured virtual hosts"
    echo "# TYPE apache_config_vhosts_total gauge"
    echo "apache_config_vhosts_total $vhost_count"
    
    # Parse ServerLimit if available
    local server_limit
    server_limit=$(grep -rihE '^\s*ServerLimit' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
    if [[ "$server_limit" != "0" ]] && [[ -n "$server_limit" ]]; then
        echo ""
        echo "# HELP apache_config_server_limit ServerLimit setting"
        echo "# TYPE apache_config_server_limit gauge"
        echo "apache_config_server_limit $server_limit"
    fi
    
    # Parse Timeout
    local timeout_val
    timeout_val=$(grep -rihE '^\s*Timeout\s' "$APACHE_CONF" "$CONF_D_DIR" 2>/dev/null | head -1 | awk '{print $2}' || echo "0")
    if [[ "$timeout_val" != "0" ]] && [[ -n "$timeout_val" ]]; then
        echo ""
        echo "# HELP apache_config_timeout Timeout setting in seconds"
        echo "# TYPE apache_config_timeout gauge"
        echo "apache_config_timeout $timeout_val"
    fi
}

#########################
### Access Log Metrics ###
#########################

collect_access_log_metrics() {
    if [[ ! -f "$ACCESS_LOG" ]] || [[ ! -r "$ACCESS_LOG" ]]; then
        echo "# Access log not readable at $ACCESS_LOG"
        return
    fi
    
    local now
    now=$(date +%s)
    
    # Only parse logs every LOG_PARSE_INTERVAL seconds
    if [[ -f "$STATE_DIR/last_parse" ]]; then
        LAST_LOG_PARSE=$(cat "$STATE_DIR/last_parse")
    fi
    
    if [[ $((now - LAST_LOG_PARSE)) -lt $LOG_PARSE_INTERVAL ]] && [[ -f "$STATE_DIR/log_metrics" ]]; then
        cat "$STATE_DIR/log_metrics"
        return
    fi
    
    echo "$now" > "$STATE_DIR/last_parse"
    
    # Parse access log for status codes and other metrics
    # Assuming combined log format: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
    
    local log_data
    log_data=$(tail -n "$LOG_TAIL_LINES" "$ACCESS_LOG" 2>/dev/null || echo "")
    
    if [[ -z "$log_data" ]]; then
        echo "# No log data available"
        return
    fi
    
    local metrics_output=""
    
    # Count by status code
    local status_counts
    status_counts=$(echo "$log_data" | awk '{print $9}' | { grep -E '^[0-9]{3}$' || true; } | sort | uniq -c | sort -rn)
    
    metrics_output+="# HELP apache_http_requests_by_status_total HTTP requests by status code (from last $LOG_TAIL_LINES log lines)
# TYPE apache_http_requests_by_status_total gauge
"
    
    # Initialize counters for status code groups
    local count_1xx=0 count_2xx=0 count_3xx=0 count_4xx=0 count_5xx=0
    
    while read -r count status; do
        if [[ -n "$status" ]] && [[ -n "$count" ]]; then
            metrics_output+="apache_http_requests_by_status_total{status=\"$status\"} $count
"
            # Aggregate by category
            case "${status:0:1}" in
                1) count_1xx=$((count_1xx + count)) ;;
                2) count_2xx=$((count_2xx + count)) ;;
                3) count_3xx=$((count_3xx + count)) ;;
                4) count_4xx=$((count_4xx + count)) ;;
                5) count_5xx=$((count_5xx + count)) ;;
            esac
        fi
    done <<< "$status_counts"
    
    metrics_output+="
# HELP apache_http_requests_by_status_class_total HTTP requests by status class
# TYPE apache_http_requests_by_status_class_total gauge
apache_http_requests_by_status_class_total{class=\"1xx\"} $count_1xx
apache_http_requests_by_status_class_total{class=\"2xx\"} $count_2xx
apache_http_requests_by_status_class_total{class=\"3xx\"} $count_3xx
apache_http_requests_by_status_class_total{class=\"4xx\"} $count_4xx
apache_http_requests_by_status_class_total{class=\"5xx\"} $count_5xx
"
    
    # Calculate total bytes sent
    local total_bytes
    total_bytes=$(echo "$log_data" | awk '{sum += $10} END {print sum+0}')
    
    metrics_output+="
# HELP apache_http_response_bytes_total Total bytes sent in responses (from last $LOG_TAIL_LINES log lines)
# TYPE apache_http_response_bytes_total gauge
apache_http_response_bytes_total $total_bytes
"
    
    # Count requests by method
    local method_counts
    method_counts=$(echo "$log_data" | awk -F'"' '{print $2}' | awk '{print $1}' | { grep -E '^(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)$' || true; } | sort | uniq -c)
    
    metrics_output+="
# HELP apache_http_requests_by_method_total HTTP requests by method (from last $LOG_TAIL_LINES log lines)
# TYPE apache_http_requests_by_method_total gauge
"
    
    while read -r count method; do
        if [[ -n "$method" ]] && [[ -n "$count" ]]; then
            metrics_output+="apache_http_requests_by_method_total{method=\"$method\"} $count
"
        fi
    done <<< "$method_counts"
    
    # Count unique IPs
    local unique_ips
    unique_ips=$(echo "$log_data" | awk '{print $1}' | sort -u | wc -l)
    
    metrics_output+="
# HELP apache_http_unique_clients Unique client IPs (from last $LOG_TAIL_LINES log lines)
# TYPE apache_http_unique_clients gauge
apache_http_unique_clients $unique_ips
"
    
    # Top URIs (for potential abuse detection)
    local top_uris
    top_uris=$(echo "$log_data" | awk -F'"' '{print $2}' | awk '{print $2}' | { grep -v '^-$' || true; } | sort | uniq -c | sort -rn | head -5)
    
    metrics_output+="
# HELP apache_http_top_uri_requests_total Top requested URIs (from last $LOG_TAIL_LINES log lines)
# TYPE apache_http_top_uri_requests_total gauge
"
    
    local rank=1
    while read -r count uri; do
        if [[ -n "$uri" ]] && [[ -n "$count" ]]; then
            # Truncate URI and escape quotes
            uri="${uri:0:100}"
            uri="${uri//\"/\\\"}"
            metrics_output+="apache_http_top_uri_requests_total{uri=\"$uri\",rank=\"$rank\"} $count
"
            rank=$((rank + 1))
        fi
    done <<< "$top_uris"
    
    # Count requests in time windows
    local recent_requests
    recent_requests=$(echo "$log_data" | wc -l)
    
    metrics_output+="
# HELP apache_http_requests_in_sample Total requests in sample window
# TYPE apache_http_requests_in_sample gauge
apache_http_requests_in_sample $recent_requests
"
    
    # Save metrics for caching
    echo "$metrics_output" > "$STATE_DIR/log_metrics"
    echo "$metrics_output"
}

#########################
### Error Log Metrics ###
#########################

collect_error_log_metrics() {
    if [[ ! -f "$ERROR_LOG" ]] || [[ ! -r "$ERROR_LOG" ]]; then
        echo "# Error log not readable at $ERROR_LOG"
        return
    fi
    
    # Count errors by level from last 1000 lines
    local log_data
    log_data=$(tail -n 1000 "$ERROR_LOG" 2>/dev/null || echo "")
    
    if [[ -z "$log_data" ]]; then
        return
    fi
    
    local emerg_count alert_count crit_count error_count warn_count notice_count info_count
    
    emerg_count=$(echo "$log_data" | grep -c '\[emerg\]' 2>/dev/null) || emerg_count=0
    alert_count=$(echo "$log_data" | grep -c '\[alert\]' 2>/dev/null) || alert_count=0
    crit_count=$(echo "$log_data" | grep -c '\[crit\]' 2>/dev/null) || crit_count=0
    error_count=$(echo "$log_data" | grep -c '\[error\]' 2>/dev/null) || error_count=0
    warn_count=$(echo "$log_data" | grep -c '\[warn\]' 2>/dev/null) || warn_count=0
    notice_count=$(echo "$log_data" | grep -c '\[notice\]' 2>/dev/null) || notice_count=0
    info_count=$(echo "$log_data" | grep -c '\[info\]' 2>/dev/null) || info_count=0
    
    cat <<EOF
# HELP apache_error_log_messages_total Error log messages by level (from last 1000 lines)
# TYPE apache_error_log_messages_total gauge
apache_error_log_messages_total{level="emerg"} $emerg_count
apache_error_log_messages_total{level="alert"} $alert_count
apache_error_log_messages_total{level="crit"} $crit_count
apache_error_log_messages_total{level="error"} $error_count
apache_error_log_messages_total{level="warn"} $warn_count
apache_error_log_messages_total{level="notice"} $notice_count
apache_error_log_messages_total{level="info"} $info_count
EOF

    # Check error log file size and age
    local log_size log_mtime now log_age
    log_size=$(stat -c %s "$ERROR_LOG" 2>/dev/null || echo "0")
    log_mtime=$(stat -c %Y "$ERROR_LOG" 2>/dev/null || echo "0")
    now=$(date +%s)
    log_age=$((now - log_mtime))
    
    cat <<EOF

# HELP apache_error_log_size_bytes Size of error log file
# TYPE apache_error_log_size_bytes gauge
apache_error_log_size_bytes $log_size

# HELP apache_error_log_last_modified_seconds Seconds since error log was last modified
# TYPE apache_error_log_last_modified_seconds gauge
apache_error_log_last_modified_seconds $log_age
EOF
}

#########################
### SSL Certificate Metrics ###
#########################

collect_ssl_metrics() {
    local domains="$SSL_CHECK_DOMAINS"
    
    # If no domains specified, try to find them from Apache config
    if [[ -z "$domains" ]]; then
        # Look for SSLCertificateFile directives
        local cert_files
        cert_files=$(grep -rh 'SSLCertificateFile\s' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | grep -v '#' | awk '{print $2}' | tr -d '"' | sort -u || echo "")
        
        if [[ -z "$cert_files" ]]; then
            echo "# No SSL certificates found in Apache config"
            return
        fi
        
        echo "# HELP apache_ssl_certificate_expiry_days Days until SSL certificate expires"
        echo "# TYPE apache_ssl_certificate_expiry_days gauge"
        echo "# HELP apache_ssl_certificate_expiry_timestamp Unix timestamp when certificate expires"
        echo "# TYPE apache_ssl_certificate_expiry_timestamp gauge"
        
        while read -r cert_file; do
            if [[ -f "$cert_file" ]]; then
                local expiry_date expiry_epoch now_epoch days_left cn
                
                expiry_date=$(openssl x509 -enddate -noout -in "$cert_file" 2>/dev/null | cut -d= -f2 || echo "")
                if [[ -n "$expiry_date" ]]; then
                    expiry_epoch=$(date -d "$expiry_date" +%s 2>/dev/null || echo "0")
                    now_epoch=$(date +%s)
                    days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
                    
                    # Get CN from certificate
                    cn=$(openssl x509 -subject -noout -in "$cert_file" 2>/dev/null | grep -oP 'CN\s*=\s*\K[^,/]+' || basename "$cert_file")
                    cn="${cn// /_}"
                    
                    echo "apache_ssl_certificate_expiry_days{certificate=\"$cn\",file=\"$cert_file\"} $days_left"
                    echo "apache_ssl_certificate_expiry_timestamp{certificate=\"$cn\",file=\"$cert_file\"} $expiry_epoch"
                fi
            fi
        done <<< "$cert_files"
        return
    fi
    
    # Check specified domains via network
    echo "# HELP apache_ssl_certificate_expiry_days Days until SSL certificate expires"
    echo "# TYPE apache_ssl_certificate_expiry_days gauge"
    echo "# HELP apache_ssl_certificate_expiry_timestamp Unix timestamp when certificate expires"
    echo "# TYPE apache_ssl_certificate_expiry_timestamp gauge"
    
    IFS=',' read -ra domain_array <<< "$domains"
    for domain in "${domain_array[@]}"; do
        domain=$(echo "$domain" | tr -d ' ')
        if [[ -n "$domain" ]]; then
            local expiry_date expiry_epoch now_epoch days_left
            
            expiry_date=$(echo | openssl s_client -servername "$domain" -connect "$domain:443" 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "")
            
            if [[ -n "$expiry_date" ]]; then
                expiry_epoch=$(date -d "$expiry_date" +%s 2>/dev/null || echo "0")
                now_epoch=$(date +%s)
                days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
                
                echo "apache_ssl_certificate_expiry_days{domain=\"$domain\"} $days_left"
                echo "apache_ssl_certificate_expiry_timestamp{domain=\"$domain\"} $expiry_epoch"
            else
                echo "apache_ssl_certificate_expiry_days{domain=\"$domain\"} -1"
            fi
        fi
    done
}

#########################
### Proxy/Upstream Metrics ###
#########################

collect_upstream_metrics() {
    # Check for proxy/balancer configurations
    local proxy_passes
    proxy_passes=$(grep -rh 'ProxyPass\s' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | grep -v '#' | grep -v 'ProxyPassReverse' | awk '{print $2}' | sort -u || echo "")
    
    local balancers
    balancers=$(grep -rhoE 'balancer://[a-zA-Z0-9_-]+' "$SITES_DIR" "$CONF_D_DIR" "$APACHE_CONF" 2>/dev/null | sort -u || echo "")
    
    if [[ -z "$proxy_passes" ]] && [[ -z "$balancers" ]]; then
        return
    fi
    
    local proxy_count=0
    if [[ -n "$proxy_passes" ]]; then
        proxy_count=$(echo "$proxy_passes" | wc -l)
    fi
    
    local balancer_count=0
    if [[ -n "$balancers" ]]; then
        balancer_count=$(echo "$balancers" | wc -l)
    fi
    
    cat <<EOF
# HELP apache_proxy_backends_configured Number of configured proxy backends
# TYPE apache_proxy_backends_configured gauge
apache_proxy_backends_configured $proxy_count

# HELP apache_balancers_configured Number of configured load balancer groups
# TYPE apache_balancers_configured gauge
apache_balancers_configured $balancer_count
EOF

    # Count members per balancer
    if [[ -n "$balancers" ]]; then
        echo ""
        echo "# HELP apache_balancer_members_total Members configured per balancer"
        echo "# TYPE apache_balancer_members_total gauge"
        
        while read -r balancer; do
            if [[ -n "$balancer" ]]; then
                local name
                name=$(echo "$balancer" | sed 's|balancer://||')
                local member_count
                member_count=$(grep -A 20 "Proxy $balancer" "$SITES_DIR"/* "$CONF_D_DIR"/* "$APACHE_CONF" 2>/dev/null | grep -c 'BalancerMember' 2>/dev/null) || member_count=0
                echo "apache_balancer_members_total{balancer=\"$name\"} $member_count"
            fi
        done <<< "$balancers"
    fi
}

#########################
### Version Metrics ###
#########################

collect_version_metrics() {
    local version="unknown"
    
    if [[ -n "$APACHE_BIN" ]]; then
        version=$($APACHE_BIN -v 2>&1 | grep -oP 'Apache/\K[0-9.]+' || echo "unknown")
    fi
    
    echo "# HELP apache_version_info Apache version information"
    echo "# TYPE apache_version_info gauge"
    echo "apache_version_info{version=\"$version\"} 1"
    
    # Check loaded modules
    if [[ -n "$APACHECTL" ]]; then
        local modules_output
        modules_output=$($APACHECTL -M 2>/dev/null || echo "")
        
        local has_ssl has_proxy has_proxy_http has_proxy_balancer has_rewrite
        local has_headers has_deflate has_expires has_status has_http2
        
        has_ssl=$(echo "$modules_output" | grep -q 'ssl_module' && echo "1" || echo "0")
        has_proxy=$(echo "$modules_output" | grep -q 'proxy_module' && echo "1" || echo "0")
        has_proxy_http=$(echo "$modules_output" | grep -q 'proxy_http_module' && echo "1" || echo "0")
        has_proxy_balancer=$(echo "$modules_output" | grep -q 'proxy_balancer_module' && echo "1" || echo "0")
        has_rewrite=$(echo "$modules_output" | grep -q 'rewrite_module' && echo "1" || echo "0")
        has_headers=$(echo "$modules_output" | grep -q 'headers_module' && echo "1" || echo "0")
        has_deflate=$(echo "$modules_output" | grep -q 'deflate_module' && echo "1" || echo "0")
        has_expires=$(echo "$modules_output" | grep -q 'expires_module' && echo "1" || echo "0")
        has_status=$(echo "$modules_output" | grep -q 'status_module' && echo "1" || echo "0")
        has_http2=$(echo "$modules_output" | grep -q 'http2_module' && echo "1" || echo "0")
        
        cat <<EOF

# HELP apache_module_loaded Whether Apache module is loaded
# TYPE apache_module_loaded gauge
apache_module_loaded{module="ssl"} $has_ssl
apache_module_loaded{module="proxy"} $has_proxy
apache_module_loaded{module="proxy_http"} $has_proxy_http
apache_module_loaded{module="proxy_balancer"} $has_proxy_balancer
apache_module_loaded{module="rewrite"} $has_rewrite
apache_module_loaded{module="headers"} $has_headers
apache_module_loaded{module="deflate"} $has_deflate
apache_module_loaded{module="expires"} $has_expires
apache_module_loaded{module="status"} $has_status
apache_module_loaded{module="http2"} $has_http2
EOF
    fi
}

#########################
### File Descriptor Metrics ###
#########################

collect_system_metrics() {
    # System-wide limits that affect Apache
    local max_files ulimit_n
    max_files=$(cat /proc/sys/fs/file-max 2>/dev/null || echo "0")
    ulimit_n=$(ulimit -n 2>/dev/null || echo "0")
    
    cat <<EOF
# HELP apache_system_file_max System-wide maximum file descriptors
# TYPE apache_system_file_max gauge
apache_system_file_max $max_files

# HELP apache_system_ulimit_n Current shell ulimit for open files
# TYPE apache_system_ulimit_n gauge
apache_system_ulimit_n $ulimit_n
EOF

    # Current system-wide open files
    local open_files
    open_files=$(cat /proc/sys/fs/file-nr 2>/dev/null | awk '{print $1}' || echo "0")
    
    echo ""
    echo "# HELP apache_system_open_files Current system-wide open files"
    echo "# TYPE apache_system_open_files gauge"
    echo "apache_system_open_files $open_files"
}

#########################
### Collect All Metrics ###
#########################

collect_all_metrics() {
    local hostname
    hostname=$(hostname -f 2>/dev/null || hostname)
    
    cat <<EOF
# Apache Metrics Exporter
# Host: $hostname
# Collected at: $(date -Iseconds)

EOF
    
    collect_version_metrics
    echo ""
    collect_server_status
    echo ""
    collect_process_metrics
    echo ""
    collect_config_metrics
    echo ""
    collect_access_log_metrics
    echo ""
    collect_error_log_metrics
    echo ""
    collect_ssl_metrics
    echo ""
    collect_upstream_metrics
    echo ""
    collect_system_metrics
}

#########################
### HTTP Server ###
#########################

handle_request() {
    local request_line=""
    local content_length=0
    
    # Read request
    while IFS= read -r line; do
        line="${line%%$'\r'}"
        [[ -z "$line" ]] && break
        [[ -z "$request_line" ]] && request_line="$line"
    done
    
    local method path
    method=$(echo "$request_line" | awk '{print $1}')
    path=$(echo "$request_line" | awk '{print $2}')
    
    case "$path" in
        /metrics|/)
            local metrics
            metrics=$(collect_all_metrics)
            local body_length=${#metrics}
            
            cat <<EOF
HTTP/1.1 200 OK
Content-Type: text/plain; charset=utf-8
Content-Length: $body_length
Connection: close

$metrics
EOF
            ;;
        /health|/healthz)
            local health_body="OK"
            cat <<EOF
HTTP/1.1 200 OK
Content-Type: text/plain
Content-Length: 2
Connection: close

OK
EOF
            ;;
        *)
            cat <<EOF
HTTP/1.1 404 Not Found
Content-Type: text/plain
Content-Length: 9
Connection: close

Not Found
EOF
            ;;
    esac
}

start_server() {
    log "Starting Apache Metrics Exporter on port $LISTEN_PORT"
    log "Metrics available at http://localhost:$LISTEN_PORT/metrics"
    log "Server status URL: $STATUS_URL"
    
    while true; do
        socat TCP-LISTEN:"$LISTEN_PORT",reuseaddr,fork EXEC:"$0 --handle-request" 2>/dev/null || {
            log "Server error, restarting in 5 seconds..."
            sleep 5
        }
    done
}

#########################
### Output ###
#########################

write_output() {
    local metrics
    metrics=$(collect_all_metrics)
    
    if [[ -n "$OUTPUT_FILE" ]]; then
        local tmp_file="${OUTPUT_FILE}.$$"
        echo "$metrics" > "$tmp_file"
        mv "$tmp_file" "$OUTPUT_FILE"
    else
        echo "$metrics"
    fi
}

#########################
### Main ###
#########################

main() {
    if [[ "${1:-}" == "--handle-request" ]]; then
        handle_request
        exit 0
    fi
    
    parse_args "$@"
    setup
    
    if [[ "$HTTP_MODE" == true ]]; then
        start_server
    elif [[ -n "$OUTPUT_FILE" ]]; then
        write_output
    else
        collect_all_metrics
    fi
}

main "$@"