#!/bin/bash

#############################################################
#### Wickr Enterprise Monitoring Setup                   ####
#### Install and configure all Prometheus exporters      ####
#### and scrape targets for Wickr Enterprise             ####
####                                                     ####
#### Components:                                         ####
####   - redis_exporter (Redis session cache)            ####
####   - mysqld_exporter (MySQL data store)              ####
####   - cAdvisor (container metrics)                    ####
####   - blackbox_exporter config (endpoint probing)     ####
####   - wickr-metrics-exporter (Admin API)              ####
####   - wickr-io-exporter (IO Gateway)                  ####
####   - Prometheus scrape config                        ####
####   - Prometheus alert rules                          ####
####                                                     ####
#### Author: Phil Connor                                 ####
#### Contact: contact@mylinux.work                       ####
#### Version: 1.0.0.20260308                             ####
#############################################################

set -euo pipefail

SCRIPT_NAME=$(basename "$0")
readonly SCRIPT_NAME

#########################
### Defaults ###
#########################
readonly DEFAULT_REDIS_EXPORTER_VERSION="1.58.0"
readonly DEFAULT_MYSQL_EXPORTER_VERSION="0.15.1"
readonly DEFAULT_CADVISOR_VERSION="0.49.1"
readonly DEFAULT_NODE_DIR="/var/lib/node_exporter"
readonly DEFAULT_PROMETHEUS_DIR="/etc/prometheus"
readonly DEFAULT_INSTALL_DIR="/usr/local/bin"

# Configuration (overridable by environment or flags)
REDIS_EXPORTER_VERSION="${REDIS_EXPORTER_VERSION:-$DEFAULT_REDIS_EXPORTER_VERSION}"
MYSQL_EXPORTER_VERSION="${MYSQL_EXPORTER_VERSION:-$DEFAULT_MYSQL_EXPORTER_VERSION}"
CADVISOR_VERSION="${CADVISOR_VERSION:-$DEFAULT_CADVISOR_VERSION}"
NODE_DIR="${NODE_DIR:-$DEFAULT_NODE_DIR}"
PROMETHEUS_DIR="${PROMETHEUS_DIR:-$DEFAULT_PROMETHEUS_DIR}"
INSTALL_DIR="${INSTALL_DIR:-$DEFAULT_INSTALL_DIR}"

# Wickr-specific settings
WICKR_ADMIN_URL="${WICKR_ADMIN_URL:-}"
WICKR_ADMIN_TOKEN="${WICKR_ADMIN_TOKEN:-}"
WICKR_ENDPOINT="${WICKR_ENDPOINT:-}"
WICKR_IO_API_URL="${WICKR_IO_API_URL:-http://localhost:8080}"
WICKR_IO_API_KEY="${WICKR_IO_API_KEY:-}"
WICKR_DOCKER_NAME="${WICKR_DOCKER_NAME:-wickr-io}"

# Database settings
MYSQL_HOST="${MYSQL_HOST:-localhost}"
MYSQL_PORT="${MYSQL_PORT:-3306}"
MYSQL_EXPORTER_USER="${MYSQL_EXPORTER_USER:-exporter}"
MYSQL_EXPORTER_PASS="${MYSQL_EXPORTER_PASS:-}"

# Redis settings
REDIS_ADDR="${REDIS_ADDR:-redis://localhost:6379}"

# Component install flags
INSTALL_REDIS_EXPORTER=true
INSTALL_MYSQL_EXPORTER=true
INSTALL_CADVISOR=true
INSTALL_BLACKBOX_CONFIG=true
INSTALL_WICKR_METRICS=true
INSTALL_WICKR_IO=true
INSTALL_ALERT_RULES=true
INSTALL_PROMETHEUS_CONFIG=true
CADVISOR_MODE="docker"
UPDATE_MODE=false
DRY_RUN=false
SKIP_DEPS=false

# Runtime
LOGFILE="/var/log/wickr-monitoring-setup.log"
ERRORS=0

#########################
### Logging ###
#########################
log_info() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $1" | tee -a "$LOGFILE"
}

log_error() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $1" | tee -a "$LOGFILE" >&2
    ERRORS=$((ERRORS + 1))
}

log_warn() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $1" | tee -a "$LOGFILE"
}

#########################
### Help ###
#########################
show_help() {
    cat << EOF
Usage: $SCRIPT_NAME [OPTIONS]

Install and configure Prometheus monitoring for a Wickr Enterprise deployment.
Sets up infrastructure exporters, custom Wickr scripts, scrape configs, and alert rules.

COMPONENT FLAGS:
    --all                       Install all components (default)
    --redis-exporter            Install redis_exporter
    --mysql-exporter            Install mysqld_exporter
    --cadvisor                  Install cAdvisor (Docker container or binary)
    --blackbox-config           Install blackbox_exporter Wickr probe config
    --wickr-metrics             Install wickr-metrics-exporter.sh (Admin API)
    --wickr-io                  Install wickr-io-exporter.sh (IO Gateway)
    --alert-rules               Install Wickr Prometheus alert rules
    --prometheus-config         Generate Wickr Prometheus scrape config
    --no-<component>            Skip a specific component

OPTIONS:
    --cadvisor-mode <mode>      cAdvisor install mode: docker or binary (default: docker)
    --update                    Update existing installations
    --dry-run                   Show what would be done without executing
    --skip-deps                 Skip dependency checks
    --help, -h                  Show this help message

WICKR SETTINGS:
    --wickr-admin-url <url>     Wickr Admin API URL (e.g. https://wickr-admin.example.com)
    --wickr-admin-token <tok>   Wickr Admin API token
    --wickr-endpoint <host>     Wickr endpoint hostname for blackbox probing
    --wickr-io-url <url>        Wickr IO REST API URL (default: http://localhost:8080)
    --wickr-io-key <key>        Wickr IO API key for statistics
    --wickr-docker-name <name>  Docker container name (default: wickr-io)

DATABASE SETTINGS:
    --mysql-host <host>         MySQL host (default: localhost)
    --mysql-port <port>         MySQL port (default: 3306)
    --mysql-user <user>         MySQL exporter user (default: exporter)
    --mysql-pass <pass>         MySQL exporter password
    --redis-addr <addr>         Redis address (default: redis://localhost:6379)

ENVIRONMENT VARIABLES:
    WICKR_ADMIN_URL, WICKR_ADMIN_TOKEN, WICKR_ENDPOINT,
    WICKR_IO_API_URL, WICKR_IO_API_KEY, WICKR_DOCKER_NAME,
    MYSQL_HOST, MYSQL_PORT, MYSQL_EXPORTER_USER, MYSQL_EXPORTER_PASS,
    REDIS_ADDR, REDIS_EXPORTER_VERSION, MYSQL_EXPORTER_VERSION,
    CADVISOR_VERSION, NODE_DIR, PROMETHEUS_DIR

EXAMPLES:
    # Install everything
    $SCRIPT_NAME --all --wickr-admin-url https://wickr.example.com --wickr-admin-token xxx

    # Just the infrastructure exporters
    $SCRIPT_NAME --redis-exporter --mysql-exporter --cadvisor

    # Just the custom Wickr scripts
    $SCRIPT_NAME --wickr-metrics --wickr-io

    # Dry run
    $SCRIPT_NAME --all --dry-run

EOF
    exit 0
}

#########################
### Parse Arguments ###
#########################
parse_arguments() {
    # If no arguments, default to --all
    [[ $# -eq 0 ]] && return

    # If any component flag is given, disable all defaults first
    local has_component_flag=false
    for arg in "$@"; do
        case "$arg" in
            --redis-exporter|--mysql-exporter|--cadvisor|--blackbox-config|\
            --wickr-metrics|--wickr-io|--alert-rules|--prometheus-config|--all)
                has_component_flag=true
                break
                ;;
        esac
    done

    if [[ "$has_component_flag" == "true" ]]; then
        INSTALL_REDIS_EXPORTER=false
        INSTALL_MYSQL_EXPORTER=false
        INSTALL_CADVISOR=false
        INSTALL_BLACKBOX_CONFIG=false
        INSTALL_WICKR_METRICS=false
        INSTALL_WICKR_IO=false
        INSTALL_ALERT_RULES=false
        INSTALL_PROMETHEUS_CONFIG=false
    fi

    while [[ $# -gt 0 ]]; do
        case $1 in
            --all)
                INSTALL_REDIS_EXPORTER=true
                INSTALL_MYSQL_EXPORTER=true
                INSTALL_CADVISOR=true
                INSTALL_BLACKBOX_CONFIG=true
                INSTALL_WICKR_METRICS=true
                INSTALL_WICKR_IO=true
                INSTALL_ALERT_RULES=true
                INSTALL_PROMETHEUS_CONFIG=true
                shift ;;
            --redis-exporter)       INSTALL_REDIS_EXPORTER=true; shift ;;
            --no-redis-exporter)    INSTALL_REDIS_EXPORTER=false; shift ;;
            --mysql-exporter)       INSTALL_MYSQL_EXPORTER=true; shift ;;
            --no-mysql-exporter)    INSTALL_MYSQL_EXPORTER=false; shift ;;
            --cadvisor)             INSTALL_CADVISOR=true; shift ;;
            --no-cadvisor)          INSTALL_CADVISOR=false; shift ;;
            --blackbox-config)      INSTALL_BLACKBOX_CONFIG=true; shift ;;
            --no-blackbox-config)   INSTALL_BLACKBOX_CONFIG=false; shift ;;
            --wickr-metrics)        INSTALL_WICKR_METRICS=true; shift ;;
            --no-wickr-metrics)     INSTALL_WICKR_METRICS=false; shift ;;
            --wickr-io)             INSTALL_WICKR_IO=true; shift ;;
            --no-wickr-io)          INSTALL_WICKR_IO=false; shift ;;
            --alert-rules)          INSTALL_ALERT_RULES=true; shift ;;
            --no-alert-rules)       INSTALL_ALERT_RULES=false; shift ;;
            --prometheus-config)    INSTALL_PROMETHEUS_CONFIG=true; shift ;;
            --no-prometheus-config) INSTALL_PROMETHEUS_CONFIG=false; shift ;;
            --cadvisor-mode)        CADVISOR_MODE="$2"; shift 2 ;;
            --wickr-admin-url)      WICKR_ADMIN_URL="$2"; shift 2 ;;
            --wickr-admin-token)    WICKR_ADMIN_TOKEN="$2"; shift 2 ;;
            --wickr-endpoint)       WICKR_ENDPOINT="$2"; shift 2 ;;
            --wickr-io-url)         WICKR_IO_API_URL="$2"; shift 2 ;;
            --wickr-io-key)         WICKR_IO_API_KEY="$2"; shift 2 ;;
            --wickr-docker-name)    WICKR_DOCKER_NAME="$2"; shift 2 ;;
            --mysql-host)           MYSQL_HOST="$2"; shift 2 ;;
            --mysql-port)           MYSQL_PORT="$2"; shift 2 ;;
            --mysql-user)           MYSQL_EXPORTER_USER="$2"; shift 2 ;;
            --mysql-pass)           MYSQL_EXPORTER_PASS="$2"; shift 2 ;;
            --redis-addr)           REDIS_ADDR="$2"; shift 2 ;;
            --update)               UPDATE_MODE=true; shift ;;
            --dry-run)              DRY_RUN=true; shift ;;
            --skip-deps)            SKIP_DEPS=true; shift ;;
            --help|-h)              show_help ;;
            *)                      log_error "Unknown option: $1"; show_help ;;
        esac
    done
}

#########################
### Pre-flight ###
#########################
check_permissions() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root"
        exit 1
    fi
}

detect_arch() {
    local arch
    arch=$(uname -m)
    case "$arch" in
        x86_64)  echo "amd64" ;;
        aarch64) echo "arm64" ;;
        armv7l)  echo "armv7" ;;
        *)       log_error "Unsupported architecture: $arch"; exit 1 ;;
    esac
}

check_dependencies() {
    if [[ "$SKIP_DEPS" == "true" ]]; then
        return
    fi

    local missing=()
    for cmd in curl wget tar jq; do
        command -v "$cmd" &>/dev/null || missing+=("$cmd")
    done

    if [[ ${#missing[@]} -gt 0 ]]; then
        log_info "Installing missing dependencies: ${missing[*]}"
        if command -v apt &>/dev/null; then
            apt -y install "${missing[@]}" 2>/dev/null
        elif command -v dnf &>/dev/null; then
            dnf -y install "${missing[@]}" 2>/dev/null
        elif command -v yum &>/dev/null; then
            yum -y install "${missing[@]}" 2>/dev/null
        else
            log_error "Cannot install dependencies — install manually: ${missing[*]}"
            exit 1
        fi
    fi
}

setup_directories() {
    mkdir -p "$(dirname "$LOGFILE")"
    touch "$LOGFILE"

    # Textfile collector directory
    local textfile_dir="${NODE_DIR}/textfile_collector"
    if [[ ! -d "$textfile_dir" ]]; then
        mkdir -p "$textfile_dir"
        chown prometheus: "$textfile_dir" 2>/dev/null || true
        log_info "Created textfile collector directory: $textfile_dir"
    fi

    # Prometheus rules directory
    mkdir -p "${PROMETHEUS_DIR}/rules" 2>/dev/null || true
}

ensure_prometheus_user() {
    if ! id prometheus &>/dev/null; then
        log_info "Creating prometheus user"
        groupadd --system prometheus 2>/dev/null || true
        useradd -s /sbin/nologin --system -g prometheus prometheus 2>/dev/null || true
    fi
}

cleanup() {
    [[ -d "/tmp/wickr-monitoring-$$" ]] && rm -rf "/tmp/wickr-monitoring-$$"
}

trap cleanup EXIT

#########################
### redis_exporter ###
#########################
install_redis_exporter() {
    log_info "Installing redis_exporter v${REDIS_EXPORTER_VERSION}"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install redis_exporter v${REDIS_EXPORTER_VERSION}"
        return
    fi

    if systemctl is-active --quiet redis_exporter 2>/dev/null && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "redis_exporter already running, skipping (use --update to reinstall)"
        return
    fi

    local arch
    arch=$(detect_arch)
    local workdir="/tmp/wickr-monitoring-$$/redis_exporter"
    mkdir -p "$workdir"

    local tarball="redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-${arch}.tar.gz"
    local url="https://github.com/oliver006/redis_exporter/releases/download/v${REDIS_EXPORTER_VERSION}/${tarball}"

    log_info "Downloading redis_exporter..."
    if ! curl -fsSL -o "${workdir}/${tarball}" "$url"; then
        log_error "Failed to download redis_exporter from $url"
        return
    fi

    tar -xzf "${workdir}/${tarball}" -C "$workdir"
    cp "${workdir}/redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-${arch}/redis_exporter" "${INSTALL_DIR}/"
    chmod +x "${INSTALL_DIR}/redis_exporter"
    chown prometheus: "${INSTALL_DIR}/redis_exporter" 2>/dev/null || true

    # Systemd unit
    cat > /etc/systemd/system/redis_exporter.service << EOF
[Unit]
Description=Prometheus Redis Exporter
Documentation=https://github.com/oliver006/redis_exporter
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=${INSTALL_DIR}/redis_exporter \\
    --redis.addr=${REDIS_ADDR} \\
    --web.listen-address=:9121
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

    systemctl daemon-reload
    systemctl enable redis_exporter
    systemctl restart redis_exporter

    log_info "redis_exporter installed (port 9121)"
}

#########################
### mysqld_exporter ###
#########################
install_mysqld_exporter() {
    log_info "Installing mysqld_exporter v${MYSQL_EXPORTER_VERSION}"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install mysqld_exporter v${MYSQL_EXPORTER_VERSION}"
        return
    fi

    if systemctl is-active --quiet mysqld_exporter 2>/dev/null && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "mysqld_exporter already running, skipping (use --update to reinstall)"
        return
    fi

    local arch
    arch=$(detect_arch)
    local workdir="/tmp/wickr-monitoring-$$/mysqld_exporter"
    mkdir -p "$workdir"

    local tarball="mysqld_exporter-${MYSQL_EXPORTER_VERSION}.linux-${arch}.tar.gz"
    local url="https://github.com/prometheus/mysqld_exporter/releases/download/v${MYSQL_EXPORTER_VERSION}/${tarball}"

    log_info "Downloading mysqld_exporter..."
    if ! curl -fsSL -o "${workdir}/${tarball}" "$url"; then
        log_error "Failed to download mysqld_exporter from $url"
        return
    fi

    tar -xzf "${workdir}/${tarball}" -C "$workdir"
    cp "${workdir}/mysqld_exporter-${MYSQL_EXPORTER_VERSION}.linux-${arch}/mysqld_exporter" "${INSTALL_DIR}/"
    chmod +x "${INSTALL_DIR}/mysqld_exporter"
    chown prometheus: "${INSTALL_DIR}/mysqld_exporter" 2>/dev/null || true

    # MySQL credentials file
    if [[ -n "$MYSQL_EXPORTER_PASS" ]]; then
        cat > /etc/.mysqld_exporter.cnf << EOF
[client]
user=${MYSQL_EXPORTER_USER}
password=${MYSQL_EXPORTER_PASS}
host=${MYSQL_HOST}
port=${MYSQL_PORT}
EOF
        chmod 600 /etc/.mysqld_exporter.cnf
        chown prometheus: /etc/.mysqld_exporter.cnf 2>/dev/null || true
        log_info "MySQL credentials written to /etc/.mysqld_exporter.cnf"
    elif [[ ! -f /etc/.mysqld_exporter.cnf ]]; then
        log_warn "No MySQL password provided — create /etc/.mysqld_exporter.cnf manually"
        cat > /etc/.mysqld_exporter.cnf << EOF
[client]
user=${MYSQL_EXPORTER_USER}
password=CHANGEME
host=${MYSQL_HOST}
port=${MYSQL_PORT}
EOF
        chmod 600 /etc/.mysqld_exporter.cnf
        chown prometheus: /etc/.mysqld_exporter.cnf 2>/dev/null || true
    fi

    # Systemd unit
    cat > /etc/systemd/system/mysqld_exporter.service << EOF
[Unit]
Description=Prometheus MySQL Exporter
Documentation=https://github.com/prometheus/mysqld_exporter
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=${INSTALL_DIR}/mysqld_exporter \\
    --config.my-cnf=/etc/.mysqld_exporter.cnf \\
    --web.listen-address=:9104
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

    systemctl daemon-reload
    systemctl enable mysqld_exporter
    systemctl restart mysqld_exporter

    log_info "mysqld_exporter installed (port 9104)"

    if [[ "$MYSQL_EXPORTER_PASS" == "" ]]; then
        log_warn "Remember to create the MySQL exporter user:"
        log_warn "  CREATE USER '${MYSQL_EXPORTER_USER}'@'%' IDENTIFIED BY 'strong-password';"
        log_warn "  GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO '${MYSQL_EXPORTER_USER}'@'%';"
        log_warn "  FLUSH PRIVILEGES;"
    fi
}

#########################
### cAdvisor ###
#########################
install_cadvisor() {
    log_info "Installing cAdvisor v${CADVISOR_VERSION} (mode: ${CADVISOR_MODE})"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install cAdvisor v${CADVISOR_VERSION} (${CADVISOR_MODE})"
        return
    fi

    case "$CADVISOR_MODE" in
        docker) install_cadvisor_docker ;;
        binary) install_cadvisor_binary ;;
        *)      log_error "Unknown cAdvisor mode: $CADVISOR_MODE"; return ;;
    esac
}

install_cadvisor_docker() {
    if ! command -v docker &>/dev/null; then
        log_error "Docker not found — install Docker first or use --cadvisor-mode binary"
        return
    fi

    local image="gcr.io/cadvisor/cadvisor:v${CADVISOR_VERSION}"

    # Check if already running
    if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "^cadvisor$" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "cAdvisor container already running, skipping"
        return
    fi

    # Remove existing container
    docker rm -f cadvisor 2>/dev/null || true

    log_info "Pulling cAdvisor image..."
    docker pull "$image"

    docker run -d \
        --name cadvisor \
        --restart unless-stopped \
        -p 8080:8080 \
        -v /:/rootfs:ro \
        -v /var/run:/var/run:ro \
        -v /sys:/sys:ro \
        -v /var/lib/docker/:/var/lib/docker:ro \
        -v /dev/disk/:/dev/disk:ro \
        --privileged \
        --device /dev/kmsg \
        "$image"

    log_info "cAdvisor container started (port 8080)"
}

install_cadvisor_binary() {
    local arch
    arch=$(detect_arch)
    local url="https://github.com/google/cadvisor/releases/download/v${CADVISOR_VERSION}/cadvisor-v${CADVISOR_VERSION}-linux-${arch}"

    log_info "Downloading cAdvisor binary..."
    if ! curl -fsSL -o "${INSTALL_DIR}/cadvisor" "$url"; then
        log_error "Failed to download cAdvisor"
        return
    fi

    chmod +x "${INSTALL_DIR}/cadvisor"

    cat > /etc/systemd/system/cadvisor.service << EOF
[Unit]
Description=cAdvisor - Container Advisor
Documentation=https://github.com/google/cadvisor
After=network.target docker.service
Wants=docker.service

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/cadvisor \\
    --port=8080 \\
    --docker_only=true \\
    --housekeeping_interval=30s \\
    --storage_duration=2m0s
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

    systemctl daemon-reload
    systemctl enable cadvisor
    systemctl restart cadvisor

    log_info "cAdvisor systemd service started (port 8080)"
}

#########################
### Blackbox Config ###
#########################
install_blackbox_config() {
    log_info "Installing blackbox_exporter Wickr probe configuration"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Wickr blackbox probe config"
        return
    fi

    if [[ ! -d "$PROMETHEUS_DIR" ]]; then
        log_warn "Prometheus config directory not found: $PROMETHEUS_DIR — skipping blackbox config"
        return
    fi

    # Check if blackbox_exporter is installed
    if ! command -v blackbox_exporter &>/dev/null && [[ ! -f "${INSTALL_DIR}/blackbox_exporter" ]]; then
        log_warn "blackbox_exporter not found — install it first (see install-prometheus-stack.sh)"
    fi

    local blackbox_config="${PROMETHEUS_DIR}/blackbox.yml"

    # Add Wickr-specific modules if not already present
    if [[ -f "$blackbox_config" ]] && grep -q "https_wickr" "$blackbox_config" 2>/dev/null; then
        log_info "Wickr blackbox modules already configured"
        return
    fi

    # If blackbox.yml doesn't exist, create it with Wickr modules
    if [[ ! -f "$blackbox_config" ]]; then
        cat > "$blackbox_config" << 'EOF'
modules:
  http_2xx:
    prober: http
    timeout: 5s

  tcp_connect:
    prober: tcp
    timeout: 5s

  https_wickr:
    prober: http
    timeout: 10s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 301, 302, 403]
      tls_config:
        insecure_skip_verify: false
EOF
        chown prometheus: "$blackbox_config" 2>/dev/null || true
        log_info "Created blackbox.yml with Wickr probe modules"
    else
        # Append Wickr module to existing config
        cat >> "$blackbox_config" << 'EOF'

  https_wickr:
    prober: http
    timeout: 10s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 301, 302, 403]
      tls_config:
        insecure_skip_verify: false
EOF
        log_info "Added https_wickr module to existing blackbox.yml"
    fi

    # Reload blackbox_exporter if running
    if systemctl is-active --quiet blackbox_exporter 2>/dev/null; then
        systemctl restart blackbox_exporter
        log_info "blackbox_exporter restarted"
    fi
}

####################################
### wickr-metrics-exporter.sh ###
####################################
install_wickr_metrics_exporter() {
    log_info "Installing wickr-metrics-exporter.sh (Admin API collector)"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install wickr-metrics-exporter.sh"
        return
    fi

    local script_path="${INSTALL_DIR}/wickr-metrics-exporter.sh"
    local download_url="https://mylinux.work/downloads/wickr-metrics-exporter.sh"

    if [[ -f "$script_path" ]] && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "wickr-metrics-exporter.sh already exists, skipping (use --update to reinstall)"
        return
    fi

    log_info "Downloading wickr-metrics-exporter.sh..."
    if ! curl -fsSL -o "$script_path" "$download_url"; then
        log_error "Failed to download wickr-metrics-exporter.sh"
        return
    fi

    chmod +x "$script_path"
    log_info "Installed $script_path"

    # Create cron job if admin URL and token are set
    if [[ -n "$WICKR_ADMIN_URL" && -n "$WICKR_ADMIN_TOKEN" ]]; then
        local cron_file="/etc/cron.d/wickr-metrics-exporter"
        cat > "$cron_file" << EOF
# Wickr Enterprise Admin API metrics collector
# Runs every minute, writes to node_exporter textfile directory
* * * * * root WICKR_ADMIN_URL=${WICKR_ADMIN_URL} WICKR_API_TOKEN=${WICKR_ADMIN_TOKEN} NODE_DIR=${NODE_DIR} ${script_path} --once >/dev/null 2>&1
EOF
        chmod 644 "$cron_file"
        log_info "Cron job installed: $cron_file"
    else
        log_warn "WICKR_ADMIN_URL and WICKR_ADMIN_TOKEN not set — skipping cron job"
        log_warn "Set them and create /etc/cron.d/wickr-metrics-exporter manually"
    fi
}

####################################
### wickr-io-exporter.sh ###
####################################
install_wickr_io_exporter() {
    log_info "Installing wickr-io-exporter.sh (IO Gateway collector)"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install wickr-io-exporter.sh"
        return
    fi

    local script_path="${INSTALL_DIR}/wickr-io-exporter.sh"
    local download_url="https://mylinux.work/downloads/wickr-io-exporter.sh"

    if [[ -f "$script_path" ]] && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "wickr-io-exporter.sh already exists, skipping (use --update to reinstall)"
        return
    fi

    log_info "Downloading wickr-io-exporter.sh..."
    if ! curl -fsSL -o "$script_path" "$download_url"; then
        log_error "Failed to download wickr-io-exporter.sh"
        return
    fi

    chmod +x "$script_path"
    log_info "Installed $script_path"

    # Create cron job
    local cron_file="/etc/cron.d/wickr-io-exporter"
    local env_vars="NODE_EXPORTER_DIR=${NODE_DIR} WICKR_DOCKER_NAME=${WICKR_DOCKER_NAME} WICKR_API_URL=${WICKR_IO_API_URL}"
    [[ -n "$WICKR_IO_API_KEY" ]] && env_vars="${env_vars} WICKR_API_KEY=${WICKR_IO_API_KEY}"

    cat > "$cron_file" << EOF
# Wickr IO Gateway metrics collector
# Runs every 5 minutes, writes to node_exporter textfile directory
*/5 * * * * root ${env_vars} ${script_path} > ${NODE_DIR}/wickr_status.prom 2>&1
EOF
    chmod 644 "$cron_file"
    log_info "Cron job installed: $cron_file"
}

#########################
### Alert Rules ###
#########################
install_alert_rules() {
    log_info "Installing Wickr Prometheus alert rules"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Wickr alert rules"
        return
    fi

    local rules_dir="${PROMETHEUS_DIR}/rules"
    mkdir -p "$rules_dir"

    # Wickr infrastructure alerts
    cat > "${rules_dir}/wickr-infrastructure.yml" << 'RULES'
groups:
  - name: wickr-mysql
    rules:
      - alert: WickrMySQLDown
        expr: mysql_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Wickr MySQL database is down"

      - alert: WickrMySQLConnectionsHigh
        expr: mysql_global_status_threads_connected / mysql_global_variables_max_connections * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MySQL connections above 80% of max"

      - alert: WickrMySQLSlowQueries
        expr: rate(mysql_global_status_slow_queries[5m]) > 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "MySQL slow queries increasing"

  - name: wickr-redis
    rules:
      - alert: WickrRedisDown
        expr: redis_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Wickr Redis cache is down"

      - alert: WickrRedisHighMemory
        expr: redis_used_memory_bytes / redis_config_maxmemory * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Redis memory usage above 90%"

      - alert: WickrRedisCacheHitRateLow
        expr: rate(redis_keyspace_hits_total[5m]) / (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 0.8
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Redis cache hit rate below 80%"

  - name: wickr-rabbitmq
    rules:
      - alert: WickrRabbitMQQueueBacklog
        expr: rabbitmq_queue_messages_ready > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "RabbitMQ queue backlog on {{ $labels.queue }}"
          description: "Queue {{ $labels.queue }} has {{ $value }} messages waiting"

      - alert: WickrRabbitMQNoConsumers
        expr: rabbitmq_queue_consumers == 0 and rabbitmq_queue_messages > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "RabbitMQ queue {{ $labels.queue }} has no consumers"
RULES

    # Wickr application alerts
    cat > "${rules_dir}/wickr-application.yml" << 'RULES'
groups:
  - name: wickr-admin-api
    rules:
      - alert: WickrAdminAPIDown
        expr: wickr_instance_up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Wickr Admin API is unreachable"

      - alert: WickrCollectorStale
        expr: (time() - wickr_collector_last_run_timestamp) > 300
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Wickr metrics collector has not run in 5 minutes"

      - alert: WickrNoActiveBots
        expr: wickr_bots_active == 0 and wickr_bots_total > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "No active Wickr bots — all bots are inactive"

  - name: wickr-io-gateway
    rules:
      - alert: WickrIOContainerDown
        expr: wickr_docker_container_status == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Wickr IO Gateway Docker container is not running"

      - alert: WickrIOAPIDown
        expr: wickr_api_status == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Wickr IO Gateway REST API is not responding"

      - alert: WickrIONetworkDown
        expr: wickr_network_connectivity == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Cannot reach Wickr cloud services"

      - alert: WickrIOSendErrors
        expr: wickr_send_errors_total > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Wickr IO Gateway reporting message send errors"

      - alert: WickrIOHighCPU
        expr: wickr_docker_cpu_percent > 90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Wickr IO container CPU usage above 90%"

  - name: wickr-blackbox
    rules:
      - alert: WickrEndpointDown
        expr: probe_success{job="blackbox-wickr"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Wickr endpoint {{ $labels.instance }} is down"

      - alert: WickrSSLExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-wickr"} - time()) / 86400 < 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "Wickr SSL certificate expires in {{ $value | humanize }} days"
RULES

    chown -R prometheus: "$rules_dir" 2>/dev/null || true
    log_info "Alert rules installed to ${rules_dir}/"

    # Reload Prometheus if running
    if systemctl is-active --quiet prometheus 2>/dev/null; then
        if promtool check rules "${rules_dir}/wickr-infrastructure.yml" &>/dev/null && \
           promtool check rules "${rules_dir}/wickr-application.yml" &>/dev/null; then
            systemctl reload prometheus 2>/dev/null || systemctl restart prometheus
            log_info "Prometheus reloaded with new alert rules"
        else
            log_warn "Alert rule validation failed — check rules manually with promtool"
        fi
    fi
}

####################################
### Prometheus Scrape Config ###
####################################
install_prometheus_config() {
    log_info "Generating Wickr Prometheus scrape configuration"

    local config_file="${PROMETHEUS_DIR}/wickr-scrape.yml"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would generate scrape config at $config_file"
        return
    fi

    # Generate a standalone scrape config snippet
    cat > "$config_file" << EOF
# Wickr Enterprise Prometheus Scrape Configuration
# Generated by $SCRIPT_NAME on $(date '+%Y-%m-%d %H:%M:%S')
#
# Include in prometheus.yml with:
#   scrape_config_files:
#     - ${config_file}
#
# Or copy the scrape_configs entries into your prometheus.yml

scrape_configs:
EOF

    # Redis
    if [[ "$INSTALL_REDIS_EXPORTER" == "true" ]]; then
        cat >> "$config_file" << 'EOF'
  - job_name: 'redis'
    scrape_interval: 15s
    static_configs:
      - targets: ['localhost:9121']

EOF
    fi

    # MySQL
    if [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]]; then
        cat >> "$config_file" << 'EOF'
  - job_name: 'mysql'
    scrape_interval: 15s
    static_configs:
      - targets: ['localhost:9104']

EOF
    fi

    # cAdvisor
    if [[ "$INSTALL_CADVISOR" == "true" ]]; then
        cat >> "$config_file" << 'EOF'
  - job_name: 'cadvisor'
    scrape_interval: 15s
    static_configs:
      - targets: ['localhost:8080']

EOF
    fi

    # Blackbox — HTTPS
    if [[ "$INSTALL_BLACKBOX_CONFIG" == "true" && -n "$WICKR_ENDPOINT" ]]; then
        cat >> "$config_file" << EOF
  - job_name: 'blackbox-wickr'
    metrics_path: /probe
    params:
      module: [https_wickr]
    static_configs:
      - targets:
          - 'https://${WICKR_ENDPOINT}'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 'localhost:9115'

  - job_name: 'blackbox-wickr-tcp'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
          - '${WICKR_ENDPOINT}:443'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 'localhost:9115'

EOF
    fi

    chown prometheus: "$config_file" 2>/dev/null || true
    log_info "Scrape config written to $config_file"

    # Check if prometheus.yml already includes this file
    local prom_config="${PROMETHEUS_DIR}/prometheus.yml"
    if [[ -f "$prom_config" ]]; then
        if grep -q "wickr-scrape.yml" "$prom_config" 2>/dev/null; then
            log_info "prometheus.yml already includes wickr-scrape.yml"
        elif grep -q "scrape_config_files" "$prom_config" 2>/dev/null; then
            log_info "Add to your scrape_config_files in prometheus.yml:"
            log_info "  - ${config_file}"
        else
            log_info "Add to prometheus.yml:"
            log_info "  scrape_config_files:"
            log_info "    - ${config_file}"
        fi
    fi
}

#########################
### Main ###
#########################
main() {
    parse_arguments "$@"
    check_permissions

    log_info "=== Wickr Enterprise Monitoring Setup ==="
    log_info "Components selected:"
    [[ "$INSTALL_REDIS_EXPORTER" == "true" ]]    && log_info "  - redis_exporter v${REDIS_EXPORTER_VERSION}"
    [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]]    && log_info "  - mysqld_exporter v${MYSQL_EXPORTER_VERSION}"
    [[ "$INSTALL_CADVISOR" == "true" ]]          && log_info "  - cAdvisor v${CADVISOR_VERSION} (${CADVISOR_MODE})"
    [[ "$INSTALL_BLACKBOX_CONFIG" == "true" ]]   && log_info "  - blackbox_exporter Wickr config"
    [[ "$INSTALL_WICKR_METRICS" == "true" ]]     && log_info "  - wickr-metrics-exporter.sh"
    [[ "$INSTALL_WICKR_IO" == "true" ]]          && log_info "  - wickr-io-exporter.sh"
    [[ "$INSTALL_ALERT_RULES" == "true" ]]       && log_info "  - Prometheus alert rules"
    [[ "$INSTALL_PROMETHEUS_CONFIG" == "true" ]]  && log_info "  - Prometheus scrape config"

    check_dependencies
    setup_directories
    ensure_prometheus_user

    # Install components
    [[ "$INSTALL_REDIS_EXPORTER" == "true" ]]    && install_redis_exporter
    [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]]    && install_mysqld_exporter
    [[ "$INSTALL_CADVISOR" == "true" ]]          && install_cadvisor
    [[ "$INSTALL_BLACKBOX_CONFIG" == "true" ]]   && install_blackbox_config
    [[ "$INSTALL_WICKR_METRICS" == "true" ]]     && install_wickr_metrics_exporter
    [[ "$INSTALL_WICKR_IO" == "true" ]]          && install_wickr_io_exporter
    [[ "$INSTALL_ALERT_RULES" == "true" ]]       && install_alert_rules
    [[ "$INSTALL_PROMETHEUS_CONFIG" == "true" ]]  && install_prometheus_config

    # Summary
    echo
    echo "=== Wickr Monitoring Setup Complete ==="
    echo

    if [[ "$DRY_RUN" == "false" ]]; then
        echo "Installed components:"
        [[ "$INSTALL_REDIS_EXPORTER" == "true" ]]  && echo "  ✓ redis_exporter        http://localhost:9121/metrics"
        [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]]  && echo "  ✓ mysqld_exporter       http://localhost:9104/metrics"
        [[ "$INSTALL_CADVISOR" == "true" ]]        && echo "  ✓ cAdvisor              http://localhost:8080/metrics"
        [[ "$INSTALL_BLACKBOX_CONFIG" == "true" ]] && echo "  ✓ blackbox config       ${PROMETHEUS_DIR}/blackbox.yml"
        [[ "$INSTALL_WICKR_METRICS" == "true" ]]   && echo "  ✓ wickr-metrics-exporter ${INSTALL_DIR}/wickr-metrics-exporter.sh"
        [[ "$INSTALL_WICKR_IO" == "true" ]]        && echo "  ✓ wickr-io-exporter     ${INSTALL_DIR}/wickr-io-exporter.sh"
        [[ "$INSTALL_ALERT_RULES" == "true" ]]     && echo "  ✓ Alert rules           ${PROMETHEUS_DIR}/rules/wickr-*.yml"
        [[ "$INSTALL_PROMETHEUS_CONFIG" == "true" ]] && echo "  ✓ Scrape config         ${PROMETHEUS_DIR}/wickr-scrape.yml"
        echo
    fi

    if [[ $ERRORS -gt 0 ]]; then
        echo "Completed with $ERRORS error(s) — check $LOGFILE for details"
    else
        echo "All components installed successfully"
    fi

    echo "Log: $LOGFILE"
    echo
}

main "$@"