linux-scripts/install-prometheus-stack.sh

#!/bin/bash

set -euo pipefail

#############################################################
#### Prometheus Stack Installer                          ####
#### For RHEL/Rocky/Alma, Oracle Linux, Debian & Ubuntu  ####
####                                                     ####
#### Author: Phil Connor                                 ####
#### Contact: contact@mylinux.work                       ####
#### License: MIT                                        ####
#### Version: 3.0                                        ####
####                                                     ####
#### Usage: ./install-prometheus-stack.sh [OPTIONS]       ####
#############################################################

# Script defaults
INSTALL_PROMETHEUS=true
INSTALL_NODE_EXPORTER=true
INSTALL_BLACKBOX=true
INSTALL_ALERTMANAGER=true
INSTALL_MYSQL_EXPORTER=false
INSTALL_GRAFANA=true
INSTALL_LOKI=false
INSTALL_ALLOY=false
WEBSERVER="nginx"
INSTALL_WEBSERVER=true
ENABLE_TLS=false
UPDATE_MODE=false
DRY_RUN=false
CONFIG_FILE=""
SKIP_DEPS=false

# System variables
domain="example.com"
bindir="/usr/local/bin"
promdir="/etc/prometheus"
logfile="/var/log/prometheus-install.log"

# MySQL Exporter variables (can be overridden by config file)
mynum=2
myuser="exporter"
mypass="password"
myhost1="db.host1.example"
myhost2="db.host2.example" 
myhost3="db.host3.example"

#########################
### Logging Functions ###
#########################
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$logfile"
}

log_error() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $1" | tee -a "$logfile" >&2
}

log_info() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $1" | tee -a "$logfile"
}

#########################
### Utility Functions ###
#########################
show_help() {
    cat << EOF
Prometheus Stack Installer

USAGE:
    $0 [OPTIONS]

OPTIONS:
    --prometheus          Install Prometheus (default: true)
    --node-exporter       Install node_exporter (default: true)
    --blackbox            Install blackbox_exporter (default: true)
    --alertmanager        Install AlertManager (default: true)
    --mysql-exporter      Install MySQL exporter (default: false)
    --grafana             Install Grafana (default: true)
    --loki                Install Loki log storage (default: false)
    --alloy               Install Alloy log/metrics collector (default: false)
    --webserver <type>    Install web server: nginx, apache, caddy (default: nginx)
    --no-webserver        Skip web server installation
    --enable-tls          Enable TLS/SSL security between components
    --all                 Install all components
    --update              Update existing installations
    --domain <domain>     Domain for reverse proxy configs (default: example.com)
    --config-file <file>  Load configuration from file
    --dry-run             Show what would be installed without doing it
    --skip-deps           Skip dependency installation
    --help                Show this help message

EXAMPLES:
    $0 --prometheus --grafana --alloy
    $0 --all --skip mysql-exporter
    $0 --update --prometheus --grafana
    $0 --dry-run --config-file prod.conf

CONFIG FILE FORMAT:
    domain=example.com
    myuser=dbuser
    mypass=dbpassword
    myhost1=db1.example.com

EOF
}

cleanup() {
    if [[ -d "/tmp/prometheus-install-$$" ]]; then
        rm -rf "/tmp/prometheus-install-$$"
    fi
}

trap cleanup EXIT

check_component_installed() {
    local component=$1
    case $component in
        "prometheus")
            systemctl is-active --quiet prometheus 2>/dev/null || [[ -f "$bindir/prometheus" ]]
            ;;
        "node_exporter") 
            systemctl is-active --quiet node_exporter 2>/dev/null || [[ -f "$bindir/node_exporter" ]]
            ;;
        "blackbox_exporter")
            systemctl is-active --quiet blackbox_exporter 2>/dev/null || [[ -f "$bindir/blackbox_exporter" ]]
            ;;
        "alertmanager")
            systemctl is-active --quiet alertmanager 2>/dev/null || [[ -f "$bindir/alertmanager" ]]
            ;;
        "grafana")
            systemctl is-active --quiet grafana-server 2>/dev/null || command -v grafana-server >/dev/null 2>&1
            ;;
        "loki")
            systemctl is-active --quiet loki 2>/dev/null || [[ -f "$bindir/loki" ]]
            ;;
        "alloy")
            systemctl is-active --quiet alloy 2>/dev/null || [[ -f "$bindir/alloy" ]]
            ;;
    esac
}

#########################
### System Detection ###
#########################
detect_os() {
    if [[ "$(command -v lsb_release)" ]]; then
        OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
        OSVER=$(lsb_release -r | awk '{print $2}' | cut -d. -f1)
    else
        OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
        OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '"' | cut -d. -f1)
    fi
    
    log_info "Detected OS: $OS version $OSVER"
}

setup_directories() {
    if [[ -d "/usr/lib/systemd/system" ]]; then
        psdir='/etc/systemd/system'
    else
        psdir='/usr/lib/systemd/system'
    fi
    
    # Create log directory
    mkdir -p "$(dirname "$logfile")"
    touch "$logfile"
}

#########################
### Package Management ###
#########################
setup_package_manager() {
    case $OS in
        "ubuntu"|"debian")
            pkgmgr="apt -y"
            ;;
        "red"|"centos"|"oracle"|"rocky"|"almalinux")
            if command -v dnf >/dev/null 2>&1; then
                pkgmgr="dnf -y"
            else
                pkgmgr="yum -y"
            fi
            ;;
        *)
            log_error "Unsupported OS: $OS"
            exit 1
            ;;
    esac
    
    log_info "Using package manager: $pkgmgr"
}

#########################
### Permission Check ###
#########################
check_permissions() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root! Login as root, or use sudo."
        exit 1
    fi
}

#########################
### User Management ###
#########################
create_prometheus_user() {
    if ! grep -q prometheus /etc/passwd; then 
        log_info "Creating prometheus user and group"
        groupadd --system prometheus
        if [[ "$OS" == "ubuntu" || "$OS" == "debian" ]]; then
            useradd -s /sbin/nologin --system -g prometheus prometheus
        else
            useradd -m -s /bin/false prometheus -g prometheus
        fi
    else
        log_info "Prometheus user already exists"
    fi
}

#########################
### Dependencies ###
#########################
install_dependencies() {
    if [[ "$SKIP_DEPS" == "true" ]]; then
        log_info "Skipping dependency installation"
        return
    fi
    
    log_info "Installing dependencies"
    
    if [[ ! "$(command -v wget)" ]]; then
        $pkgmgr install wget
    fi
    
    if [[ ! "$(command -v curl)" ]]; then
        $pkgmgr install curl
    fi
    
    if [[ ! "$(command -v tar)" ]]; then
        $pkgmgr install tar
    fi
    
    if [[ ! "$(command -v unzip)" ]]; then
        $pkgmgr install unzip
    fi
}

##########################
### Install Prometheus ###
##########################
install_prometheus() {
    log_info "Installing Prometheus"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Prometheus"
        return
    fi
    
    if check_component_installed "prometheus" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "Prometheus already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/prometheus"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Create directories
    mkdir -p "$promdir" /var/lib/prometheus
    chown prometheus /var/lib/prometheus/
    
    for dir in backups rules templates consoles console_libraries; do 
        mkdir -p "$promdir/${dir}" 
        chown -R prometheus. "$promdir/${dir}"
        chmod -R 755 "$promdir/${dir}"
    done
    
    # Download latest Prometheus
    log_info "Downloading Prometheus"
    curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | \
        grep browser_download_url | \
        grep linux-amd64 | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download Prometheus"
            exit 1
        }
    
    tar -xzf prometheus*.tar.gz
    cd prometheus-*/
    
    # Install binaries
    mv prometheus promtool "$bindir/"
    
    # Install config if not exists or in update mode
    if [[ ! -f "$promdir/prometheus.yml" ]] || [[ "$UPDATE_MODE" == "true" ]]; then
        if [[ -f "$promdir/prometheus.yml" ]]; then
            cp "$promdir/prometheus.yml" "$promdir/backups/prometheus.yml.$(date +%Y%m%d_%H%M%S)"
        fi
        mv prometheus.yml "$promdir/"
    fi
    
    mv consoles/ console_libraries/ "$promdir/" || true
    chown -R prometheus. /var/lib/prometheus/ "$promdir/"
    
    # SELinux context for RHEL 8+
    if [[ "$OS" == "red" && "$OSVER" -ge 8 ]]; then
        restorecon -rv "$bindir/prometheus" || true
    fi
    
    # Create systemd service
    create_prometheus_service
    
    systemctl daemon-reload
    systemctl enable prometheus
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart prometheus
    else
        systemctl start prometheus
    fi
    
    log_info "Prometheus installation completed"
}

create_prometheus_service() {
    cat > "$psdir/prometheus.service" << 'EOF'
[Unit]
Description=Prometheus Time Series Collection and Processing Server
Documentation=https://prometheus.io/docs/introduction/overview/
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=prometheus
Group=prometheus

ExecReload=/bin/kill -HUP $MAINPID
ExecStart=/usr/local/bin/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/data \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries \
    --web.listen-address=0.0.0.0:9090 \
    --web.external-url= \
    --enable-feature=new-service-discovery-manager,exemplar-storage,extra-scrape-metrics

Restart=always
RestartSec=5s
SyslogIdentifier=prometheus

[Install]
WantedBy=multi-user.target
EOF
    
    # Create default config if it doesn't exist
    if [[ ! -f "$promdir/prometheus.yml" ]]; then
        create_prometheus_config
    fi
}

create_prometheus_config() {
    cat > "$promdir/prometheus.yml" << 'EOF'
# Global config
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 
  scrape_timeout: 15s  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape: Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label 'job=<job_name>' to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'server_metrics'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9100']
        labels:
          alias: Prometheus Server
EOF
}

#############################
### Install node_exporter ###
#############################
install_node_exporter() {
    log_info "Installing node_exporter"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install node_exporter"
        return
    fi
    
    if check_component_installed "node_exporter" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "node_exporter already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/node_exporter"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Download latest node_exporter
    log_info "Downloading node_exporter"
    curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | \
        grep browser_download_url | \
        grep linux-amd64 | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download node_exporter"
            exit 1
        }
    
    tar -xzf node_exporter*.tar.gz
    cd node_exporter-*/
    mv node_exporter "$bindir/"
    chown prometheus. "$bindir/node_exporter"
    
    # SELinux context for RHEL 8+
    if [[ "$OS" == "red" && "$OSVER" -ge 8 ]]; then
        restorecon -rv "$bindir/node_exporter" || true
    fi
    
    # Create systemd service
    create_node_exporter_service
    
    systemctl daemon-reload
    systemctl enable node_exporter
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart node_exporter
    else
        systemctl start node_exporter
    fi
    
    log_info "node_exporter installation completed"
}

create_node_exporter_service() {
    cat > "$psdir/node_exporter.service" << 'EOF'
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=root
Group=root
Type=simple
ExecStart=/usr/local/bin/node_exporter $OPTIONS

[Install]
WantedBy=multi-user.target
EOF
    
    # Create default options
    echo 'OPTIONS="--collector.ethtool --collector.interrupts --collector.processes --collector.systemd --collector.tcpstat"' > /etc/default/node_exporter
}

########################
### Install BlackBox ###
########################
install_blackbox() {
    log_info "Installing blackbox_exporter"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install blackbox_exporter"
        return
    fi
    
    if check_component_installed "blackbox_exporter" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "blackbox_exporter already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/blackbox"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Download latest blackbox_exporter
    log_info "Downloading blackbox_exporter"
    curl -s https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | \
        grep browser_download_url | \
        grep linux-amd64 | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download blackbox_exporter"
            exit 1
        }
    
    tar -xzf blackbox_exporter*.tar.gz
    cd blackbox_exporter-*/
    mv blackbox_exporter "$bindir/"
    chown prometheus. "$bindir/blackbox_exporter"
    
    # Install config
    mkdir -p "$promdir"
    if [[ ! -f "$promdir/blackbox.yml" ]] || [[ "$UPDATE_MODE" == "true" ]]; then
        if [[ -f "$promdir/blackbox.yml" ]]; then
            cp "$promdir/blackbox.yml" "$promdir/backups/blackbox.yml.$(date +%Y%m%d_%H%M%S)"
        fi
        mv blackbox.yml "$promdir/"
    fi
    
    chown -R prometheus. "$promdir/"
    
    # SELinux context for RHEL 8+
    if [[ "$OS" == "red" && "$OSVER" -ge 8 ]]; then
        restorecon -rv "$bindir/blackbox_exporter" || true
    fi
    
    # Create systemd service
    create_blackbox_service
    
    systemctl daemon-reload
    systemctl enable blackbox_exporter
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart blackbox_exporter
    else
        systemctl start blackbox_exporter
    fi
    
    # Add to prometheus config if not already present
    add_blackbox_to_prometheus_config
    
    log_info "blackbox_exporter installation completed"
}

create_blackbox_service() {
    cat > "$psdir/blackbox_exporter.service" << 'EOF'
[Unit]
Description=Prometheus Blackbox Exporter Http/Https Monitoring
After=network.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/blackbox_exporter \
    --config.file /etc/prometheus/blackbox.yml \
    --web.listen-address=":9115"

Restart=always

[Install]
WantedBy=multi-user.target
EOF
}

add_blackbox_to_prometheus_config() {
    if ! grep -q "job_name.*blackbox" "$promdir/prometheus.yml" 2>/dev/null; then
        log_info "Adding blackbox configuration to Prometheus"
        cat >> "$promdir/prometheus.yml" << 'EOF'

  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
      #### Local Targets ####
         - http://localhost:9090

      #### Remote Targets ####
         #- https://google.com

    relabel_configs:
     - source_labels: [__address__]
       target_label: __param_target
     - source_labels: [__param_target]
       target_label: instance
     - target_label: __address__
       replacement: localhost:9115
EOF
        
        # Restart prometheus to reload config
        if systemctl is-active --quiet prometheus; then
            systemctl reload prometheus || systemctl restart prometheus
        fi
    fi
}

#######################
### Install Grafana ###
#######################
install_grafana() {
    log_info "Installing Grafana"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Grafana"
        return
    fi
    
    if check_component_installed "grafana" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "Grafana already installed, skipping"
        return
    fi
    
    case $OS in
        "ubuntu"|"debian")
            install_grafana_debian
            ;;
        "red"|"centos"|"oracle"|"rocky"|"almalinux")
            install_grafana_rhel
            ;;
        *)
            log_error "Unsupported OS for Grafana installation: $OS"
            return 1
            ;;
    esac
    
    systemctl daemon-reload
    systemctl enable grafana-server
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart grafana-server
    else
        systemctl start grafana-server
    fi
    
    log_info "Grafana installation completed"
}

install_grafana_debian() {
    # Add Grafana APT repository
    $pkgmgr update
    $pkgmgr install -y software-properties-common wget
    wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key
    echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list
    $pkgmgr update
    $pkgmgr install grafana
}

install_grafana_rhel() {
    # Add Grafana YUM repository
    cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
    
    $pkgmgr install grafana
}

###################
### Install Loki ###
###################
install_loki() {
    log_info "Installing Loki"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Loki"
        return
    fi
    
    if check_component_installed "loki" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "Loki already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/loki"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Download latest Loki
    log_info "Downloading Loki"
    curl -s https://api.github.com/repos/grafana/loki/releases/latest | \
        grep browser_download_url | \
        grep loki-linux-amd64.zip | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download Loki"
            exit 1
        }
    
    unzip loki-linux-amd64.zip
    mv loki-linux-amd64 "$bindir/loki"
    chown prometheus. "$bindir/loki"
    chmod +x "$bindir/loki"
    
    # Create Loki directories
    mkdir -p /var/lib/loki/{wal,chunks,index}
    chown -R prometheus. /var/lib/loki
    
    # Create Loki config directory
    mkdir -p "$promdir"
    chown -R prometheus. "$promdir"
    
    # Create Loki config
    create_loki_config
    
    # Create systemd service
    create_loki_service
    
    systemctl daemon-reload
    systemctl enable loki
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart loki
    else
        systemctl start loki
    fi
    
    log_info "Loki installation completed"
}

create_loki_config() {
    if [[ ! -f "$promdir/loki.yml" ]] || [[ "$UPDATE_MODE" == "true" ]]; then
        if [[ -f "$promdir/loki.yml" ]]; then
            cp "$promdir/loki.yml" "$promdir/backups/loki.yml.$(date +%Y%m%d_%H%M%S)"
        fi
        
        cat > "$promdir/loki.yml" << 'EOF'
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path_prefix: /var/lib/loki
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
#
# Statistics help us better understand how Loki is used, and they show us performance
# levels for most users. This helps us prioritize features and documentation.
# For more information on what's sent, look at
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
# Refer to the buildReport method to see what goes into a report.
#
# If you would like to disable reporting, uncomment the following lines:
analytics:
  reporting_enabled: false
EOF
        
        chown prometheus. "$promdir/loki.yml"
    fi
}

create_loki_service() {
    cat > "$psdir/loki.service" << 'EOF'
[Unit]
Description=Loki log aggregation system
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=/usr/local/bin/loki -config.file /etc/prometheus/loki.yml
Restart=always
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF
}

####################
### Install Alloy ###
####################
install_alloy() {
    log_info "Installing Grafana Alloy"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install Grafana Alloy"
        return
    fi
    
    if check_component_installed "alloy" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "Alloy already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/alloy"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Download latest Alloy
    log_info "Downloading Grafana Alloy"
    curl -s https://api.github.com/repos/grafana/alloy/releases/latest | \
        grep browser_download_url | \
        grep -E "alloy-.*-linux-amd64\.zip" | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download Grafana Alloy"
            exit 1
        }
    
    unzip alloy-*-linux-amd64.zip
    mv alloy-*-linux-amd64 "$bindir/alloy"
    chown prometheus. "$bindir/alloy"
    chmod +x "$bindir/alloy"
    
    # Create config directory
    mkdir -p "$promdir/alloy"
    chown -R prometheus. "$promdir/alloy"
    
    # Create basic Alloy config
    create_alloy_config
    
    # Create systemd service
    create_alloy_service
    
    systemctl daemon-reload
    systemctl enable alloy
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart alloy
    else
        systemctl start alloy
    fi
    
    log_info "Grafana Alloy installation completed"
}

create_alloy_config() {
    cat > "$promdir/alloy/config.alloy" << 'EOF'
// Basic Alloy configuration for log collection
logging {
  level  = "info"
  format = "logfmt"
}

// Loki logs endpoint
loki.write "default" {
  endpoint {
    url = "http://localhost:3100/loki/api/v1/push"
  }
}

// Local file logs
local.file_match "varlog" {
  path_targets = ["/var/log/*.log"]
}

loki.source.file "varlog" {
  targets    = local.file_match.varlog.targets
  forward_to = [loki.write.default.receiver]
}

// Prometheus metrics
prometheus.scrape "default" {
  targets = [
    {"__address__" = "localhost:9090", "job" = "prometheus"},
    {"__address__" = "localhost:9100", "job" = "node"},
  ]
  forward_to = [prometheus.remote_write.default.receiver]
}

prometheus.remote_write "default" {
  endpoint {
    url = "http://localhost:9090/api/v1/write"
  }
}
EOF
    
    chown prometheus. "$promdir/alloy/config.alloy"
}

create_alloy_service() {
    cat > "$psdir/alloy.service" << 'EOF'
[Unit]
Description=Grafana Alloy
Documentation=https://grafana.com/docs/alloy/
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=/usr/local/bin/alloy run /etc/prometheus/alloy/config.alloy
Restart=always
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF
}

########################
### Install Web Server ###
########################
install_webserver() {
    log_info "Installing and configuring $WEBSERVER"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install and configure $WEBSERVER"
        return
    fi
    
    case $WEBSERVER in
        "nginx")
            install_nginx
            ;;
        "apache")
            install_apache
            ;;
        "caddy")
            install_caddy
            ;;
        *)
            log_error "Unsupported web server: $WEBSERVER"
            exit 1
            ;;
    esac
    
    log_info "$WEBSERVER installation and configuration completed"
}

install_nginx() {
    $pkgmgr install nginx
    
    # Determine nginx config directory
    if [[ -d "/etc/nginx/sites-available" ]]; then
        sitesa="/etc/nginx/sites-available"
        sitese="/etc/nginx/sites-enabled/"
    elif [[ -d "/etc/nginx/conf.d" ]]; then
        sitesa="/etc/nginx/conf.d"
        sitese=""
    fi
    
    create_nginx_configs
    
    # Test nginx config
    if ! nginx -t; then
        log_error "Nginx configuration test failed"
        exit 1
    fi
    
    systemctl enable nginx
    systemctl restart nginx
}

install_apache() {
    case $OS in
        "ubuntu"|"debian")
            $pkgmgr install apache2
            sitesa="/etc/apache2/sites-available"
            sitese="/etc/apache2/sites-enabled"
            service_name="apache2"
            ;;
        "red"|"centos"|"oracle"|"rocky"|"almalinux")
            $pkgmgr install httpd
            sitesa="/etc/httpd/conf.d"
            sitese=""
            service_name="httpd"
            ;;
    esac
    
    # Enable required modules
    if [[ "$OS" == "ubuntu" || "$OS" == "debian" ]]; then
        a2enmod proxy proxy_http headers
    fi
    
    create_apache_configs
    
    # Test apache config
    if ! $service_name -t; then
        log_error "Apache configuration test failed"
        exit 1
    fi
    
    systemctl enable $service_name
    systemctl restart $service_name
}

install_caddy() {
    # Install Caddy
    case $OS in
        "ubuntu"|"debian")
            curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
            curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
            $pkgmgr update
            $pkgmgr install caddy
            ;;
        "red"|"centos"|"oracle"|"rocky"|"almalinux")
            $pkgmgr install 'dnf-command(copr)'
            $pkgmgr copr enable @caddy/caddy
            $pkgmgr install caddy
            ;;
    esac
    
    create_caddy_config
    
    systemctl enable caddy
    systemctl restart caddy
}

create_nginx_configs() {
    # Prometheus config
    cat > "$sitesa/prometheus.conf" << EOF
server {
    listen 80;
    listen [::]:80;
    
    server_name prometheus.$domain;
    
    location / {
        proxy_pass http://localhost:9090/;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
}
EOF
    
    # Grafana config
    cat > "$sitesa/grafana.conf" << EOF
server {
    listen 80;
    listen [::]:80;
    
    server_name metrics.$domain;
    
    location / {
        proxy_pass http://localhost:3000/;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
}
EOF
    
    # AlertManager config
    cat > "$sitesa/alertmanager.conf" << EOF
server {
    listen 80;
    listen [::]:80;
    
    server_name alerts.$domain;
    
    location / {
        proxy_pass http://localhost:9093/;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
}
EOF
    
    # Loki config
    cat > "$sitesa/loki.conf" << EOF
server {
    listen 80;
    listen [::]:80;
    
    server_name loki.$domain;
    
    location / {
        proxy_pass http://localhost:3100/;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
}
EOF
    
    # Enable sites if using sites-available/sites-enabled structure
    if [[ -n "$sitese" ]]; then
        ln -sf "$sitesa/prometheus.conf" "$sitese" 2>/dev/null || true
        ln -sf "$sitesa/grafana.conf" "$sitese" 2>/dev/null || true
        ln -sf "$sitesa/alertmanager.conf" "$sitese" 2>/dev/null || true
        ln -sf "$sitesa/loki.conf" "$sitese" 2>/dev/null || true
    fi
}

create_apache_configs() {
    local protocol="http"
    local ssl_config=""
    
    if [[ "$ENABLE_TLS" == "true" ]]; then
        protocol="https"
        ssl_config="
        SSLEngine on
        SSLCertificateFile /etc/ssl/certs/prometheus.crt
        SSLCertificateKeyFile /etc/ssl/private/prometheus.key"
    fi
    
    # Prometheus config
    cat > "$sitesa/prometheus.conf" << EOF
<VirtualHost *:80>
    ServerName prometheus.$domain
    ProxyPreserveHost On
    ProxyRequests Off
    ProxyPass / http://localhost:9090/
    ProxyPassReverse / http://localhost:9090/
    ProxyPassReverse / $protocol://prometheus.$domain/
</VirtualHost>
EOF
    
    # Grafana config
    cat > "$sitesa/grafana.conf" << EOF
<VirtualHost *:80>
    ServerName metrics.$domain
    ProxyPreserveHost On
    ProxyRequests Off
    ProxyPass / http://localhost:3000/
    ProxyPassReverse / http://localhost:3000/
    ProxyPassReverse / $protocol://metrics.$domain/
</VirtualHost>
EOF
    
    # AlertManager config
    cat > "$sitesa/alertmanager.conf" << EOF
<VirtualHost *:80>
    ServerName alerts.$domain
    ProxyPreserveHost On
    ProxyRequests Off
    ProxyPass / http://localhost:9093/
    ProxyPassReverse / http://localhost:9093/
    ProxyPassReverse / $protocol://alerts.$domain/
</VirtualHost>
EOF
    
    # Loki config
    cat > "$sitesa/loki.conf" << EOF
<VirtualHost *:80>
    ServerName loki.$domain
    ProxyPreserveHost On
    ProxyRequests Off
    ProxyPass / http://localhost:3100/
    ProxyPassReverse / http://localhost:3100/
    ProxyPassReverse / $protocol://loki.$domain/
</VirtualHost>
EOF
    
    # Enable sites if using sites-available/sites-enabled structure
    if [[ -n "$sitese" ]]; then
        a2ensite prometheus grafana alertmanager loki 2>/dev/null || true
    fi
}

create_caddy_config() {
    local protocol="http"
    local tls_config=""
    
    if [[ "$ENABLE_TLS" == "true" ]]; then
        protocol="https"
        tls_config="
    tls /etc/ssl/certs/prometheus.crt /etc/ssl/private/prometheus.key"
    fi
    
    cat > /etc/caddy/Caddyfile << EOF
# Prometheus
prometheus.$domain {$tls_config
    reverse_proxy localhost:9090
}

# Grafana
metrics.$domain {$tls_config
    reverse_proxy localhost:3000
}

# AlertManager
alerts.$domain {$tls_config
    reverse_proxy localhost:9093
}

# Loki
loki.$domain {$tls_config
    reverse_proxy localhost:3100
}
EOF
}

#############################
### Install AlertManager ###
#############################
install_alertmanager() {
    log_info "Installing AlertManager"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install AlertManager"
        return
    fi
    
    if check_component_installed "alertmanager" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "AlertManager already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/alertmanager"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Create alertmanager user if doesn't exist
    if ! grep -q alertmanager /etc/passwd; then
        groupadd --system alertmanager
        if [[ "$OS" == "ubuntu" || "$OS" == "debian" ]]; then
            useradd -s /sbin/nologin --system -g alertmanager alertmanager
        else
            useradd -m -s /bin/false alertmanager -g alertmanager
        fi
        mkdir -p /var/lib/alertmanager
        chown alertmanager:alertmanager /var/lib/alertmanager
    fi
    
    # Download latest AlertManager
    log_info "Downloading AlertManager"
    curl -s https://api.github.com/repos/prometheus/alertmanager/releases/latest | \
        grep browser_download_url | \
        grep linux-amd64 | \
        cut -d '"' -f 4 | \
        wget -qi - || {
            log_error "Failed to download AlertManager"
            exit 1
        }
    
    tar -xzf alertmanager*.tar.gz
    cd alertmanager-*/
    mv amtool alertmanager "$bindir/"
    chown alertmanager:alertmanager "$bindir/alertmanager" "$bindir/amtool"
    
    # Install config if not exists or in update mode
    if [[ ! -f "$promdir/alertmanager.yml" ]] || [[ "$UPDATE_MODE" == "true" ]]; then
        if [[ -f "$promdir/alertmanager.yml" ]]; then
            cp "$promdir/alertmanager.yml" "$promdir/backups/alertmanager.yml.$(date +%Y%m%d_%H%M%S)"
        fi
        mv alertmanager.yml "$promdir/"
    fi
    
    chown -R alertmanager:alertmanager "$promdir/alertmanager.yml"
    
    # Create systemd service
    create_alertmanager_service
    
    systemctl daemon-reload
    systemctl enable alertmanager
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart alertmanager
    else
        systemctl start alertmanager
    fi
    
    log_info "AlertManager installation completed"
}

create_alertmanager_service() {
    cat > "$psdir/alertmanager.service" << 'EOF'
[Unit]
Description=Prometheus AlertManager Service
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
ExecStart=/usr/local/bin/alertmanager \
    --config.file /etc/prometheus/alertmanager.yml \
    --storage.path /var/lib/alertmanager/ \
    --cluster.advertise-address=0.0.0.0:9093

[Install]
WantedBy=multi-user.target
EOF
}

##############################
### Install MySQL Exporter ###
##############################
install_mysql_exporter() {
    log_info "Installing MySQL Exporter"
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY RUN] Would install MySQL Exporter"
        return
    fi
    
    if check_component_installed "mysqld_exporter" && [[ "$UPDATE_MODE" == "false" ]]; then
        log_info "MySQL Exporter already installed, skipping"
        return
    fi
    
    local workdir="/tmp/prometheus-install-$$/mysql_exporter"
    mkdir -p "$workdir"
    cd "$workdir"
    
    # Download latest mysqld_exporter
    log_info "Downloading MySQL Exporter"
    curl -s https://api.github.com/repos/prometheus/mysqld_exporter/releases/latest | \
        grep browser_download_url | \
        grep linux-amd64.tar.gz | \
        awk '{gsub(/"/, "", $2); print $2}' | \
        wget -qi - || {
            log_error "Failed to download MySQL Exporter"
            exit 1
        }
    
    tar -xzf mysqld_exporter-*
    cd mysqld_exporter-*/
    mv mysqld_exporter* "$bindir/mysqld_exporter"
    chown prometheus. "$bindir/mysqld_exporter"
    
    # Create MySQL config
    create_mysql_exporter_config
    
    # Create systemd service
    create_mysql_exporter_service
    
    systemctl daemon-reload
    systemctl enable mysqld_exporter
    
    if [[ "$UPDATE_MODE" == "true" ]]; then
        systemctl restart mysqld_exporter
    else
        systemctl start mysqld_exporter
    fi
    
    # Add to prometheus config
    add_mysql_to_prometheus_config
    
    log_info "MySQL Exporter installation completed"
}

create_mysql_exporter_config() {
    touch /etc/.mysqld_exporter.cnf
    chown prometheus. /etc/.mysqld_exporter.cnf
    chmod 600 /etc/.mysqld_exporter.cnf
    
    # Generate config based on number of hosts
    {
        for ((i=1; i<=mynum; i++)); do
            echo "[client]"
            echo "user=$myuser"
            echo "password=$mypass"
            eval "echo \"host=\$myhost$i\""
            echo
        done
    } > /etc/.mysqld_exporter.cnf
}

create_mysql_exporter_service() {
    cat > "$psdir/mysqld_exporter.service" << EOF
[Unit]
Description=MySQL Exporter Service
Wants=network.target
After=network.target

[Service]
User=prometheus
Group=prometheus
Environment="DATA_SOURCE_NAME=mysqld_exporter:$mypass@$myuser:(/var/lib/mysql/mysql.sock)"
Type=simple
ExecStart=/usr/local/bin/mysqld_exporter
Restart=always

[Install]
WantedBy=multi-user.target
EOF
}

add_mysql_to_prometheus_config() {
    if ! grep -q "job_name.*mysql" "$promdir/prometheus.yml" 2>/dev/null; then
        log_info "Adding MySQL configuration to Prometheus"
        cat >> "$promdir/prometheus.yml" << 'EOF'

  - job_name: 'mysql_metrics'
    scrape_interval: 5s
    static_configs:
      - targets:
         - localhost:9104
EOF
        
        # Restart prometheus to reload config
        if systemctl is-active --quiet prometheus; then
            systemctl reload prometheus || systemctl restart prometheus
        fi
    fi
}

#########################
### Parse Arguments ###
#########################
parse_arguments() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            --prometheus)
                INSTALL_PROMETHEUS=true
                shift
                ;;
            --no-prometheus)
                INSTALL_PROMETHEUS=false
                shift
                ;;
            --node-exporter)
                INSTALL_NODE_EXPORTER=true
                shift
                ;;
            --no-node-exporter)
                INSTALL_NODE_EXPORTER=false
                shift
                ;;
            --blackbox)
                INSTALL_BLACKBOX=true
                shift
                ;;
            --no-blackbox)
                INSTALL_BLACKBOX=false
                shift
                ;;
            --alertmanager)
                INSTALL_ALERTMANAGER=true
                shift
                ;;
            --no-alertmanager)
                INSTALL_ALERTMANAGER=false
                shift
                ;;
            --mysql-exporter)
                INSTALL_MYSQL_EXPORTER=true
                shift
                ;;
            --no-mysql-exporter)
                INSTALL_MYSQL_EXPORTER=false
                shift
                ;;
            --grafana)
                INSTALL_GRAFANA=true
                shift
                ;;
            --no-grafana)
                INSTALL_GRAFANA=false
                shift
                ;;
            --loki)
                INSTALL_LOKI=true
                shift
                ;;
            --no-loki)
                INSTALL_LOKI=false
                shift
                ;;
            --alloy)
                INSTALL_ALLOY=true
                shift
                ;;
            --no-alloy)
                INSTALL_ALLOY=false
                shift
                ;;
            --webserver)
                WEBSERVER="$2"
                INSTALL_WEBSERVER=true
                shift 2
                ;;
            --no-webserver)
                INSTALL_WEBSERVER=false
                shift
                ;;
            --enable-tls)
                ENABLE_TLS=true
                shift
                ;;
            --all)
                INSTALL_PROMETHEUS=true
                INSTALL_NODE_EXPORTER=true
                INSTALL_BLACKBOX=true
                INSTALL_ALERTMANAGER=true
                INSTALL_MYSQL_EXPORTER=true
                INSTALL_GRAFANA=true
                INSTALL_LOKI=true
                INSTALL_ALLOY=true
                INSTALL_WEBSERVER=true
                shift
                ;;
            --update)
                UPDATE_MODE=true
                shift
                ;;
            --domain)
                domain="$2"
                shift 2
                ;;
            --config-file)
                CONFIG_FILE="$2"
                shift 2
                ;;
            --dry-run)
                DRY_RUN=true
                shift
                ;;
            --skip-deps)
                SKIP_DEPS=true
                shift
                ;;
            --help)
                show_help
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                show_help
                exit 1
                ;;
        esac
    done
}

#########################
### Load Config File ###
#########################
load_config_file() {
    if [[ -n "$CONFIG_FILE" && -f "$CONFIG_FILE" ]]; then
        log_info "Loading configuration from $CONFIG_FILE"
        # shellcheck source=/dev/null
        source "$CONFIG_FILE"
    fi
}

##########################
### Main Installation ###
##########################
main() {
    log_info "Starting Prometheus stack installation"
    log_info "Command line: $0 $*"
    
    parse_arguments "$@"
    load_config_file
    
    check_permissions
    detect_os
    setup_directories
    setup_package_manager
    create_prometheus_user
    install_dependencies
    
    # Install components based on flags
    if [[ "$INSTALL_PROMETHEUS" == "true" ]]; then
        install_prometheus
    fi
    
    if [[ "$INSTALL_NODE_EXPORTER" == "true" ]]; then
        install_node_exporter
    fi
    
    if [[ "$INSTALL_BLACKBOX" == "true" ]]; then
        install_blackbox
    fi
    
    if [[ "$INSTALL_ALERTMANAGER" == "true" ]]; then
        install_alertmanager
    fi
    
    if [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]]; then
        install_mysql_exporter
    fi
    
    if [[ "$INSTALL_GRAFANA" == "true" ]]; then
        install_grafana
    fi
    
    if [[ "$INSTALL_LOKI" == "true" ]]; then
        install_loki
    fi
    
    if [[ "$INSTALL_ALLOY" == "true" ]]; then
        install_alloy
    fi
    
    if [[ "$INSTALL_WEBSERVER" == "true" ]]; then
        install_webserver
    fi
    
    log_info "Installation completed successfully"
    
    if [[ "$DRY_RUN" == "false" ]]; then
        echo
        echo "=== Installation Summary ==="
        echo "Components installed:"
        [[ "$INSTALL_PROMETHEUS" == "true" ]] && echo "  ✓ Prometheus (http://localhost:9090)"
        [[ "$INSTALL_NODE_EXPORTER" == "true" ]] && echo "  ✓ Node Exporter (http://localhost:9100)"
        [[ "$INSTALL_BLACKBOX" == "true" ]] && echo "  ✓ Blackbox Exporter (http://localhost:9115)"
        [[ "$INSTALL_ALERTMANAGER" == "true" ]] && echo "  ✓ AlertManager (http://localhost:9093)"
        [[ "$INSTALL_MYSQL_EXPORTER" == "true" ]] && echo "  ✓ MySQL Exporter (http://localhost:9104)"
        [[ "$INSTALL_GRAFANA" == "true" ]] && echo "  ✓ Grafana (http://localhost:3000)"
        [[ "$INSTALL_LOKI" == "true" ]] && echo "  ✓ Loki (http://localhost:3100)"
        [[ "$INSTALL_ALLOY" == "true" ]] && echo "  ✓ Grafana Alloy"
        [[ "$INSTALL_WEBSERVER" == "true" ]] && echo "  ✓ $WEBSERVER Web Server"
        echo
        echo "Check logs at: $logfile"
        echo "Default Grafana credentials: admin/admin"
    fi
}

# Run main function
main "$@"