Add all 44 scripts, update CI: error severity baseline, PowerShell validation, multi-distro testing

Amp-Thread-ID: https://ampcode.com/threads/T-019cc404-c628-759e-a50b-f5eeea35b91f
Co-authored-by: Amp <amp@ampcode.com>

cert-check.sh

@@ -0,0 +1,428 @@
+#!/bin/bash
+
+################################################
+#### AD Certificate checker and renewal     ####
+#### for Amazon, Ubuntu and RedHat servers  ####
+####                                        ####
+#### Author: Phil Connor                    ####
+#### License: MIT                            ####
+#### Contact: contact@mylinux.work           ####
+#### Version: 3.00-081425                   ####
+################################################
+
+set -o pipefail
+
+SCRIPT_NAME=$(basename "$0")
+readonly SCRIPT_NAME
+
+# Default configuration
+readonly DEFAULT_PEM_PATH="/etc/pki/ca-trust/source/anchors/ad-cert.pem"
+readonly DEFAULT_DAYS_THRESHOLD=30
+readonly DEFAULT_DOMAIN="example"
+readonly DEFAULT_NODE_DIR="/var/lib/node_exporter"
+
+# Configuration variables (can be overridden by environment)
+PEM_PATH=${PEM_PATH:-$DEFAULT_PEM_PATH}
+DAYS_THRESHOLD=${DAYS_THRESHOLD:-$DEFAULT_DAYS_THRESHOLD}
+DOMAIN=${DOMAIN:-$DEFAULT_DOMAIN}
+NODE_DIR=${NODE_DIR:-$DEFAULT_NODE_DIR}
+SERVER_TYPE=${SERVER_TYPE:-}
+DEBUG=${DEBUG:-}
+
+# Runtime flags
+MONITOR_ONLY=false
+RENEW_ONLY=false
+
+handle_error() {
+    local exit_code=$1
+    local line_number=$2
+    echo "Error: $SCRIPT_NAME failed at line $line_number with exit code $exit_code" >&2
+    exit "$exit_code"
+}
+
+trap 'handle_error $? $LINENO' ERR
+
+debug_echo() {
+    if [[ -n "$DEBUG" ]]; then
+        echo "[DEBUG] $*" >&2
+    fi
+}
+
+show_help() {
+    cat << EOF
+Usage: $SCRIPT_NAME [OPTIONS]
+
+SSL certificate checker and renewal script for Prometheus monitoring.
+
+OPTIONS:
+    --monitor       Only generate Prometheus metrics (no renewal)
+    --renew         Only handle certificate renewal (no monitoring)
+    --all           Run both monitoring and renewal (default)
+    --help, -h      Show this help message
+
+ENVIRONMENT VARIABLES:
+    PEM_PATH        Path to certificate file (default: $DEFAULT_PEM_PATH)
+    DAYS_THRESHOLD  Days before expiry to trigger renewal (default: $DEFAULT_DAYS_THRESHOLD)
+    DOMAIN          Domain name (default: $DEFAULT_DOMAIN)
+    NODE_DIR        Node exporter directory (default: $DEFAULT_NODE_DIR)
+    SERVER_TYPE     Server type (artifactory, bitbucket, cloudaccess, jira)
+    DEBUG           Enable debug output
+
+EXAMPLES:
+    $SCRIPT_NAME --monitor
+    SERVER_TYPE=bitbucket $SCRIPT_NAME --renew
+    DEBUG=1 $SCRIPT_NAME --all
+EOF
+}
+
+validate_certificate_file() {
+    local cert_file="$1"
+    
+    if [[ ! -f "$cert_file" ]]; then
+        debug_echo "Certificate file not found: $cert_file"
+        return 1
+    fi
+    
+    if ! openssl x509 -noout -text -in "$cert_file" >/dev/null 2>&1; then
+        echo "Error: Invalid certificate file: $cert_file" >&2
+        return 1
+    fi
+    
+    return 0
+}
+
+download_certificate() {
+    local domain="$1"
+    local output_file="$2"
+    local server_url="us.${domain}.net:636"
+    
+    debug_echo "Downloading certificate from $server_url"
+    
+    if ! timeout 30 openssl s_client -connect "$server_url" -servername "us.${domain}.net" < /dev/null 2>/dev/null | \
+         sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > "$output_file"; then
+        echo "Error: Failed to download certificate from $server_url" >&2
+        return 1
+    fi
+    
+    return 0
+}
+
+calculate_certificate_dates() {
+    local cert_file="$1"
+    local -n days_left_ref=$2
+    local -n days_gone_ref=$3
+    
+    local beg_date end_date beg_sec end_sec now_sec
+    
+    beg_date=$(openssl x509 -noout -startdate -in "$cert_file")
+    end_date=$(openssl x509 -noout -enddate -in "$cert_file")
+    
+    beg_sec=$(date --date="${beg_date##*=}" +%s)
+    end_sec=$(date --date="${end_date##*=}" +%s)
+    now_sec=$(date +%s)
+    
+    days_gone_ref=$(( (now_sec - beg_sec) / 86400 ))
+    days_left_ref=$(( (end_sec - now_sec) / 86400 ))
+    
+    debug_echo "Certificate valid from $(date -d @"$beg_sec") to $(date -d @"$end_sec")"
+    debug_echo "Days gone: $days_gone_ref, Days left: $days_left_ref"
+}
+
+generate_prometheus_metrics() {
+    local days_left="$1"
+    local days_gone="$2"
+    local output_file="$NODE_DIR/adcert_check.prom"
+    
+    debug_echo "Generating Prometheus metrics to $output_file"
+    
+    mkdir -p "$NODE_DIR"
+    
+    {
+        echo '# HELP linux_ad_cert_expire AD Certificate expiration days'
+        echo '# TYPE linux_ad_cert_expire gauge'
+        if [[ $days_left -lt 0 ]]; then
+            echo "linux_ad_cert_expire{status=\"expired\",days_gone=\"$days_gone\"} 0"
+        else
+            echo "linux_ad_cert_expire{status=\"valid\"} $days_left"
+        fi
+    } > "$output_file"
+}
+
+get_keystore_password() {
+    local password_url="$1"
+    local storepass=""
+    
+    # Try Vault HTTP API first if URL provided
+    if [[ -n "$password_url" ]]; then
+        debug_echo "Retrieving keystore password from $password_url"
+        storepass=$(curl -sf -X GET "$password_url" 2>/dev/null | jq -r '.data.password // empty' 2>/dev/null || true)
+    fi
+    
+    # Fall back to Vault CLI
+    if [[ -z "$storepass" ]]; then
+        debug_echo "Falling back to Vault CLI for keystore password"
+        storepass=$(vault kv get -field=password secret/keystore 2>/dev/null || true)
+    fi
+    
+    # Fall back to default
+    if [[ -z "$storepass" ]]; then
+        debug_echo "Using default keystore password"
+        storepass="changeit"
+    fi
+    
+    echo "$storepass"
+}
+
+execute_keytool_command() {
+    local java_bin="$1"
+    local keystore="$2"
+    local action="$3"
+    local cert_file="$4"
+    local password_url="$5"
+    
+    local storepass
+    storepass=$(get_keystore_password "$password_url")
+    
+    case "$action" in
+        "delete")
+            "$java_bin/keytool" -delete -alias ad -keystore "$keystore" -storepass "$storepass" 2>/dev/null || true
+            ;;
+        "import")
+            "$java_bin/keytool" -import -noprompt -alias ad -keystore "$keystore" -file "$cert_file" -storepass "$storepass"
+            ;;
+    esac
+}
+
+handle_artifactory_renewal() {
+    local java_bin keystore
+    local vault_url="http://vault.${DOMAIN}.net/v1/secret/secret/artifactory/keytool"
+    
+    # Check app-specific paths first, then fall back to auto-detection
+    java_bin="/opt/jfrog/artifactory/app/third-party/java/bin"
+    keystore="/opt/jfrog/artifactory/app/third-party/java/lib/security/cacerts"
+    
+    if [[ ! -x "$java_bin/keytool" || ! -f "$keystore" ]]; then
+        debug_echo "Artifactory default paths not found, searching for Java"
+        if ! find_java_keystore java_bin keystore; then
+            echo "Error: Could not find Java keytool or keystore for Artifactory" >&2
+            return 1
+        fi
+    fi
+    
+    execute_keytool_command "$java_bin" "$keystore" "delete" "$PEM_PATH" "$vault_url"
+    execute_keytool_command "$java_bin" "$keystore" "import" "$PEM_PATH" "$vault_url"
+    systemctl restart artifactory
+}
+
+handle_bitbucket_renewal() {
+    local java_bin keystore
+    local vault_url="http://vault.${DOMAIN}.net/v1/secret/secret/bitbucket/keytool"
+    
+    # Check app-specific paths first, then fall back to auto-detection
+    java_bin="/mnt/ebs/bitbucket/8.19.3/jre/bin"
+    keystore="/mnt/ebs/bitbucket/8.19.3/jre/lib/security/cacerts"
+    
+    if [[ ! -x "$java_bin/keytool" || ! -f "$keystore" ]]; then
+        debug_echo "Bitbucket default paths not found, searching for Java"
+        if ! find_java_keystore java_bin keystore; then
+            echo "Error: Could not find Java keytool or keystore for Bitbucket" >&2
+            return 1
+        fi
+    fi
+    
+    if [[ -n "$DEBUG" ]]; then
+        debug_echo "Would execute: $java_bin/keytool -delete -alias ad -keystore $keystore"
+        debug_echo "Would execute: curl -X GET $vault_url"
+        debug_echo "Would execute: $java_bin/keytool -import -alias ad -keystore $keystore -file $PEM_PATH"
+        debug_echo "Would execute: systemctl restart atlbitbucket"
+    else
+        execute_keytool_command "$java_bin" "$keystore" "delete" "$PEM_PATH" "$vault_url"
+        execute_keytool_command "$java_bin" "$keystore" "import" "$PEM_PATH" "$vault_url"
+        systemctl restart atlbitbucket
+    fi
+}
+
+handle_cloudaccess_renewal() {
+    docker restart cloudaccess_server_
+}
+
+handle_jira_renewal() {
+    local java_bin keystore
+    local vault_url="http://vault.${DOMAIN}.net/v1/secret/secret/jira/keytool"
+    
+    # Check app-specific paths first, then fall back to auto-detection
+    java_bin="/mnt/ebs/jira/jre/bin"
+    keystore="/mnt/ebs/jira/jre/lib/security/cacerts"
+    
+    if [[ ! -x "$java_bin/keytool" || ! -f "$keystore" ]]; then
+        debug_echo "Jira default paths not found, searching for Java"
+        if ! find_java_keystore java_bin keystore; then
+            echo "Error: Could not find Java keytool or keystore for Jira" >&2
+            return 1
+        fi
+    fi
+    
+    execute_keytool_command "$java_bin" "$keystore" "delete" "$PEM_PATH" "$vault_url"
+    execute_keytool_command "$java_bin" "$keystore" "import" "$PEM_PATH" "$vault_url"
+    systemctl restart jira
+}
+
+find_java_keystore() {
+    local -n java_bin_ref=$1
+    local -n keystore_ref=$2
+    
+    # Common Java installation paths
+    local java_paths=(
+        "/opt/jfrog/artifactory/app/third-party/java"
+        "/mnt/ebs/bitbucket/*/jre"
+        "/mnt/ebs/jira/jre"
+        "/usr/lib/jvm/java-*-openjdk"
+        "/usr/lib/jvm/default-java"
+        "/opt/java"
+        "/usr/java/latest"
+    )
+    
+    # Check JAVA_HOME first
+    if [[ -n "$JAVA_HOME" && -x "$JAVA_HOME/bin/keytool" ]]; then
+        java_bin_ref="$JAVA_HOME/bin"
+        keystore_ref="$JAVA_HOME/lib/security/cacerts"
+        if [[ -f "$keystore_ref" ]]; then
+            debug_echo "Found Java via JAVA_HOME: $java_bin_ref"
+            return 0
+        fi
+    fi
+    
+    # Search common paths with glob expansion
+    for path_pattern in "${java_paths[@]}"; do
+        for java_dir in $path_pattern; do
+            if [[ -d "$java_dir" ]]; then
+                local bin_dir="$java_dir/bin"
+                local cacerts="$java_dir/lib/security/cacerts"
+                
+                if [[ -x "$bin_dir/keytool" && -f "$cacerts" ]]; then
+                    java_bin_ref="$bin_dir"
+                    keystore_ref="$cacerts"
+                    debug_echo "Found Java at: $java_dir"
+                    return 0
+                fi
+            fi
+        done
+    done
+    
+    # Fallback: try system keytool
+    if command -v keytool >/dev/null 2>&1; then
+        java_bin_ref="$(dirname "$(command -v keytool)")"
+        # Try common system keystore locations
+        local system_keystores=(
+            "/etc/ssl/certs/java/cacerts"
+            "/usr/lib/jvm/default-java/lib/security/cacerts"
+            "/etc/pki/ca-trust/extracted/java/cacerts"
+        )
+        for keystore in "${system_keystores[@]}"; do
+            if [[ -f "$keystore" ]]; then
+                keystore_ref="$keystore"
+                debug_echo "Found system Java at: $java_bin_ref"
+                return 0
+            fi
+        done
+    fi
+    
+    return 1
+}
+
+handle_server_renewal() {
+    if [[ -z "$SERVER_TYPE" ]]; then
+        echo "Error: SERVER_TYPE environment variable must be set for renewal" >&2
+        echo "Valid values: artifactory, bitbucket, cloudaccess, jira" >&2
+        return 1
+    fi
+    
+    debug_echo "Handling renewal for server type: $SERVER_TYPE"
+    
+    case "$SERVER_TYPE" in
+        "artifactory") handle_artifactory_renewal ;;
+        "bitbucket") handle_bitbucket_renewal ;;
+        "cloudaccess") handle_cloudaccess_renewal ;;
+        "jira") handle_jira_renewal ;;
+        *) 
+            echo "Error: Unknown server type: $SERVER_TYPE" >&2
+            echo "Valid values: artifactory, bitbucket, cloudaccess, jira" >&2
+            return 1
+            ;;
+    esac
+}
+
+parse_arguments() {
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            --monitor)
+                MONITOR_ONLY=true
+                shift
+                ;;
+            --renew)
+                RENEW_ONLY=true
+                shift
+                ;;
+            --all)
+                MONITOR_ONLY=false
+                RENEW_ONLY=false
+                shift
+                ;;
+            --help|-h)
+                show_help
+                exit 0
+                ;;
+            *)
+                echo "Error: Unknown option: $1" >&2
+                show_help >&2
+                exit 1
+                ;;
+        esac
+    done
+}
+
+main() {
+    parse_arguments "$@"
+    
+    # Check if certificate file exists, if not exit silently
+    if [[ ! -f "$PEM_PATH" ]]; then
+        debug_echo "Certificate file not found: $PEM_PATH"
+        exit 0
+    fi
+    
+    # Download fresh certificate
+    if ! download_certificate "$DOMAIN" "$PEM_PATH"; then
+        exit 1
+    fi
+    
+    # Validate the downloaded certificate
+    if ! validate_certificate_file "$PEM_PATH"; then
+        exit 1
+    fi
+    
+    # Calculate certificate expiration dates
+    local days_left days_gone
+    calculate_certificate_dates "$PEM_PATH" days_left days_gone
+    
+    # Handle monitoring (unless renew-only mode)
+    if [[ "$RENEW_ONLY" != true ]]; then
+        generate_prometheus_metrics "$days_left" "$days_gone"
+        debug_echo "Generated Prometheus metrics"
+    fi
+    
+    # Handle renewal (unless monitor-only mode)
+    if [[ "$MONITOR_ONLY" != true && $days_left -le $DAYS_THRESHOLD ]]; then
+        debug_echo "Certificate expires in $days_left days (threshold: $DAYS_THRESHOLD)"
+        if ! handle_server_renewal; then
+            exit 1
+        fi
+        debug_echo "Certificate renewal completed"
+    fi
+    
+    debug_echo "Script completed successfully"
+}
+
+# Execute main function if script is run directly
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main "$@"
+fi