#!/usr/bin/env bash ######################################################################################### #### jenkins-smoke-tests.sh — Verify Jenkins instance health after upgrades/changes #### #### Zero external dependencies. Runs in air-gapped environments. #### #### Requires: bash 4+, curl, openssl (optional) #### #### #### #### Author: Phil Connor #### #### Contact: contact@mylinux.work #### #### License: MIT #### #### Version 1.01 #### #### #### #### Usage: #### #### export JENKINS_URL="https://jenkins.example.com" #### #### export JENKINS_USER="admin" #### #### export JENKINS_TOKEN="your-api-token" #### #### ./jenkins-smoke-tests.sh #### #### #### #### See --help for all options. #### ######################################################################################### set -euo pipefail # ── Defaults ────────────────────────────────────────────────────────── JENKINS_URL="${JENKINS_URL:-}" JENKINS_USER="${JENKINS_USER:-}" JENKINS_TOKEN="${JENKINS_TOKEN:-}" CURL_TIMEOUT="${CURL_TIMEOUT:-10}" CURL_INSECURE="${CURL_INSECURE:-false}" SKIP_PLUGINS="${SKIP_PLUGINS:-false}" SKIP_DISK="${SKIP_DISK:-false}" OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}" # text, tap, junit JUNIT_FILE="${JUNIT_FILE:-smoke-results.xml}" VERBOSE="${VERBOSE:-false}" COLOR="${COLOR:-auto}" JENKINS_HOME="${JENKINS_HOME:-/var/lib/jenkins}" # ── State ───────────────────────────────────────────────────────────── PASS=0 FAIL=0 SKIP=0 TOTAL=0 RESULTS=() START_TIME="" JENKINS_VERSION="" CRUMB_VALUE="" # ── Colors ──────────────────────────────────────────────────────────── setup_colors() { if [[ "$COLOR" == "never" ]]; then RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET="" return fi if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' BOLD='\033[1m' RESET='\033[0m' else RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET="" fi } # ── Logging ─────────────────────────────────────────────────────────── log() { echo -e "${BLUE}[INFO]${RESET} $*"; } warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; } err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; } verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; } # ── Test Result Recording ───────────────────────────────────────────── record_pass() { local name="$1" local detail="${2:-}" ((PASS++)) || true ((TOTAL++)) || true RESULTS+=("PASS|${name}|${detail}") if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name}" else echo -e " ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}" fi } record_fail() { local name="$1" local detail="${2:-}" ((FAIL++)) || true ((TOTAL++)) || true RESULTS+=("FAIL|${name}|${detail}") if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "not ok ${TOTAL} - ${name}" [[ -n "$detail" ]] && echo " # ${detail}" else echo -e " ${RED}✗${RESET} ${name}${detail:+ — ${detail}}" fi } record_skip() { local name="$1" local reason="${2:-}" ((SKIP++)) || true ((TOTAL++)) || true RESULTS+=("SKIP|${name}|${reason}") if [[ "$OUTPUT_FORMAT" == "tap" ]]; then echo "ok ${TOTAL} - ${name} # SKIP ${reason}" else echo -e " ${YELLOW}⊘${RESET} ${name}${reason:+ — ${reason}}" fi } # ── curl wrapper ────────────────────────────────────────────────────── api_curl() { local endpoint="$1" shift local curl_opts=(-s -S --max-time "$CURL_TIMEOUT") [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k) [[ -n "$JENKINS_USER" && -n "$JENKINS_TOKEN" ]] && curl_opts+=(-u "${JENKINS_USER}:${JENKINS_TOKEN}") local url="${JENKINS_URL}${endpoint}" verbose "curl GET ${url} $*" curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null } api_curl_status() { local endpoint="$1" shift local curl_opts=(-s -S -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT") [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k) [[ -n "$JENKINS_USER" && -n "$JENKINS_TOKEN" ]] && curl_opts+=(-u "${JENKINS_USER}:${JENKINS_TOKEN}") local url="${JENKINS_URL}${endpoint}" curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null } api_curl_headers() { local endpoint="$1" shift local curl_opts=(-s -S -D - -o /dev/null --max-time "$CURL_TIMEOUT") [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k) [[ -n "$JENKINS_USER" && -n "$JENKINS_TOKEN" ]] && curl_opts+=(-u "${JENKINS_USER}:${JENKINS_TOKEN}") local url="${JENKINS_URL}${endpoint}" curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null } # ── JSON parsing (no jq required) ──────────────────────────────────── json_value() { local key="$1" local json="$2" echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"?\K[^\",}]+" || true; } | head -1 } json_value_string() { local key="$1" local json="$2" echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"\K[^\"]*" || true; } | head -1 } json_count() { local key="$1" local json="$2" echo "$json" | { grep -oP "\"${key}\"\s*:" || true; } | wc -l } # ══════════════════════════════════════════════════════════════════════ # TEST SUITES # ══════════════════════════════════════════════════════════════════════ # ── 1. Connectivity ────────────────────────────────────────────────── test_connectivity() { echo "" echo -e "${BOLD}Connectivity${RESET}" # 1a. HTTP(S) reachable local curl_opts=(-s -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT" -L) [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k) local http_code http_code=$(curl "${curl_opts[@]}" "${JENKINS_URL}/" 2>/dev/null) || http_code="000" if [[ "$http_code" =~ ^(200|403)$ ]]; then record_pass "Jenkins reachable" "HTTP ${http_code}" else record_fail "Jenkins reachable" "HTTP ${http_code}" fi # 1b. Login page accessible local login_code login_code=$(curl "${curl_opts[@]}" "${JENKINS_URL}/login" 2>/dev/null) || login_code="000" if [[ "$login_code" == "200" ]]; then record_pass "Login page accessible" "HTTP ${login_code}" else record_fail "Login page accessible" "HTTP ${login_code}" fi # 1c. TLS certificate validity (if HTTPS) if [[ "$JENKINS_URL" == https://* ]]; then local host host=$(echo "$JENKINS_URL" | sed 's|https://||' | cut -d/ -f1 | cut -d: -f1) local port port=$(echo "$JENKINS_URL" | grep -oP ':\K[0-9]+$' || echo "443") local expiry expiry=$(echo | openssl s_client -servername "$host" -connect "${host}:${port}" 2>/dev/null | \ openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) || expiry="" if [[ -n "$expiry" ]]; then local expiry_epoch expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null) || expiry_epoch=0 local now_epoch now_epoch=$(date +%s) local days_left=$(( (expiry_epoch - now_epoch) / 86400 )) if [[ $days_left -gt 30 ]]; then record_pass "TLS certificate valid" "${days_left} days remaining" elif [[ $days_left -gt 0 ]]; then record_pass "TLS certificate valid" "${days_left} days remaining (renew soon)" else record_fail "TLS certificate valid" "expired or expiring in ${days_left} days" fi else record_skip "TLS certificate check" "could not retrieve certificate" fi else record_skip "TLS certificate check" "not using HTTPS" fi } # ── 2. API Authentication ──────────────────────────────────────────── test_api_auth() { echo "" echo -e "${BOLD}API Authentication${RESET}" # 2a. JSON API reachable local api_status api_status=$(api_curl_status "/api/json") if [[ "$api_status" == "200" ]]; then record_pass "JSON API reachable" "HTTP ${api_status}" elif [[ "$api_status" == "403" ]]; then record_fail "JSON API reachable" "HTTP 403 — authentication failed" return else record_fail "JSON API reachable" "HTTP ${api_status}" return fi # 2b. Authentication — extract version from headers local headers headers=$(api_curl_headers "/api/json") JENKINS_VERSION=$(echo "$headers" | { grep -i '^X-Jenkins:' || true; } | tr -d '\r' | awk '{print $2}') if [[ -n "$JENKINS_VERSION" ]]; then record_pass "API authentication" "Jenkins ${JENKINS_VERSION}" else record_pass "API authentication" "authenticated (version unknown)" fi # 2c. CRUMB (CSRF protection token) local crumb_json crumb_json=$(api_curl "/crumbIssuer/api/json" 2>/dev/null) || crumb_json="" if [[ -n "$crumb_json" ]]; then local crumb_header crumb_header=$(json_value_string "crumbRequestField" "$crumb_json") CRUMB_VALUE=$(json_value_string "crumb" "$crumb_json") verbose "CSRF crumb: ${crumb_header}=${CRUMB_VALUE}" if [[ -n "$CRUMB_VALUE" ]]; then record_pass "CSRF crumb available" "crumb retrieved" else record_skip "CSRF crumb available" "crumb issuer returned empty" fi else record_skip "CSRF crumb available" "crumb issuer not enabled" fi } # ── 3. System Health ───────────────────────────────────────────────── test_system_health() { echo "" echo -e "${BOLD}System Health${RESET}" # 3a. Jenkins version from headers (already captured, but confirm) if [[ -n "$JENKINS_VERSION" ]]; then record_pass "Jenkins version" "${JENKINS_VERSION}" else local headers headers=$(api_curl_headers "/" 2>/dev/null) || headers="" JENKINS_VERSION=$(echo "$headers" | { grep -i '^X-Jenkins:' || true; } | tr -d '\r' | awk '{print $2}') if [[ -n "$JENKINS_VERSION" ]]; then record_pass "Jenkins version" "${JENKINS_VERSION}" else record_skip "Jenkins version" "could not determine version" fi fi # 3b. Executor status local computer_json computer_json=$(api_curl "/computer/api/json" 2>/dev/null) || computer_json="" if [[ -n "$computer_json" ]]; then local total_executors total_executors=$(json_value "totalExecutors" "$computer_json") local busy_executors busy_executors=$(json_value "busyExecutors" "$computer_json") if [[ -n "$total_executors" ]]; then local free_executors=$(( total_executors - busy_executors )) record_pass "Executor status" "${busy_executors}/${total_executors} busy, ${free_executors} idle" else record_skip "Executor status" "could not parse executor count" fi else record_fail "Executor status" "could not reach /computer/api/json" fi # 3c. Build queue local queue_json queue_json=$(api_curl "/queue/api/json" 2>/dev/null) || queue_json="" if [[ -n "$queue_json" ]]; then local queue_items queue_items=$(echo "$queue_json" | { grep -oP '"id"\s*:' || true; } | wc -l) if [[ $queue_items -eq 0 ]]; then record_pass "Build queue" "empty" elif [[ $queue_items -lt 10 ]]; then record_pass "Build queue" "${queue_items} item(s) queued" else record_fail "Build queue" "${queue_items} items queued (possible bottleneck)" fi else record_fail "Build queue" "could not reach /queue/api/json" fi # 3d. System info (admin only) local sysinfo_status sysinfo_status=$(api_curl_status "/manage/systemInfo") if [[ "$sysinfo_status" == "200" ]]; then record_pass "System info accessible" "admin access confirmed" elif [[ "$sysinfo_status" == "403" ]]; then record_skip "System info accessible" "admin access required" else record_fail "System info accessible" "HTTP ${sysinfo_status}" fi } # ── 4. Agents/Nodes ────────────────────────────────────────────────── test_agents() { echo "" echo -e "${BOLD}Agents${RESET}" local computer_json computer_json=$(api_curl "/computer/api/json?depth=1" 2>/dev/null) || computer_json="" if [[ -z "$computer_json" ]]; then record_fail "Agent list" "could not reach /computer/api/json" return fi # Count nodes local node_count node_count=$(echo "$computer_json" | { grep -oP '"displayName"\s*:' || true; } | wc -l) if [[ $node_count -eq 0 ]]; then record_skip "Agent list" "no nodes found" return fi record_pass "Agent list" "${node_count} node(s) registered" # Master node online local master_offline # The built-in node is typically first and named "master" or "(built-in)" # Check if any node has offline=false master_offline=$(echo "$computer_json" | { grep -oP '"offline"\s*:\s*\K(true|false)' || true; } | head -1) if [[ "$master_offline" == "false" ]]; then record_pass "Built-in node online" "controller node available" elif [[ "$master_offline" == "true" ]]; then record_fail "Built-in node online" "controller node offline" else record_skip "Built-in node online" "could not determine status" fi # Count online vs offline local online_count online_count=$(echo "$computer_json" | { grep -oP '"offline"\s*:\s*false' || true; } | wc -l) local offline_count offline_count=$(echo "$computer_json" | { grep -oP '"offline"\s*:\s*true' || true; } | wc -l) if [[ $offline_count -eq 0 ]]; then record_pass "Agent availability" "${online_count}/${node_count} online" elif [[ $online_count -gt 0 ]]; then record_fail "Agent availability" "${online_count}/${node_count} online, ${offline_count} offline" else record_fail "Agent availability" "all ${node_count} agents offline" fi # Temporarily offline agents (manually taken offline) local temp_offline temp_offline=$(echo "$computer_json" | { grep -oP '"temporarilyOffline"\s*:\s*true' || true; } | wc -l) if [[ $temp_offline -gt 0 ]]; then record_pass "Manually offline agents" "${temp_offline} agent(s) temporarily disabled" fi } # ── 5. Jobs ─────────────────────────────────────────────────────────── test_jobs() { echo "" echo -e "${BOLD}Jobs${RESET}" local jobs_json jobs_json=$(api_curl "/api/json?tree=jobs[name,color,url,inQueue]" 2>/dev/null) || jobs_json="" if [[ -z "$jobs_json" ]]; then record_fail "Job list" "could not reach /api/json" return fi local job_count job_count=$(echo "$jobs_json" | { grep -oP '"name"\s*:' || true; } | wc -l) if [[ $job_count -eq 0 ]]; then record_skip "Job list" "no jobs configured" return fi record_pass "Job list" "${job_count} job(s) found" # Count by status (color field) local blue_count red_count yellow_count disabled_count notbuilt_count aborted_count blue_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"blue[^"]*"' || true; } | wc -l) red_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"red[^"]*"' || true; } | wc -l) yellow_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"yellow[^"]*"' || true; } | wc -l) disabled_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"disabled[^"]*"' || true; } | wc -l) notbuilt_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"notbuilt[^"]*"' || true; } | wc -l) aborted_count=$(echo "$jobs_json" | { grep -oP '"color"\s*:\s*"aborted[^"]*"' || true; } | wc -l) local status_parts=() [[ $blue_count -gt 0 ]] && status_parts+=("${blue_count} passing") [[ $red_count -gt 0 ]] && status_parts+=("${red_count} failing") [[ $yellow_count -gt 0 ]] && status_parts+=("${yellow_count} unstable") [[ $disabled_count -gt 0 ]] && status_parts+=("${disabled_count} disabled") [[ $notbuilt_count -gt 0 ]] && status_parts+=("${notbuilt_count} not built") [[ $aborted_count -gt 0 ]] && status_parts+=("${aborted_count} aborted") local status_summary status_summary=$(IFS=", "; echo "${status_parts[*]}") if [[ $red_count -gt 0 ]]; then record_fail "Job health" "${status_summary}" else record_pass "Job health" "${status_summary}" fi # Stuck builds (in queue) local in_queue_count in_queue_count=$(echo "$jobs_json" | { grep -oP '"inQueue"\s*:\s*true' || true; } | wc -l) if [[ $in_queue_count -gt 0 ]]; then record_pass "Queued jobs" "${in_queue_count} job(s) waiting in queue" fi } # ── 6. Plugins ──────────────────────────────────────────────────────── test_plugins() { if [[ "$SKIP_PLUGINS" == "true" ]]; then echo "" echo -e "${BOLD}Plugins${RESET}" record_skip "Plugin check" "SKIP_PLUGINS=true" return fi echo "" echo -e "${BOLD}Plugins${RESET}" local plugins_json plugins_json=$(api_curl "/pluginManager/api/json?depth=1" 2>/dev/null) || plugins_json="" if [[ -z "$plugins_json" ]]; then record_fail "Plugin list" "could not reach /pluginManager/api/json" return fi # Count installed plugins local plugin_count plugin_count=$(echo "$plugins_json" | { grep -oP '"shortName"\s*:' || true; } | wc -l) if [[ $plugin_count -eq 0 ]]; then record_skip "Plugin list" "no plugins installed" return fi record_pass "Plugin list" "${plugin_count} plugin(s) installed" # Count active vs inactive local active_count active_count=$(echo "$plugins_json" | { grep -oP '"active"\s*:\s*true' || true; } | wc -l) local inactive_count inactive_count=$(echo "$plugins_json" | { grep -oP '"active"\s*:\s*false' || true; } | wc -l) if [[ $inactive_count -gt 0 ]]; then record_pass "Plugin status" "${active_count} active, ${inactive_count} inactive" else record_pass "Plugin status" "${active_count} active" fi # Plugins with updates available local update_count update_count=$(echo "$plugins_json" | { grep -oP '"hasUpdate"\s*:\s*true' || true; } | wc -l) if [[ $update_count -eq 0 ]]; then record_pass "Plugin updates" "all plugins up to date" elif [[ $update_count -lt 5 ]]; then record_pass "Plugin updates" "${update_count} update(s) available" else record_fail "Plugin updates" "${update_count} updates available (review recommended)" fi } # ── 7. Disk & Resources ────────────────────────────────────────────── test_disk() { if [[ "$SKIP_DISK" == "true" ]]; then echo "" echo -e "${BOLD}Disk & Resources${RESET}" record_skip "Disk space check" "SKIP_DISK=true" return fi echo "" echo -e "${BOLD}Disk & Resources${RESET}" # Check if JENKINS_HOME exists locally if [[ ! -d "$JENKINS_HOME" ]]; then record_skip "Disk space check" "JENKINS_HOME not found locally at ${JENKINS_HOME}" return fi # Get disk usage percentage for JENKINS_HOME partition local disk_usage disk_usage=$(df "$JENKINS_HOME" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') || disk_usage="" if [[ -z "$disk_usage" ]]; then record_skip "Disk space check" "could not determine disk usage" return fi local disk_avail disk_avail=$(df -h "$JENKINS_HOME" 2>/dev/null | awk 'NR==2 {print $4}') || disk_avail="unknown" if [[ $disk_usage -lt 70 ]]; then record_pass "Disk space" "${disk_usage}% used (${disk_avail} free)" elif [[ $disk_usage -lt 85 ]]; then record_pass "Disk space" "${disk_usage}% used (${disk_avail} free) — monitor closely" elif [[ $disk_usage -lt 95 ]]; then record_fail "Disk space" "${disk_usage}% used (${disk_avail} free) — cleanup needed" else record_fail "Disk space" "${disk_usage}% used (${disk_avail} free) — critical" fi # JENKINS_HOME size local home_size home_size=$(du -sh "$JENKINS_HOME" 2>/dev/null | awk '{print $1}') || home_size="" if [[ -n "$home_size" ]]; then record_pass "JENKINS_HOME size" "${home_size} at ${JENKINS_HOME}" fi } # ══════════════════════════════════════════════════════════════════════ # OUTPUT # ══════════════════════════════════════════════════════════════════════ print_summary() { local end_time end_time=$(date +%s) local duration=$(( end_time - START_TIME )) echo "" echo -e "${BOLD}────────────────────────────────────────${RESET}" echo -e "${BOLD}Summary${RESET} ${JENKINS_URL}" echo -e " ${GREEN}${PASS} passed${RESET} ${RED}${FAIL} failed${RESET} ${YELLOW}${SKIP} skipped${RESET} (${duration}s)" echo -e "${BOLD}────────────────────────────────────────${RESET}" if [[ $FAIL -eq 0 ]]; then echo -e "${GREEN}${BOLD}All tests passed.${RESET}" else echo -e "${RED}${BOLD}${FAIL} test(s) failed.${RESET}" fi } print_tap_header() { echo "TAP version 13" } print_tap_footer() { echo "1..${TOTAL}" echo "# pass ${PASS}" echo "# fail ${FAIL}" echo "# skip ${SKIP}" } write_junit() { local end_time end_time=$(date +%s) local duration=$(( end_time - START_TIME )) cat > "$JUNIT_FILE" < JUNIT_EOF for result in "${RESULTS[@]}"; do local status name detail status=$(echo "$result" | cut -d'|' -f1) name=$(echo "$result" | cut -d'|' -f2) detail=$(echo "$result" | cut -d'|' -f3) # XML-escape the values name=$(echo "$name" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') detail=$(echo "$detail" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') case "$status" in PASS) echo " " >> "$JUNIT_FILE" [[ -n "$detail" ]] && echo " ${detail}" >> "$JUNIT_FILE" echo " " >> "$JUNIT_FILE" ;; FAIL) echo " " >> "$JUNIT_FILE" echo " FAILED: ${name} — ${detail}" >> "$JUNIT_FILE" echo " " >> "$JUNIT_FILE" ;; SKIP) echo " " >> "$JUNIT_FILE" echo " " >> "$JUNIT_FILE" echo " " >> "$JUNIT_FILE" ;; esac done echo " " >> "$JUNIT_FILE" echo "" >> "$JUNIT_FILE" log "JUnit report written to ${JUNIT_FILE}" } # ══════════════════════════════════════════════════════════════════════ # MAIN # ══════════════════════════════════════════════════════════════════════ usage() { cat <