linux-scripts/gitlab-smoke-tests.sh

#!/usr/bin/env bash

#########################################################################################
#### gitlab-smoke-tests.sh — Verify GitLab instance health after upgrades or changes ####
#### Zero external dependencies. Runs in air-gapped environments.                    ####
#### Requires: bash 4+, curl, git, openssl (optional)                                ####
####                                                                                 ####
#### Author: Phil Connor                                                             ####
#### Contact: contact@mylinux.work                                                   ####
#### License: MIT                                                                    ####
#### Version 1.02                                                                    ####
####                                                                                 ####
#### Usage:                                                                          ####
####   export GITLAB_URL="https://gitlab.example.com"                                ####
####   export GITLAB_TOKEN="glpat-xxxxxxxxxxxxxxxxxxxx"                              ####
####   export GITLAB_HEALTH_TOKEN="your-health-token"  # optional                    ####
####   ./gitlab-smoke-tests.sh                                                       ####
####                                                                                 ####
#### See --help for all options.                                                     ####
#########################################################################################

set -euo pipefail

# ── Defaults ──────────────────────────────────────────────────────────
GITLAB_URL="${GITLAB_URL:-}"
GITLAB_TOKEN="${GITLAB_TOKEN:-}"
GITLAB_USER="${GITLAB_USER:-root}"
SMOKE_PROJECT_PREFIX="${SMOKE_PROJECT_PREFIX:-smoke-test}"
CURL_TIMEOUT="${CURL_TIMEOUT:-10}"
CURL_INSECURE="${CURL_INSECURE:-false}"
SKIP_GIT="${SKIP_GIT:-false}"
SKIP_REGISTRY="${SKIP_REGISTRY:-false}"
SKIP_CLEANUP="${SKIP_CLEANUP:-false}"
GITLAB_HEALTH_TOKEN="${GITLAB_HEALTH_TOKEN:-}"
OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"  # text, tap, junit
JUNIT_FILE="${JUNIT_FILE:-smoke-results.xml}"
VERBOSE="${VERBOSE:-false}"
COLOR="${COLOR:-auto}"

# ── State ─────────────────────────────────────────────────────────────
PASS=0
FAIL=0
SKIP=0
TOTAL=0
RESULTS=()
CLEANUP_PROJECT_ID=""
TMPDIR_SMOKE=""
START_TIME=""
GIT_CLONE_OK="false"

# ── Colors ────────────────────────────────────────────────────────────
setup_colors() {
    if [[ "$COLOR" == "never" ]]; then
        RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
        return
    fi
    if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
        RED='\033[0;31m'
        GREEN='\033[0;32m'
        YELLOW='\033[0;33m'
        BLUE='\033[0;34m'
        BOLD='\033[1m'
        RESET='\033[0m'
    else
        RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
    fi
}

# ── Logging ───────────────────────────────────────────────────────────
log()     { echo -e "${BLUE}[INFO]${RESET} $*"; }
warn()    { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; }
err()     { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }

# ── Test Result Recording ─────────────────────────────────────────────
record_pass() {
    local name="$1"
    local detail="${2:-}"
    ((PASS++)) || true
    ((TOTAL++)) || true
    RESULTS+=("PASS|${name}|${detail}")
    if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
        echo "ok ${TOTAL} - ${name}"
    else
        echo -e "  ${GREEN}✓${RESET} ${name}${detail:+ — ${detail}}"
    fi
}

record_fail() {
    local name="$1"
    local detail="${2:-}"
    ((FAIL++)) || true
    ((TOTAL++)) || true
    RESULTS+=("FAIL|${name}|${detail}")
    if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
        echo "not ok ${TOTAL} - ${name}"
        [[ -n "$detail" ]] && echo "  # ${detail}"
    else
        echo -e "  ${RED}✗${RESET} ${name}${detail:+ — ${detail}}"
    fi
}

record_skip() {
    local name="$1"
    local reason="${2:-}"
    ((SKIP++)) || true
    ((TOTAL++)) || true
    RESULTS+=("SKIP|${name}|${reason}")
    if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
        echo "ok ${TOTAL} - ${name} # SKIP ${reason}"
    else
        echo -e "  ${YELLOW}⊘${RESET} ${name}${reason:+ — ${reason}}"
    fi
}

# ── curl wrapper ──────────────────────────────────────────────────────
api_curl() {
    local method="$1"
    local endpoint="$2"
    shift 2
    local curl_opts=(-s -S --max-time "$CURL_TIMEOUT" -X "$method")

    [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
    [[ -n "$GITLAB_TOKEN" ]] && curl_opts+=(-H "PRIVATE-TOKEN: ${GITLAB_TOKEN}")
    curl_opts+=(-H "Content-Type: application/json")

    local url="${GITLAB_URL}/api/v4${endpoint}"
    verbose "curl ${method} ${url} $*"

    curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null
}

api_curl_status() {
    local method="$1"
    local endpoint="$2"
    shift 2
    local curl_opts=(-s -S -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT" -X "$method")

    [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)
    [[ -n "$GITLAB_TOKEN" ]] && curl_opts+=(-H "PRIVATE-TOKEN: ${GITLAB_TOKEN}")
    curl_opts+=(-H "Content-Type: application/json")

    local url="${GITLAB_URL}/api/v4${endpoint}"
    curl "${curl_opts[@]}" "$@" "$url" 2>/dev/null
}

# ── JSON parsing (no jq required) ────────────────────────────────────
# Extract a top-level string/number value from flat JSON
json_value() {
    local key="$1"
    local json="$2"
    echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"?\K[^\",}]+" || true; } | head -1
}

json_value_string() {
    local key="$1"
    local json="$2"
    echo "$json" | { grep -oP "\"${key}\"\s*:\s*\"\K[^\"]*" || true; } | head -1
}

# ── Cleanup ───────────────────────────────────────────────────────────
cleanup() {
    if [[ -n "$CLEANUP_PROJECT_ID" && "$SKIP_CLEANUP" != "true" ]]; then
        verbose "Cleaning up smoke test project (ID: ${CLEANUP_PROJECT_ID})"
        api_curl DELETE "/projects/${CLEANUP_PROJECT_ID}" >/dev/null 2>&1 || true
    fi
    if [[ -n "$TMPDIR_SMOKE" && -d "$TMPDIR_SMOKE" ]]; then
        rm -rf "$TMPDIR_SMOKE"
    fi
}

trap cleanup EXIT

# ══════════════════════════════════════════════════════════════════════
#                          TEST SUITES
# ══════════════════════════════════════════════════════════════════════

# ── 1. Connectivity ──────────────────────────────────────────────────
test_connectivity() {
    echo ""
    echo -e "${BOLD}Connectivity${RESET}"

    # 1a. HTTP(S) reachable
    local curl_opts=(-s -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT")
    [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)

    local health_qs=""
    [[ -n "$GITLAB_HEALTH_TOKEN" ]] && health_qs="?token=${GITLAB_HEALTH_TOKEN}"

    local http_code
    http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/health${health_qs}" 2>/dev/null) || http_code="000"

    if [[ "$http_code" == "200" ]]; then
        record_pass "GitLab health endpoint reachable" "HTTP ${http_code}"
    else
        record_fail "GitLab health endpoint reachable" "HTTP ${http_code}"
    fi

    # 1b. Readiness check
    http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/readiness${health_qs}" 2>/dev/null) || http_code="000"
    if [[ "$http_code" == "200" ]]; then
        record_pass "GitLab readiness check" "HTTP ${http_code}"
    else
        record_fail "GitLab readiness check" "HTTP ${http_code}"
    fi

    # 1c. Liveness check
    http_code=$(curl "${curl_opts[@]}" "${GITLAB_URL}/-/liveness${health_qs}" 2>/dev/null) || http_code="000"
    if [[ "$http_code" == "200" ]]; then
        record_pass "GitLab liveness check" "HTTP ${http_code}"
    else
        record_fail "GitLab liveness check" "HTTP ${http_code}"
    fi

    # 1d. TLS certificate validity (if HTTPS)
    if [[ "$GITLAB_URL" == https://* ]]; then
        local host
        host=$(echo "$GITLAB_URL" | sed 's|https://||' | cut -d/ -f1 | cut -d: -f1)
        local port
        port=$(echo "$GITLAB_URL" | grep -oP ':\K[0-9]+$' || echo "443")

        local expiry
        expiry=$(echo | openssl s_client -servername "$host" -connect "${host}:${port}" 2>/dev/null | \
                 openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) || expiry=""

        if [[ -n "$expiry" ]]; then
            local expiry_epoch
            expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null) || expiry_epoch=0
            local now_epoch
            now_epoch=$(date +%s)
            local days_left=$(( (expiry_epoch - now_epoch) / 86400 ))

            if [[ $days_left -gt 30 ]]; then
                record_pass "TLS certificate valid" "${days_left} days remaining"
            elif [[ $days_left -gt 0 ]]; then
                record_pass "TLS certificate valid" "${days_left} days remaining (renew soon)"
            else
                record_fail "TLS certificate valid" "expired or expiring in ${days_left} days"
            fi
        else
            record_skip "TLS certificate check" "could not retrieve certificate"
        fi
    else
        record_skip "TLS certificate check" "not using HTTPS"
    fi
}

# ── 2. API ────────────────────────────────────────────────────────────
test_api() {
    echo ""
    echo -e "${BOLD}API${RESET}"

    # 2a. Version endpoint
    local version_json
    version_json=$(api_curl GET "/version" 2>/dev/null) || version_json=""

    local gl_version
    gl_version=$(json_value_string "version" "$version_json")
    local gl_revision
    gl_revision=$(json_value_string "revision" "$version_json")

    if [[ -n "$gl_version" ]]; then
        record_pass "API version endpoint" "GitLab ${gl_version} (${gl_revision})"
    else
        record_fail "API version endpoint" "no version returned"
    fi

    # 2b. Authentication
    local auth_status
    auth_status=$(api_curl_status GET "/user")
    if [[ "$auth_status" == "200" ]]; then
        local user_json
        user_json=$(api_curl GET "/user")
        local username
        username=$(json_value_string "username" "$user_json")
        record_pass "API authentication" "authenticated as ${username}"
    elif [[ "$auth_status" == "401" ]]; then
        record_fail "API authentication" "token rejected (HTTP 401)"
    else
        record_fail "API authentication" "HTTP ${auth_status}"
    fi

    # 2c. List projects (verify database queries work)
    local projects_status
    projects_status=$(api_curl_status GET "/projects?per_page=1")
    if [[ "$projects_status" == "200" ]]; then
        record_pass "API list projects" "database responding"
    else
        record_fail "API list projects" "HTTP ${projects_status}"
    fi

    # 2d. List users
    local users_status
    users_status=$(api_curl_status GET "/users?per_page=1")
    if [[ "$users_status" == "200" ]]; then
        record_pass "API list users" "user directory accessible"
    else
        record_fail "API list users" "HTTP ${users_status}"
    fi

    # 2e. Sidekiq health (job processing)
    local sidekiq_json
    sidekiq_json=$(api_curl GET "/sidekiq/compound_metrics" 2>/dev/null) || sidekiq_json=""

    if [[ -n "$sidekiq_json" && "$sidekiq_json" != *"error"* ]]; then
        local processes
        processes=$(echo "$sidekiq_json" | { grep -oP '"hostname"\s*:' || true; } | wc -l)
        record_pass "Sidekiq running" "${processes} process(es) responding"
    else
        record_fail "Sidekiq running" "could not query Sidekiq metrics"
    fi

    # 2f. Runners endpoint
    local runners_status
    runners_status=$(api_curl_status GET "/runners/all?per_page=1")
    if [[ "$runners_status" == "200" ]]; then
        record_pass "API runners endpoint" "runner management accessible"
    elif [[ "$runners_status" == "403" ]]; then
        record_skip "API runners endpoint" "token lacks admin scope"
    else
        record_fail "API runners endpoint" "HTTP ${runners_status}"
    fi

    # 2g. Search endpoint
    local search_status
    search_status=$(api_curl_status GET "/search?scope=projects&search=test")
    if [[ "$search_status" == "200" ]]; then
        record_pass "API search" "search index responding"
    elif [[ "$search_status" == "403" ]]; then
        record_skip "API search" "search disabled or token lacks scope"
    else
        record_fail "API search" "HTTP ${search_status}"
    fi
}

# ── 3. Git Operations ────────────────────────────────────────────────
test_git() {
    if [[ "$SKIP_GIT" == "true" ]]; then
        echo ""
        echo -e "${BOLD}Git Operations${RESET}"
        record_skip "Git clone" "SKIP_GIT=true"
        record_skip "Git push" "SKIP_GIT=true"
        return
    fi

    echo ""
    echo -e "${BOLD}Git Operations${RESET}"

    # Create a test project via API
    local project_name
    project_name="${SMOKE_PROJECT_PREFIX}-$(date +%s)"
    local create_json
    create_json=$(api_curl POST "/projects" -d "{\"name\":\"${project_name}\",\"visibility\":\"private\",\"initialize_with_readme\":true}")

    local project_id
    project_id=$(json_value "id" "$create_json")
    local http_url
    http_url=$(json_value_string "http_url_to_repo" "$create_json")

    if [[ -z "$project_id" || "$project_id" == "null" ]]; then
        record_fail "Create test project" "API returned: $(echo "$create_json" | head -c 200)"
        record_skip "Git clone" "no test project"
        record_skip "Git push" "no test project"
        return
    fi

    CLEANUP_PROJECT_ID="$project_id"
    record_pass "Create test project" "${project_name} (ID: ${project_id})"

    # Clone
    TMPDIR_SMOKE=$(mktemp -d)

    # Fallback: if http_url_to_repo wasn't parsed, construct it
    if [[ -z "$http_url" ]]; then
        http_url="${GITLAB_URL}/${GITLAB_USER}/${project_name}.git"
        verbose "http_url_to_repo not found in API response, constructed: ${http_url}"
    fi
    verbose "Clone URL (from API): ${http_url}"

    # Replace the hostname in the API-returned URL with GITLAB_URL
    # (the API may return an internal hostname that's unreachable remotely)
    local api_origin
    api_origin=$(echo "$http_url" | grep -oP 'https?://[^/]+')
    if [[ -n "$api_origin" && "$api_origin" != "$GITLAB_URL" ]]; then
        http_url="${http_url/$api_origin/$GITLAB_URL}"
        verbose "Rewrote clone URL to: ${http_url}"
    fi

    local clone_url
    # Inject token into URL for HTTPS clone
    if [[ "$http_url" == https://* ]]; then
        clone_url="https://oauth2:${GITLAB_TOKEN}@${http_url#https://}"
    elif [[ "$http_url" == http://* ]]; then
        clone_url="http://oauth2:${GITLAB_TOKEN}@${http_url#http://}"
    else
        clone_url="$http_url"
    fi

    local git_opts=()
    [[ "$CURL_INSECURE" == "true" ]] && git_opts+=(-c http.sslVerify=false)

    # Brief wait for repository initialization (initialize_with_readme is async)
    sleep 2

    verbose "Running: git clone <url> ${TMPDIR_SMOKE}/repo"
    local clone_err clone_rc
    clone_err=$(git ${git_opts[@]+"${git_opts[@]}"} clone "$clone_url" "${TMPDIR_SMOKE}/repo" 2>&1) && clone_rc=0 || clone_rc=$?
    if [[ "$clone_rc" -eq 0 ]]; then
        GIT_CLONE_OK="true"
        record_pass "Git clone (HTTPS)" "Gitaly responding"
    else
        local short_err redacted_url
        short_err=$(echo "$clone_err" | grep -i -E 'fatal|error' | head -1 | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
        redacted_url=$(echo "$http_url" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
        verbose "Full clone output: $(echo "$clone_err" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")"
        local redacted_clone
        redacted_clone=$(echo "$clone_url" | sed "s|${GITLAB_TOKEN}|[REDACTED]|g")
        verbose "Attempted URL: ${redacted_clone}"
        record_fail "Git clone (HTTPS)" "${short_err:-clone failed (exit $clone_rc)}"
        return
    fi

    # Push a commit
    pushd "${TMPDIR_SMOKE}/repo" >/dev/null
    git config user.email "smoke-test@example.com"
    git config user.name "Smoke Test"
    echo "smoke test $(date -u +%Y-%m-%dT%H:%M:%SZ)" > smoke-test.txt
    git add smoke-test.txt
    git commit -m "smoke test commit" >/dev/null 2>&1

    if git ${git_opts[@]+"${git_opts[@]}"} push origin main >/dev/null 2>&1 || \
       git ${git_opts[@]+"${git_opts[@]}"} push origin master >/dev/null 2>&1; then
        record_pass "Git push (HTTPS)" "write to Gitaly succeeded"
    else
        record_fail "Git push (HTTPS)" "push failed"
    fi
    popd >/dev/null
}

# ── 4. Container Registry ────────────────────────────────────────────
test_registry() {
    if [[ "$SKIP_REGISTRY" == "true" ]]; then
        echo ""
        echo -e "${BOLD}Container Registry${RESET}"
        record_skip "Registry API" "SKIP_REGISTRY=true"
        return
    fi

    echo ""
    echo -e "${BOLD}Container Registry${RESET}"

    # Check if registry is enabled via application settings API
    local registry_enabled=""
    local settings_json
    settings_json=$(api_curl GET "/application/settings" 2>/dev/null) || settings_json=""

    if [[ -n "$settings_json" ]]; then
        registry_enabled=$(json_value "container_registry_enabled" "$settings_json" 2>/dev/null || echo "")
    fi

    if [[ "$registry_enabled" == "false" ]]; then
        record_skip "Registry API reachable" "container registry disabled in application settings"
        record_skip "Registry project endpoint" "container registry disabled in application settings"
        return
    fi

    # Try the registry v2 API endpoint
    local host
    host=$(echo "$GITLAB_URL" | sed 's|https\?://||' | cut -d/ -f1)

    local curl_opts=(-s -o /dev/null -w "%{http_code}" --max-time "$CURL_TIMEOUT")
    [[ "$CURL_INSECURE" == "true" ]] && curl_opts+=(-k)

    local registry_status
    registry_status=$(curl "${curl_opts[@]}" "${GITLAB_URL}:5050/v2/" 2>/dev/null) || \
    registry_status=$(curl "${curl_opts[@]}" "https://${host}:5050/v2/" 2>/dev/null) || \
    registry_status=$(curl "${curl_opts[@]}" "https://registry.${host}/v2/" 2>/dev/null) || \
    registry_status="000"

    if [[ "$registry_status" == "200" || "$registry_status" == "401" ]]; then
        record_pass "Registry API reachable" "HTTP ${registry_status}"
    elif [[ "$registry_status" == "000" ]]; then
        if [[ "$registry_enabled" == "true" ]]; then
            record_fail "Registry API reachable" "enabled in settings but not reachable at standard ports/hosts"
        else
            record_skip "Registry API reachable" "not found at standard ports/hosts (settings unreadable — may need admin token)"
        fi
    else
        record_fail "Registry API reachable" "HTTP ${registry_status}"
    fi

    # Check registry via GitLab API (project-level)
    if [[ -n "$CLEANUP_PROJECT_ID" ]]; then
        local reg_status
        reg_status=$(api_curl_status GET "/projects/${CLEANUP_PROJECT_ID}/registry/repositories")
        if [[ "$reg_status" == "200" ]]; then
            record_pass "Registry project endpoint" "project registry accessible"
        elif [[ "$reg_status" == "404" ]]; then
            record_skip "Registry project endpoint" "container registry not enabled for project"
        else
            record_fail "Registry project endpoint" "HTTP ${reg_status}"
        fi
    fi
}

# ── 5. CI/CD ──────────────────────────────────────────────────────────
test_cicd() {
    echo ""
    echo -e "${BOLD}CI/CD${RESET}"

    # Check runners
    local runners_json
    runners_json=$(api_curl GET "/runners/all?per_page=100" 2>/dev/null) || runners_json=""

    if [[ "$runners_json" == "["* ]]; then
        local runner_count
        runner_count=$(echo "$runners_json" | { grep -oP '"id"\s*:' || true; } | wc -l)
        local online_count
        online_count=$(echo "$runners_json" | { grep -oP '"status"\s*:\s*"online"' || true; } | wc -l)

        if [[ $online_count -gt 0 ]]; then
            record_pass "CI/CD runners online" "${online_count}/${runner_count} runners online"
        elif [[ $runner_count -gt 0 ]]; then
            record_fail "CI/CD runners online" "0/${runner_count} runners online"
        else
            record_skip "CI/CD runners online" "no runners registered"
        fi
    else
        record_skip "CI/CD runners" "could not query runners (admin token required)"
    fi

    # Check CI/CD settings via API
    local cicd_status
    cicd_status=$(api_curl_status GET "/application/settings")
    if [[ "$cicd_status" == "200" ]]; then
        record_pass "CI/CD settings accessible" "application settings readable"
    elif [[ "$cicd_status" == "403" ]]; then
        record_skip "CI/CD settings accessible" "admin token required"
    else
        record_fail "CI/CD settings accessible" "HTTP ${cicd_status}"
    fi
}

# ── 6. Background Migrations ─────────────────────────────────────────
test_migrations() {
    echo ""
    echo -e "${BOLD}Background Migrations${RESET}"

    # Batched background migrations (admin only)
    local migrations_json
    migrations_json=$(api_curl GET "/admin/batched_background_migrations?database=main" 2>/dev/null) || migrations_json=""

    if [[ "$migrations_json" == "["* ]]; then
        local total_mig
        total_mig=$(echo "$migrations_json" | { grep -oP '"id"\s*:' || true; } | wc -l)

        local failed_mig
        failed_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"failed"' || true; } | wc -l)
        local active_mig
        active_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"active"' || true; } | wc -l)
        local paused_mig
        paused_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"paused"' || true; } | wc -l)
        local finalized_mig
        finalized_mig=$(echo "$migrations_json" | { grep -oP '"status"\s*:\s*"finished"' || true; } | wc -l)

        if [[ $failed_mig -gt 0 ]]; then
            record_fail "Background migrations" "${failed_mig} failed, ${active_mig} active, ${paused_mig} paused, ${finalized_mig} finished of ${total_mig}"
        elif [[ $paused_mig -gt 0 ]]; then
            record_fail "Background migrations" "${paused_mig} paused, ${active_mig} active, ${finalized_mig} finished of ${total_mig}"
        elif [[ $active_mig -gt 0 ]]; then
            record_pass "Background migrations" "${active_mig} active, ${finalized_mig} finished of ${total_mig} (in progress)"
        else
            record_pass "Background migrations" "all ${total_mig} finished"
        fi
    else
        local mig_status
        mig_status=$(api_curl_status GET "/admin/batched_background_migrations?database=main")
        if [[ "$mig_status" == "403" ]]; then
            record_skip "Background migrations" "admin token required"
        else
            record_skip "Background migrations" "could not query (HTTP ${mig_status})"
        fi
    fi
}

# ── 7. Storage & Components ──────────────────────────────────────────
test_components() {
    echo ""
    echo -e "${BOLD}Components${RESET}"

    # Metadata endpoint
    local metadata_json
    metadata_json=$(api_curl GET "/metadata" 2>/dev/null) || metadata_json=""

    if [[ -n "$metadata_json" ]]; then
        local gl_version
        gl_version=$(json_value_string "version" "$metadata_json")
        local enterprise
        enterprise=$(json_value "enterprise" "$metadata_json")

        if [[ -n "$gl_version" ]]; then
            local edition="CE"
            [[ "$enterprise" == "true" ]] && edition="EE"
            record_pass "GitLab metadata" "${gl_version} ${edition}"
        else
            record_pass "GitLab metadata" "endpoint reachable"
        fi
    else
        record_skip "GitLab metadata" "metadata endpoint not available"
    fi

    # Statistics (admin)
    local stats_json
    stats_json=$(api_curl GET "/application/statistics" 2>/dev/null) || stats_json=""

    if [[ -n "$stats_json" && "$stats_json" != *"error"* && "$stats_json" != *"403"* ]]; then
        local active_users
        active_users=$(json_value "active_users" "$stats_json")
        local projects
        projects=$(json_value "projects" "$stats_json")
        local groups
        groups=$(json_value "groups" "$stats_json")

        if [[ -n "$active_users" ]]; then
            record_pass "Instance statistics" "${active_users} users, ${projects} projects, ${groups} groups"
        else
            record_pass "Instance statistics" "endpoint reachable"
        fi
    else
        record_skip "Instance statistics" "admin token required"
    fi

    # Gitaly check — only report pass if clone actually succeeded
    if [[ "$GIT_CLONE_OK" == "true" ]]; then
        record_pass "Gitaly storage" "project created and cloned successfully"
    elif [[ -n "$CLEANUP_PROJECT_ID" ]]; then
        record_skip "Gitaly storage" "project created but clone was not tested or failed"
    fi

    # PostgreSQL (inferred from API responsiveness)
    local pg_test
    pg_test=$(api_curl_status GET "/projects?per_page=1&order_by=updated_at")
    if [[ "$pg_test" == "200" ]]; then
        record_pass "PostgreSQL" "database queries succeeding"
    else
        record_fail "PostgreSQL" "sorted query failed (HTTP ${pg_test})"
    fi

    # Redis (inferred from session/cache)
    local redis_test
    redis_test=$(api_curl_status GET "/user")
    if [[ "$redis_test" == "200" ]]; then
        record_pass "Redis" "session/cache operational (auth succeeded)"
    else
        record_skip "Redis" "cannot verify independently"
    fi
}

# ══════════════════════════════════════════════════════════════════════
#                          OUTPUT
# ══════════════════════════════════════════════════════════════════════

print_summary() {
    local end_time
    end_time=$(date +%s)
    local duration=$(( end_time - START_TIME ))

    echo ""
    echo -e "${BOLD}────────────────────────────────────────${RESET}"
    echo -e "${BOLD}Summary${RESET}  ${GITLAB_URL}"
    echo -e "  ${GREEN}${PASS} passed${RESET}  ${RED}${FAIL} failed${RESET}  ${YELLOW}${SKIP} skipped${RESET}  (${duration}s)"
    echo -e "${BOLD}────────────────────────────────────────${RESET}"

    if [[ $FAIL -eq 0 ]]; then
        echo -e "${GREEN}${BOLD}All tests passed.${RESET}"
    else
        echo -e "${RED}${BOLD}${FAIL} test(s) failed.${RESET}"
    fi
}

print_tap_header() {
    echo "TAP version 13"
}

print_tap_footer() {
    echo "1..${TOTAL}"
    echo "# pass ${PASS}"
    echo "# fail ${FAIL}"
    echo "# skip ${SKIP}"
}

write_junit() {
    local end_time
    end_time=$(date +%s)
    local duration=$(( end_time - START_TIME ))

    cat > "$JUNIT_FILE" <<JUNIT_EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="${TOTAL}" failures="${FAIL}" skipped="${SKIP}" time="${duration}">
  <testsuite name="gitlab-smoke-tests" tests="${TOTAL}" failures="${FAIL}" skipped="${SKIP}" time="${duration}">
JUNIT_EOF

    for result in "${RESULTS[@]}"; do
        local status name detail
        status=$(echo "$result" | cut -d'|' -f1)
        name=$(echo "$result" | cut -d'|' -f2)
        detail=$(echo "$result" | cut -d'|' -f3)

        # XML-escape the values
        name=$(echo "$name" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')
        detail=$(echo "$detail" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')

        case "$status" in
            PASS)
                echo "    <testcase name=\"${name}\" classname=\"smoke\">" >> "$JUNIT_FILE"
                [[ -n "$detail" ]] && echo "      <system-out>${detail}</system-out>" >> "$JUNIT_FILE"
                echo "    </testcase>" >> "$JUNIT_FILE"
                ;;
            FAIL)
                echo "    <testcase name=\"${name}\" classname=\"smoke\">" >> "$JUNIT_FILE"
                echo "      <failure message=\"${detail}\">FAILED: ${name} — ${detail}</failure>" >> "$JUNIT_FILE"
                echo "    </testcase>" >> "$JUNIT_FILE"
                ;;
            SKIP)
                echo "    <testcase name=\"${name}\" classname=\"smoke\">" >> "$JUNIT_FILE"
                echo "      <skipped message=\"${detail}\"/>" >> "$JUNIT_FILE"
                echo "    </testcase>" >> "$JUNIT_FILE"
                ;;
        esac
    done

    echo "  </testsuite>" >> "$JUNIT_FILE"
    echo "</testsuites>" >> "$JUNIT_FILE"

    log "JUnit report written to ${JUNIT_FILE}"
}

# ══════════════════════════════════════════════════════════════════════
#                          MAIN
# ══════════════════════════════════════════════════════════════════════

usage() {
    cat <<EOF
Usage: $(basename "$0") [OPTIONS]

Smoke-test a GitLab instance. Zero external dependencies — bash, curl, git, openssl only.
Designed for air-gapped environments.

Required environment variables:
  GITLAB_URL          GitLab base URL (https://gitlab.example.com)
  GITLAB_TOKEN        Personal access token (api scope; admin for full coverage)

Optional environment variables:
  GITLAB_HEALTH_TOKEN Health check access token (for /-/health, /-/readiness, /-/liveness)

Options:
  --skip-git          Skip git clone/push tests
  --skip-registry     Skip container registry tests
  --skip-cleanup      Don't delete the test project after run
  --insecure          Allow self-signed TLS certificates (-k)
  --timeout N         curl timeout in seconds (default: 10)
  --format FORMAT     Output: text (default), tap, junit
  --junit-file FILE   JUnit output path (default: smoke-results.xml)
  --verbose           Show debug output
  --no-color          Disable colored output
  --help              Show this help

Examples:
  # Basic run
  export GITLAB_URL="https://gitlab.example.com"
  export GITLAB_TOKEN="glpat-xxxxxxxxxxxxxxxxxxxx"
  ./$(basename "$0")

  # Air-gapped with self-signed cert, JUnit output
  GITLAB_URL=https://gitlab.local GITLAB_TOKEN=glpat-xxx \\
    ./$(basename "$0") --insecure --format junit

  # Quick API-only check
  GITLAB_URL=https://gitlab.local GITLAB_TOKEN=glpat-xxx \\
    ./$(basename "$0") --skip-git --skip-registry

  # TAP output for CI pipeline
  GITLAB_URL=https://gitlab.local GITLAB_TOKEN=glpat-xxx \\
    ./$(basename "$0") --format tap
EOF
}

main() {
    # Parse arguments
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --skip-git)       SKIP_GIT=true ;;
            --skip-registry)  SKIP_REGISTRY=true ;;
            --skip-cleanup)   SKIP_CLEANUP=true ;;
            --insecure)       CURL_INSECURE=true ;;
            --timeout)        CURL_TIMEOUT="$2"; shift ;;
            --format)         OUTPUT_FORMAT="$2"; shift ;;
            --junit-file)     JUNIT_FILE="$2"; shift ;;
            --verbose)        VERBOSE=true ;;
            --no-color)       COLOR=never ;;
            --help|-h)        usage; exit 0 ;;
            *)                err "Unknown option: $1"; usage; exit 1 ;;
        esac
        shift
    done

    setup_colors

    # Validate required vars
    if [[ -z "$GITLAB_URL" ]]; then
        err "GITLAB_URL is required"
        echo ""
        usage
        exit 1
    fi

    if [[ -z "$GITLAB_TOKEN" ]]; then
        err "GITLAB_TOKEN is required"
        echo ""
        usage
        exit 1
    fi

    # Strip trailing slash
    GITLAB_URL="${GITLAB_URL%/}"

    START_TIME=$(date +%s)

    if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
        print_tap_header
    else
        echo ""
        echo -e "${BOLD}GitLab Smoke Tests${RESET}"
        echo -e "Target: ${GITLAB_URL}"
        echo -e "Time:   $(date -u +%Y-%m-%dT%H:%M:%SZ)"
        echo ""
    fi

    # Run test suites
    test_connectivity
    test_api
    test_git
    test_registry
    test_cicd
    test_migrations
    test_components

    # Output
    if [[ "$OUTPUT_FORMAT" == "tap" ]]; then
        print_tap_footer
    elif [[ "$OUTPUT_FORMAT" == "junit" ]]; then
        print_summary
        write_junit
    else
        print_summary
    fi

    # Exit code
    [[ $FAIL -eq 0 ]] && exit 0 || exit 1
}

main "$@"