#!/bin/bash ################################################################################ # Script Name: alertmanager-silence-manager.sh # Version: 1.0 # Description: CLI tool for managing Prometheus Alertmanager silences. # Create, bulk-create, extend, expire, list, audit, and export # silences via the Alertmanager API v2. Supports dry-run mode, # pattern-based operations, and YAML bulk silence files. # # Author: Phil Connor # Contact: contact@mylinux.work # Website: https://mylinux.work # License: MIT # # Prerequisites: # - curl # - jq # - Alertmanager running and accessible # # Usage: # # Create a single silence # ./alertmanager-silence-manager.sh create --matcher 'alertname=HighCPU' --duration 2h --comment "Maintenance" # # # Bulk create from YAML # ./alertmanager-silence-manager.sh bulk-create --file maintenance.yaml # # # List active silences # ./alertmanager-silence-manager.sh list --state active # # # Extend a silence # ./alertmanager-silence-manager.sh extend --id abc12345 --duration 1h # # # Expire a silence # ./alertmanager-silence-manager.sh expire --id abc12345 # # # Export active silences to YAML # ./alertmanager-silence-manager.sh export --output silences.yaml # # # Audit silences # ./alertmanager-silence-manager.sh audit # # Configuration: # ALERTMANAGER_URL Alertmanager base URL (default: http://localhost:9093) # SILENCE_AUTHOR Author name for silences (default: current user) # SILENCE_COMMENT_PREFIX Prefix for all silence comments (default: none) # ################################################################################ # ============================================================================ # CONFIGURATION VARIABLES # ============================================================================ AM_URL="${ALERTMANAGER_URL:-http://localhost:9093}" AUTHOR="${SILENCE_AUTHOR:-$(whoami)}" COMMENT_PREFIX="${SILENCE_COMMENT_PREFIX:-}" DRY_RUN=false # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # ============================================================================ # HELPER FUNCTIONS # ============================================================================ show_usage() { cat < [OPTIONS] Manage Prometheus Alertmanager silences via the API. COMMANDS: create Create a single silence bulk-create Create silences from a YAML file list List silences in table format extend Extend a silence by duration expire Expire silences by ID or pattern export Export active silences to YAML audit Show detailed silence audit info CREATE OPTIONS: --matcher STR Label matcher (e.g., 'alertname=HighCPU'), repeatable --duration STR Duration (e.g., 2h, 30m, 1d) --comment STR Silence comment/reason --dry-run Preview without creating BULK-CREATE OPTIONS: --file PATH Path to YAML silence definitions --dry-run Preview without creating LIST OPTIONS: --state STR Filter: active, pending, expired, all (default: active) EXTEND OPTIONS: --id ID Silence ID to extend --duration STR Additional duration (e.g., 1h) --dry-run Preview without extending EXPIRE OPTIONS: --id ID Silence ID to expire --match STR Pattern match on comment (e.g., 'comment=~maintenance.*') --dry-run Preview without expiring EXPORT OPTIONS: --output PATH Output file (default: stdout) ENVIRONMENT VARIABLES: ALERTMANAGER_URL Base URL (default: http://localhost:9093) SILENCE_AUTHOR Author name (default: current user) SILENCE_COMMENT_PREFIX Prefix for comments EXAMPLES: $0 create --matcher 'alertname=HighCPU' --matcher 'instance=web-01' --duration 2h --comment "Maintenance" $0 bulk-create --file maintenance-window.yaml --dry-run $0 list --state active $0 extend --id 4a2f8c3e --duration 1h $0 expire --match 'comment=~maintenance.*' $0 export --output silences-backup.yaml $0 audit EOF exit 0 } log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } log_dry_run() { echo -e "${YELLOW}[DRY RUN]${NC} $*"; } check_requirements() { local missing=0 if ! command -v curl >/dev/null 2>&1; then log_error "curl not found" missing=1 fi if ! command -v jq >/dev/null 2>&1; then log_error "jq not found" missing=1 fi return $missing } check_connectivity() { local status status=$(curl -sf --connect-timeout 5 --max-time 10 -o /dev/null -w "%{http_code}" "${AM_URL}/api/v2/status" 2>/dev/null) if [ "$status" != "200" ]; then log_error "Cannot reach Alertmanager at ${AM_URL} (HTTP $status)" exit 1 fi } # Query Alertmanager API # Args: $1 - method, $2 - endpoint, $3 - data (optional) am_api() { local method="$1" endpoint="$2" data="$3" if [ -n "$data" ]; then curl -sf --connect-timeout 5 --max-time 15 \ -X "$method" \ -H "Content-Type: application/json" \ -d "$data" \ "${AM_URL}${endpoint}" 2>/dev/null else curl -sf --connect-timeout 5 --max-time 15 \ -X "$method" \ "${AM_URL}${endpoint}" 2>/dev/null fi } # Parse duration string to seconds # Supports: 30s, 5m, 2h, 1d parse_duration() { local input="$1" local num unit num=$(echo "$input" | sed 's/[^0-9]//g') unit=$(echo "$input" | sed 's/[0-9]//g') case "$unit" in s) echo "$num" ;; m) echo $((num * 60)) ;; h) echo $((num * 3600)) ;; d) echo $((num * 86400)) ;; *) log_error "Invalid duration unit: $unit (use s/m/h/d)"; return 1 ;; esac } # Get ISO 8601 timestamp for now + offset seconds # Args: $1 - offset in seconds (default: 0) iso_timestamp() { local offset="${1:-0}" date -u -d "+${offset} seconds" "+%Y-%m-%dT%H:%M:%S.000Z" 2>/dev/null || \ date -u -v "+${offset}S" "+%Y-%m-%dT%H:%M:%S.000Z" 2>/dev/null } # Parse a matcher string like 'alertname=HighCPU' or 'instance=~web-0[1-3]' # Returns JSON object parse_matcher() { local input="$1" local name value is_regex is_equal if [[ "$input" == *"=~"* ]]; then name="${input%%=~*}" value="${input#*=~}" is_regex="true" is_equal="true" elif [[ "$input" == *"!~"* ]]; then name="${input%%!~*}" value="${input#*!~}" is_regex="true" is_equal="false" elif [[ "$input" == *"!="* ]]; then name="${input%%!=*}" value="${input#*!=}" is_regex="false" is_equal="false" elif [[ "$input" == *"="* ]]; then name="${input%%=*}" value="${input#*=}" is_regex="false" is_equal="true" else log_error "Invalid matcher format: $input" return 1 fi jq -n --arg n "$name" --arg v "$value" \ --argjson r "$is_regex" --argjson e "$is_equal" \ '{name: $n, value: $v, isRegex: $r, isEqual: $e}' } # Truncate string to max length truncate() { local str="$1" max="${2:-30}" if [ ${#str} -gt "$max" ]; then echo "${str:0:$((max-2))}.." else echo "$str" fi } # ============================================================================ # CREATE COMMAND # ============================================================================ cmd_create() { local matchers_json="[]" local duration="" local comment="" while [[ $# -gt 0 ]]; do case $1 in --matcher) local m m=$(parse_matcher "$2") || exit 1 matchers_json=$(echo "$matchers_json" | jq --argjson m "$m" '. + [$m]') shift 2 ;; --duration) duration="$2"; shift 2 ;; --comment) comment="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; *) log_error "Unknown option for create: $1"; exit 1 ;; esac done if [ "$(echo "$matchers_json" | jq 'length')" -eq 0 ]; then log_error "At least one --matcher is required" exit 1 fi if [ -z "$duration" ]; then log_error "--duration is required" exit 1 fi if [ -z "$comment" ]; then log_error "--comment is required" exit 1 fi local duration_secs duration_secs=$(parse_duration "$duration") || exit 1 local starts_at ends_at starts_at=$(iso_timestamp 0) ends_at=$(iso_timestamp "$duration_secs") local full_comment="${COMMENT_PREFIX}${comment}" local payload payload=$(jq -n \ --argjson matchers "$matchers_json" \ --arg startsAt "$starts_at" \ --arg endsAt "$ends_at" \ --arg createdBy "$AUTHOR" \ --arg comment "$full_comment" \ '{matchers: $matchers, startsAt: $startsAt, endsAt: $endsAt, createdBy: $createdBy, comment: $comment}') if [ "$DRY_RUN" = true ]; then log_dry_run "Would create silence:" echo "$payload" | jq . return fi local response response=$(am_api POST "/api/v2/silences" "$payload") if [ $? -eq 0 ] && [ -n "$response" ]; then local sid sid=$(echo "$response" | jq -r '.silenceID // empty') if [ -n "$sid" ]; then log_info "Silence created: ${sid}" else log_info "Silence created" fi else log_error "Failed to create silence" exit 1 fi } # ============================================================================ # BULK-CREATE COMMAND # ============================================================================ cmd_bulk_create() { local file="" while [[ $# -gt 0 ]]; do case $1 in --file) file="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; *) log_error "Unknown option for bulk-create: $1"; exit 1 ;; esac done if [ -z "$file" ] || [ ! -f "$file" ]; then log_error "Valid --file is required" exit 1 fi local success=0 failed=0 local in_silence=false local in_matchers=false local matchers_json="[]" local duration="" comment="" local current_matcher_name="" current_matcher_value="" current_matcher_regex="false" flush_silence() { if [ -z "$duration" ] || [ "$(echo "$matchers_json" | jq 'length')" -eq 0 ]; then return fi local duration_secs starts_at ends_at full_comment payload duration_secs=$(parse_duration "$duration") || { ((failed++)); return; } starts_at=$(iso_timestamp 0) ends_at=$(iso_timestamp "$duration_secs") full_comment="${COMMENT_PREFIX}${comment}" payload=$(jq -n \ --argjson matchers "$matchers_json" \ --arg startsAt "$starts_at" \ --arg endsAt "$ends_at" \ --arg createdBy "$AUTHOR" \ --arg comment "$full_comment" \ '{matchers: $matchers, startsAt: $startsAt, endsAt: $endsAt, createdBy: $createdBy, comment: $comment}') if [ "$DRY_RUN" = true ]; then log_dry_run "Would create silence:" echo "$payload" | jq . echo "" ((success++)) return fi local response response=$(am_api POST "/api/v2/silences" "$payload") if [ $? -eq 0 ] && [ -n "$response" ]; then local sid sid=$(echo "$response" | jq -r '.silenceID // empty') log_info "Silence created: ${sid:-ok} (${comment})" ((success++)) else log_error "Failed to create silence: ${comment}" ((failed++)) fi } flush_matcher() { if [ -n "$current_matcher_name" ]; then local m m=$(jq -n --arg n "$current_matcher_name" --arg v "$current_matcher_value" \ --argjson r "$current_matcher_regex" \ '{name: $n, value: $v, isRegex: $r, isEqual: true}') matchers_json=$(echo "$matchers_json" | jq --argjson m "$m" '. + [$m]') current_matcher_name="" current_matcher_value="" current_matcher_regex="false" fi } while IFS= read -r line || [ -n "$line" ]; do # Strip leading/trailing whitespace for comparison local trimmed trimmed=$(echo "$line" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//') # Skip comments and empty lines [[ "$trimmed" =~ ^# ]] && continue [ -z "$trimmed" ] && continue # New silence block if [[ "$trimmed" == "- matchers:" ]]; then flush_matcher if [ "$in_silence" = true ]; then flush_silence fi in_silence=true in_matchers=true matchers_json="[]" duration="" comment="" continue fi # Matcher entry if [[ "$trimmed" == "- name:"* ]] && [ "$in_matchers" = true ]; then flush_matcher current_matcher_name=$(echo "$trimmed" | sed 's/^- name:[[:space:]]*//') continue fi if [[ "$trimmed" == "value:"* ]] && [ "$in_matchers" = true ]; then current_matcher_value=$(echo "$trimmed" | sed 's/^value:[[:space:]]*//') continue fi if [[ "$trimmed" == "isRegex:"* ]] && [ "$in_matchers" = true ]; then current_matcher_regex=$(echo "$trimmed" | sed 's/^isRegex:[[:space:]]*//') continue fi # Duration if [[ "$trimmed" == "duration:"* ]]; then flush_matcher in_matchers=false duration=$(echo "$trimmed" | sed 's/^duration:[[:space:]]*//') continue fi # Comment if [[ "$trimmed" == "comment:"* ]]; then comment=$(echo "$trimmed" | sed 's/^comment:[[:space:]]*//' | sed 's/^"//' | sed 's/"$//') continue fi done < "$file" # Flush last silence flush_matcher if [ "$in_silence" = true ]; then flush_silence fi echo "" log_info "Bulk create complete: ${success} succeeded, ${failed} failed" } # ============================================================================ # LIST COMMAND # ============================================================================ cmd_list() { local state="active" while [[ $# -gt 0 ]]; do case $1 in --state) state="$2"; shift 2 ;; *) log_error "Unknown option for list: $1"; exit 1 ;; esac done local silences_json silences_json=$(am_api GET "/api/v2/silences") if [ -z "$silences_json" ]; then log_error "Failed to fetch silences" exit 1 fi # Filter by state local filtered if [ "$state" = "all" ]; then filtered="$silences_json" else filtered=$(echo "$silences_json" | jq --arg s "$state" '[.[] | select(.status.state == $s)]') fi local count count=$(echo "$filtered" | jq 'length') if [ "$count" -eq 0 ]; then log_info "No ${state} silences found" return fi printf "${BLUE}%-10s %-10s %-12s %-30s %-20s %-20s %s${NC}\n" \ "ID" "STATE" "AUTHOR" "MATCHERS" "STARTS" "ENDS" "COMMENT" printf '%.0s-' {1..120} echo "" echo "$filtered" | jq -r '.[] | [ .id[0:8], .status.state, .createdBy, ([.matchers[] | "\(.name)=\(.value)"] | join(", ")), .startsAt[0:19], .endsAt[0:19], .comment ] | @tsv' | while IFS=$'\t' read -r id st author matchers starts ends comment; do printf "%-10s %-10s %-12s %-30s %-20s %-20s %s\n" \ "$id" "$st" "$(truncate "$author" 12)" "$(truncate "$matchers" 30)" \ "$starts" "$ends" "$(truncate "$comment" 40)" done echo "" log_info "${count} silence(s) found" } # ============================================================================ # EXTEND COMMAND # ============================================================================ cmd_extend() { local silence_id="" duration="" while [[ $# -gt 0 ]]; do case $1 in --id) silence_id="$2"; shift 2 ;; --duration) duration="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; *) log_error "Unknown option for extend: $1"; exit 1 ;; esac done if [ -z "$silence_id" ]; then log_error "--id is required" exit 1 fi if [ -z "$duration" ]; then log_error "--duration is required" exit 1 fi local duration_secs duration_secs=$(parse_duration "$duration") || exit 1 # Find the silence (match by prefix) local silences_json silence silences_json=$(am_api GET "/api/v2/silences") silence=$(echo "$silences_json" | jq --arg id "$silence_id" '[.[] | select(.id | startswith($id)) | select(.status.state == "active")] | first // empty') if [ -z "$silence" ] || [ "$silence" = "null" ]; then log_error "Active silence not found matching ID: ${silence_id}" exit 1 fi local full_id full_id=$(echo "$silence" | jq -r '.id') # Build new silence with extended endsAt local new_ends_at new_ends_at=$(iso_timestamp "$duration_secs") local payload payload=$(echo "$silence" | jq --arg endsAt "$new_ends_at" \ 'del(.id, .status, .updatedAt) | .endsAt = $endsAt') if [ "$DRY_RUN" = true ]; then log_dry_run "Would extend silence ${full_id} to ${new_ends_at}:" echo "$payload" | jq . return fi local response response=$(am_api POST "/api/v2/silences" "$payload") if [ $? -eq 0 ] && [ -n "$response" ]; then local new_id new_id=$(echo "$response" | jq -r '.silenceID // empty') log_info "Silence extended: ${new_id:-ok} (new end: ${new_ends_at})" else log_error "Failed to extend silence" exit 1 fi } # ============================================================================ # EXPIRE COMMAND # ============================================================================ cmd_expire() { local silence_id="" match_pattern="" while [[ $# -gt 0 ]]; do case $1 in --id) silence_id="$2"; shift 2 ;; --match) match_pattern="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; *) log_error "Unknown option for expire: $1"; exit 1 ;; esac done if [ -z "$silence_id" ] && [ -z "$match_pattern" ]; then log_error "Either --id or --match is required" exit 1 fi local silences_json silences_json=$(am_api GET "/api/v2/silences") if [ -z "$silences_json" ]; then log_error "Failed to fetch silences" exit 1 fi local ids_to_expire=() if [ -n "$silence_id" ]; then # Match by ID prefix local matched matched=$(echo "$silences_json" | jq -r --arg id "$silence_id" \ '.[] | select(.id | startswith($id)) | select(.status.state == "active") | .id') while IFS= read -r id; do [ -n "$id" ] && ids_to_expire+=("$id") done <<< "$matched" fi if [ -n "$match_pattern" ]; then # Parse pattern: 'comment=~regex' local field pattern if [[ "$match_pattern" == *"=~"* ]]; then field="${match_pattern%%=~*}" pattern="${match_pattern#*=~}" else log_error "Match pattern must use =~ syntax (e.g., 'comment=~maintenance.*')" exit 1 fi local matched if [ "$field" = "comment" ]; then matched=$(echo "$silences_json" | jq -r --arg p "$pattern" \ '.[] | select(.status.state == "active") | select(.comment | test($p)) | .id') else matched=$(echo "$silences_json" | jq -r --arg f "$field" --arg p "$pattern" \ '.[] | select(.status.state == "active") | select(.matchers[] | select(.name == $f) | .value | test($p)) | .id') fi while IFS= read -r id; do [ -n "$id" ] && ids_to_expire+=("$id") done <<< "$matched" fi if [ ${#ids_to_expire[@]} -eq 0 ]; then log_warn "No matching active silences found" return fi local success=0 failed=0 for id in "${ids_to_expire[@]}"; do if [ "$DRY_RUN" = true ]; then log_dry_run "Would expire silence: ${id}" ((success++)) continue fi if curl -sf --connect-timeout 5 --max-time 10 \ -X DELETE "${AM_URL}/api/v2/silence/${id}" >/dev/null 2>&1; then log_info "Expired silence: ${id}" ((success++)) else log_error "Failed to expire silence: ${id}" ((failed++)) fi done echo "" log_info "Expire complete: ${success} succeeded, ${failed} failed" } # ============================================================================ # EXPORT COMMAND # ============================================================================ cmd_export() { local output="" while [[ $# -gt 0 ]]; do case $1 in --output) output="$2"; shift 2 ;; *) log_error "Unknown option for export: $1"; exit 1 ;; esac done local silences_json silences_json=$(am_api GET "/api/v2/silences") if [ -z "$silences_json" ]; then log_error "Failed to fetch silences" exit 1 fi local active active=$(echo "$silences_json" | jq '[.[] | select(.status.state == "active")]') local count count=$(echo "$active" | jq 'length') if [ "$count" -eq 0 ]; then log_warn "No active silences to export" return fi local yaml_output yaml_output=$(echo "$active" | jq -r ' "silences:", (.[] | " - matchers:", (.matchers[] | " - name: \(.name)", " value: \(.value)", " isRegex: \(.isRegex)" ), " duration: \( ((.endsAt | fromdateiso8601) - (.startsAt | fromdateiso8601)) | if . >= 86400 then "\(. / 86400 | floor)d" elif . >= 3600 then "\(. / 3600 | floor)h" elif . >= 60 then "\(. / 60 | floor)m" else "\(.)s" end )", " comment: \"\(.comment)\"" ) ') if [ -n "$output" ]; then echo "$yaml_output" > "$output" log_info "Exported ${count} silence(s) to ${output}" else echo "$yaml_output" fi } # ============================================================================ # AUDIT COMMAND # ============================================================================ cmd_audit() { local silences_json silences_json=$(am_api GET "/api/v2/silences") if [ -z "$silences_json" ]; then log_error "Failed to fetch silences" exit 1 fi local active active=$(echo "$silences_json" | jq '[.[] | select(.status.state == "active")]') local count count=$(echo "$active" | jq 'length') if [ "$count" -eq 0 ]; then log_info "No active silences" return fi printf "${BLUE}%-10s %-12s %-20s %-20s %-10s %-28s %s${NC}\n" \ "ID" "AUTHOR" "CREATED" "EXPIRES" "DURATION" "MATCHERS" "COMMENT" printf '%.0s-' {1..130} echo "" echo "$active" | jq -r '.[] | [ .id[0:8], .createdBy, .startsAt[0:19], .endsAt[0:19], ( ((.endsAt | fromdateiso8601) - (.startsAt | fromdateiso8601)) | if . >= 86400 then "\(. / 86400 | floor)d" elif . >= 3600 then "\(. / 3600 | floor)h" elif . >= 60 then "\(. / 60 | floor)m" else "\(.)s" end ), ([.matchers[] | "\(.name)=\(.value)"] | join(", ")), .comment ] | @tsv' | while IFS=$'\t' read -r id author created expires duration matchers comment; do printf "%-10s %-12s %-20s %-20s %-10s %-28s %s\n" \ "$id" "$(truncate "$author" 12)" "$created" "$expires" "$duration" \ "$(truncate "$matchers" 28)" "$(truncate "$comment" 40)" done echo "" log_info "${count} active silence(s)" } # ============================================================================ # MAIN # ============================================================================ main() { if [ $# -eq 0 ]; then show_usage fi check_requirements || exit 1 local command="$1" shift case "$command" in -h|--help) show_usage ;; create|bulk-create|list|extend|expire|export|audit) ;; *) log_error "Unknown command: $command"; echo ""; show_usage ;; esac # All commands except --help require connectivity check_connectivity case "$command" in create) cmd_create "$@" ;; bulk-create) cmd_bulk_create "$@" ;; list) cmd_list "$@" ;; extend) cmd_extend "$@" ;; expire) cmd_expire "$@" ;; export) cmd_export "$@" ;; audit) cmd_audit "$@" ;; esac } main "$@"