Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Script Name: clickhouse-exporter.sh
|
||||
# Version: 1.0
|
||||
# Description: Prometheus textfile exporter for ClickHouse. Pulls metrics from
|
||||
# the native Prometheus endpoint (/metrics on port 9363) and writes
|
||||
# a filtered subset to a .prom file for node_exporter's textfile
|
||||
# collector. Keeps original ClickHouse metric names for community
|
||||
# dashboard compatibility.
|
||||
#
|
||||
# Author: Phil Connor
|
||||
# Contact: contact@mylinux.work
|
||||
# Website: https://mylinux.work
|
||||
# License: MIT
|
||||
#
|
||||
# Prerequisites:
|
||||
# - curl
|
||||
# - ClickHouse Prometheus endpoint enabled (port 9363)
|
||||
#
|
||||
# Usage:
|
||||
# ./clickhouse-exporter.sh
|
||||
# ./clickhouse-exporter.sh --textfile
|
||||
# ./clickhouse-exporter.sh --http
|
||||
# CLICKHOUSE_URL="http://ch-node:9363" ./clickhouse-exporter.sh --textfile
|
||||
#
|
||||
# Parameters:
|
||||
# --textfile Write to textfile collector directory
|
||||
# --http Run as HTTP server
|
||||
# --install Create cron job for automatic collection
|
||||
# --help Show usage
|
||||
#
|
||||
# Environment:
|
||||
# CLICKHOUSE_URL ClickHouse Prometheus endpoint (default: http://localhost:9363)
|
||||
# METRICS_PATH Metrics path (default: /metrics)
|
||||
# TEXTFILE_DIR Textfile collector directory (default: /var/lib/node_exporter/textfile_collector)
|
||||
# CURL_TIMEOUT Request timeout in seconds (default: 10)
|
||||
#
|
||||
# Metrics Exported (Tier 2 — ~30 key metrics, original ClickHouse names):
|
||||
#
|
||||
# Gauges (ClickHouseMetrics_*):
|
||||
# - ClickHouseMetrics_Query
|
||||
# - ClickHouseMetrics_Merge
|
||||
# - ClickHouseMetrics_MemoryTracking
|
||||
# - ClickHouseMetrics_TCPConnection
|
||||
# - ClickHouseMetrics_HTTPConnection
|
||||
# - ClickHouseMetrics_OpenFileForRead
|
||||
# - ClickHouseMetrics_OpenFileForWrite
|
||||
# - ClickHouseMetrics_ReplicasMaxQueueSize
|
||||
# - ClickHouseMetrics_BackgroundMergesAndMutationsPoolTask
|
||||
# - ClickHouseMetrics_DelayedInserts
|
||||
#
|
||||
# Counters (ClickHouseProfileEvents_*):
|
||||
# - ClickHouseProfileEvents_Query
|
||||
# - ClickHouseProfileEvents_SelectQuery
|
||||
# - ClickHouseProfileEvents_InsertQuery
|
||||
# - ClickHouseProfileEvents_FailedQuery
|
||||
# - ClickHouseProfileEvents_InsertedRows
|
||||
# - ClickHouseProfileEvents_InsertedBytes
|
||||
# - ClickHouseProfileEvents_MergedRows
|
||||
# - ClickHouseProfileEvents_ReadCompressedBytes
|
||||
# - ClickHouseProfileEvents_CompressedReadBufferBytes
|
||||
# - ClickHouseProfileEvents_ReplicatedPartFetches
|
||||
# - ClickHouseProfileEvents_ReplicatedPartFailedFetches
|
||||
# - ClickHouseProfileEvents_DiskReadElapsedMicroseconds
|
||||
# - ClickHouseProfileEvents_DiskWriteElapsedMicroseconds
|
||||
# - ClickHouseProfileEvents_NetworkSendBytes
|
||||
# - ClickHouseProfileEvents_NetworkReceiveBytes
|
||||
# - ClickHouseProfileEvents_ZooKeeperTransactions
|
||||
# - ClickHouseProfileEvents_DNSError
|
||||
#
|
||||
# Async Metrics (ClickHouseAsyncMetrics_*):
|
||||
# - ClickHouseAsyncMetrics_Uptime
|
||||
# - ClickHouseAsyncMetrics_MaxPartCountForPartition
|
||||
# - ClickHouseAsyncMetrics_MemoryResident
|
||||
# - ClickHouseAsyncMetrics_ReplicasMaxAbsoluteDelay
|
||||
#
|
||||
# Exporter:
|
||||
# - clickhouse_exporter_up
|
||||
# - clickhouse_exporter_duration_seconds
|
||||
# - clickhouse_exporter_last_run_timestamp
|
||||
#
|
||||
################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --- Configuration ---
|
||||
readonly VERSION="1.0"
|
||||
readonly SCRIPT_NAME="$(basename "$0")"
|
||||
CLICKHOUSE_URL="${CLICKHOUSE_URL:-http://localhost:9363}"
|
||||
METRICS_PATH="${METRICS_PATH:-/metrics}"
|
||||
TEXTFILE_DIR="${TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}"
|
||||
CURL_TIMEOUT="${CURL_TIMEOUT:-10}"
|
||||
TEXTFILE_MODE=false
|
||||
HTTP_MODE=false
|
||||
HTTP_PORT=9201
|
||||
OUTPUT=""
|
||||
START_TIME=""
|
||||
|
||||
# Tier 2 metric filter — grep pattern (one metric name per line)
|
||||
readonly METRIC_FILTER='ClickHouseMetrics_Query[[:space:]]
|
||||
ClickHouseMetrics_Merge[[:space:]]
|
||||
ClickHouseMetrics_MemoryTracking[[:space:]]
|
||||
ClickHouseMetrics_TCPConnection[[:space:]]
|
||||
ClickHouseMetrics_HTTPConnection[[:space:]]
|
||||
ClickHouseMetrics_OpenFileForRead[[:space:]]
|
||||
ClickHouseMetrics_OpenFileForWrite[[:space:]]
|
||||
ClickHouseMetrics_ReplicasMaxQueueSize[[:space:]]
|
||||
ClickHouseMetrics_BackgroundMergesAndMutationsPoolTask[[:space:]]
|
||||
ClickHouseMetrics_DelayedInserts[[:space:]]
|
||||
ClickHouseProfileEvents_Query[[:space:]]
|
||||
ClickHouseProfileEvents_SelectQuery[[:space:]]
|
||||
ClickHouseProfileEvents_InsertQuery[[:space:]]
|
||||
ClickHouseProfileEvents_FailedQuery[[:space:]]
|
||||
ClickHouseProfileEvents_InsertedRows[[:space:]]
|
||||
ClickHouseProfileEvents_InsertedBytes[[:space:]]
|
||||
ClickHouseProfileEvents_MergedRows[[:space:]]
|
||||
ClickHouseProfileEvents_ReadCompressedBytes[[:space:]]
|
||||
ClickHouseProfileEvents_CompressedReadBufferBytes[[:space:]]
|
||||
ClickHouseProfileEvents_ReplicatedPartFetches[[:space:]]
|
||||
ClickHouseProfileEvents_ReplicatedPartFailedFetches[[:space:]]
|
||||
ClickHouseProfileEvents_DiskReadElapsedMicroseconds[[:space:]]
|
||||
ClickHouseProfileEvents_DiskWriteElapsedMicroseconds[[:space:]]
|
||||
ClickHouseProfileEvents_NetworkSendBytes[[:space:]]
|
||||
ClickHouseProfileEvents_NetworkReceiveBytes[[:space:]]
|
||||
ClickHouseProfileEvents_ZooKeeperTransactions[[:space:]]
|
||||
ClickHouseProfileEvents_DNSError[[:space:]]
|
||||
ClickHouseAsyncMetrics_Uptime[[:space:]]
|
||||
ClickHouseAsyncMetrics_MaxPartCountForPartition[[:space:]]
|
||||
ClickHouseAsyncMetrics_MemoryResident[[:space:]]
|
||||
ClickHouseAsyncMetrics_ReplicasMaxAbsoluteDelay[[:space:]]'
|
||||
|
||||
# --- Functions ---
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $SCRIPT_NAME [OPTIONS]
|
||||
|
||||
ClickHouse Prometheus Textfile Exporter
|
||||
|
||||
Pulls metrics from the ClickHouse native Prometheus endpoint and writes
|
||||
a filtered subset (~30 key metrics) to a .prom file for node_exporter.
|
||||
Original ClickHouse metric names are preserved for community dashboard
|
||||
compatibility.
|
||||
|
||||
Options:
|
||||
--textfile Write metrics to textfile collector directory
|
||||
--http Run as HTTP server on port $HTTP_PORT
|
||||
-p, --port HTTP port (default: $HTTP_PORT)
|
||||
--install Create cron job for automatic collection
|
||||
--help Show this help message
|
||||
|
||||
Environment Variables:
|
||||
CLICKHOUSE_URL ClickHouse Prometheus endpoint (default: http://localhost:9363)
|
||||
METRICS_PATH Metrics path (default: /metrics)
|
||||
TEXTFILE_DIR Output directory (default: /var/lib/node_exporter/textfile_collector)
|
||||
CURL_TIMEOUT Request timeout in seconds (default: 10)
|
||||
|
||||
Examples:
|
||||
$SCRIPT_NAME
|
||||
$SCRIPT_NAME --textfile
|
||||
CLICKHOUSE_URL="http://ch-01:9363" $SCRIPT_NAME --textfile
|
||||
$SCRIPT_NAME --http -p 9201
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
check_dependencies() {
|
||||
if ! command -v curl &>/dev/null; then
|
||||
echo "# ERROR: curl is required" >&2
|
||||
echo "# Install with: apt install curl OR dnf install curl" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
collect_metrics() {
|
||||
local raw
|
||||
raw=$(curl -sf --max-time "$CURL_TIMEOUT" \
|
||||
"${CLICKHOUSE_URL}${METRICS_PATH}" 2>/dev/null) || {
|
||||
OUTPUT+="# HELP clickhouse_exporter_up ClickHouse Prometheus endpoint reachability (1=up, 0=down)
|
||||
# TYPE clickhouse_exporter_up gauge
|
||||
clickhouse_exporter_up 0
|
||||
"
|
||||
return 1
|
||||
}
|
||||
|
||||
OUTPUT+="# HELP clickhouse_exporter_up ClickHouse Prometheus endpoint reachability (1=up, 0=down)
|
||||
# TYPE clickhouse_exporter_up gauge
|
||||
clickhouse_exporter_up 1
|
||||
"
|
||||
|
||||
# Filter raw metrics to Tier 2 subset
|
||||
# Include HELP and TYPE lines for matched metrics, plus the value lines
|
||||
local filtered
|
||||
filtered=$(echo "$raw" | grep -E "$METRIC_FILTER" || true)
|
||||
|
||||
if [[ -z "$filtered" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# For each matched metric, also grab its HELP and TYPE lines
|
||||
local seen_metrics=""
|
||||
while IFS= read -r line; do
|
||||
# Extract metric name (before space or brace)
|
||||
local metric_name
|
||||
metric_name=$(echo "$line" | awk '{print $1}' | sed 's/{.*//')
|
||||
|
||||
# Add HELP/TYPE lines if we haven't seen this metric yet
|
||||
if [[ ! "$seen_metrics" == *"|${metric_name}|"* ]]; then
|
||||
local help_line type_line
|
||||
help_line=$(echo "$raw" | grep "^# HELP ${metric_name} " || true)
|
||||
type_line=$(echo "$raw" | grep "^# TYPE ${metric_name} " || true)
|
||||
if [[ -n "$help_line" ]]; then
|
||||
OUTPUT+="${help_line}
|
||||
"
|
||||
fi
|
||||
if [[ -n "$type_line" ]]; then
|
||||
OUTPUT+="${type_line}
|
||||
"
|
||||
fi
|
||||
seen_metrics+="|${metric_name}|"
|
||||
fi
|
||||
|
||||
OUTPUT+="${line}
|
||||
"
|
||||
done <<< "$filtered"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Output ---
|
||||
|
||||
write_output() {
|
||||
if [[ "$TEXTFILE_MODE" == true ]]; then
|
||||
local output_file="${TEXTFILE_DIR}/clickhouse.prom"
|
||||
local temp_file="${output_file}.$$"
|
||||
|
||||
mkdir -p "$TEXTFILE_DIR"
|
||||
echo "$OUTPUT" > "$temp_file"
|
||||
mv "$temp_file" "$output_file"
|
||||
echo "# Wrote metrics to ${output_file}" >&2
|
||||
else
|
||||
echo "$OUTPUT"
|
||||
fi
|
||||
}
|
||||
|
||||
serve_http() {
|
||||
if ! command -v nc &>/dev/null && ! command -v ncat &>/dev/null; then
|
||||
echo "# ERROR: nc (netcat) or ncat required for HTTP mode" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "# ClickHouse exporter listening on port ${HTTP_PORT}" >&2
|
||||
echo "# Metrics endpoint: http://localhost:${HTTP_PORT}/metrics" >&2
|
||||
|
||||
local nc_cmd="nc"
|
||||
if command -v ncat &>/dev/null; then
|
||||
nc_cmd="ncat"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
OUTPUT=""
|
||||
START_TIME=$(date +%s%N)
|
||||
|
||||
collect_metrics
|
||||
|
||||
local end_time duration
|
||||
end_time=$(date +%s%N)
|
||||
duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
|
||||
OUTPUT+="# HELP clickhouse_exporter_duration_seconds Time to collect and filter metrics
|
||||
# TYPE clickhouse_exporter_duration_seconds gauge
|
||||
clickhouse_exporter_duration_seconds ${duration}
|
||||
# HELP clickhouse_exporter_last_run_timestamp Unix timestamp of last successful run
|
||||
# TYPE clickhouse_exporter_last_run_timestamp gauge
|
||||
clickhouse_exporter_last_run_timestamp $(date +%s)
|
||||
"
|
||||
|
||||
local content_length=${#OUTPUT}
|
||||
local response="HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\nContent-Length: ${content_length}\r\nConnection: close\r\n\r\n${OUTPUT}"
|
||||
|
||||
echo -e "$response" | $nc_cmd -l -p "$HTTP_PORT" -q 1 2>/dev/null || \
|
||||
echo -e "$response" | $nc_cmd -l "$HTTP_PORT" -c 2>/dev/null || \
|
||||
echo -e "$response" | $nc_cmd -l -p "$HTTP_PORT" 2>/dev/null || true
|
||||
done
|
||||
}
|
||||
|
||||
install_cron() {
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "# ERROR: --install requires root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local script_path
|
||||
script_path=$(readlink -f "$0")
|
||||
|
||||
cat > /etc/cron.d/clickhouse-exporter <<EOF
|
||||
# ClickHouse Prometheus Textfile Exporter -- runs every 3 minutes
|
||||
CLICKHOUSE_URL=${CLICKHOUSE_URL}
|
||||
TEXTFILE_DIR=${TEXTFILE_DIR}
|
||||
*/3 * * * * root ${script_path} --textfile 2>/dev/null
|
||||
EOF
|
||||
|
||||
chmod 644 /etc/cron.d/clickhouse-exporter
|
||||
echo "# Installed cron job: /etc/cron.d/clickhouse-exporter" >&2
|
||||
echo "# Metrics will be written to: ${TEXTFILE_DIR}/clickhouse.prom" >&2
|
||||
}
|
||||
|
||||
# --- Main ---
|
||||
|
||||
main() {
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--textfile) TEXTFILE_MODE=true ;;
|
||||
--http) HTTP_MODE=true ;;
|
||||
-p|--port) shift; HTTP_PORT="${1:-$HTTP_PORT}" ;;
|
||||
--install)
|
||||
check_dependencies
|
||||
install_cron
|
||||
exit 0
|
||||
;;
|
||||
--help|-h) usage ;;
|
||||
*) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
check_dependencies
|
||||
|
||||
if [[ "$HTTP_MODE" == true ]]; then
|
||||
serve_http
|
||||
exit 0
|
||||
fi
|
||||
|
||||
START_TIME=$(date +%s%N)
|
||||
|
||||
collect_metrics
|
||||
|
||||
local end_time duration
|
||||
end_time=$(date +%s%N)
|
||||
duration=$(echo "scale=2; ($end_time - $START_TIME) / 1000000000" | bc 2>/dev/null || echo "0")
|
||||
OUTPUT+="# HELP clickhouse_exporter_duration_seconds Time to collect and filter metrics
|
||||
# TYPE clickhouse_exporter_duration_seconds gauge
|
||||
clickhouse_exporter_duration_seconds ${duration}
|
||||
# HELP clickhouse_exporter_last_run_timestamp Unix timestamp of last successful run
|
||||
# TYPE clickhouse_exporter_last_run_timestamp gauge
|
||||
clickhouse_exporter_last_run_timestamp $(date +%s)
|
||||
"
|
||||
|
||||
write_output
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user