Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
Executable
+341
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#########################################################################################
|
||||
#### seo-exporter.sh — Pull Google Search Console metrics into Prometheus ####
|
||||
#### Queries GSC API for clicks, impressions, CTR, position, top pages/queries ####
|
||||
#### Pushes Prometheus-format metrics to Pushgateway — runs on cron, no daemon ####
|
||||
#### ####
|
||||
#### Author: Phil Connor ####
|
||||
#### Contact: contact@mylinux.work ####
|
||||
#### License: MIT ####
|
||||
#### Version 1.00 ####
|
||||
#### ####
|
||||
#### Usage: ####
|
||||
#### ./seo-exporter.sh ####
|
||||
#### ./seo-exporter.sh --site https://mylinux.work/ ####
|
||||
#### ####
|
||||
#### See --help for all options. ####
|
||||
#########################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
CREDENTIALS_FILE="${CREDENTIALS_FILE:-/etc/seo-exporter/credentials.json}"
|
||||
SITE_URL="${SITE_URL:-https://mylinux.work/}"
|
||||
PUSHGATEWAY_URL="${PUSHGATEWAY_URL:-http://localhost:9091}"
|
||||
JOB_NAME="${JOB_NAME:-seo_exporter}"
|
||||
DAYS_BACK="${DAYS_BACK:-7}"
|
||||
TOP_N="${TOP_N:-50}"
|
||||
VERBOSE="${VERBOSE:-false}"
|
||||
COLOR="${COLOR:-auto}"
|
||||
|
||||
# ── State ─────────────────────────────────────────────────────────────
|
||||
SCRIPT_NAME="$(basename "$0")"
|
||||
readonly SCRIPT_NAME
|
||||
ACCESS_TOKEN=""
|
||||
METRICS=""
|
||||
|
||||
# ── Colors ────────────────────────────────────────────────────────────
|
||||
setup_colors() {
|
||||
if [[ "$COLOR" == "never" ]]; then
|
||||
BLUE="" RED="" DIM="" RESET=""
|
||||
return
|
||||
fi
|
||||
if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
DIM='\033[2m'
|
||||
RESET='\033[0m'
|
||||
else
|
||||
BLUE="" RED="" DIM="" RESET=""
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────
|
||||
log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
|
||||
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
|
||||
verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${DIM}[DEBUG]${RESET} $*"; fi; }
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# JWT / OAUTH2
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
base64url() {
|
||||
openssl base64 -A | tr '+/' '-_' | tr -d '='
|
||||
}
|
||||
|
||||
get_access_token() {
|
||||
local creds="$CREDENTIALS_FILE"
|
||||
|
||||
if [[ ! -f "$creds" ]]; then
|
||||
err "Credentials file not found: $creds"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local client_email token_uri private_key
|
||||
client_email=$(jq -r '.client_email' "$creds")
|
||||
token_uri=$(jq -r '.token_uri' "$creds")
|
||||
private_key=$(jq -r '.private_key' "$creds")
|
||||
|
||||
local now exp
|
||||
now=$(date +%s)
|
||||
exp=$((now + 3600))
|
||||
|
||||
# Build JWT header and claims
|
||||
local header claims
|
||||
header=$(printf '{"alg":"RS256","typ":"JWT"}' | base64url)
|
||||
claims=$(printf '{"iss":"%s","scope":"https://www.googleapis.com/auth/webmasters.readonly","aud":"%s","iat":%d,"exp":%d}' \
|
||||
"$client_email" "$token_uri" "$now" "$exp" | base64url)
|
||||
|
||||
# Sign with private key
|
||||
local signature
|
||||
signature=$(printf '%s.%s' "$header" "$claims" \
|
||||
| openssl dgst -sha256 -sign <(printf '%s' "$private_key") \
|
||||
| base64url)
|
||||
|
||||
local jwt="${header}.${claims}.${signature}"
|
||||
|
||||
# Exchange JWT for access token
|
||||
local response
|
||||
response=$(curl -s -X POST "$token_uri" \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
-d "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=${jwt}")
|
||||
|
||||
ACCESS_TOKEN=$(echo "$response" | jq -r '.access_token // empty')
|
||||
|
||||
if [[ -z "$ACCESS_TOKEN" ]]; then
|
||||
err "Failed to get access token"
|
||||
err "Response: $response"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
verbose "Access token acquired (${#ACCESS_TOKEN} chars)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# GSC API QUERIES
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
gsc_query() {
|
||||
local body="$1"
|
||||
local encoded_site
|
||||
encoded_site=$(printf '%s' "$SITE_URL" | jq -sRr @uri)
|
||||
|
||||
curl -s \
|
||||
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"https://searchconsole.googleapis.com/webmasters/v3/sites/${encoded_site}/searchAnalytics/query"
|
||||
}
|
||||
|
||||
query_aggregate() {
|
||||
local end_date start_date
|
||||
end_date=$(date -d "-3 days" +%Y-%m-%d)
|
||||
start_date=$(date -d "-$((3 + DAYS_BACK)) days" +%Y-%m-%d)
|
||||
|
||||
local body
|
||||
body=$(jq -n \
|
||||
--arg start "$start_date" \
|
||||
--arg end "$end_date" \
|
||||
'{"startDate": $start, "endDate": $end}')
|
||||
|
||||
gsc_query "$body"
|
||||
}
|
||||
|
||||
query_dimension() {
|
||||
local dimension="$1"
|
||||
local limit="${2:-$TOP_N}"
|
||||
local end_date start_date
|
||||
end_date=$(date -d "-3 days" +%Y-%m-%d)
|
||||
start_date=$(date -d "-$((3 + DAYS_BACK)) days" +%Y-%m-%d)
|
||||
|
||||
local body
|
||||
body=$(jq -n \
|
||||
--arg start "$start_date" \
|
||||
--arg end "$end_date" \
|
||||
--arg dim "$dimension" \
|
||||
--argjson limit "$limit" \
|
||||
'{"startDate": $start, "endDate": $end, "dimensions": [$dim], "rowLimit": $limit}')
|
||||
|
||||
gsc_query "$body"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# METRICS BUILDER
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
add_metric() {
|
||||
local name="$1" type="$2" help="$3" value="$4" labels="${5:-}"
|
||||
if [[ -z "$labels" ]]; then
|
||||
METRICS+="# HELP ${name} ${help}
|
||||
# TYPE ${name} ${type}
|
||||
${name} ${value}
|
||||
"
|
||||
else
|
||||
# Only add HELP/TYPE if not already present
|
||||
if [[ "$METRICS" != *"# TYPE ${name}"* ]]; then
|
||||
METRICS+="# HELP ${name} ${help}
|
||||
# TYPE ${name} ${type}
|
||||
"
|
||||
fi
|
||||
METRICS+="${name}{${labels}} ${value}
|
||||
"
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# COLLECT METRICS
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
collect_metrics() {
|
||||
METRICS=""
|
||||
|
||||
# Aggregate metrics
|
||||
log "Fetching aggregate metrics..."
|
||||
local agg
|
||||
agg=$(query_aggregate)
|
||||
|
||||
local clicks impressions ctr position
|
||||
clicks=$(echo "$agg" | jq '.rows[0].clicks // 0')
|
||||
impressions=$(echo "$agg" | jq '.rows[0].impressions // 0')
|
||||
ctr=$(echo "$agg" | jq '(.rows[0].ctr // 0) * 100 | . * 100 | round / 100')
|
||||
position=$(echo "$agg" | jq '(.rows[0].position // 0) * 10 | round / 10')
|
||||
|
||||
add_metric "seo_clicks_total" "gauge" "Total clicks" "$clicks"
|
||||
add_metric "seo_impressions_total" "gauge" "Total impressions" "$impressions"
|
||||
add_metric "seo_ctr_average" "gauge" "Average CTR percent" "$ctr"
|
||||
add_metric "seo_position_average" "gauge" "Average position" "$position"
|
||||
|
||||
log "clicks=${clicks} impressions=${impressions} ctr=${ctr}% position=${position}"
|
||||
|
||||
# Per-page metrics (top N)
|
||||
log "Fetching per-page metrics (top ${TOP_N})..."
|
||||
local pages
|
||||
pages=$(query_dimension "page")
|
||||
|
||||
while IFS= read -r row; do
|
||||
[[ -z "$row" ]] && continue
|
||||
local page pg_clicks pg_imp pg_pos pg_ctr
|
||||
page=$(echo "$row" | jq -r '.keys[0]' | sed "s|${SITE_URL}|/|")
|
||||
pg_clicks=$(echo "$row" | jq '.clicks // 0')
|
||||
pg_imp=$(echo "$row" | jq '.impressions // 0')
|
||||
pg_pos=$(echo "$row" | jq '(.position // 0) * 10 | round / 10')
|
||||
pg_ctr=$(echo "$row" | jq '(.ctr // 0) * 100 | . * 100 | round / 100')
|
||||
|
||||
add_metric "seo_page_clicks" "gauge" "Clicks per page" "$pg_clicks" "page=\"${page}\""
|
||||
add_metric "seo_page_impressions" "gauge" "Impressions per page" "$pg_imp" "page=\"${page}\""
|
||||
add_metric "seo_page_position" "gauge" "Position per page" "$pg_pos" "page=\"${page}\""
|
||||
add_metric "seo_page_ctr" "gauge" "CTR per page percent" "$pg_ctr" "page=\"${page}\""
|
||||
done < <(echo "$pages" | jq -c '.rows[]? // empty')
|
||||
|
||||
# Per-query metrics (top N)
|
||||
log "Fetching per-query metrics (top ${TOP_N})..."
|
||||
local queries
|
||||
queries=$(query_dimension "query")
|
||||
|
||||
while IFS= read -r row; do
|
||||
[[ -z "$row" ]] && continue
|
||||
local query q_clicks q_imp q_pos
|
||||
query=$(echo "$row" | jq -r '.keys[0]' | sed 's/"/\\"/g')
|
||||
q_clicks=$(echo "$row" | jq '.clicks // 0')
|
||||
q_imp=$(echo "$row" | jq '.impressions // 0')
|
||||
q_pos=$(echo "$row" | jq '(.position // 0) * 10 | round / 10')
|
||||
|
||||
add_metric "seo_query_clicks" "gauge" "Clicks per query" "$q_clicks" "query=\"${query}\""
|
||||
add_metric "seo_query_impressions" "gauge" "Impressions per query" "$q_imp" "query=\"${query}\""
|
||||
add_metric "seo_query_position" "gauge" "Position per query" "$q_pos" "query=\"${query}\""
|
||||
done < <(echo "$queries" | jq -c '.rows[]? // empty')
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# PUSH TO PUSHGATEWAY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
push_metrics() {
|
||||
log "Pushing metrics to ${PUSHGATEWAY_URL}..."
|
||||
|
||||
local http_code
|
||||
http_code=$(printf '%s' "$METRICS" | curl -s -o /dev/null -w "%{http_code}" \
|
||||
--data-binary @- \
|
||||
"${PUSHGATEWAY_URL}/metrics/job/${JOB_NAME}")
|
||||
|
||||
if [[ "$http_code" == "200" ]] || [[ "$http_code" == "202" ]]; then
|
||||
log "Push successful (HTTP ${http_code})"
|
||||
else
|
||||
err "Push failed (HTTP ${http_code})"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# USAGE
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
${SCRIPT_NAME} — Pull Google Search Console metrics into Prometheus
|
||||
|
||||
USAGE:
|
||||
${SCRIPT_NAME} [OPTIONS]
|
||||
|
||||
OPTIONS:
|
||||
--site URL Site URL in GSC (default: ${SITE_URL})
|
||||
--credentials FILE Path to service account JSON (default: ${CREDENTIALS_FILE})
|
||||
--pushgateway URL Pushgateway URL (default: ${PUSHGATEWAY_URL})
|
||||
--days N Days of data to fetch (default: ${DAYS_BACK})
|
||||
--top N Top N pages/queries to track (default: ${TOP_N})
|
||||
--verbose Enable debug output
|
||||
--no-color Disable colored output
|
||||
--help Show this help
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
CREDENTIALS_FILE Service account JSON path
|
||||
SITE_URL Site URL in GSC
|
||||
PUSHGATEWAY_URL Pushgateway URL
|
||||
DAYS_BACK Days of data to fetch
|
||||
TOP_N Number of top pages/queries
|
||||
EOF
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# ARGUMENT PARSING
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--site) SITE_URL="$2"; shift 2 ;;
|
||||
--credentials) CREDENTIALS_FILE="$2"; shift 2 ;;
|
||||
--pushgateway) PUSHGATEWAY_URL="$2"; shift 2 ;;
|
||||
--days) DAYS_BACK="$2"; shift 2 ;;
|
||||
--top) TOP_N="$2"; shift 2 ;;
|
||||
--verbose) VERBOSE="true"; shift ;;
|
||||
--no-color) COLOR="never"; shift ;;
|
||||
--help|-h) setup_colors; usage; exit 0 ;;
|
||||
*)
|
||||
err "Unknown option: $1"
|
||||
echo "Run ${SCRIPT_NAME} --help for usage" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MAIN
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
setup_colors
|
||||
|
||||
log "SEO Exporter — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
log "Site: ${SITE_URL}"
|
||||
|
||||
get_access_token
|
||||
collect_metrics
|
||||
push_metrics
|
||||
|
||||
log "Done."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user