Sync all scripts from website downloads — 352 scripts total

Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.
This commit is contained in:
2026-05-25 03:31:08 +02:00
parent dbd6bf0324
commit a1a17e81a1
332 changed files with 174509 additions and 1106 deletions
+605
View File
@@ -0,0 +1,605 @@
#!/usr/bin/env bash
#########################################################################################
#### packer-build-pipeline.sh — Build, test, and promote machine images with Packer ####
#### Validate templates, build images, run post-build tests, and tag for promotion ####
#### Requires: bash 4+, packer ####
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
#### License: MIT ####
#### Version 1.00 ####
#### ####
#### Usage: ####
#### ./packer-build-pipeline.sh --build --template ./image.pkr.hcl ####
#### ####
#### See --help for all options. ####
#########################################################################################
set -euo pipefail
# ---------------------------------------------------------------------------
# Color variables — pre-initialized empty, set by setup_colors()
# ---------------------------------------------------------------------------
RED=""
GREEN=""
YELLOW=""
BLUE=""
CYAN=""
BOLD=""
DIM=""
RESET=""
setup_colors() {
if [[ "${COLOR}" == "never" ]]; then
return
fi
if [[ "${COLOR}" == "always" ]] || [[ -t 1 ]]; then
RED="\033[0;31m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
BLUE="\033[0;34m"
CYAN="\033[0;36m"
BOLD="\033[1m"
DIM="\033[2m"
RESET="\033[0m"
fi
}
# ---------------------------------------------------------------------------
# Standard helpers
# ---------------------------------------------------------------------------
log() { printf "%b[+]%b %s\n" "$GREEN" "$RESET" "$*"; }
warn() { printf "%b[!]%b %s\n" "$YELLOW" "$RESET" "$*" >&2; }
err() { printf "%b[-]%b %s\n" "$RED" "$RESET" "$*" >&2; }
verbose() { [[ "$VERBOSE" == "true" ]] && printf "%b[~]%b %s\n" "$DIM" "$RESET" "$*"; return 0; }
die() { err "$*"; exit 1; }
section_header() {
printf "\n%b%b══ %b%s%b\n" "$CYAN" "$BOLD" "$BLUE" "$*" "$RESET"
}
field() {
printf " %-24s %s\n" "$1" "$2"
}
field_color() {
local label="$1" color="$2" value="$3"
printf " %-24s %b%s%b\n" "$label" "$color" "$value" "$RESET"
}
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
RUN_MODE=""
TEMPLATE_PATH="${PACKER_TEMPLATE:-}"
VAR_FILE="${PACKER_VAR_FILE:-}"
BUILD_DIR="${PACKER_BUILD_DIR:-.}"
OUTPUT_DIR="${PACKER_OUTPUT_DIR:-./packer-output}"
TEST_SCRIPT="${PACKER_TEST_SCRIPT:-}"
PROMOTE_TAG="${PACKER_PROMOTE_TAG:-production}"
PACKER_PATH="${PACKER_PATH:-packer}"
ON_ERROR="${PACKER_ON_ERROR:-cleanup}"
FORCE_BUILD=false
VERBOSE="${VERBOSE:-false}"
COLOR="${COLOR:-auto}"
# ---------------------------------------------------------------------------
# State
# ---------------------------------------------------------------------------
readonly SCRIPT_NAME="${0##*/}"
START_TIME=""
BUILD_ID=""
export BUILD_STATUS=""
# ---------------------------------------------------------------------------
# Dependency checks
# ---------------------------------------------------------------------------
require_packer() {
if ! command -v "$PACKER_PATH" &>/dev/null; then
die "Packer not found at '${PACKER_PATH}'. Install from https://packer.io or set PACKER_PATH."
fi
verbose "Packer found: $(command -v "$PACKER_PATH") ($("$PACKER_PATH" --version 2>/dev/null || echo 'unknown'))"
}
# ---------------------------------------------------------------------------
# Manifest helpers
# ---------------------------------------------------------------------------
write_manifest() {
local manifest_dir="$1" artifact_id="$2" artifact_type="${3:-unknown}" status="${4:-success}"
local manifest_file="${manifest_dir}/manifest.json"
local timestamp
timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
cat > "$manifest_file" <<MANIFEST_EOF
{
"build_id": "${BUILD_ID}",
"timestamp": "${timestamp}",
"template": "${TEMPLATE_PATH}",
"artifact_id": "${artifact_id}",
"artifact_type": "${artifact_type}",
"status": "${status}",
"promote_tag": ""
}
MANIFEST_EOF
verbose "Manifest written: ${manifest_file}"
}
read_manifest() {
local manifest_file="$1"
if [[ ! -f "$manifest_file" ]]; then
die "Manifest not found: ${manifest_file}"
fi
# Return the artifact_id from the manifest
local artifact_id
artifact_id="$(grep '"artifact_id"' "$manifest_file" | head -1 | sed 's/.*: *"\(.*\)".*/\1/')"
if [[ -z "$artifact_id" ]]; then
die "Could not read artifact_id from manifest: ${manifest_file}"
fi
printf "%s" "$artifact_id"
}
find_latest_manifest() {
local latest_dir
latest_dir="$(find "$OUTPUT_DIR" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -r | head -1)"
if [[ -z "$latest_dir" ]]; then
die "No build directories found in ${OUTPUT_DIR}"
fi
local manifest="${latest_dir}/manifest.json"
if [[ ! -f "$manifest" ]]; then
die "No manifest.json in latest build directory: ${latest_dir}"
fi
printf "%s" "$manifest"
}
# ---------------------------------------------------------------------------
# Elapsed time helper
# ---------------------------------------------------------------------------
elapsed() {
local end_time
end_time="$(date +%s)"
echo "$(( end_time - START_TIME ))s"
}
# ---------------------------------------------------------------------------
# Mode: validate
# ---------------------------------------------------------------------------
do_validate() {
require_packer
section_header "Validate Packer Template"
if [[ -z "$TEMPLATE_PATH" ]]; then
die "No template specified. Use --template <path> or set PACKER_TEMPLATE."
fi
if [[ ! -f "$TEMPLATE_PATH" ]]; then
die "Template not found: ${TEMPLATE_PATH}"
fi
field "Template:" "$TEMPLATE_PATH"
[[ -n "$VAR_FILE" ]] && field "Var file:" "$VAR_FILE"
# Initialize plugins
log "Running packer init..."
if ! "$PACKER_PATH" init "$TEMPLATE_PATH" 2>&1; then
warn "packer init returned non-zero (plugins may already be installed)"
fi
# Build validate command
local -a validate_cmd=("$PACKER_PATH" "validate")
if [[ -n "$VAR_FILE" ]]; then
validate_cmd+=("-var-file=${VAR_FILE}")
fi
validate_cmd+=("$TEMPLATE_PATH")
log "Running packer validate..."
verbose "Command: ${validate_cmd[*]}"
local validate_output
if validate_output="$("${validate_cmd[@]}" 2>&1)"; then
field_color "Result:" "$GREEN" "PASS"
verbose "$validate_output"
BUILD_STATUS="validated"
else
err "$validate_output"
field_color "Result:" "$RED" "FAIL"
BUILD_STATUS="validation-failed"
return 1
fi
}
# ---------------------------------------------------------------------------
# Mode: build
# ---------------------------------------------------------------------------
do_build() {
require_packer
section_header "Build Packer Image"
if [[ -z "$TEMPLATE_PATH" ]]; then
die "No template specified. Use --template <path> or set PACKER_TEMPLATE."
fi
if [[ ! -f "$TEMPLATE_PATH" ]]; then
die "Template not found: ${TEMPLATE_PATH}"
fi
BUILD_ID="$(date +%Y%m%d-%H%M%S)"
local build_output_dir="${OUTPUT_DIR}/${BUILD_ID}"
local log_file="${build_output_dir}/build.log"
mkdir -p "$build_output_dir"
field "Template:" "$TEMPLATE_PATH"
field "Build ID:" "$BUILD_ID"
field "Output dir:" "$build_output_dir"
field "On error:" "$ON_ERROR"
[[ -n "$VAR_FILE" ]] && field "Var file:" "$VAR_FILE"
[[ "$FORCE_BUILD" == "true" ]] && field "Force:" "yes"
# Initialize plugins
log "Running packer init..."
"$PACKER_PATH" init "$TEMPLATE_PATH" >> "$log_file" 2>&1 || true
# Build packer build args
local -a build_cmd=("$PACKER_PATH" "build")
if [[ -n "$VAR_FILE" ]]; then
build_cmd+=("-var-file=${VAR_FILE}")
fi
build_cmd+=("-on-error=${ON_ERROR}")
if [[ "$FORCE_BUILD" == "true" ]]; then
build_cmd+=("-force")
fi
build_cmd+=("-color=false")
build_cmd+=("$TEMPLATE_PATH")
log "Starting packer build..."
verbose "Command: ${build_cmd[*]}"
local build_start build_end build_rc=0
build_start="$(date +%s)"
if "${build_cmd[@]}" 2>&1 | tee -a "$log_file"; then
build_rc=0
else
build_rc=$?
fi
build_end="$(date +%s)"
local build_duration="$(( build_end - build_start ))s"
if [[ "$build_rc" -ne 0 ]]; then
field_color "Result:" "$RED" "FAIL (exit code ${build_rc})"
BUILD_STATUS="build-failed"
write_manifest "$build_output_dir" "" "unknown" "failed"
return 1
fi
# Parse artifact info from build output
local artifact_id="" artifact_type="unknown"
# Try to find AMI IDs (aws-style)
local ami_match
ami_match="$(grep -oP 'ami-[0-9a-f]+' "$log_file" | tail -1 || true)"
if [[ -n "$ami_match" ]]; then
artifact_id="$ami_match"
artifact_type="aws-ami"
fi
# Try generic artifact lines if no AMI found
if [[ -z "$artifact_id" ]]; then
local artifact_line
artifact_line="$(grep -i 'artifact' "$log_file" | grep -i 'id\|name\|created' | tail -1 || true)"
if [[ -n "$artifact_line" ]]; then
# Extract the last colon-separated value or the last word
artifact_id="$(echo "$artifact_line" | sed 's/.*[: ]//' | tr -d '[:space:]')"
artifact_type="generic"
fi
fi
# Fall back to build ID as artifact reference
if [[ -z "$artifact_id" ]]; then
artifact_id="build-${BUILD_ID}"
artifact_type="local"
warn "Could not parse artifact ID from build output — using ${artifact_id}"
fi
write_manifest "$build_output_dir" "$artifact_id" "$artifact_type" "success"
section_header "Build Results"
field "Build time:" "$build_duration"
field_color "Artifact ID:" "$GREEN" "$artifact_id"
field "Artifact type:" "$artifact_type"
field "Log file:" "$log_file"
field "Manifest:" "${build_output_dir}/manifest.json"
BUILD_STATUS="built"
log "Build completed successfully"
}
# ---------------------------------------------------------------------------
# Mode: test
# ---------------------------------------------------------------------------
do_test() {
section_header "Post-Build Tests"
if [[ -z "$TEST_SCRIPT" ]]; then
die "No test script specified. Use --test-script <path> or set PACKER_TEST_SCRIPT."
fi
if [[ ! -f "$TEST_SCRIPT" ]]; then
die "Test script not found: ${TEST_SCRIPT}"
fi
if [[ ! -x "$TEST_SCRIPT" ]]; then
die "Test script is not executable: ${TEST_SCRIPT}"
fi
local manifest_file
manifest_file="$(find_latest_manifest)"
local artifact_id
artifact_id="$(read_manifest "$manifest_file")"
field "Test script:" "$TEST_SCRIPT"
field "Artifact ID:" "$artifact_id"
field "Manifest:" "$manifest_file"
log "Running test script..."
verbose "Command: ${TEST_SCRIPT} ${artifact_id}"
local test_output test_rc=0
if test_output="$("$TEST_SCRIPT" "$artifact_id" 2>&1)"; then
test_rc=0
else
test_rc=$?
fi
if [[ -n "$test_output" ]]; then
printf "\n%s\n" "$test_output"
fi
if [[ "$test_rc" -eq 0 ]]; then
field_color "Result:" "$GREEN" "PASS"
BUILD_STATUS="tested"
else
field_color "Result:" "$RED" "FAIL (exit code ${test_rc})"
BUILD_STATUS="test-failed"
return 1
fi
}
# ---------------------------------------------------------------------------
# Mode: promote
# ---------------------------------------------------------------------------
do_promote() {
section_header "Promote Artifact"
local manifest_file
manifest_file="$(find_latest_manifest)"
local artifact_id
artifact_id="$(read_manifest "$manifest_file")"
field "Artifact ID:" "$artifact_id"
field "Promote tag:" "$PROMOTE_TAG"
field "Manifest:" "$manifest_file"
# Update the manifest with the promotion tag
local tmp_manifest
tmp_manifest="$(mktemp)"
sed "s/\"promote_tag\": \".*\"/\"promote_tag\": \"${PROMOTE_TAG}\"/" "$manifest_file" > "$tmp_manifest"
mv "$tmp_manifest" "$manifest_file"
log "Updated manifest with promote tag: ${PROMOTE_TAG}"
# If the artifact looks like an AWS AMI, show the tagging command
if [[ "$artifact_id" =~ ^ami- ]]; then
section_header "AWS AMI Tagging"
log "To tag this AMI for promotion, run:"
printf "\n %baws ec2 create-tags \\\\\n --resources %s \\\\\n --tags Key=Environment,Value=%s%b\n\n" \
"$DIM" "$artifact_id" "$PROMOTE_TAG" "$RESET"
warn "Command not executed automatically — review and run manually."
fi
field_color "Status:" "$GREEN" "Promoted as ${PROMOTE_TAG}"
BUILD_STATUS="promoted"
log "Promotion complete"
}
# ---------------------------------------------------------------------------
# Mode: full pipeline
# ---------------------------------------------------------------------------
do_full() {
section_header "Full Pipeline: Validate → Build → Test → Promote"
log "Stage 1/4: Validate"
if ! do_validate; then
die "Pipeline aborted: validation failed"
fi
log "Stage 2/4: Build"
if ! do_build; then
die "Pipeline aborted: build failed"
fi
if [[ -n "$TEST_SCRIPT" ]]; then
log "Stage 3/4: Test"
if ! do_test; then
die "Pipeline aborted: tests failed"
fi
else
log "Stage 3/4: Test (skipped — no test script configured)"
fi
log "Stage 4/4: Promote"
if ! do_promote; then
die "Pipeline aborted: promotion failed"
fi
section_header "Pipeline Summary"
field "Template:" "$TEMPLATE_PATH"
field "Build ID:" "$BUILD_ID"
field "Promote tag:" "$PROMOTE_TAG"
field_color "Status:" "$GREEN" "All stages completed successfully"
field "Duration:" "$(elapsed)"
}
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
show_help() {
cat <<EOF
${SCRIPT_NAME} — Build, test, and promote machine images with Packer
USAGE
${SCRIPT_NAME} <MODE> [OPTIONS]
MODES
--validate Validate the Packer template (init + validate)
--build Build the image (init + build + capture artifacts)
--test Run post-build tests against the built artifact
--promote Tag/promote the latest built artifact
--full Run the full pipeline: validate → build → test → promote
OPTIONS
--template PATH Packer template file (.pkr.hcl or .json)
--var-file PATH Variables file to pass to packer
--build-dir DIR Working directory for packer (default: .)
--output-dir DIR Output directory for logs/manifests (default: ./packer-output)
--test-script PATH Post-build test script (receives artifact ID as \$1)
--promote-tag TAG Promotion tag name (default: production)
--on-error ACTION Packer on-error behavior: cleanup, abort, ask (default: cleanup)
--force Force build even if artifacts exist
--verbose Enable verbose output
--no-color Disable color output
--help Show this help message
ENVIRONMENT VARIABLES
PACKER_TEMPLATE Default template path
PACKER_VAR_FILE Default var-file path
PACKER_BUILD_DIR Default build directory
PACKER_OUTPUT_DIR Default output directory
PACKER_TEST_SCRIPT Default test script path
PACKER_PROMOTE_TAG Default promotion tag
PACKER_PATH Path to packer binary
PACKER_ON_ERROR Default on-error behavior
VERBOSE Set to 'true' for verbose output
COLOR Set to 'never' to disable colors
EXAMPLES
${SCRIPT_NAME} --validate --template ./image.pkr.hcl
${SCRIPT_NAME} --build --template ./image.pkr.hcl --var-file vars.pkrvars.hcl
${SCRIPT_NAME} --build --template ./image.pkr.hcl --on-error abort --force
${SCRIPT_NAME} --test --test-script ./tests/verify-image.sh
${SCRIPT_NAME} --promote --promote-tag staging
${SCRIPT_NAME} --full --template ./image.pkr.hcl --test-script ./tests/verify.sh
PIPELINE FLOW
validate → Syntax/config checks on template
build → packer init + packer build; logs + manifest saved
test → Run external test script with artifact ID
promote → Tag artifact in manifest (show AWS command if AMI)
EOF
}
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--validate)
RUN_MODE="validate"
shift
;;
--build)
RUN_MODE="build"
shift
;;
--test)
RUN_MODE="test"
shift
;;
--promote)
RUN_MODE="promote"
shift
;;
--full)
RUN_MODE="full"
shift
;;
--template)
TEMPLATE_PATH="${2:-}"
[[ -z "$TEMPLATE_PATH" ]] && die "--template requires a PATH argument"
shift 2
;;
--var-file)
VAR_FILE="${2:-}"
[[ -z "$VAR_FILE" ]] && die "--var-file requires a PATH argument"
shift 2
;;
--build-dir)
BUILD_DIR="${2:-}"
[[ -z "$BUILD_DIR" ]] && die "--build-dir requires a DIR argument"
shift 2
;;
--output-dir)
OUTPUT_DIR="${2:-}"
[[ -z "$OUTPUT_DIR" ]] && die "--output-dir requires a DIR argument"
shift 2
;;
--test-script)
TEST_SCRIPT="${2:-}"
[[ -z "$TEST_SCRIPT" ]] && die "--test-script requires a PATH argument"
shift 2
;;
--promote-tag)
PROMOTE_TAG="${2:-}"
[[ -z "$PROMOTE_TAG" ]] && die "--promote-tag requires a TAG argument"
shift 2
;;
--on-error)
ON_ERROR="${2:-}"
[[ -z "$ON_ERROR" ]] && die "--on-error requires an ACTION argument"
case "$ON_ERROR" in
cleanup|abort|ask) ;;
*) die "--on-error must be one of: cleanup, abort, ask" ;;
esac
shift 2
;;
--force)
FORCE_BUILD=true
shift
;;
--verbose)
VERBOSE="true"
shift
;;
--no-color)
COLOR="never"
shift
;;
--help|-h)
RUN_MODE="help"
shift
;;
*)
die "Unknown option: $1 (see --help)"
;;
esac
done
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
parse_args "$@"
setup_colors
START_TIME="$(date +%s)"
case "$RUN_MODE" in
validate) do_validate ;;
build) do_build ;;
test) do_test ;;
promote) do_promote ;;
full) do_full ;;
help) show_help ;;
"") show_help; die "No mode specified — use --validate, --build, --test, --promote, or --full" ;;
*) die "Unknown mode: ${RUN_MODE}" ;;
esac
}
main "$@"