Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist, same-site referer bypass, Blackbox-Exporter allowed bot, and all new exporters, cheat sheets, and automation scripts.
This commit is contained in:
@@ -0,0 +1,847 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#########################################################################################
|
||||
#### s3-bucket-manager.sh — Copy, sync, delete, audit, and manage AWS S3 buckets ####
|
||||
#### Works with any AWS credential method — SSO, assume-role, env vars, profiles ####
|
||||
#### Requires: bash 4+, aws-cli v2, jq ####
|
||||
#### ####
|
||||
#### Author: Phil Connor ####
|
||||
#### Contact: contact@mylinux.work ####
|
||||
#### License: MIT ####
|
||||
#### Version 1.01 ####
|
||||
#### ####
|
||||
#### Usage: ####
|
||||
#### export AWS_PROFILE="production" ####
|
||||
#### ./s3-bucket-manager.sh --list-buckets ####
|
||||
#### ./s3-bucket-manager.sh --copy s3://src-bucket s3://dst-bucket ####
|
||||
#### ####
|
||||
#### See --help for all options. ####
|
||||
#########################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
AWS_REGION="${AWS_REGION:-}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
DELETE_OLDER_THAN="${DELETE_OLDER_THAN:-}"
|
||||
DELETE_PREFIX="${DELETE_PREFIX:-}"
|
||||
INCLUDE_PATTERN="${INCLUDE_PATTERN:-}"
|
||||
EXCLUDE_PATTERN="${EXCLUDE_PATTERN:-}"
|
||||
STORAGE_CLASS="${STORAGE_CLASS:-}"
|
||||
ACL="${ACL:-}"
|
||||
SSE="${SSE:-}"
|
||||
VERBOSE="${VERBOSE:-false}"
|
||||
COLOR="${COLOR:-auto}"
|
||||
PARALLEL="${PARALLEL:-true}"
|
||||
|
||||
# ── State ─────────────────────────────────────────────────────────────
|
||||
SCRIPT_NAME="$(basename "$0")"
|
||||
readonly SCRIPT_NAME
|
||||
RUN_MODE=""
|
||||
SOURCE_PATH=""
|
||||
DEST_PATH=""
|
||||
TARGET_BUCKET=""
|
||||
START_TIME=""
|
||||
WARNINGS=0
|
||||
|
||||
# ── Colors ────────────────────────────────────────────────────────────
|
||||
setup_colors() {
|
||||
if [[ "$COLOR" == "never" ]]; then
|
||||
RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
|
||||
return
|
||||
fi
|
||||
if [[ "$COLOR" == "always" ]] || [[ -t 1 ]]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
BOLD='\033[1m'
|
||||
RESET='\033[0m'
|
||||
else
|
||||
RED="" GREEN="" YELLOW="" BLUE="" BOLD="" RESET=""
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────
|
||||
log() { echo -e "${BLUE}[INFO]${RESET} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; ((WARNINGS++)) || true; }
|
||||
err() { echo -e "${RED}[ERROR]${RESET} $*" >&2; }
|
||||
verbose() { if [[ "$VERBOSE" == "true" ]]; then echo -e "${BLUE}[DEBUG]${RESET} $*"; fi; }
|
||||
|
||||
# ── AWS CLI wrapper ───────────────────────────────────────────────────
|
||||
aws_cmd() {
|
||||
local args=("$@")
|
||||
[[ -n "$AWS_REGION" ]] && args+=(--region "$AWS_REGION")
|
||||
verbose "aws ${args[*]}"
|
||||
aws "${args[@]}"
|
||||
}
|
||||
|
||||
# ── Credential check ─────────────────────────────────────────────────
|
||||
check_deps() {
|
||||
for cmd in aws jq; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
err "${cmd} is required but not installed"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify credentials are valid
|
||||
local identity
|
||||
identity=$(aws sts get-caller-identity 2>&1) || {
|
||||
err "AWS credentials not configured, expired, or invalid"
|
||||
echo "" >&2
|
||||
echo "Supported credential methods:" >&2
|
||||
echo " • AWS_PROFILE — named profile from ~/.aws/credentials" >&2
|
||||
echo " • AWS SSO — run 'aws sso login --profile your-profile'" >&2
|
||||
echo " • Environment vars — AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN" >&2
|
||||
echo " • Instance profile — automatic on EC2/ECS" >&2
|
||||
echo " • AWS_ROLE_ARN — assume role via STS" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
local account arn
|
||||
account=$(echo "$identity" | jq -r '.Account')
|
||||
arn=$(echo "$identity" | jq -r '.Arn')
|
||||
verbose "Account: ${account}"
|
||||
verbose "Identity: ${arn}"
|
||||
|
||||
# Check for session expiry (if using temporary credentials)
|
||||
if [[ -n "${AWS_SESSION_TOKEN:-}" ]]; then
|
||||
verbose "Using temporary credentials (session token present)"
|
||||
fi
|
||||
|
||||
# Determine region if not set
|
||||
if [[ -z "$AWS_REGION" ]]; then
|
||||
AWS_REGION=$(aws configure get region 2>/dev/null || echo "")
|
||||
if [[ -n "$AWS_REGION" ]]; then
|
||||
verbose "Using region from config: ${AWS_REGION}"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Authenticated as ${arn}"
|
||||
}
|
||||
|
||||
# ── Human-readable sizes ─────────────────────────────────────────────
|
||||
human_size() {
|
||||
local bytes="$1"
|
||||
if [[ "$bytes" -ge 1099511627776 ]]; then
|
||||
printf "%.1f TiB" "$(echo "$bytes / 1099511627776" | bc -l)"
|
||||
elif [[ "$bytes" -ge 1073741824 ]]; then
|
||||
printf "%.1f GiB" "$(echo "$bytes / 1073741824" | bc -l)"
|
||||
elif [[ "$bytes" -ge 1048576 ]]; then
|
||||
printf "%.1f MiB" "$(echo "$bytes / 1048576" | bc -l)"
|
||||
elif [[ "$bytes" -ge 1024 ]]; then
|
||||
printf "%.1f KiB" "$(echo "$bytes / 1024" | bc -l)"
|
||||
else
|
||||
echo "${bytes} B"
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# LIST BUCKETS
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_list_buckets() {
|
||||
log "Listing S3 buckets..."
|
||||
|
||||
local buckets_json
|
||||
buckets_json=$(aws_cmd s3api list-buckets --output json)
|
||||
|
||||
local bucket_count
|
||||
bucket_count=$(echo "$buckets_json" | jq '.Buckets | length')
|
||||
|
||||
if [[ "$bucket_count" -eq 0 ]]; then
|
||||
log "No buckets found"
|
||||
return
|
||||
fi
|
||||
|
||||
echo ""
|
||||
printf " %-40s %-22s %s\n" "BUCKET" "CREATED" "REGION"
|
||||
echo " $(printf '%.0s─' {1..80})"
|
||||
|
||||
echo "$buckets_json" | jq -c '.Buckets[]' | while IFS= read -r bucket; do
|
||||
local name created region
|
||||
name=$(echo "$bucket" | jq -r '.Name')
|
||||
created=$(echo "$bucket" | jq -r '.CreationDate' | cut -c1-19)
|
||||
|
||||
region=$(aws_cmd s3api get-bucket-location \
|
||||
--bucket "$name" \
|
||||
--query 'LocationConstraint' \
|
||||
--output text 2>/dev/null) || region="error"
|
||||
[[ "$region" == "None" || "$region" == "null" ]] && region="us-east-1"
|
||||
|
||||
printf " %-40s %-22s %s\n" "$name" "$created" "$region"
|
||||
done
|
||||
|
||||
echo ""
|
||||
log "Total: ${bucket_count} bucket(s)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# LIST OBJECTS
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_list_objects() {
|
||||
if [[ -z "$TARGET_BUCKET" ]]; then
|
||||
err "Bucket required. Use --list s3://bucket-name[/prefix]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local bucket prefix
|
||||
bucket="${TARGET_BUCKET#s3://}"
|
||||
prefix=""
|
||||
if [[ "$bucket" == */* ]]; then
|
||||
prefix="${bucket#*/}"
|
||||
bucket="${bucket%%/*}"
|
||||
fi
|
||||
|
||||
log "Listing objects in s3://${bucket}/${prefix}..."
|
||||
|
||||
local list_args=(s3api list-objects-v2 --bucket "$bucket" --output json)
|
||||
[[ -n "$prefix" ]] && list_args+=(--prefix "$prefix")
|
||||
|
||||
local objects_json
|
||||
objects_json=$(aws_cmd "${list_args[@]}" 2>/dev/null) || {
|
||||
err "Failed to list objects in s3://${bucket}/${prefix}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
local obj_count
|
||||
obj_count=$(echo "$objects_json" | jq '.Contents // [] | length')
|
||||
|
||||
if [[ "$obj_count" -eq 0 ]]; then
|
||||
log "No objects found"
|
||||
return
|
||||
fi
|
||||
|
||||
echo ""
|
||||
printf " %12s %-22s %s\n" "SIZE" "MODIFIED" "KEY"
|
||||
echo " $(printf '%.0s─' {1..90})"
|
||||
|
||||
echo "$objects_json" | jq -c '.Contents[]' | while IFS= read -r obj; do
|
||||
local key size modified
|
||||
key=$(echo "$obj" | jq -r '.Key')
|
||||
size=$(echo "$obj" | jq -r '.Size')
|
||||
modified=$(echo "$obj" | jq -r '.LastModified' | cut -c1-19)
|
||||
|
||||
printf " %12s %-22s %s\n" "$(human_size "$size")" "$modified" "$key"
|
||||
done
|
||||
|
||||
local total_size
|
||||
total_size=$(echo "$objects_json" | jq '[.Contents[].Size] | add // 0')
|
||||
|
||||
echo ""
|
||||
log "Objects: ${obj_count}, Total size: $(human_size "$total_size")"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# COPY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_copy() {
|
||||
if [[ -z "$SOURCE_PATH" || -z "$DEST_PATH" ]]; then
|
||||
err "Source and destination required. Use --copy s3://src s3://dst"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Copying ${SOURCE_PATH} → ${DEST_PATH}..."
|
||||
|
||||
local cp_args=(s3 cp "$SOURCE_PATH" "$DEST_PATH" --recursive)
|
||||
|
||||
[[ "$DRY_RUN" == "true" ]] && cp_args+=(--dryrun)
|
||||
[[ -n "$INCLUDE_PATTERN" ]] && cp_args+=(--include "$INCLUDE_PATTERN")
|
||||
[[ -n "$EXCLUDE_PATTERN" ]] && cp_args+=(--exclude "$EXCLUDE_PATTERN")
|
||||
[[ -n "$STORAGE_CLASS" ]] && cp_args+=(--storage-class "$STORAGE_CLASS")
|
||||
[[ -n "$ACL" ]] && cp_args+=(--acl "$ACL")
|
||||
[[ -n "$SSE" ]] && cp_args+=(--sse "$SSE")
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
log "${YELLOW}DRY RUN${RESET} — no objects will be copied"
|
||||
fi
|
||||
|
||||
aws_cmd "${cp_args[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" != "true" ]]; then
|
||||
echo ""
|
||||
echo -e " ${GREEN}✓${RESET} Copy complete"
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SYNC
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_sync() {
|
||||
if [[ -z "$SOURCE_PATH" || -z "$DEST_PATH" ]]; then
|
||||
err "Source and destination required. Use --sync s3://src s3://dst"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Syncing ${SOURCE_PATH} → ${DEST_PATH}..."
|
||||
|
||||
local sync_args=(s3 sync "$SOURCE_PATH" "$DEST_PATH")
|
||||
|
||||
[[ "$DRY_RUN" == "true" ]] && sync_args+=(--dryrun)
|
||||
[[ -n "$INCLUDE_PATTERN" ]] && sync_args+=(--include "$INCLUDE_PATTERN")
|
||||
[[ -n "$EXCLUDE_PATTERN" ]] && sync_args+=(--exclude "$EXCLUDE_PATTERN")
|
||||
[[ -n "$STORAGE_CLASS" ]] && sync_args+=(--storage-class "$STORAGE_CLASS")
|
||||
[[ -n "$ACL" ]] && sync_args+=(--acl "$ACL")
|
||||
[[ -n "$SSE" ]] && sync_args+=(--sse "$SSE")
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
log "${YELLOW}DRY RUN${RESET} — no objects will be synced"
|
||||
fi
|
||||
|
||||
aws_cmd "${sync_args[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" != "true" ]]; then
|
||||
echo ""
|
||||
echo -e " ${GREEN}✓${RESET} Sync complete"
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# DELETE
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_delete() {
|
||||
if [[ -z "$TARGET_BUCKET" ]]; then
|
||||
err "Bucket required. Use --delete s3://bucket-name[/prefix]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local bucket prefix
|
||||
bucket="${TARGET_BUCKET#s3://}"
|
||||
prefix=""
|
||||
if [[ "$bucket" == */* ]]; then
|
||||
prefix="${bucket#*/}"
|
||||
bucket="${bucket%%/*}"
|
||||
fi
|
||||
|
||||
# Safety: require prefix or --all or --older-than
|
||||
if [[ -z "$prefix" && -z "$DELETE_PREFIX" && -z "$DELETE_OLDER_THAN" ]]; then
|
||||
err "Refusing to delete all objects without explicit confirmation"
|
||||
err "Use one of:"
|
||||
err " --delete s3://bucket/prefix — delete by prefix"
|
||||
err " --delete s3://bucket --prefix pfx — delete by prefix"
|
||||
err " --delete s3://bucket --older-than 30d — delete by age"
|
||||
err " --empty s3://bucket — empty entire bucket"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ -n "$DELETE_PREFIX" ]] && prefix="$DELETE_PREFIX"
|
||||
|
||||
if [[ -n "$DELETE_OLDER_THAN" ]]; then
|
||||
do_delete_by_age "$bucket" "$prefix"
|
||||
else
|
||||
do_delete_by_prefix "$bucket" "$prefix"
|
||||
fi
|
||||
}
|
||||
|
||||
do_delete_by_prefix() {
|
||||
local bucket="$1"
|
||||
local prefix="$2"
|
||||
|
||||
log "Deleting objects from s3://${bucket}/${prefix}..."
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
log "${YELLOW}DRY RUN${RESET} — listing objects that would be deleted"
|
||||
aws_cmd s3 rm "s3://${bucket}/${prefix}" --recursive --dryrun
|
||||
return
|
||||
fi
|
||||
|
||||
aws_cmd s3 rm "s3://${bucket}/${prefix}" --recursive
|
||||
|
||||
echo ""
|
||||
echo -e " ${GREEN}✓${RESET} Delete complete"
|
||||
}
|
||||
|
||||
do_delete_by_age() {
|
||||
local bucket="$1"
|
||||
local prefix="$2"
|
||||
|
||||
# Parse age (e.g., 30d, 12h, 90d)
|
||||
local age_value age_unit cutoff_epoch
|
||||
age_value="${DELETE_OLDER_THAN%%[dhDH]*}"
|
||||
age_unit="${DELETE_OLDER_THAN##*[0-9]}"
|
||||
age_unit="${age_unit,,}" # lowercase
|
||||
|
||||
case "$age_unit" in
|
||||
d) cutoff_epoch=$(date -d "-${age_value} days" +%s 2>/dev/null) || \
|
||||
cutoff_epoch=$(date -v-"${age_value}"d +%s 2>/dev/null) ;;
|
||||
h) cutoff_epoch=$(date -d "-${age_value} hours" +%s 2>/dev/null) || \
|
||||
cutoff_epoch=$(date -v-"${age_value}"H +%s 2>/dev/null) ;;
|
||||
*) err "Invalid age format '${DELETE_OLDER_THAN}'. Use Nd or Nh (e.g., 30d, 12h)"; exit 1 ;;
|
||||
esac
|
||||
|
||||
local cutoff_date
|
||||
cutoff_date=$(date -d "@${cutoff_epoch}" +%Y-%m-%dT%H:%M:%S 2>/dev/null) || \
|
||||
cutoff_date=$(date -r "${cutoff_epoch}" +%Y-%m-%dT%H:%M:%S 2>/dev/null)
|
||||
|
||||
log "Deleting objects older than ${DELETE_OLDER_THAN} (before ${cutoff_date})..."
|
||||
|
||||
local list_args=(s3api list-objects-v2 --bucket "$bucket" --output json)
|
||||
[[ -n "$prefix" ]] && list_args+=(--prefix "$prefix")
|
||||
|
||||
local objects_json
|
||||
objects_json=$(aws_cmd "${list_args[@]}" 2>/dev/null)
|
||||
|
||||
echo "$objects_json" | jq -c '.Contents // [] | .[]' | while IFS= read -r obj; do
|
||||
local key modified size modified_epoch
|
||||
key=$(echo "$obj" | jq -r '.Key')
|
||||
modified=$(echo "$obj" | jq -r '.LastModified')
|
||||
size=$(echo "$obj" | jq -r '.Size')
|
||||
|
||||
modified_epoch=$(date -d "$modified" +%s 2>/dev/null) || \
|
||||
modified_epoch=$(date -jf "%Y-%m-%dT%H:%M:%S" "${modified%%.*}" +%s 2>/dev/null) || modified_epoch=0
|
||||
|
||||
if [[ $modified_epoch -lt $cutoff_epoch ]]; then
|
||||
local age_days=$(( ($(date +%s) - modified_epoch) / 86400 ))
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo -e " ${YELLOW}⊘${RESET} ${key} — ${age_days}d old, $(human_size "$size") (would delete)"
|
||||
else
|
||||
if aws_cmd s3api delete-object --bucket "$bucket" --key "$key" >/dev/null 2>&1; then
|
||||
echo -e " ${GREEN}✓${RESET} ${key} — deleted (${age_days}d old)"
|
||||
else
|
||||
echo -e " ${RED}✗${RESET} ${key} — delete failed"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
log "${YELLOW}DRY RUN${RESET} — no objects were deleted. Use --force to delete."
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# EMPTY BUCKET
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_empty() {
|
||||
if [[ -z "$TARGET_BUCKET" ]]; then
|
||||
err "Bucket required. Use --empty s3://bucket-name"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local bucket="${TARGET_BUCKET#s3://}"
|
||||
bucket="${bucket%%/*}"
|
||||
|
||||
log "Emptying s3://${bucket}..."
|
||||
|
||||
# Check for versioning
|
||||
local versioning
|
||||
versioning=$(aws_cmd s3api get-bucket-versioning \
|
||||
--bucket "$bucket" \
|
||||
--query 'Status' \
|
||||
--output text 2>/dev/null) || versioning=""
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
local count
|
||||
count=$(aws_cmd s3api list-objects-v2 \
|
||||
--bucket "$bucket" \
|
||||
--query 'KeyCount' \
|
||||
--output text 2>/dev/null) || count="?"
|
||||
log "${YELLOW}DRY RUN${RESET} — would delete ${count} objects from s3://${bucket}"
|
||||
if [[ "$versioning" == "Enabled" ]]; then
|
||||
log "Bucket has versioning enabled — would also delete all version markers"
|
||||
fi
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$versioning" == "Enabled" ]]; then
|
||||
log "Bucket has versioning enabled — deleting all versions and markers..."
|
||||
# Delete all object versions
|
||||
local versions
|
||||
versions=$(aws_cmd s3api list-object-versions \
|
||||
--bucket "$bucket" \
|
||||
--output json 2>/dev/null)
|
||||
|
||||
# Delete versions
|
||||
echo "$versions" | jq -c '.Versions // [] | .[]' 2>/dev/null | while IFS= read -r ver; do
|
||||
local key version_id
|
||||
key=$(echo "$ver" | jq -r '.Key')
|
||||
version_id=$(echo "$ver" | jq -r '.VersionId')
|
||||
aws_cmd s3api delete-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" \
|
||||
--version-id "$version_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Delete markers
|
||||
echo "$versions" | jq -c '.DeleteMarkers // [] | .[]' 2>/dev/null | while IFS= read -r marker; do
|
||||
local key version_id
|
||||
key=$(echo "$marker" | jq -r '.Key')
|
||||
version_id=$(echo "$marker" | jq -r '.VersionId')
|
||||
aws_cmd s3api delete-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" \
|
||||
--version-id "$version_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
else
|
||||
aws_cmd s3 rm "s3://${bucket}" --recursive
|
||||
fi
|
||||
|
||||
echo -e " ${GREEN}✓${RESET} Bucket s3://${bucket} emptied"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# BUCKET SIZE
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_size() {
|
||||
if [[ -z "$TARGET_BUCKET" ]]; then
|
||||
err "Bucket required. Use --size s3://bucket-name"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local bucket="${TARGET_BUCKET#s3://}"
|
||||
bucket="${bucket%%/*}"
|
||||
|
||||
log "Calculating size of s3://${bucket}..."
|
||||
|
||||
# Use CloudWatch for accurate billing-level metrics (last 2 days)
|
||||
local cw_size
|
||||
cw_size=$(aws_cmd cloudwatch get-metric-statistics \
|
||||
--namespace AWS/S3 \
|
||||
--metric-name BucketSizeBytes \
|
||||
--dimensions "Name=BucketName,Value=${bucket}" "Name=StorageType,Value=StandardStorage" \
|
||||
--start-time "$(date -d '-2 days' -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-2d -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--period 86400 \
|
||||
--statistics Average \
|
||||
--query 'sort_by(Datapoints, &Timestamp)[-1].Average' \
|
||||
--output text 2>/dev/null) || cw_size=""
|
||||
|
||||
local cw_count
|
||||
cw_count=$(aws_cmd cloudwatch get-metric-statistics \
|
||||
--namespace AWS/S3 \
|
||||
--metric-name NumberOfObjects \
|
||||
--dimensions "Name=BucketName,Value=${bucket}" "Name=StorageType,Value=AllStorageTypes" \
|
||||
--start-time "$(date -d '-2 days' -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-2d -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--period 86400 \
|
||||
--statistics Average \
|
||||
--query 'sort_by(Datapoints, &Timestamp)[-1].Average' \
|
||||
--output text 2>/dev/null) || cw_count=""
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}Bucket: s3://${bucket}${RESET}"
|
||||
|
||||
if [[ -n "$cw_size" && "$cw_size" != "None" ]]; then
|
||||
local size_int="${cw_size%%.*}"
|
||||
local count_int="${cw_count%%.*}"
|
||||
local monthly_cost
|
||||
monthly_cost=$(echo "${size_int} / 1073741824 * 0.023" | bc -l 2>/dev/null | head -c 8) || monthly_cost="?"
|
||||
|
||||
echo -e " Size: $(human_size "$size_int") (CloudWatch metric)"
|
||||
echo -e " Objects: ${count_int}"
|
||||
echo -e " Est. cost: \$${monthly_cost}/month (Standard)"
|
||||
else
|
||||
# Fallback: list and sum (slower, but works without CloudWatch)
|
||||
log "CloudWatch metrics not available — counting objects manually..."
|
||||
local summary
|
||||
summary=$(aws_cmd s3 ls "s3://${bucket}" --recursive --summarize 2>/dev/null | tail -2)
|
||||
|
||||
local obj_count total_size
|
||||
obj_count=$(echo "$summary" | grep 'Total Objects:' | awk '{print $3}')
|
||||
total_size=$(echo "$summary" | grep 'Total Size:' | awk '{print $3}')
|
||||
|
||||
echo -e " Size: $(human_size "${total_size:-0}")"
|
||||
echo -e " Objects: ${obj_count:-0}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# AUDIT
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_audit() {
|
||||
log "Auditing S3 buckets..."
|
||||
|
||||
local buckets_json
|
||||
buckets_json=$(aws_cmd s3api list-buckets --output json)
|
||||
|
||||
local bucket_count
|
||||
bucket_count=$(echo "$buckets_json" | jq '.Buckets | length')
|
||||
|
||||
if [[ "$bucket_count" -eq 0 ]]; then
|
||||
log "No buckets found"
|
||||
return
|
||||
fi
|
||||
|
||||
echo ""
|
||||
printf " %-35s %-12s %-12s %-10s %-10s %s\n" \
|
||||
"BUCKET" "VERSIONING" "ENCRYPTION" "PUBLIC" "LIFECYCLE" "LOGGING"
|
||||
echo " $(printf '%.0s─' {1..100})"
|
||||
|
||||
echo "$buckets_json" | jq -r '.Buckets[].Name' | while IFS= read -r name; do
|
||||
local versioning encryption public_access lifecycle logging
|
||||
|
||||
# Versioning
|
||||
versioning=$(aws_cmd s3api get-bucket-versioning \
|
||||
--bucket "$name" \
|
||||
--query 'Status' \
|
||||
--output text 2>/dev/null) || versioning="error"
|
||||
[[ "$versioning" == "None" || -z "$versioning" ]] && versioning="off"
|
||||
|
||||
# Encryption
|
||||
if aws_cmd s3api get-bucket-encryption --bucket "$name" >/dev/null 2>&1; then
|
||||
encryption="on"
|
||||
else
|
||||
encryption="${RED}off${RESET}"
|
||||
fi
|
||||
|
||||
# Public access block
|
||||
local public_json
|
||||
public_json=$(aws_cmd s3api get-public-access-block \
|
||||
--bucket "$name" 2>/dev/null)
|
||||
|
||||
if [[ -n "$public_json" ]]; then
|
||||
local all_blocked
|
||||
all_blocked=$(echo "$public_json" | jq '
|
||||
.PublicAccessBlockConfiguration |
|
||||
(.BlockPublicAcls and .IgnorePublicAcls and .BlockPublicPolicy and .RestrictPublicBuckets)
|
||||
')
|
||||
if [[ "$all_blocked" == "true" ]]; then
|
||||
public_access="blocked"
|
||||
else
|
||||
public_access="${RED}partial${RESET}"
|
||||
fi
|
||||
else
|
||||
public_access="${RED}none${RESET}"
|
||||
fi
|
||||
|
||||
# Lifecycle rules
|
||||
local lc_json
|
||||
lc_json=$(aws_cmd s3api get-bucket-lifecycle-configuration \
|
||||
--bucket "$name" 2>/dev/null)
|
||||
if [[ -n "$lc_json" ]]; then
|
||||
local rule_count
|
||||
rule_count=$(echo "$lc_json" | jq '.Rules | length')
|
||||
lifecycle="${rule_count} rules"
|
||||
else
|
||||
lifecycle="none"
|
||||
fi
|
||||
|
||||
# Logging
|
||||
local log_json
|
||||
log_json=$(aws_cmd s3api get-bucket-logging \
|
||||
--bucket "$name" \
|
||||
--query 'LoggingEnabled' \
|
||||
--output text 2>/dev/null) || log_json=""
|
||||
if [[ -n "$log_json" && "$log_json" != "None" ]]; then
|
||||
logging="on"
|
||||
else
|
||||
logging="${YELLOW}off${RESET}"
|
||||
fi
|
||||
|
||||
printf " %-35s %-12s %-12b %-10b %-10s %b\n" \
|
||||
"$name" "$versioning" "$encryption" "$public_access" "$lifecycle" "$logging"
|
||||
done
|
||||
|
||||
echo ""
|
||||
log "Audited ${bucket_count} bucket(s)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# PRESIGN
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
do_presign() {
|
||||
if [[ -z "$SOURCE_PATH" ]]; then
|
||||
err "S3 path required. Use --presign s3://bucket/key"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local expires="${PRESIGN_EXPIRES:-3600}"
|
||||
|
||||
log "Generating presigned URL (expires in ${expires}s)..."
|
||||
|
||||
local url
|
||||
url=$(aws_cmd s3 presign "$SOURCE_PATH" --expires-in "$expires") || {
|
||||
err "Failed to generate presigned URL"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}Presigned URL:${RESET}"
|
||||
echo "$url"
|
||||
echo ""
|
||||
echo -e "Expires in: ${expires}s ($(( expires / 60 )) minutes)"
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MAIN
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
show_help() {
|
||||
cat <<EOF
|
||||
Usage: $SCRIPT_NAME [MODE] [OPTIONS]
|
||||
|
||||
Manage AWS S3 buckets — copy, sync, delete, audit, and more.
|
||||
Works with any AWS credential method (profiles, SSO, env vars, instance roles).
|
||||
|
||||
MODES:
|
||||
--list-buckets List all S3 buckets
|
||||
--list s3://bucket[/prefix] List objects in a bucket
|
||||
--copy SRC DST Copy objects (s3→s3, local→s3, s3→local)
|
||||
--sync SRC DST Sync source to destination (like rsync)
|
||||
--delete s3://bucket/prefix Delete objects by prefix
|
||||
--empty s3://bucket Empty an entire bucket (including versions)
|
||||
--size s3://bucket Calculate bucket size and cost
|
||||
--audit Security and config audit of all buckets
|
||||
--presign s3://bucket/key Generate a presigned download URL
|
||||
|
||||
OPTIONS:
|
||||
--dry-run Show what would happen without doing it
|
||||
--force Disable dry-run for delete operations
|
||||
--older-than AGE Delete objects older than AGE (e.g., 30d, 12h)
|
||||
--prefix PREFIX Filter by key prefix
|
||||
--include PATTERN Include pattern (e.g., "*.log")
|
||||
--exclude PATTERN Exclude pattern (e.g., "*.tmp")
|
||||
--storage-class CLASS Set storage class (STANDARD, GLACIER, etc.)
|
||||
--sse ALGO Server-side encryption (AES256, aws:kms)
|
||||
--expires SECONDS Presign URL expiry (default: 3600)
|
||||
--verbose Debug output
|
||||
--no-color Disable colored output
|
||||
--help, -h Show this help
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
AWS_PROFILE AWS CLI profile
|
||||
AWS_REGION AWS region
|
||||
AWS_ACCESS_KEY_ID Access key (temporary or permanent)
|
||||
AWS_SECRET_ACCESS_KEY Secret key
|
||||
AWS_SESSION_TOKEN Session token (for temporary credentials)
|
||||
DRY_RUN Dry-run mode (true/false)
|
||||
DELETE_OLDER_THAN Age threshold for delete (e.g., 30d)
|
||||
STORAGE_CLASS Storage class for copy/sync
|
||||
SSE Server-side encryption algorithm
|
||||
|
||||
EXAMPLES:
|
||||
# List all buckets
|
||||
./$(basename "$0") --list-buckets
|
||||
|
||||
# List objects in a bucket
|
||||
./$(basename "$0") --list s3://my-bucket/logs/
|
||||
|
||||
# Copy between buckets
|
||||
./$(basename "$0") --copy s3://source-bucket s3://dest-bucket
|
||||
|
||||
# Copy with storage class change
|
||||
./$(basename "$0") --copy s3://source-bucket s3://archive-bucket --storage-class GLACIER
|
||||
|
||||
# Sync local directory to S3
|
||||
./$(basename "$0") --sync ./backups/ s3://backup-bucket/daily/
|
||||
|
||||
# Sync between buckets (only changed files)
|
||||
./$(basename "$0") --sync s3://primary-bucket s3://dr-bucket
|
||||
|
||||
# Dry-run delete by prefix
|
||||
./$(basename "$0") --delete s3://my-bucket/tmp/ --dry-run
|
||||
|
||||
# Delete logs older than 30 days
|
||||
./$(basename "$0") --delete s3://my-bucket --prefix logs/ --older-than 30d --force
|
||||
|
||||
# Empty a bucket (required before deleting the bucket)
|
||||
./$(basename "$0") --empty s3://old-bucket --force
|
||||
|
||||
# Calculate bucket size and cost
|
||||
./$(basename "$0") --size s3://my-bucket
|
||||
|
||||
# Security audit of all buckets
|
||||
./$(basename "$0") --audit
|
||||
|
||||
# Generate a presigned URL (1 hour)
|
||||
./$(basename "$0") --presign s3://my-bucket/reports/q4.pdf
|
||||
|
||||
# Generate a presigned URL (24 hours)
|
||||
./$(basename "$0") --presign s3://my-bucket/file.zip --expires 86400
|
||||
|
||||
# With temporary credentials
|
||||
export AWS_ACCESS_KEY_ID="ASIA..."
|
||||
export AWS_SECRET_ACCESS_KEY="..."
|
||||
export AWS_SESSION_TOKEN="..."
|
||||
./$(basename "$0") --list-buckets
|
||||
|
||||
# With SSO profile
|
||||
aws sso login --profile prod
|
||||
AWS_PROFILE=prod ./$(basename "$0") --audit
|
||||
EOF
|
||||
}
|
||||
|
||||
PRESIGN_EXPIRES=""
|
||||
|
||||
main() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--list-buckets) RUN_MODE="list-buckets" ;;
|
||||
--list) RUN_MODE="list-objects"; TARGET_BUCKET="$2"; shift ;;
|
||||
--copy) RUN_MODE="copy"; SOURCE_PATH="$2"; DEST_PATH="$3"; shift 2 ;;
|
||||
--sync) RUN_MODE="sync"; SOURCE_PATH="$2"; DEST_PATH="$3"; shift 2 ;;
|
||||
--delete) RUN_MODE="delete"; TARGET_BUCKET="$2"; shift ;;
|
||||
--empty) RUN_MODE="empty"; TARGET_BUCKET="$2"; shift ;;
|
||||
--size) RUN_MODE="size"; TARGET_BUCKET="$2"; shift ;;
|
||||
--audit) RUN_MODE="audit" ;;
|
||||
--presign) RUN_MODE="presign"; SOURCE_PATH="$2"; shift ;;
|
||||
--dry-run) DRY_RUN="true" ;;
|
||||
--force) DRY_RUN="false" ;;
|
||||
--older-than) DELETE_OLDER_THAN="$2"; shift ;;
|
||||
--prefix) DELETE_PREFIX="$2"; shift ;;
|
||||
--include) INCLUDE_PATTERN="$2"; shift ;;
|
||||
--exclude) EXCLUDE_PATTERN="$2"; shift ;;
|
||||
--storage-class) STORAGE_CLASS="$2"; shift ;;
|
||||
--sse) SSE="$2"; shift ;;
|
||||
--expires) PRESIGN_EXPIRES="$2"; shift ;;
|
||||
--verbose) VERBOSE="true" ;;
|
||||
--no-color) COLOR="never" ;;
|
||||
--help|-h) show_help; exit 0 ;;
|
||||
*) err "Unknown option: $1"; echo ""; show_help; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
setup_colors
|
||||
|
||||
if [[ -z "$RUN_MODE" ]]; then
|
||||
err "No mode specified. Use --help for usage."
|
||||
echo ""
|
||||
show_help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}S3 Bucket Manager${RESET}"
|
||||
echo -e "Mode: ${RUN_MODE}"
|
||||
echo -e "Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo ""
|
||||
|
||||
check_deps
|
||||
|
||||
case "$RUN_MODE" in
|
||||
list-buckets) do_list_buckets ;;
|
||||
list-objects) do_list_objects ;;
|
||||
copy) do_copy ;;
|
||||
sync) do_sync ;;
|
||||
delete) do_delete ;;
|
||||
empty) do_empty ;;
|
||||
size) do_size ;;
|
||||
audit) do_audit ;;
|
||||
presign) do_presign ;;
|
||||
esac
|
||||
|
||||
local end_time
|
||||
end_time=$(date +%s)
|
||||
local duration=$(( end_time - START_TIME ))
|
||||
echo ""
|
||||
log "Completed in ${duration}s"
|
||||
|
||||
if [[ $WARNINGS -gt 0 ]]; then
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user