Files
linux-scripts/immich-migration.sh
chiefgeek a1a17e81a1 Sync all scripts from website downloads — 352 scripts total
Includes updated JS challenge scripts with Claude-User whitelist,
same-site referer bypass, Blackbox-Exporter allowed bot, and all
new exporters, cheat sheets, and automation scripts.
2026-05-25 03:31:08 +02:00

531 lines
20 KiB
Bash

#!/usr/bin/env bash
##########################################################################################
#### immich-migration.sh — Pre-process and bulk upload photos/videos to Immich ####
#### Google Takeout EXIF repair, date fixing, HEIC conversion, duplicate detection, ####
#### folder-based album creation, progress tracking, and immich-cli upload ####
#### Requires: bash 4+, immich-cli, exiftool, jq ####
#### ####
#### Author: Phil Connor ####
#### Contact: contact@mylinux.work ####
#### License: MIT ####
#### Version 1.00 ####
#### ####
#### Usage: ####
#### ./immich-migration.sh --source ~/Photos --server URL --api-key KEY ####
#### ####
#### See --help for all options. ####
##########################################################################################
set -euo pipefail
# ── Colors ────────────────────────────────────────────────────────────
RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET=""
setup_colors() {
if [[ -t 1 ]]; then
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
fi
}
# ── Logging ───────────────────────────────────────────────────────────
log_info() { printf "${CYAN}[INFO]${RESET} %s\n" "$1"; }
log_ok() { printf "${GREEN}[OK]${RESET} %s\n" "$1"; }
log_warn() { printf "${YELLOW}[WARN]${RESET} %s\n" "$1"; }
log_error() { printf "${RED}[ERROR]${RESET} %s\n" "$1" >&2; }
log_step() { printf "\n${BOLD}── %s ──${RESET}\n\n" "$1"; }
write_log() {
[[ -n "${LOG_FILE:-}" ]] && printf "%s %-8s %s\n" "$(date +%Y-%m-%dT%H:%M:%S)" "$1" "$2" >> "$LOG_FILE"
}
# ── Defaults ──────────────────────────────────────────────────────────
SOURCE_DIR=""
SERVER_URL=""
API_KEY=""
DRY_RUN=false
ALBUM_FROM_FOLDER=false
FIX_DATES=false
SKIP_DUPLICATES=false
SKIP_HEIC_CONVERT=true
LOG_FILE=""
IMMICH_CMD=""
UPLOAD_LOG=""
WORK_DIR=""
COUNT_TOTAL=0
COUNT_UPLOADED=0
COUNT_SKIPPED=0
COUNT_FAILED=0
START_TIME=0
# ── Usage ─────────────────────────────────────────────────────────────
usage() {
cat <<EOF
${BOLD}Immich Migration Script${RESET} — pre-process and bulk upload to Immich
${BOLD}Usage:${RESET}
$(basename "$0") --source DIR --server URL --api-key KEY [options]
${BOLD}Required:${RESET}
--source DIR Source directory containing photos/videos
--server URL Immich server URL (e.g., https://immich.example.com)
--api-key KEY Immich API key
${BOLD}Options:${RESET}
--dry-run Preview operations without uploading or modifying
--album-from-folder Create albums from parent directory names
--fix-dates Attempt to fix missing EXIF dates from filenames
--skip-duplicates Skip files already in the upload log (SHA256)
--skip-heic-convert Skip HEIC → JPEG conversion (default: skip)
--no-skip-heic-convert Enable HEIC → JPEG conversion
--log FILE Write results to log file
-h, --help Show this help
${BOLD}Examples:${RESET}
$(basename "$0") --source ~/takeout --server https://immich.local --api-key abc123 --fix-dates
$(basename "$0") --source ~/Photos --server https://immich.local --api-key abc123 --album-from-folder --dry-run
EOF
exit 0
}
# ── Argument Parsing ──────────────────────────────────────────────────
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--source) SOURCE_DIR="$2"; shift 2 ;;
--server) SERVER_URL="$2"; shift 2 ;;
--api-key) API_KEY="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--album-from-folder) ALBUM_FROM_FOLDER=true; shift ;;
--fix-dates) FIX_DATES=true; shift ;;
--skip-duplicates) SKIP_DUPLICATES=true; shift ;;
--skip-heic-convert) SKIP_HEIC_CONVERT=true; shift ;;
--no-skip-heic-convert) SKIP_HEIC_CONVERT=false; shift ;;
--log) LOG_FILE="$2"; shift 2 ;;
-h|--help) usage ;;
*) log_error "Unknown option: $1"; usage ;;
esac
done
[[ -z "$SOURCE_DIR" ]] && { log_error "--source is required"; exit 1; }
[[ -z "$SERVER_URL" ]] && { log_error "--server is required"; exit 1; }
[[ -z "$API_KEY" ]] && { log_error "--api-key is required"; exit 1; }
[[ ! -d "$SOURCE_DIR" ]] && { log_error "Source directory not found: $SOURCE_DIR"; exit 1; }
SOURCE_DIR="$(cd "$SOURCE_DIR" && pwd)"
UPLOAD_LOG="${SOURCE_DIR}/.immich-upload.log"
}
# ── Dependency Checks ─────────────────────────────────────────────────
check_deps() {
log_step "Checking Dependencies"
if command -v immich &>/dev/null; then
IMMICH_CMD="immich"
elif command -v immich-cli &>/dev/null; then
IMMICH_CMD="immich-cli"
else
log_error "immich-cli not found. Install with: npm install -g @immich/cli"
exit 1
fi
log_ok "immich-cli found: $IMMICH_CMD"
for cmd in exiftool jq; do
if ! command -v "$cmd" &>/dev/null; then
log_error "$cmd not found. Install with: apt install $( [[ $cmd == exiftool ]] && echo libimage-exiftool-perl || echo "$cmd" )"
exit 1
fi
log_ok "$cmd found"
done
if [[ "$SKIP_HEIC_CONVERT" == false ]]; then
if command -v magick &>/dev/null || command -v convert &>/dev/null; then
log_ok "ImageMagick found (HEIC conversion enabled)"
elif command -v heif-convert &>/dev/null; then
log_ok "heif-convert found (HEIC conversion enabled)"
else
log_warn "No HEIC converter found — disabling HEIC conversion"
SKIP_HEIC_CONVERT=true
fi
fi
}
# ── Server Connectivity ──────────────────────────────────────────────
check_server() {
log_step "Checking Server Connectivity"
local http_code
http_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${SERVER_URL}/api/server/ping" \
-H "x-api-key: ${API_KEY}" 2>/dev/null) || true
if [[ "$http_code" == "200" ]]; then
log_ok "Server reachable: ${SERVER_URL}"
else
log_error "Cannot reach Immich server at ${SERVER_URL} (HTTP ${http_code})"
exit 1
fi
}
# ── File Discovery ────────────────────────────────────────────────────
discover_files() {
log_step "Scanning Source Directory"
local file_list
file_list=$(mktemp)
find "$SOURCE_DIR" -type f \( \
-iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.gif' \
-o -iname '*.bmp' -o -iname '*.tiff' -o -iname '*.tif' -o -iname '*.webp' \
-o -iname '*.heic' -o -iname '*.heif' -o -iname '*.avif' \
-o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.avi' -o -iname '*.mkv' \
-o -iname '*.m4v' -o -iname '*.3gp' -o -iname '*.wmv' -o -iname '*.mpg' \
-o -iname '*.raw' -o -iname '*.cr2' -o -iname '*.nef' -o -iname '*.arw' \
-o -iname '*.dng' -o -iname '*.orf' -o -iname '*.rw2' \
\) | sort > "$file_list"
COUNT_TOTAL=$(wc -l < "$file_list")
log_info "Found ${BOLD}${COUNT_TOTAL}${RESET} media files"
echo "$file_list"
}
# ── Google Takeout JSON Merge ─────────────────────────────────────────
process_takeout_json() {
local file_list="$1"
log_step "Google Takeout — JSON Sidecar Processing"
local json_count=0
local merged_count=0
while IFS= read -r media_file; do
local json_file=""
if [[ -f "${media_file}.json" ]]; then
json_file="${media_file}.json"
elif [[ -f "${media_file%.*}.json" ]]; then
json_file="${media_file%.*}.json"
fi
[[ -z "$json_file" ]] && continue
((json_count++)) || true
local taken_ts geo_lat geo_lng description
taken_ts=$(jq -r '.photoTakenTime.timestamp // empty' "$json_file" 2>/dev/null) || true
geo_lat=$(jq -r '.geoData.latitude // empty' "$json_file" 2>/dev/null) || true
geo_lng=$(jq -r '.geoData.longitude // empty' "$json_file" 2>/dev/null) || true
description=$(jq -r '.description // empty' "$json_file" 2>/dev/null) || true
local exif_args=()
if [[ -n "$taken_ts" && "$taken_ts" != "0" ]]; then
local taken_date
taken_date=$(date -d "@${taken_ts}" +"%Y:%m:%d %H:%M:%S" 2>/dev/null) || \
taken_date=$(date -r "${taken_ts}" +"%Y:%m:%d %H:%M:%S" 2>/dev/null) || true
if [[ -n "$taken_date" ]]; then
exif_args+=("-DateTimeOriginal=$taken_date" "-CreateDate=$taken_date")
fi
fi
if [[ -n "$geo_lat" && -n "$geo_lng" && "$geo_lat" != "0" && "$geo_lng" != "0" ]]; then
local lat_ref="N" lng_ref="E"
[[ $(echo "$geo_lat < 0" | bc -l 2>/dev/null || echo 0) == "1" ]] && lat_ref="S" && geo_lat="${geo_lat#-}"
[[ $(echo "$geo_lng < 0" | bc -l 2>/dev/null || echo 0) == "1" ]] && lng_ref="W" && geo_lng="${geo_lng#-}"
exif_args+=("-GPSLatitude=$geo_lat" "-GPSLatitudeRef=$lat_ref"
"-GPSLongitude=$geo_lng" "-GPSLongitudeRef=$lng_ref")
fi
if [[ -n "$description" ]]; then
exif_args+=("-ImageDescription=$description" "-Description=$description")
fi
if [[ ${#exif_args[@]} -eq 0 ]]; then
continue
fi
if [[ "$DRY_RUN" == true ]]; then
log_info "[DRY-RUN] Would merge JSON metadata → $(basename "$media_file")"
else
if exiftool -overwrite_original -quiet "${exif_args[@]}" "$media_file" 2>/dev/null; then
((merged_count++)) || true
else
log_warn "Failed to merge metadata for: $(basename "$media_file")"
fi
fi
done < "$file_list"
log_info "Takeout sidecars found: ${json_count} | Merged: ${merged_count}"
}
# ── Date Fixing from Filenames ────────────────────────────────────────
fix_dates_from_filenames() {
local file_list="$1"
log_step "Fixing Missing EXIF Dates"
local checked=0
local fixed=0
while IFS= read -r file; do
local existing_date
existing_date=$(exiftool -s3 -DateTimeOriginal "$file" 2>/dev/null) || true
if [[ -n "$existing_date" && "$existing_date" != "0000:00:00 00:00:00" ]]; then
continue
fi
((checked++)) || true
local basename_file
basename_file=$(basename "$file")
local extracted_date=""
# IMG_YYYYMMDD_HHMMSS
if [[ "$basename_file" =~ ([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])_([0-9]{2})([0-9]{2})([0-9]{2}) ]]; then
extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}"
# Screenshot_YYYY-MM-DD-HH-MM-SS or YYYY-MM-DD_HH-MM-SS
elif [[ "$basename_file" =~ ([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])[-_]([0-9]{2})[-.]([0-9]{2})[-.]([0-9]{2}) ]]; then
extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}"
# YYYY-MM-DD (date only, no time)
elif [[ "$basename_file" =~ ([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) ]]; then
extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} 12:00:00"
fi
if [[ -z "$extracted_date" ]]; then
continue
fi
if [[ "$DRY_RUN" == true ]]; then
log_info "[DRY-RUN] Would set date ${extracted_date} on $(basename "$file")"
else
if exiftool -overwrite_original -quiet \
"-DateTimeOriginal=$extracted_date" \
"-CreateDate=$extracted_date" \
"$file" 2>/dev/null; then
((fixed++)) || true
write_log "FIXED" "$file"
else
log_warn "Failed to set date on: $(basename "$file")"
fi
fi
done < "$file_list"
log_info "Files missing dates: ${checked} | Fixed from filename: ${fixed}"
}
# ── HEIC Conversion ──────────────────────────────────────────────────
convert_heic_files() {
local file_list="$1"
if [[ "$SKIP_HEIC_CONVERT" == true ]]; then
return
fi
log_step "HEIC → JPEG Conversion"
local heic_count=0
local converted=0
while IFS= read -r file; do
local ext="${file##*.}"
ext="${ext,,}"
[[ "$ext" != "heic" && "$ext" != "heif" ]] && continue
((heic_count++)) || true
local jpeg_file="${file%.*}.jpg"
if [[ -f "$jpeg_file" ]]; then
continue
fi
if [[ "$DRY_RUN" == true ]]; then
log_info "[DRY-RUN] Would convert $(basename "$file") → JPEG"
else
local success=false
if command -v magick &>/dev/null; then
magick "$file" "$jpeg_file" 2>/dev/null && success=true
elif command -v convert &>/dev/null; then
convert "$file" "$jpeg_file" 2>/dev/null && success=true
elif command -v heif-convert &>/dev/null; then
heif-convert "$file" "$jpeg_file" &>/dev/null && success=true
fi
if [[ "$success" == true ]]; then
exiftool -overwrite_original -quiet -TagsFromFile "$file" "$jpeg_file" 2>/dev/null || true
((converted++)) || true
else
log_warn "Failed to convert: $(basename "$file")"
fi
fi
done < "$file_list"
log_info "HEIC files found: ${heic_count} | Converted: ${converted}"
}
# ── Duplicate Detection ──────────────────────────────────────────────
check_duplicate() {
local file="$1"
local checksum
checksum=$(sha256sum "$file" 2>/dev/null | awk '{print $1}') || return 1
if [[ -f "$UPLOAD_LOG" ]] && grep -q "^${checksum}" "$UPLOAD_LOG" 2>/dev/null; then
return 0
fi
return 1
}
record_upload() {
local file="$1"
local checksum
checksum=$(sha256sum "$file" 2>/dev/null | awk '{print $1}') || return 0
echo "${checksum} ${file}" >> "$UPLOAD_LOG"
}
# ── Progress Display ─────────────────────────────────────────────────
show_progress() {
local current="$1"
local total="$2"
local pct=0
[[ "$total" -gt 0 ]] && pct=$(( current * 100 / total ))
local elapsed=$(( $(date +%s) - START_TIME ))
local mins=$(( elapsed / 60 ))
local secs=$(( elapsed % 60 ))
printf "\r${DIM}[%d/%d] %d%% complete | elapsed: %dm%02ds${RESET}" \
"$current" "$total" "$pct" "$mins" "$secs"
}
# ── Upload Files ──────────────────────────────────────────────────────
upload_files() {
local file_list="$1"
log_step "Uploading to Immich"
if [[ "$DRY_RUN" == true ]]; then
log_info "[DRY-RUN] Would upload ${COUNT_TOTAL} files to ${SERVER_URL}"
log_info "[DRY-RUN] Album mode: $( [[ "$ALBUM_FROM_FOLDER" == true ]] && echo "from folder names" || echo "none" )"
return
fi
local current=0
while IFS= read -r file; do
((current++)) || true
show_progress "$current" "$COUNT_TOTAL"
if [[ "$SKIP_DUPLICATES" == true ]] && check_duplicate "$file"; then
((COUNT_SKIPPED++)) || true
write_log "SKIPPED" "$file"
continue
fi
local upload_args=("upload" "--server" "$SERVER_URL" "--key" "$API_KEY")
if [[ "$ALBUM_FROM_FOLDER" == true ]]; then
local album_name
album_name=$(basename "$(dirname "$file")")
if [[ -n "$album_name" && "$album_name" != "." ]]; then
upload_args+=("--album" "$album_name")
fi
fi
upload_args+=("$file")
if $IMMICH_CMD "${upload_args[@]}" &>/dev/null; then
((COUNT_UPLOADED++)) || true
record_upload "$file"
write_log "UPLOADED" "$file"
else
((COUNT_FAILED++)) || true
write_log "FAILED" "$file"
log_warn "Failed: $(basename "$file")"
fi
done < "$file_list"
printf "\n"
}
# ── Summary ───────────────────────────────────────────────────────────
print_summary() {
local elapsed=$(( $(date +%s) - START_TIME ))
local mins=$(( elapsed / 60 ))
local secs=$(( elapsed % 60 ))
log_step "Migration Summary"
printf " ${BOLD}Source:${RESET} %s\n" "$SOURCE_DIR"
printf " ${BOLD}Server:${RESET} %s\n" "$SERVER_URL"
printf " ${BOLD}Total:${RESET} %d files\n" "$COUNT_TOTAL"
printf " ${GREEN}Uploaded:${RESET} %d\n" "$COUNT_UPLOADED"
printf " ${YELLOW}Skipped:${RESET} %d (duplicate)\n" "$COUNT_SKIPPED"
printf " ${RED}Failed:${RESET} %d\n" "$COUNT_FAILED"
printf " ${BOLD}Duration:${RESET} %dm%02ds\n" "$mins" "$secs"
if [[ -n "$LOG_FILE" ]]; then
printf " ${BOLD}Log:${RESET} %s\n" "$LOG_FILE"
fi
if [[ "$DRY_RUN" == true ]]; then
printf "\n ${YELLOW}(dry-run — no files were uploaded or modified)${RESET}\n"
fi
printf "\n"
}
# ── Cleanup ───────────────────────────────────────────────────────────
cleanup() {
[[ -n "${WORK_DIR:-}" && -d "${WORK_DIR:-}" ]] && rm -rf "$WORK_DIR"
}
# ── Main ──────────────────────────────────────────────────────────────
main() {
setup_colors
parse_args "$@"
printf "\n${BOLD}Immich Migration Script${RESET}\n"
printf "${DIM}Source: %s${RESET}\n" "$SOURCE_DIR"
printf "${DIM}Server: %s${RESET}\n" "$SERVER_URL"
[[ "$DRY_RUN" == true ]] && printf "${YELLOW}Mode: DRY-RUN${RESET}\n"
printf "\n"
START_TIME=$(date +%s)
trap cleanup EXIT
check_deps
check_server
local file_list
file_list=$(discover_files)
if [[ "$COUNT_TOTAL" -eq 0 ]]; then
log_warn "No media files found in ${SOURCE_DIR}"
exit 0
fi
process_takeout_json "$file_list"
if [[ "$FIX_DATES" == true ]]; then
fix_dates_from_filenames "$file_list"
fi
convert_heic_files "$file_list"
upload_files "$file_list"
rm -f "$file_list"
print_summary
if [[ "$COUNT_FAILED" -gt 0 ]]; then
exit 1
fi
}
main "$@"