#!/usr/bin/env bash ########################################################################################## #### immich-migration.sh — Pre-process and bulk upload photos/videos to Immich #### #### Google Takeout EXIF repair, date fixing, HEIC conversion, duplicate detection, #### #### folder-based album creation, progress tracking, and immich-cli upload #### #### Requires: bash 4+, immich-cli, exiftool, jq #### #### #### #### Author: Phil Connor #### #### Contact: contact@mylinux.work #### #### License: MIT #### #### Version 1.00 #### #### #### #### Usage: #### #### ./immich-migration.sh --source ~/Photos --server URL --api-key KEY #### #### #### #### See --help for all options. #### ########################################################################################## set -euo pipefail # ── Colors ──────────────────────────────────────────────────────────── RED="" GREEN="" YELLOW="" BLUE="" CYAN="" BOLD="" DIM="" RESET="" setup_colors() { if [[ -t 1 ]]; then RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' DIM='\033[2m' RESET='\033[0m' fi } # ── Logging ─────────────────────────────────────────────────────────── log_info() { printf "${CYAN}[INFO]${RESET} %s\n" "$1"; } log_ok() { printf "${GREEN}[OK]${RESET} %s\n" "$1"; } log_warn() { printf "${YELLOW}[WARN]${RESET} %s\n" "$1"; } log_error() { printf "${RED}[ERROR]${RESET} %s\n" "$1" >&2; } log_step() { printf "\n${BOLD}── %s ──${RESET}\n\n" "$1"; } write_log() { [[ -n "${LOG_FILE:-}" ]] && printf "%s %-8s %s\n" "$(date +%Y-%m-%dT%H:%M:%S)" "$1" "$2" >> "$LOG_FILE" } # ── Defaults ────────────────────────────────────────────────────────── SOURCE_DIR="" SERVER_URL="" API_KEY="" DRY_RUN=false ALBUM_FROM_FOLDER=false FIX_DATES=false SKIP_DUPLICATES=false SKIP_HEIC_CONVERT=true LOG_FILE="" IMMICH_CMD="" UPLOAD_LOG="" WORK_DIR="" COUNT_TOTAL=0 COUNT_UPLOADED=0 COUNT_SKIPPED=0 COUNT_FAILED=0 START_TIME=0 # ── Usage ───────────────────────────────────────────────────────────── usage() { cat </dev/null; then IMMICH_CMD="immich" elif command -v immich-cli &>/dev/null; then IMMICH_CMD="immich-cli" else log_error "immich-cli not found. Install with: npm install -g @immich/cli" exit 1 fi log_ok "immich-cli found: $IMMICH_CMD" for cmd in exiftool jq; do if ! command -v "$cmd" &>/dev/null; then log_error "$cmd not found. Install with: apt install $( [[ $cmd == exiftool ]] && echo libimage-exiftool-perl || echo "$cmd" )" exit 1 fi log_ok "$cmd found" done if [[ "$SKIP_HEIC_CONVERT" == false ]]; then if command -v magick &>/dev/null || command -v convert &>/dev/null; then log_ok "ImageMagick found (HEIC conversion enabled)" elif command -v heif-convert &>/dev/null; then log_ok "heif-convert found (HEIC conversion enabled)" else log_warn "No HEIC converter found — disabling HEIC conversion" SKIP_HEIC_CONVERT=true fi fi } # ── Server Connectivity ────────────────────────────────────────────── check_server() { log_step "Checking Server Connectivity" local http_code http_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${SERVER_URL}/api/server/ping" \ -H "x-api-key: ${API_KEY}" 2>/dev/null) || true if [[ "$http_code" == "200" ]]; then log_ok "Server reachable: ${SERVER_URL}" else log_error "Cannot reach Immich server at ${SERVER_URL} (HTTP ${http_code})" exit 1 fi } # ── File Discovery ──────────────────────────────────────────────────── discover_files() { log_step "Scanning Source Directory" local file_list file_list=$(mktemp) find "$SOURCE_DIR" -type f \( \ -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.gif' \ -o -iname '*.bmp' -o -iname '*.tiff' -o -iname '*.tif' -o -iname '*.webp' \ -o -iname '*.heic' -o -iname '*.heif' -o -iname '*.avif' \ -o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.avi' -o -iname '*.mkv' \ -o -iname '*.m4v' -o -iname '*.3gp' -o -iname '*.wmv' -o -iname '*.mpg' \ -o -iname '*.raw' -o -iname '*.cr2' -o -iname '*.nef' -o -iname '*.arw' \ -o -iname '*.dng' -o -iname '*.orf' -o -iname '*.rw2' \ \) | sort > "$file_list" COUNT_TOTAL=$(wc -l < "$file_list") log_info "Found ${BOLD}${COUNT_TOTAL}${RESET} media files" echo "$file_list" } # ── Google Takeout JSON Merge ───────────────────────────────────────── process_takeout_json() { local file_list="$1" log_step "Google Takeout — JSON Sidecar Processing" local json_count=0 local merged_count=0 while IFS= read -r media_file; do local json_file="" if [[ -f "${media_file}.json" ]]; then json_file="${media_file}.json" elif [[ -f "${media_file%.*}.json" ]]; then json_file="${media_file%.*}.json" fi [[ -z "$json_file" ]] && continue ((json_count++)) || true local taken_ts geo_lat geo_lng description taken_ts=$(jq -r '.photoTakenTime.timestamp // empty' "$json_file" 2>/dev/null) || true geo_lat=$(jq -r '.geoData.latitude // empty' "$json_file" 2>/dev/null) || true geo_lng=$(jq -r '.geoData.longitude // empty' "$json_file" 2>/dev/null) || true description=$(jq -r '.description // empty' "$json_file" 2>/dev/null) || true local exif_args=() if [[ -n "$taken_ts" && "$taken_ts" != "0" ]]; then local taken_date taken_date=$(date -d "@${taken_ts}" +"%Y:%m:%d %H:%M:%S" 2>/dev/null) || \ taken_date=$(date -r "${taken_ts}" +"%Y:%m:%d %H:%M:%S" 2>/dev/null) || true if [[ -n "$taken_date" ]]; then exif_args+=("-DateTimeOriginal=$taken_date" "-CreateDate=$taken_date") fi fi if [[ -n "$geo_lat" && -n "$geo_lng" && "$geo_lat" != "0" && "$geo_lng" != "0" ]]; then local lat_ref="N" lng_ref="E" [[ $(echo "$geo_lat < 0" | bc -l 2>/dev/null || echo 0) == "1" ]] && lat_ref="S" && geo_lat="${geo_lat#-}" [[ $(echo "$geo_lng < 0" | bc -l 2>/dev/null || echo 0) == "1" ]] && lng_ref="W" && geo_lng="${geo_lng#-}" exif_args+=("-GPSLatitude=$geo_lat" "-GPSLatitudeRef=$lat_ref" "-GPSLongitude=$geo_lng" "-GPSLongitudeRef=$lng_ref") fi if [[ -n "$description" ]]; then exif_args+=("-ImageDescription=$description" "-Description=$description") fi if [[ ${#exif_args[@]} -eq 0 ]]; then continue fi if [[ "$DRY_RUN" == true ]]; then log_info "[DRY-RUN] Would merge JSON metadata → $(basename "$media_file")" else if exiftool -overwrite_original -quiet "${exif_args[@]}" "$media_file" 2>/dev/null; then ((merged_count++)) || true else log_warn "Failed to merge metadata for: $(basename "$media_file")" fi fi done < "$file_list" log_info "Takeout sidecars found: ${json_count} | Merged: ${merged_count}" } # ── Date Fixing from Filenames ──────────────────────────────────────── fix_dates_from_filenames() { local file_list="$1" log_step "Fixing Missing EXIF Dates" local checked=0 local fixed=0 while IFS= read -r file; do local existing_date existing_date=$(exiftool -s3 -DateTimeOriginal "$file" 2>/dev/null) || true if [[ -n "$existing_date" && "$existing_date" != "0000:00:00 00:00:00" ]]; then continue fi ((checked++)) || true local basename_file basename_file=$(basename "$file") local extracted_date="" # IMG_YYYYMMDD_HHMMSS if [[ "$basename_file" =~ ([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])_([0-9]{2})([0-9]{2})([0-9]{2}) ]]; then extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}" # Screenshot_YYYY-MM-DD-HH-MM-SS or YYYY-MM-DD_HH-MM-SS elif [[ "$basename_file" =~ ([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])[-_]([0-9]{2})[-.]([0-9]{2})[-.]([0-9]{2}) ]]; then extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}" # YYYY-MM-DD (date only, no time) elif [[ "$basename_file" =~ ([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) ]]; then extracted_date="${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} 12:00:00" fi if [[ -z "$extracted_date" ]]; then continue fi if [[ "$DRY_RUN" == true ]]; then log_info "[DRY-RUN] Would set date ${extracted_date} on $(basename "$file")" else if exiftool -overwrite_original -quiet \ "-DateTimeOriginal=$extracted_date" \ "-CreateDate=$extracted_date" \ "$file" 2>/dev/null; then ((fixed++)) || true write_log "FIXED" "$file" else log_warn "Failed to set date on: $(basename "$file")" fi fi done < "$file_list" log_info "Files missing dates: ${checked} | Fixed from filename: ${fixed}" } # ── HEIC Conversion ────────────────────────────────────────────────── convert_heic_files() { local file_list="$1" if [[ "$SKIP_HEIC_CONVERT" == true ]]; then return fi log_step "HEIC → JPEG Conversion" local heic_count=0 local converted=0 while IFS= read -r file; do local ext="${file##*.}" ext="${ext,,}" [[ "$ext" != "heic" && "$ext" != "heif" ]] && continue ((heic_count++)) || true local jpeg_file="${file%.*}.jpg" if [[ -f "$jpeg_file" ]]; then continue fi if [[ "$DRY_RUN" == true ]]; then log_info "[DRY-RUN] Would convert $(basename "$file") → JPEG" else local success=false if command -v magick &>/dev/null; then magick "$file" "$jpeg_file" 2>/dev/null && success=true elif command -v convert &>/dev/null; then convert "$file" "$jpeg_file" 2>/dev/null && success=true elif command -v heif-convert &>/dev/null; then heif-convert "$file" "$jpeg_file" &>/dev/null && success=true fi if [[ "$success" == true ]]; then exiftool -overwrite_original -quiet -TagsFromFile "$file" "$jpeg_file" 2>/dev/null || true ((converted++)) || true else log_warn "Failed to convert: $(basename "$file")" fi fi done < "$file_list" log_info "HEIC files found: ${heic_count} | Converted: ${converted}" } # ── Duplicate Detection ────────────────────────────────────────────── check_duplicate() { local file="$1" local checksum checksum=$(sha256sum "$file" 2>/dev/null | awk '{print $1}') || return 1 if [[ -f "$UPLOAD_LOG" ]] && grep -q "^${checksum}" "$UPLOAD_LOG" 2>/dev/null; then return 0 fi return 1 } record_upload() { local file="$1" local checksum checksum=$(sha256sum "$file" 2>/dev/null | awk '{print $1}') || return 0 echo "${checksum} ${file}" >> "$UPLOAD_LOG" } # ── Progress Display ───────────────────────────────────────────────── show_progress() { local current="$1" local total="$2" local pct=0 [[ "$total" -gt 0 ]] && pct=$(( current * 100 / total )) local elapsed=$(( $(date +%s) - START_TIME )) local mins=$(( elapsed / 60 )) local secs=$(( elapsed % 60 )) printf "\r${DIM}[%d/%d] %d%% complete | elapsed: %dm%02ds${RESET}" \ "$current" "$total" "$pct" "$mins" "$secs" } # ── Upload Files ────────────────────────────────────────────────────── upload_files() { local file_list="$1" log_step "Uploading to Immich" if [[ "$DRY_RUN" == true ]]; then log_info "[DRY-RUN] Would upload ${COUNT_TOTAL} files to ${SERVER_URL}" log_info "[DRY-RUN] Album mode: $( [[ "$ALBUM_FROM_FOLDER" == true ]] && echo "from folder names" || echo "none" )" return fi local current=0 while IFS= read -r file; do ((current++)) || true show_progress "$current" "$COUNT_TOTAL" if [[ "$SKIP_DUPLICATES" == true ]] && check_duplicate "$file"; then ((COUNT_SKIPPED++)) || true write_log "SKIPPED" "$file" continue fi local upload_args=("upload" "--server" "$SERVER_URL" "--key" "$API_KEY") if [[ "$ALBUM_FROM_FOLDER" == true ]]; then local album_name album_name=$(basename "$(dirname "$file")") if [[ -n "$album_name" && "$album_name" != "." ]]; then upload_args+=("--album" "$album_name") fi fi upload_args+=("$file") if $IMMICH_CMD "${upload_args[@]}" &>/dev/null; then ((COUNT_UPLOADED++)) || true record_upload "$file" write_log "UPLOADED" "$file" else ((COUNT_FAILED++)) || true write_log "FAILED" "$file" log_warn "Failed: $(basename "$file")" fi done < "$file_list" printf "\n" } # ── Summary ─────────────────────────────────────────────────────────── print_summary() { local elapsed=$(( $(date +%s) - START_TIME )) local mins=$(( elapsed / 60 )) local secs=$(( elapsed % 60 )) log_step "Migration Summary" printf " ${BOLD}Source:${RESET} %s\n" "$SOURCE_DIR" printf " ${BOLD}Server:${RESET} %s\n" "$SERVER_URL" printf " ${BOLD}Total:${RESET} %d files\n" "$COUNT_TOTAL" printf " ${GREEN}Uploaded:${RESET} %d\n" "$COUNT_UPLOADED" printf " ${YELLOW}Skipped:${RESET} %d (duplicate)\n" "$COUNT_SKIPPED" printf " ${RED}Failed:${RESET} %d\n" "$COUNT_FAILED" printf " ${BOLD}Duration:${RESET} %dm%02ds\n" "$mins" "$secs" if [[ -n "$LOG_FILE" ]]; then printf " ${BOLD}Log:${RESET} %s\n" "$LOG_FILE" fi if [[ "$DRY_RUN" == true ]]; then printf "\n ${YELLOW}(dry-run — no files were uploaded or modified)${RESET}\n" fi printf "\n" } # ── Cleanup ─────────────────────────────────────────────────────────── cleanup() { [[ -n "${WORK_DIR:-}" && -d "${WORK_DIR:-}" ]] && rm -rf "$WORK_DIR" } # ── Main ────────────────────────────────────────────────────────────── main() { setup_colors parse_args "$@" printf "\n${BOLD}Immich Migration Script${RESET}\n" printf "${DIM}Source: %s${RESET}\n" "$SOURCE_DIR" printf "${DIM}Server: %s${RESET}\n" "$SERVER_URL" [[ "$DRY_RUN" == true ]] && printf "${YELLOW}Mode: DRY-RUN${RESET}\n" printf "\n" START_TIME=$(date +%s) trap cleanup EXIT check_deps check_server local file_list file_list=$(discover_files) if [[ "$COUNT_TOTAL" -eq 0 ]]; then log_warn "No media files found in ${SOURCE_DIR}" exit 0 fi process_takeout_json "$file_list" if [[ "$FIX_DATES" == true ]]; then fix_dates_from_filenames "$file_list" fi convert_heic_files "$file_list" upload_files "$file_list" rm -f "$file_list" print_summary if [[ "$COUNT_FAILED" -gt 0 ]]; then exit 1 fi } main "$@"