diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 2e1c68b..1673c17 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -66,8 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" - acquire_lock if ! command -v docker &>/dev/null; then @@ -75,7 +73,6 @@ if ! command -v docker &>/dev/null; then notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning" exit 1 fi -success "Docker found" # detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS detect_hosts @@ -86,6 +83,8 @@ if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then exit 0 fi +echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} containers configured" + # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -114,10 +113,10 @@ retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do - info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if "$@"; then - success "Succeeded on attempt $attempt" + log "Succeeded on attempt $attempt" return 0 else warn "Attempt $attempt failed" @@ -225,7 +224,7 @@ check_dependency_delay() { local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then - info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." + echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..." sleep "$CONTAINER_DELAY" fi } @@ -240,7 +239,7 @@ retry_docker() { info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if docker_cmd "$@"; then - success "Succeeded on attempt $attempt" + log "Succeeded on attempt $attempt" return 0 else warn "Attempt $attempt failed" diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index b2ea2a8..da7df64 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -66,8 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" - acquire_lock if ! command -v docker &>/dev/null; then @@ -75,7 +73,6 @@ if ! command -v docker &>/dev/null; then notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning" exit 1 fi -success "Docker found" # detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS detect_hosts @@ -86,6 +83,8 @@ if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then exit 0 fi +echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#WEEKLY_RESTART_CONTAINERS[@]} containers configured" + # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -127,10 +126,10 @@ retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do - info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if docker_cmd "$@"; then - success "Succeeded on attempt $attempt" + log "Succeeded on attempt $attempt" return 0 else warn "Attempt $attempt failed" diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index 2d514d8..271a2ff 100644 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -21,7 +21,7 @@ # NORMAL — remote up, internet up # Own containers only. DDNS ON. Silent operation. # -# FAILOVER — remote down, internet up +# FALLBACK — remote down, internet up # Start remote containers locally — tiered by outage duration. # Remote DDNS started immediately (Tier 1). # Own containers keep running — failover is additive. @@ -40,20 +40,21 @@ # Script controls DDNS exclusively — network state NEVER auto-starts DDNS. # DDNS only starts after full handback sequence confirms containers are up. # One DDNS per domain active at all times — never two, never zero for long. -# 1 minute TTL + 1 minute check interval = minimal user impact on failover. +# 1 minute TTL + 90s strike window = minimal user impact on failover. # # ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── -# When remote returns after FAILOVER: +# When remote returns after FALLBACK: # 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks # 2. Pre-flight checks — version parity, remote array, remote Docker daemon -# 3. Stop remote DDNS first — prevents split brain DNS during rsync -# 4. Stop remote containers — clean state before rsync -# 5. Tiered rsync writeback — skip if outage under threshold -# 6. Start remote containers — in dependency order with verification -# 7. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up -# 8. Return to NORMAL +# 3. Staged reverse-tier handback — Tier 4 → Tier 3 → Tier 2 → Tier 1 +# Each stage: stop local tier containers → rsync writeback → start on remote +# Emby (Tier 1) stays on fallback serving users until all higher tiers are done +# 4. DDNS handoff — immediately before Tier 1 goes down (not at start) +# 5. Tier 1 handback — stop local → rsync → start remote +# 6. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up +# 7. Return to NORMAL # -# ── TIERED FAILOVER ─────────────────────────────────────────────────────────────────────────── +# ── TIERED FALLBACK ─────────────────────────────────────────────────────────────────────────── # Tier 1 — Immediate — vital services + Live TV — cannot wait # Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr) # Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr) @@ -72,16 +73,16 @@ # # ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── # HOST*_DDNS_CONTAINERS — DDNS containers this host manages -# FAILOVER_HOST*_STOP_ON_NO_NET — containers stopped on internet loss -# FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other +# FALLBACK_HOST*_STOP_ON_NO_NET — containers stopped on internet loss +# FALLBACK_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other # HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes # HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this -# FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier +# FALLBACK_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.) -# FALLBACK_CHECK_INTERVAL — seconds between checks -# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback +# FALLBACK_CHECK_INTERVAL — seconds between checks (default 30) +# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback (default 3 = 90s) # FALLBACK_STATE_FILE — /boot/config path — survives reboots # FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs # EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8) @@ -332,7 +333,7 @@ remote_ddns_stop() { } # ============================================================================================== -# ── TIERED FAILOVER HELPERS ─────────────────────────────────────────────────────────────────── +# ── TIERED FALLBACK HELPERS ─────────────────────────────────────────────────────────────────── # ============================================================================================== # All routing uses MY_ID — not hostname string comparison. # MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc. @@ -436,150 +437,152 @@ fi # ============================================================================================== # ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Called when remote returns after FAILOVER state. +# Called when remote returns after FALLBACK state. # CRITICAL sequencing — do not reorder without understanding the consequences. # Each step must succeed before proceeding — aborts and retries next cycle on failure. run_handback() { echo "" echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - warn "$REMOTE_SERVER_NAME has returned — beginning handback" + warn "$REMOTE_SERVER_NAME has returned — beginning staged handback" # ── Step 1: Pre-flight checks ──────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" - # Version parity — refuse handback if unRAID versions mismatch if ! check_unraid_version_parity; then warn "Version parity check failed — aborting handback, will retry next cycle" state_set handback_strikes 0 return 1 fi - # Remote array must be mounted before rsync if ! check_remote_array; then warn "Remote array not ready — aborting handback, will retry next cycle" state_set handback_strikes 0 return 1 fi - # Remote Docker daemon must be responsive before issuing container commands if ! check_remote_docker_daemon; then warn "Remote Docker daemon not ready — aborting handback, will retry next cycle" state_set handback_strikes 0 return 1 fi - log "Pre-flight checks passed ✅" + echo " Pre-flight checks passed ✅" - # ── Step 2: Stop remote DDNS FIRST ────────────────────────────────────────────────────── - # CRITICAL — prevents split-brain DNS while data is being synced + local outage_minutes + outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 )) + local tier1_wb_delay t2_delay t3_delay t4_delay + tier1_wb_delay=$(get_tier1_writeback_delay) + t2_delay=$(get_tier2_delay) + t3_delay=$(get_tier3_delay) + t4_delay=$(get_tier4_delay) + + warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1" + + # ── Staged handback helper ──────────────────────────────────────────────────────────────── + # Each tier: stop local containers → rsync writeback → start on remote. + # Skips tiers that were never activated. Emby (Tier 1) stays on fallback + # serving users until all higher tiers complete. + handback_tier() { + local tier="$1" threshold="$2" label="$3" + + [[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0 + + echo "" + echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━" + + local containers + read -r -a containers <<< "$(get_tier_containers "$tier")" + + echo " Stopping local Tier $tier containers..." + for container in "${containers[@]}"; do + [[ -n "$container" ]] && local_stop "$container" + done + + if ! check_rsync_enabled "FALLBACK"; then + warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback" + else + local jobs + read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")" + if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then + echo " Rsync writeback — outage ${outage_minutes}min >= ${threshold}min" + for job in "${jobs[@]}"; do + [[ -z "$job" ]] && continue + log "Syncing: $job" + [[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \ + || warn "DRY RUN — would rsync: $job" + done + else + log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min" + fi + fi + + echo " Starting Tier $tier on $REMOTE_SERVER_NAME..." + for container in "${containers[@]}"; do + [[ -n "$container" ]] && remote_start "$container" + done + + state_set "tier${tier}_started" "false" + warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME" + } + + # ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ────────────────────────────────── + handback_tier 4 "$t4_delay" "media shares + edge cases" + handback_tier 3 "$t3_delay" "secondary services" + handback_tier 2 "$t2_delay" "productivity services" + + # ── Step 5: DDNS handoff — immediately before Tier 1 goes down ────────────────────────── + # Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks. + # Prevents split-brain DNS during Tier 1 rsync window. echo "" echo "━━━ $ICON_NET DDNS Handoff ━━━" remote_ddns_stop - sleep 5 # brief pause to ensure DDNS stop propagates before rsync + sleep 5 - # ── Step 3: Stop remote containers ────────────────────────────────────────────────────── - # Clean state before rsync — no dirty writes during transfer window + # ── Step 6: Tier 1 handback — vital services + Emby ───────────────────────────────────── echo "" - echo "━━━ $ICON_STOP Stop Remote Containers ━━━" + echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━" - local ALL_FALLBACK_CONTAINERS=() - - # Collect all started tiers in reverse order (highest tier stops first) - if [[ "$(state_get tier4_started)" == "true" ]]; then - read -r -a t4 <<< "$(get_tier_containers 4)" - ALL_FALLBACK_CONTAINERS+=("${t4[@]}") - fi - if [[ "$(state_get tier3_started)" == "true" ]]; then - read -r -a t3 <<< "$(get_tier_containers 3)" - ALL_FALLBACK_CONTAINERS+=("${t3[@]}") - fi - if [[ "$(state_get tier2_started)" == "true" ]]; then - read -r -a t2 <<< "$(get_tier_containers 2)" - ALL_FALLBACK_CONTAINERS+=("${t2[@]}") - fi - - # Tier 1 always started — stop last (DDNS already handled above) read -r -a t1 <<< "$(get_tier_containers 1)" + + echo " Stopping local Tier 1 containers..." for container in "${t1[@]}"; do + [[ -z "$container" ]] && continue local is_ddns=false for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do [[ "$container" == "$ddns" ]] && is_ddns=true && break done - [[ "$is_ddns" == false ]] && ALL_FALLBACK_CONTAINERS+=("$container") + [[ "$is_ddns" == false ]] && local_stop "$container" done - for container in "${ALL_FALLBACK_CONTAINERS[@]}"; do - [[ -z "$container" ]] && continue - local_stop "$container" - done - - log "All fallback containers stopped locally" - - # ── Step 4: Rsync writeback ────────────────────────────────────────────────────────────── - # Tiered writeback with skip window — short outages do not benefit from writeback. - # After a short outage primary's clean nightly state is more reliable than dirty - # accumulation — skip writeback entirely for outages under the threshold. - echo "" - echo "━━━ $ICON_SYNC Rsync Writeback ━━━" - - local outage_minutes - outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 )) - local tier1_wb_delay - tier1_wb_delay=$(get_tier1_writeback_delay) - - warn "Outage duration: ${outage_minutes}min" - if ! check_rsync_enabled "FALLBACK"; then - warn "FALLBACK_RSYNC_ENABLED=false — skipping all writeback jobs" - warn "Handback will complete without syncing state back to primary" + warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback" else - run_writeback_tier() { - local tier="$1" threshold="$2" label="$3" - local jobs - read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")" - [[ ${#jobs[@]} -eq 0 ]] && return - - if [[ "$outage_minutes" -ge "$threshold" ]]; then - warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min" - for job in "${jobs[@]}"; do - [[ -z "$job" ]] && continue - log "Syncing: $job" - if [[ "$DRY_RUN" == false ]]; then - if [[ "$(basename "$job")" == "Emby" ]]; then - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback - else - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" - fi + local t1_jobs + read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)" + if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then + echo " Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min" + for job in "${t1_jobs[@]}"; do + [[ -z "$job" ]] && continue + log "Syncing: $job" + if [[ "$DRY_RUN" == false ]]; then + if [[ "$(basename "$job")" == "Emby" ]]; then + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback else - warn "DRY RUN — would rsync: $job" + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" fi - done - else - log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner" - fi - } - - local t2_delay t3_delay t4_delay - t2_delay=$(get_tier2_delay) - t3_delay=$(get_tier3_delay) - t4_delay=$(get_tier4_delay) - - run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack" - run_writeback_tier 2 "$t2_delay" "productivity services" - run_writeback_tier 3 "$t3_delay" "secondary services" - run_writeback_tier 4 "$t4_delay" "media shares + edge cases" + else + warn "DRY RUN — would rsync: $job" + fi + done + else + log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min" + fi fi - log "Writeback complete" - - # ── Step 5: Start remote containers ───────────────────────────────────────────────────── - # Start in tier order with dependency awareness — databases before apps - echo "" - echo "━━━ $ICON_START Start Remote Containers ━━━" - - # Tier 1 — excluding DDNS (handled separately as final step) + echo " Starting Tier 1 on $REMOTE_SERVER_NAME..." + sleep 10 # give databases time to initialise before apps for container in "${t1[@]}"; do [[ -z "$container" ]] && continue local is_ddns=false @@ -589,36 +592,17 @@ run_handback() { [[ "$is_ddns" == false ]] && remote_start "$container" done - sleep 10 # give databases time to initialise before apps - - [[ "$(state_get tier2_started)" == "true" ]] && \ - for container in "${t2[@]}"; do - [[ -n "$container" ]] && remote_start "$container" - done - - [[ "$(state_get tier3_started)" == "true" ]] && \ - for container in "${t3[@]}"; do - [[ -n "$container" ]] && remote_start "$container" - done - - [[ "$(state_get tier4_started)" == "true" ]] && \ - for container in "${t4[@]}"; do - [[ -n "$container" ]] && remote_start "$container" - done - - log "Remote containers started" - - # ── Step 6: Start remote DDNS LAST ────────────────────────────────────────────────────── - # DNS cuts over ONLY after containers confirmed up — this is the final step + # ── Step 7: DNS Cutover — final step ──────────────────────────────────────────────────── + # DNS cuts over ONLY after Tier 1 containers confirmed up echo "" echo "━━━ $ICON_NET DNS Cutover ━━━" - sleep 5 # brief pause to ensure containers are accepting connections + sleep 5 for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do remote_start "$container" done warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME" - # ── Step 7: Return to NORMAL ───────────────────────────────────────────────────────────── + # ── Step 8: Return to NORMAL ───────────────────────────────────────────────────────────── echo "" state_set state "NORMAL" state_set fallback_start "0" @@ -627,6 +611,9 @@ run_handback() { state_set tier3_started "false" state_set tier4_started "false" + # Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback + local_ddns_start + warn "$ICON_DONE Handback complete — returned to NORMAL" notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \ "Fallback" "normal" @@ -671,7 +658,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do log "$ICON_SUCCESS NORMAL — all systems up" elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then - # Remote is down — enter FAILOVER + # Remote is down — enter FALLBACK immediately echo "" echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback" @@ -713,7 +700,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do fi # ════════════════════════════════════════════════════════════════ - # FAILOVER STATE + # FALLBACK STATE # ════════════════════════════════════════════════════════════════ elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh index dc47a6b..8f10a15 100644 --- a/Fallback/fallback_test.sh +++ b/Fallback/fallback_test.sh @@ -14,7 +14,7 @@ # Phase 1 — Pre-flight verify both servers reachable, daemons healthy, # version parity, fallback.sh exists, state is NORMAL # Phase 2 — Block Remote iptables rule drops all traffic to remote IP -# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FAILOVER +# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FALLBACK # Phase 4 — Container Start verify Tier 1 failover containers started locally # Phase 5 — Restore remove iptables rule, remote becomes reachable # Phase 6 — Handback wait for fallback.sh to complete handback to NORMAL diff --git a/Initial_run/partnership_onboard.sh b/Initial_run/partnership_onboard.sh index 5d6ba62..7d08bfb 100755 --- a/Initial_run/partnership_onboard.sh +++ b/Initial_run/partnership_onboard.sh @@ -20,12 +20,17 @@ # Step 2: partnership_manager.sh --onboard # — verifies both servers, reconfigures auth WebUIs → owner IP, # writes ACTIVE state, creates FolderView3 folder (if enabled) +# Step 3: arr_sync.sh — bidirectional library bootstrap: partner gets our full +# Lidarr/Sonarr/Radarr library, we get theirs. Both sides +# start tracking the merged library from day one. +# Non-fatal — partnership is valid even if arrs aren't live yet. # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# Initial_run/partnership_onboard.sh — full setup -# Initial_run/partnership_onboard.sh --dry-run — preview without changes -# Initial_run/partnership_onboard.sh --log — verbose output -# Initial_run/partnership_onboard.sh --skip-ssh — skip ssh_setup.sh (key already set up) +# Initial_run/partnership_onboard.sh — full setup +# Initial_run/partnership_onboard.sh --dry-run — preview without changes +# Initial_run/partnership_onboard.sh --log — verbose output +# Initial_run/partnership_onboard.sh --skip-ssh — skip ssh_setup.sh (key already set up) +# Initial_run/partnership_onboard.sh --skip-arr-sync — skip arr sync (arrs not live yet) # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -35,11 +40,13 @@ source "$SCRIPTS_ROOT/load_config.sh" # ── Parse --skip-ssh before parse_args ──────────────────────────────────────────────────────── SKIP_SSH=false +SKIP_ARR_SYNC=false FILTERED_ARGS=() for arg in "$@"; do case "$arg" in - --skip-ssh) SKIP_SSH=true ;; - *) FILTERED_ARGS+=("$arg") ;; + --skip-ssh) SKIP_SSH=true ;; + --skip-arr-sync) SKIP_ARR_SYNC=true ;; + *) FILTERED_ARGS+=("$arg") ;; esac done @@ -74,7 +81,7 @@ EXTRA_FLAGS=() # ━━━ Step 1: SSH Key Setup ━━━ # ============================================================================================== echo "" -echo "━━━ Step 1/2 — SSH Key Setup ━━━" +echo "━━━ Step 1/3 — SSH Key Setup ━━━" if [[ "$SKIP_SSH" == true ]]; then warn "Skipping SSH setup (--skip-ssh)" @@ -92,7 +99,7 @@ fi # ━━━ Step 2: Partnership Onboard ━━━ # ============================================================================================== echo "" -echo "━━━ Step 2/2 — Partnership Onboard ━━━" +echo "━━━ Step 2/3 — Partnership Onboard ━━━" if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then log "Partnership onboard complete ✅" @@ -102,6 +109,37 @@ else ONBOARD_OK=false fi +# ============================================================================================== +# ━━━ Step 3: Arr Library Bootstrap ━━━ +# ============================================================================================== +echo "" +echo "━━━ Step 3/3 — Arr Library Bootstrap ━━━" + +ARR_SYNC_OK=true +if [[ "$ONBOARD_OK" == false ]]; then + warn "Skipping arr sync — onboard did not complete" + ARR_SYNC_OK=false +elif [[ "$SKIP_ARR_SYNC" == true ]]; then + warn "Skipping arr sync (--skip-arr-sync)" + ARR_SYNC_OK=false +else + ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" + if [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then + warn "arr_sync.sh not found — skipping library bootstrap" + warn "Run Media/arr_sync.sh manually once arrs are live" + ARR_SYNC_OK=false + else + log "Syncing arr libraries with $REMOTE_SERVER_NAME..." + if bash "$ARR_SYNC_SCRIPT" "${EXTRA_FLAGS[@]}"; then + log "Arr library bootstrap complete ✅" + else + warn "Arr sync completed with errors — partnership is still valid" + warn "Re-run Media/arr_sync.sh once all arr containers are live" + ARR_SYNC_OK=false + fi + fi +fi + # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== @@ -113,20 +151,28 @@ echo "$ICON_NET Partner: $REMOTE_SERVER_NAME" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" +_ssh_status() { [[ "$SKIP_SSH" == true ]] && echo "skipped" || echo "✅"; } +_arr_status() { + if [[ "$SKIP_ARR_SYNC" == true ]]; then echo "skipped (--skip-arr-sync)" + elif [[ "$ONBOARD_OK" == false ]]; then echo "skipped (onboard failed)" + elif [[ "$ARR_SYNC_OK" == true ]]; then echo "✅" + else echo "⚠️ errors — re-run arr_sync.sh once arrs are live" + fi +} + +echo " Step 1 — SSH key: $(_ssh_status)" +echo " Step 2 — Onboard: $( [[ "$ONBOARD_OK" == true ]] && echo "✅" || echo "❌" )" +echo " Step 3 — Arr bootstrap: $(_arr_status)" +echo "" + if [[ "$ONBOARD_OK" == true ]]; then - echo " Step 1 — SSH key: ✅" - echo " Step 2 — Onboard: ✅" - echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else - warn "$ICON_DONE DONE — partnership established ✅" - warn "Next: verify with 'Partnership/partnership_manager.sh --status'" + log "$ICON_DONE DONE — partnership established ✅" + log "Next: verify with 'Partnership/partnership_manager.sh --status'" fi else - echo " Step 1 — SSH key: $( [[ "$SKIP_SSH" == true ]] && echo "skipped" || echo "✅" )" - echo " Step 2 — Onboard: ❌" - echo "" error "Setup incomplete — resolve onboard errors above and re-run" fi diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh new file mode 100755 index 0000000..df8aa3c --- /dev/null +++ b/Media/arr_sync.sh @@ -0,0 +1,692 @@ +#!/bin/bash +# ============================================================================================== +# ================================= ARR SYNC =================================================== +# ============================================================================================== +# Bidirectional arr library sync across all nodes in the ecosystem. +# Syncs Lidarr, Sonarr, and Radarr libraries so every node tracks the same content. +# Run before rsync — once arrs agree on library, rsync spreads the files. +# +# ── DESIGN ──────────────────────────────────────────────────────────────────────────────────── +# Full mesh: every node syncs with every other node — no primary, no hierarchy. +# Union model: if any node tracks an item, all nodes get it (unless blocklisted). +# Convergence: any node can add content; after one full cycle all nodes agree. +# Upgrade-aware: server1 upgrades a file → arr tracks new path → rsync spreads it → +# arr_cleanup removes old file on all nodes because arr no longer tracks it. +# +# ── NODE DISCOVERY ──────────────────────────────────────────────────────────────────────────── +# Reads HOST* vars from master.conf. Add HOST3= and it joins the sync automatically. +# No scripts change when adding a new node. +# +# ── REMOTE API KEY ACCESS ───────────────────────────────────────────────────────────────────── +# Remote API keys are not stored anywhere. Script SSHes to each remote node and reads +# the key directly from that arr's config.xml in its appdata directory. +# Only the API response (JSON) is returned — key never leaves the remote node. +# Self-maintaining: remote key regeneration is picked up automatically. +# +# ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── +# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added anywhere. +# Read from ALL nodes via SSH at start of each run — immediate effect before rsync. +# Propagates to all nodes via rsync on the next cycle. +# Manage via --blocklist-add / --blocklist-remove / --blocklist-list. +# +# ── GRACEFUL SKIP ───────────────────────────────────────────────────────────────────────────── +# Arr not configured locally → skip cleanly, no error. +# Arr not reachable on a remote → skip that node for that arr type, continue with others. +# Partial mesh works — nodes that share an arr type sync with each other. +# +# ── WHAT GETS SYNCED ────────────────────────────────────────────────────────────────────────── +# Library items (tracked artists/series/movies) keyed on stable IDs: +# Lidarr — MusicBrainz artist ID (foreignArtistId) +# Sonarr — TVDB series ID (tvdbId) +# Radarr — TMDB movie ID (tmdbId) +# When adding to a remote node, that node's own quality profile, metadata profile, +# and root folder path are used — settings are never copied from the source node. +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# arr_sync.sh — sync all arr types, all nodes +# arr_sync.sh --dry-run — preview only, no changes +# arr_sync.sh --log — verbose output +# arr_sync.sh --status — show config and exit +# arr_sync.sh --blocklist-add lidarr "reason" — tombstone an ID on all nodes +# arr_sync.sh --blocklist-remove lidarr — un-tombstone an ID +# arr_sync.sh --blocklist-list — show all blocklisted IDs +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ARR_SYNC_ENABLED — global on/off toggle (default true) +# ARR_SYNC_BLOCKLIST — path to TSV blocklist (default: DATA_DIR/arr_sync_blocklist.tsv) +# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default 10) +# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default 60) +# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default /mnt/user/appdata) +# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default 8686) +# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default 8989) +# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default 7878) +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts) +# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr +# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" + +# ── Blocklist flag pre-processing ───────────────────────────────────────────────────────────── +# Handle before parse_args — these are action flags, not standard options +BLOCKLIST_ACTION="" +BLOCKLIST_ARR="" +BLOCKLIST_ID="" +BLOCKLIST_REASON="" +FILTERED_ARGS=() +_skip_next=false +for _arg in "$@"; do + if [[ "$_skip_next" == true ]]; then _skip_next=false; continue; fi + case "$_arg" in + --blocklist-add) BLOCKLIST_ACTION="add" ;; + --blocklist-remove) BLOCKLIST_ACTION="remove" ;; + --blocklist-list) BLOCKLIST_ACTION="list" ;; + *) FILTERED_ARGS+=("$_arg") ;; + esac +done +unset _arg _skip_next + +parse_args "${FILTERED_ARGS[@]}" + +# Consume blocklist positional args from remaining FILTERED_ARGS +# --blocklist-add lidarr "reason" +# --blocklist-remove lidarr +if [[ -n "$BLOCKLIST_ACTION" ]] && [[ "$BLOCKLIST_ACTION" != "list" ]]; then + for _a in "${FILTERED_ARGS[@]}"; do + [[ "$_a" == --* ]] && continue + if [[ -z "$BLOCKLIST_ARR" ]]; then BLOCKLIST_ARR="$_a" + elif [[ -z "$BLOCKLIST_ID" ]]; then BLOCKLIST_ID="$_a" + elif [[ -z "$BLOCKLIST_REASON" ]]; then BLOCKLIST_REASON="$_a" + fi + done + unset _a +fi + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +for _tool in curl jq; do + if ! command -v "$_tool" >/dev/null 2>&1; then + error "$_tool not found — required for arr API calls" + exit 1 + fi +done +unset _tool + +detect_hosts + +# ── Runtime config with defaults ────────────────────────────────────────────────────────────── +ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}" +ARR_SYNC_BLOCKLIST="${ARR_SYNC_BLOCKLIST:-${DATA_DIR}/arr_sync_blocklist.tsv}" +ARR_SYNC_CONNECT_TIMEOUT="${ARR_SYNC_CONNECT_TIMEOUT:-10}" +ARR_SYNC_API_TIMEOUT="${ARR_SYNC_API_TIMEOUT:-60}" +DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}" +ARR_SYNC_LIDARR_PORT="${ARR_SYNC_LIDARR_PORT:-8686}" +ARR_SYNC_SONARR_PORT="${ARR_SYNC_SONARR_PORT:-8989}" +ARR_SYNC_RADARR_PORT="${ARR_SYNC_RADARR_PORT:-7878}" + +if [[ "$ARR_SYNC_ENABLED" != "true" ]]; then + log "ARR_SYNC_ENABLED=false — exiting" + exit 0 +fi + +# ── Arr type definitions ─────────────────────────────────────────────────────────────────────── +# Each arr type maps to its port, API version, endpoint, stable ID field, and display name field +declare -A _PORT=([lidarr]="$ARR_SYNC_LIDARR_PORT" [sonarr]="$ARR_SYNC_SONARR_PORT" [radarr]="$ARR_SYNC_RADARR_PORT") +declare -A _VER=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3") +declare -A _EP=( [lidarr]="artist" [sonarr]="series" [radarr]="movie") +declare -A _ID=( [lidarr]="foreignArtistId" [sonarr]="tvdbId" [radarr]="tmdbId") +declare -A _NAME=([lidarr]="artistName" [sonarr]="title" [radarr]="title") +# ID type: "string" for MusicBrainz UUID, "int" for TVDB/TMDB numeric IDs +declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int") +ARR_TYPES=(lidarr sonarr radarr) + +# ── Remote node discovery ────────────────────────────────────────────────────────────────────── +REMOTE_NODES=() +for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do + [[ "$_hv" == "$MY_ID" ]] && continue + [[ -z "${!_hv:-}" ]] && continue + REMOTE_NODES+=("$_hv") +done +unset _hv + +if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then + warn "No remote nodes defined in master.conf — nothing to sync" + exit 0 +fi + +echo " Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo " Remote nodes: ${REMOTE_NODES[*]}" +echo " Blocklist: $ARR_SYNC_BLOCKLIST" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +declare -A BLOCKLIST_MAP # key: "arr_type:stable_id" → display name + +_load_blocklist() { + BLOCKLIST_MAP=() + local count=0 + + _parse_blocklist_lines() { + while IFS=$'\t' read -r arr_type stable_id display_name rest; do + [[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue + BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name" + (( count++ )) + done + } + + # Local blocklist + [[ -f "$ARR_SYNC_BLOCKLIST" ]] && _parse_blocklist_lines < "$ARR_SYNC_BLOCKLIST" + + # Remote blocklists — read via SSH so tombstones are effective immediately + for _node_id in "${REMOTE_NODES[@]}"; do + local _node_ip + _node_ip=$(_resolve_node_ip "$_node_id") || continue + local _remote_lines + _remote_lines=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$_node_ip" "cat '$ARR_SYNC_BLOCKLIST' 2>/dev/null" 2>/dev/null) || continue + _parse_blocklist_lines <<< "$_remote_lines" + done + unset _node_id _node_ip _remote_lines + + log "Loaded blocklist: $count entries from $(( ${#REMOTE_NODES[@]} + 1 )) nodes" + unset -f _parse_blocklist_lines +} + +_is_blocklisted() { + [[ -n "${BLOCKLIST_MAP["${1}:${2}"]:-}" ]] +} + +_blocklist_add() { + local arr_type="$1" stable_id="$2" display_name="$3" reason="${4:-manually excluded}" + mkdir -p "$(dirname "$ARR_SYNC_BLOCKLIST")" + if ! grep -qP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" 2>/dev/null; then + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$arr_type" "$stable_id" "$display_name" \ + "$MY_ID" "$(date -Iseconds)" "$reason" \ + >> "$ARR_SYNC_BLOCKLIST" + BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name" + log "Blocklisted: [$arr_type] $display_name ($stable_id)" + else + log "Already blocklisted: [$arr_type] $stable_id" + fi +} + +_blocklist_remove() { + local arr_type="$1" stable_id="$2" + if [[ -f "$ARR_SYNC_BLOCKLIST" ]]; then + local tmp + tmp=$(mktemp) + grep -vP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" > "$tmp" && \ + mv "$tmp" "$ARR_SYNC_BLOCKLIST" || rm -f "$tmp" + unset "BLOCKLIST_MAP[${arr_type}:${stable_id}]" + log "Removed from blocklist: [$arr_type] $stable_id" + fi +} + +# ── Blocklist management mode ────────────────────────────────────────────────────────────────── +if [[ -n "$BLOCKLIST_ACTION" ]]; then + case "$BLOCKLIST_ACTION" in + list) + echo "" + echo "━━━━━ ARR SYNC BLOCKLIST ━━━━━" + if [[ ! -f "$ARR_SYNC_BLOCKLIST" ]] || [[ ! -s "$ARR_SYNC_BLOCKLIST" ]]; then + echo " (empty)" + else + echo "" + printf '%-8s %-40s %-30s %s\n' "Arr" "Stable ID" "Name" "Reason" + printf '%-8s %-40s %-30s %s\n' "---" "---------" "----" "------" + while IFS=$'\t' read -r arr_type stable_id display_name tombstoned_by ts reason; do + [[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue + printf '%-8s %-40s %-30s %s\n' "$arr_type" "$stable_id" "$display_name" "$reason" + done < "$ARR_SYNC_BLOCKLIST" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 + ;; + add) + if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then + error "Usage: arr_sync.sh --blocklist-add [reason]" + error " arr_type: lidarr | sonarr | radarr" + exit 1 + fi + _blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "${BLOCKLIST_ID}" "${BLOCKLIST_REASON:-manually excluded}" + echo " Blocklisted [$BLOCKLIST_ARR] $BLOCKLIST_ID — will propagate via rsync on next cycle" + exit 0 + ;; + remove) + if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then + error "Usage: arr_sync.sh --blocklist-remove " + exit 1 + fi + _blocklist_remove "$BLOCKLIST_ARR" "$BLOCKLIST_ID" + echo " Removed [$BLOCKLIST_ARR] $BLOCKLIST_ID from blocklist" + exit 0 + ;; + esac +fi + +# ============================================================================================== +# ── STATUS ──────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY ARR SYNC STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Remote nodes: ${REMOTE_NODES[*]}" + echo "$ICON_GEAR Appdata base: $DOCKER_APPDATA_BASE" + echo "$ICON_GEAR Blocklist: $ARR_SYNC_BLOCKLIST" + echo "$ICON_GEAR Connect timeout: ${ARR_SYNC_CONNECT_TIMEOUT}s" + echo "$ICON_GEAR API timeout: ${ARR_SYNC_API_TIMEOUT}s" + echo "" + echo " Arr Port URL" + for _arr in "${ARR_TYPES[@]}"; do + local _url="" _key="" + case "$_arr" in + lidarr) _url="${LIDARR_URL:-not configured}" ;; + sonarr) _url="${SONARR_URL:-not configured}" ;; + radarr) _url="${RADARR_URL:-not configured}" ;; + esac + printf ' %-8s %-6s %s\n' "$_arr" "${_PORT[$_arr]}" "$_url" + done + unset _arr _url + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── REMOTE HELPERS ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +_resolve_node_ip() { + local node_id="$1" + local node_name="${!node_id}" + local ts_name="${node_name,,}" + local ip + ip=$(tailscale ip -4 "$ts_name" 2>/dev/null) + [[ -z "$ip" ]] && ip=$(tailscale status 2>/dev/null | \ + awk -v n="$ts_name" '$2 ~ "^" n { print $1; exit }') + [[ -z "$ip" ]] && return 1 + echo "$ip" +} + +# Check if arr is reachable on remote node +_remote_arr_up() { + local node_ip="$1" port="$2" api_ver="$3" config_xml="$4" + ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" \ + "KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) + [[ -z \"\$KEY\" ]] && exit 1 + curl -sf --max-time 5 -H \"X-Api-Key: \$KEY\" \ + 'http://localhost:${port}/api/${api_ver}/system/status' >/dev/null" 2>/dev/null +} + +# Fetch full library from remote arr — returns raw JSON array +_remote_library() { + local node_ip="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5" + ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" \ + "KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) + [[ -z \"\$KEY\" ]] && exit 1 + curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \ + -H \"X-Api-Key: \$KEY\" \ + 'http://localhost:${port}/api/${api_ver}/${endpoint}'" 2>/dev/null +} + +# Fetch remote arr defaults: qualityProfileId, rootFolderPath, metadataProfileId (Lidarr) +_remote_defaults() { + local node_ip="$1" port="$2" api_ver="$3" arr_type="$4" config_xml="$5" + local meta_field="" + [[ "$arr_type" == "lidarr" ]] && \ + meta_field=', metadataProfileId: ($mp | map(select(.name == "Standard")) | .[0].id // .[0].id // 1)' + + ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" bash </dev/null +KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) +[[ -z "\$KEY" ]] && exit 1 +QP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/qualityprofile") +RF=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/rootfolder") +MP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/metadataprofile" 2>/dev/null || echo '[]') +jq -n --argjson qp "\$QP" --argjson rf "\$RF" --argjson mp "\$MP" \ + '{qualityProfileId: (\$qp | .[0].id // 1), rootFolderPath: (\$rf | .[0].path // "")${meta_field}}' +REMOTE +} + +# Add item to remote arr — payload is base64-encoded to avoid SSH quoting issues +_remote_add() { + local node_ip="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5" + local encoded="$6" # base64-encoded JSON body + + ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" bash </dev/null +KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) +[[ -z "\$KEY" ]] && exit 1 +BODY=\$(printf '%s' '${encoded}' | base64 -d) +curl -sf -o /dev/null -w '%{http_code}' -X POST \ + -H "X-Api-Key: \$KEY" \ + -H 'Content-Type: application/json' \ + -d "\$BODY" \ + "http://localhost:${port}/api/${api_ver}/${endpoint}" +REMOTE +} + +# ============================================================================================== +# ── LOCAL HELPERS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +_local_library() { + local url="$1" api_key="$2" api_ver="$3" endpoint="$4" + curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_ver}/${endpoint}" 2>/dev/null +} + +_local_defaults() { + local url="$1" api_key="$2" api_ver="$3" arr_type="$4" + local qp rf mp meta_field="" + qp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/qualityprofile" | jq '.[0].id // 1') + rf=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/rootfolder" | jq -r '.[0].path // ""') + if [[ "$arr_type" == "lidarr" ]]; then + mp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/metadataprofile" | \ + jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1') + jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \ + '{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}' + else + jq -n --argjson qp "$qp" --arg rf "$rf" \ + '{qualityProfileId: $qp, rootFolderPath: $rf}' + fi +} + +_local_add() { + local url="$1" api_key="$2" api_ver="$3" endpoint="$4" payload="$5" + curl -sf -o /dev/null -w '%{http_code}' -X POST \ + -H "X-Api-Key: $api_key" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + "${url}/api/${api_ver}/${endpoint}" 2>/dev/null +} + +# ============================================================================================== +# ── PAYLOAD BUILDER ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Builds the minimal POST body to add an item to an arr. +# Uses the TARGET node's own defaults — never copies source node settings. + +_build_payload() { + local arr_type="$1" stable_id="$2" display_name="$3" monitored="$4" defaults_json="$5" + case "$arr_type" in + lidarr) + jq -n \ + --arg id "$stable_id" \ + --arg nm "$display_name" \ + --argjson mn "$monitored" \ + --argjson df "$defaults_json" \ + '{ + foreignArtistId: $id, + artistName: $nm, + monitored: $mn, + qualityProfileId: $df.qualityProfileId, + metadataProfileId: ($df.metadataProfileId // 1), + rootFolderPath: $df.rootFolderPath, + addOptions: {monitor: "all", searchForMissingAlbums: false} + }' + ;; + sonarr) + jq -n \ + --argjson id "$stable_id" \ + --arg nm "$display_name" \ + --argjson mn "$monitored" \ + --argjson df "$defaults_json" \ + '{ + tvdbId: $id, + title: $nm, + monitored: $mn, + qualityProfileId: $df.qualityProfileId, + rootFolderPath: $df.rootFolderPath, + seasons: [], + addOptions: {searchForMissingEpisodes: false, monitor: "all"} + }' + ;; + radarr) + jq -n \ + --argjson id "$stable_id" \ + --arg nm "$display_name" \ + --argjson mn "$monitored" \ + --argjson df "$defaults_json" \ + '{ + tmdbId: $id, + title: $nm, + monitored: $mn, + qualityProfileId: $df.qualityProfileId, + rootFolderPath: $df.rootFolderPath, + addOptions: {searchForMovie: false} + }' + ;; + esac +} + +# ============================================================================================== +# ── CORE SYNC — one arr type across all remote nodes ────────────────────────────────────────── +# ============================================================================================== + +_sync_arr() { + local arr_type="$1" + local port="${_PORT[$arr_type]}" + local api_ver="${_VER[$arr_type]}" + local endpoint="${_EP[$arr_type]}" + local id_field="${_ID[$arr_type]}" + local name_field="${_NAME[$arr_type]}" + local id_type="${_ID_TYPE[$arr_type]}" + local config_xml="${DOCKER_APPDATA_BASE}/$(echo "${arr_type^}")/config.xml" + + # Resolve local credentials + local local_url local_key + case "$arr_type" in + lidarr) local_url="${LIDARR_URL:-}"; local_key="${LIDARR_API_KEY:-}" ;; + sonarr) local_url="${SONARR_URL:-}"; local_key="${SONARR_API_KEY:-}" ;; + radarr) local_url="${RADARR_URL:-}"; local_key="${RADARR_API_KEY:-}" ;; + esac + + if [[ -z "$local_url" ]] || [[ -z "$local_key" ]]; then + log "${arr_type^}: not configured on $MY_ID — skipping" + return 0 + fi + + echo "" + echo "━━━ ${arr_type^} ━━━" + + # ── Fetch local library ──────────────────────────────────────────────────────────────────── + local local_json + local_json=$(_local_library "$local_url" "$local_key" "$api_ver" "$endpoint") + if [[ -z "$local_json" ]] || ! echo "$local_json" | jq -e '.' >/dev/null 2>&1; then + warn "${arr_type^}: could not fetch local library — skipping" + return 0 + fi + + # Build local ID map: stable_id → "display_name|monitored" + declare -A local_ids + local local_count=0 + local _jq_id + [[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)" + while IFS=$'\t' read -r stable_id display_name monitored; do + [[ -z "$stable_id" ]] && continue + local_ids["$stable_id"]="${display_name}|${monitored}" + (( local_count++ )) + done < <(echo "$local_json" | jq -r \ + ".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null) + unset _jq_id + + log "${arr_type^}: $local_count items in local library" + + local total_added_local=0 total_added_remote=0 total_skipped=0 + + # ── Sync with each remote node ───────────────────────────────────────────────────────────── + for node_id in "${REMOTE_NODES[@]}"; do + local node_name="${!node_id}" + log "${arr_type^}: syncing with $node_name..." + + local node_ip + node_ip=$(_resolve_node_ip "$node_id") || { + warn "${arr_type^}: cannot resolve Tailscale IP for $node_name — skipping" + continue + } + + if ! _remote_arr_up "$node_ip" "$port" "$api_ver" "$config_xml"; then + log "${arr_type^}: not reachable on $node_name — skipping" + continue + fi + + local remote_json + remote_json=$(_remote_library "$node_ip" "$port" "$api_ver" "$endpoint" "$config_xml") + if [[ -z "$remote_json" ]] || ! echo "$remote_json" | jq -e '.' >/dev/null 2>&1; then + warn "${arr_type^}: could not fetch library from $node_name — skipping" + continue + fi + + # Build remote ID map + declare -A remote_ids + local remote_count=0 + local _jq_id + [[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)" + while IFS=$'\t' read -r stable_id display_name monitored; do + [[ -z "$stable_id" ]] && continue + remote_ids["$stable_id"]="${display_name}|${monitored}" + (( remote_count++ )) + done < <(echo "$remote_json" | jq -r \ + ".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null) + unset _jq_id + + log "${arr_type^}: $remote_count items on $node_name" + + # ── Remote → Local: items on remote not in local ─────────────────────────────────────── + local to_add_local=() + for stable_id in "${!remote_ids[@]}"; do + [[ -n "${local_ids[$stable_id]:-}" ]] && continue + if _is_blocklisted "$arr_type" "$stable_id"; then + log "BLOCKLISTED [$arr_type] $stable_id — skipping" + (( total_skipped++ )) + continue + fi + to_add_local+=("$stable_id") + done + + if [[ "${#to_add_local[@]}" -gt 0 ]]; then + local local_defs + local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type") + if [[ -z "$local_defs" ]]; then + warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name" + else + for stable_id in "${to_add_local[@]}"; do + IFS='|' read -r display_name monitored <<< "${remote_ids[$stable_id]}" + local payload + payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \ + "$monitored" "$local_defs") + if [[ "$DRY_RUN" == true ]]; then + log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)" + (( total_added_local++ )) + else + local http_code + http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \ + "$endpoint" "$payload") + if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then + log "Added to local ${arr_type^}: $display_name" + local_ids["$stable_id"]="${display_name}|${monitored}" + (( total_added_local++ )) + else + warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)" + fi + fi + done + fi + fi + + # ── Local → Remote: items on local not on remote ─────────────────────────────────────── + local to_add_remote=() + for stable_id in "${!local_ids[@]}"; do + [[ -n "${remote_ids[$stable_id]:-}" ]] && continue + if _is_blocklisted "$arr_type" "$stable_id"; then + (( total_skipped++ )) + continue + fi + to_add_remote+=("$stable_id") + done + + if [[ "${#to_add_remote[@]}" -gt 0 ]]; then + local remote_defs + remote_defs=$(_remote_defaults "$node_ip" "$port" "$api_ver" "$arr_type" "$config_xml") + if [[ -z "$remote_defs" ]]; then + warn "${arr_type^}: could not fetch defaults from $node_name — skipping remote adds" + else + for stable_id in "${to_add_remote[@]}"; do + IFS='|' read -r display_name monitored <<< "${local_ids[$stable_id]}" + local payload + payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \ + "$monitored" "$remote_defs") + if [[ "$DRY_RUN" == true ]]; then + log "DRY RUN: would add to $node_name ${arr_type^}: $display_name ($stable_id)" + (( total_added_remote++ )) + else + local encoded http_code + encoded=$(printf '%s' "$payload" | base64 -w0) + http_code=$(_remote_add "$node_ip" "$port" "$api_ver" "$endpoint" \ + "$config_xml" "$encoded") + if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then + log "Added to $node_name ${arr_type^}: $display_name" + (( total_added_remote++ )) + else + warn "Failed to add to $node_name ${arr_type^}: $display_name (HTTP $http_code)" + fi + fi + done + fi + fi + + echo " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted" + + unset remote_ids + declare -A remote_ids + done + + echo " Total added to local: $total_added_local | to remotes: $total_added_remote | blocklisted: $total_skipped" + unset local_ids + declare -A local_ids +} + +# ============================================================================================== +# ━━━ Main ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Loading Blocklist ━━━" +_load_blocklist + +START=$(date +%s) + +for arr_type in "${ARR_TYPES[@]}"; do + _sync_arr "$arr_type" +done + +END=$(date +%s) + +echo "" +echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━" +echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +exit 0 diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 1fb847f..691a783 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -43,9 +43,8 @@ # The user accepts full responsibility — this is 100% intentional by design. # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Lidarr runs on HOST1 only — music library is HOST1's source of truth. -# detect_hosts() sets MY_ID — if MY_ID != HOST1 script exits cleanly with no action. -# LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT aliased by detect_hosts() automatically. +# Runs on any node where Lidarr is configured — skips cleanly if LIDARR_URL/API_KEY not set. +# detect_hosts() aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT from HOST*_LIDARR_*. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — large scans take time, wait for previous run to finish @@ -155,9 +154,9 @@ acquire_lock "wait" # detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT detect_hosts -# Lidarr is HOST1 only — exit cleanly on any other host -if [[ "$MY_ID" != "HOST1" ]]; then - log "Lidarr runs on HOST1 only — skipping on $MY_ID ($LOCAL_SERVER_NAME)" +# Skip if Lidarr is not configured on this host +if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then + log "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" exit 0 fi @@ -182,8 +181,7 @@ if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then exit 1 fi -log "Lidarr URL: $LIDARR_URL" -log "Music root: $LIDARR_MUSIC_ROOT" +echo " $MY_ID ($LOCAL_SERVER_NAME) — $LIDARR_URL" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" [[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" @@ -310,7 +308,7 @@ fi # Safety Layer 3 — API version check check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "Lidarr" || exit 1 -log "Querying Lidarr API: $LIDARR_URL" +echo " Querying Lidarr API..." # Fetch all artists ARTIST_RESPONSE=$(lidarr_api "artist") || { @@ -371,7 +369,7 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then exit 1 fi -warn "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists" +echo " $ARTIST_COUNT artists | $TRACKED_COUNT tracked files" # Safety Layer 6 — percentage drop vs last known count if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then @@ -399,9 +397,7 @@ echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE" # ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning Music Root ━━━" -log "Root: $LIDARR_MUSIC_ROOT" -log "Orphan age: ${LIDARR_ORPHAN_AGE} days" -log "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}" +echo " Root: $LIDARR_MUSIC_ROOT | Orphan age: ${LIDARR_ORPHAN_AGE} days" echo "" START=$(date +%s) diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index cbeceae..930fe5b 100644 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -24,6 +24,10 @@ # Lidarr runs on HOST1 only. detect_hosts() sets LIDARR_URL — if empty (HOST2) the # script exits cleanly with no action rather than failing. # +# ── OUTPUT ──────────────────────────────────────────────────────────────────────────────────── +# Minimal by default — section headers + per-section summary always visible. +# --log shows per-item detail (each album, each artist, each file fetched). +# # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock — prevents concurrent runs during large library scans # curl + jq check — fail fast if tools missing @@ -50,7 +54,7 @@ # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # lidarr_missing_art.sh — fetch all missing artwork # lidarr_missing_art.sh --dry-run — preview without downloading -# lidarr_missing_art.sh --log — verbose output +# lidarr_missing_art.sh --log — verbose per-item output # lidarr_missing_art.sh --status — show config and exit # ============================================================================================== @@ -70,7 +74,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for API calls" @@ -80,7 +83,6 @@ if ! command -v jq >/dev/null 2>&1; then error "jq not found — required for JSON parsing" exit 1 fi -success "curl and jq found" acquire_lock @@ -91,7 +93,8 @@ if [[ -z "$LIDARR_URL" ]]; then log "Lidarr not configured for $MY_ID — nothing to do" exit 0 fi -success "Identity: $MY_ID ($LOCAL_SERVER_NAME)" + +echo " $MY_ID ($LOCAL_SERVER_NAME) — tools OK" # ============================================================================================== # ━━━ Status ━━━ @@ -115,6 +118,10 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written" +# ── Temp dir for subshell fetch/fail counters ───────────────────────────────────────────────── +LIDARR_TMP=$(mktemp -d) +trap 'rm -rf "$LIDARR_TMP"' EXIT + # ============================================================================================== # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -143,7 +150,7 @@ download_if_valid() { [[ -f "$dest" ]] && return 0 if [[ "$DRY_RUN" == true ]]; then - info "DRY RUN — would fetch: $(basename "$dest")" + log "DRY RUN — would fetch: $(basename "$dest")" return 0 fi @@ -155,14 +162,14 @@ download_if_valid() { size=$(stat -c%s "$tmp" 2>/dev/null || echo 0) if (( size > LIDARR_ART_MIN_SIZE )); then mv "$tmp" "$dest" - log "Fetched: $dest" + log " Fetched: $(basename "$dest")" return 0 fi rm -f "$tmp" sleep 1 done - warn "Failed to fetch valid image: $(basename "$dest")" + warn "Failed to fetch: $(basename "$dest")" return 1 } @@ -193,19 +200,15 @@ if ! curl_json "$LIDARR_URL/api/v1/system/status?apikey=$LIDARR_API_KEY" | jq -e notify "lidarr_missing_art failed — Lidarr API unreachable on $(hostname)" "Lidarr Missing Art" "warning" exit 1 fi -success "Lidarr API reachable" +echo " Reachable — $LIDARR_URL" START=$(date +%s) ALBUMS_CHECKED=0 ALBUMS_COMPLETE=0 -ALBUM_FETCHES=0 -ALBUM_FAILS=0 ARTISTS_CHECKED=0 ARTISTS_COMPLETE=0 -ARTIST_FETCHES=0 -ARTIST_FAILS=0 # ============================================================================================== # ━━━ Albums ━━━ @@ -222,7 +225,7 @@ if [[ -z "$albums" || "$albums" == "null" ]]; then fi total_albums=$(echo "$albums" | jq '. | length') -info "$total_albums albums to process" +echo " Processing $total_albums albums..." while IFS=$'\t' read -r local_path mbid artist_name album_name; do (( ALBUMS_CHECKED++ )) @@ -242,6 +245,8 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do wait_for_slot ( + _fetches=0 _fails=0 + JSON="" if [[ -n "$mbid" && "$mbid" != "null" ]]; then JSON=$(curl_json "http://webservice.fanart.tv/v3/music/albums/$mbid?api_key=$FANART_API_KEY") @@ -250,29 +255,43 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do if [[ ! -f "$local_path/cover.jpg" ]]; then IMG=$(echo "$JSON" | jq -r '.[].albumcover[0].url // empty') - if ! download_if_valid "$IMG" "$local_path/cover.jpg"; then + if download_if_valid "$IMG" "$local_path/cover.jpg"; then + (( _fetches++ )) + else query=$(printf "%s %s" "$artist_name" "$album_name" | sed 's/ /+/g') itunes=$(curl_json "https://itunes.apple.com/search?term=$query&entity=album&limit=1" | jq -r '.results[0].artworkUrl100 // empty' | sed 's/100x100/600x600/') - download_if_valid "$itunes" "$local_path/cover.jpg" + if download_if_valid "$itunes" "$local_path/cover.jpg"; then + (( _fetches++ )) + else + (( _fails++ )) + fi fi fi if [[ ! -f "$local_path/cdart.png" ]]; then IMG=$(echo "$JSON" | jq -r '.[].cdart[0].url // empty') - download_if_valid "$IMG" "$local_path/cdart.png" + if download_if_valid "$IMG" "$local_path/cdart.png"; then (( _fetches++ )); else (( _fails++ )); fi fi if [[ ! -f "$local_path/back.jpg" ]]; then IMG=$(echo "$JSON" | jq -r '.[].albumback[0].url // empty') - download_if_valid "$IMG" "$local_path/back.jpg" + if download_if_valid "$IMG" "$local_path/back.jpg"; then (( _fetches++ )); else (( _fails++ )); fi fi + + (( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fetches" + (( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fails" ) & done < <(echo "$albums" | jq -r '.[] | [.path, .foreignAlbumId, .artist.artistName, .title] | @tsv') wait +ALBUM_FETCHED=$(wc -l < "$LIDARR_TMP/album_fetches" 2>/dev/null || echo 0) +ALBUM_FAILED=$(wc -l < "$LIDARR_TMP/album_fails" 2>/dev/null || echo 0) +ALBUM_MISSING=$(( ALBUMS_CHECKED - ALBUMS_COMPLETE )) +echo " Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Fetched: $ALBUM_FETCHED | Failed: $ALBUM_FAILED" + # ============================================================================================== # ━━━ Artists ━━━ # ============================================================================================== @@ -288,7 +307,7 @@ if [[ -z "$artists" || "$artists" == "null" ]]; then fi total_artists=$(echo "$artists" | jq '. | length') -info "$total_artists artists to process" +echo " Processing $total_artists artists..." while IFS=$'\t' read -r local_path mbid name; do (( ARTISTS_CHECKED++ )) @@ -310,43 +329,63 @@ while IFS=$'\t' read -r local_path mbid name; do wait_for_slot ( + _fetches=0 _fails=0 + JSON=$(curl_json "http://webservice.fanart.tv/v3/music/$mbid?api_key=$FANART_API_KEY") sleep "$LIDARR_ART_SLEEP_BETWEEN" if [[ ! -f "$local_path/folder.jpg" ]]; then IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty') - if ! download_if_valid "$IMG" "$local_path/folder.jpg"; then + if download_if_valid "$IMG" "$local_path/folder.jpg"; then + (( _fetches++ )) + else IMG=$(deezer_artist_image "$name") - if ! download_if_valid "$IMG" "$local_path/folder.jpg"; then + if download_if_valid "$IMG" "$local_path/folder.jpg"; then + (( _fetches++ )) + else IMG=$(lastfm_artist_image "$name") - download_if_valid "$IMG" "$local_path/folder.jpg" + if download_if_valid "$IMG" "$local_path/folder.jpg"; then + (( _fetches++ )) + else + (( _fails++ )) + fi fi fi fi if [[ ! -f "$local_path/fanart.jpg" ]]; then IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty') - if ! download_if_valid "$IMG" "$local_path/fanart.jpg"; then + if download_if_valid "$IMG" "$local_path/fanart.jpg"; then + (( _fetches++ )) + else IMG=$(deezer_artist_image "$name") - download_if_valid "$IMG" "$local_path/fanart.jpg" + if download_if_valid "$IMG" "$local_path/fanart.jpg"; then (( _fetches++ )); else (( _fails++ )); fi fi fi if [[ ! -f "$local_path/logo.png" ]]; then IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty') - download_if_valid "$IMG" "$local_path/logo.png" + if download_if_valid "$IMG" "$local_path/logo.png"; then (( _fetches++ )); else (( _fails++ )); fi fi if [[ ! -f "$local_path/banner.jpg" ]]; then IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty') - download_if_valid "$IMG" "$local_path/banner.jpg" + if download_if_valid "$IMG" "$local_path/banner.jpg"; then (( _fetches++ )); else (( _fails++ )); fi fi + + (( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fetches" + (( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fails" ) & done < <(echo "$artists" | jq -r '.[] | [.path, .foreignArtistId, .artistName] | @tsv') wait +ARTIST_FETCHED=$(wc -l < "$LIDARR_TMP/artist_fetches" 2>/dev/null || echo 0) +ARTIST_FAILED=$(wc -l < "$LIDARR_TMP/artist_fails" 2>/dev/null || echo 0) +ARTIST_MISSING=$(( ARTISTS_CHECKED - ARTISTS_COMPLETE )) +echo " Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Fetched: $ARTIST_FETCHED | Failed: $ARTIST_FAILED" + END=$(date +%s) # ============================================================================================== @@ -356,8 +395,8 @@ echo "" echo "━━━━━ $ICON_SUMMARY LIDARR MISSING ART SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked, $ALBUMS_COMPLETE already complete" -echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked, $ARTISTS_COMPLETE already complete" +echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched | $ALBUM_FAILED failed" +echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched | $ARTIST_FAILED failed" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no files written" diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 6ddbf5c..550a547 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -164,8 +164,7 @@ if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then exit 1 fi -log "Radarr URL: $RADARR_URL" -log "Movies root: $RADARR_MOVIES_ROOT" +echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" [[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" @@ -298,7 +297,7 @@ fi # Safety Layer 3 — API version check check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1 -log "Querying Radarr API: $RADARR_URL" +echo " Querying Radarr API..." # Fetch all movies MOVIES_RESPONSE=$(radarr_api "movie") || { @@ -363,16 +362,14 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then exit 1 fi -warn "Radarr tracks $TRACKED_COUNT movie files across $MOVIE_COUNT movies" +echo " $MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files" # ============================================================================================== # ━━━ Scan Movies Root ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━" -log "Root: $RADARR_MOVIES_ROOT" -log "Orphan age: ${RADARR_ORPHAN_AGE} days" -log "Protected: ${RADARR_PROTECTED_PATTERNS[*]}" +echo " Root: $RADARR_MOVIES_ROOT | Orphan age: ${RADARR_ORPHAN_AGE} days" echo "" START=$(date +%s) diff --git a/Media/radarr_tmdb_removed.sh b/Media/radarr_tmdb_removed.sh new file mode 100644 index 0000000..c3312b1 --- /dev/null +++ b/Media/radarr_tmdb_removed.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Radarr — TMDb Removed ========================================== +# ============================================================================================== +# Removes movies from Radarr that have been dropped from TMDb. +# Radarr marks these with status="deleted" — they generate system health errors and +# can never be monitored or downloaded. 99% are future/announced movies that were +# delisted before release. +# +# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# Queries Radarr API for movies with status="deleted" (TMDb removal marker) +# Reports each found entry with file status and size +# Removes the movie record from Radarr +# Optionally deletes associated files (disabled by default — most have none) +# Optionally adds to Radarr's import exclusion list (default: true) +# +# ── SAFE DEFAULTS ───────────────────────────────────────────────────────────────────────────── +# Files are NOT deleted by default — use --delete-files to also remove from disk +# Import exclusion added by default — prevents Radarr re-adding dropped movies +# Per-deletion output always visible — deletions are never silently swallowed +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# Radarr runs on HOST1 only. detect_hosts() sets RADARR_URL — if empty (HOST2) +# the script exits cleanly. +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RADARR_DROPPED_ADD_EXCLUSION — add removed movies to import exclusion (default: true) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# radarr_tmdb_removed.sh — remove records, keep files, add exclusion +# radarr_tmdb_removed.sh --delete-files — also delete files from disk +# radarr_tmdb_removed.sh --dry-run — preview without removing anything +# radarr_tmdb_removed.sh --log — verbose output +# radarr_tmdb_removed.sh --status — show config and exit +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +# ── Parse --delete-files before standard parse_args ─────────────────────────────────────────── +DELETE_FILES=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--delete-files" ]]; then + DELETE_FILES=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found" + exit 1 +fi +if ! command -v jq >/dev/null 2>&1; then + error "jq not found" + exit 1 +fi + +acquire_lock + +detect_hosts + +if [[ -z "$RADARR_URL" ]]; then + log "Radarr not configured for $MY_ID — nothing to do" + exit 0 +fi + +ADD_EXCLUSION="${RADARR_DROPPED_ADD_EXCLUSION:-true}" + +echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL" +[[ "$DELETE_FILES" == true ]] && warn "DELETE FILES MODE — files will be removed from disk" +[[ "$DELETE_FILES" == false ]] && echo " Files: records only (use --delete-files to also remove from disk)" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Radarr URL: $RADARR_URL" + echo "$ICON_GEAR Add exclusion: $ADD_EXCLUSION" + echo "$ICON_GEAR Delete files: $DELETE_FILES" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Query Radarr ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Querying Radarr ━━━" + +if ! curl -sf --connect-timeout 5 --max-time 10 \ + "$RADARR_URL/api/v3/system/status?apikey=$RADARR_API_KEY" | jq -e '.version' >/dev/null 2>&1; then + error "Radarr API unreachable at $RADARR_URL" + exit 1 +fi + +MOVIES=$(curl -sf --connect-timeout 5 --max-time 30 \ + "$RADARR_URL/api/v3/movie?apikey=$RADARR_API_KEY" 2>/dev/null) + +if [[ -z "$MOVIES" || "$MOVIES" == "null" ]]; then + error "Radarr movie API returned empty" + exit 1 +fi + +TOTAL=$(echo "$MOVIES" | jq '. | length') +DROPPED=$(echo "$MOVIES" | jq '[.[] | select(.status == "deleted")] | length') +echo " $TOTAL movies total — $DROPPED dropped from TMDb" + +if [[ "$DROPPED" -eq 0 ]]; then + log "No TMDb-removed movies found — nothing to do" + echo "" + echo "━━━━━ $ICON_SUMMARY RADARR TMDB REMOVED SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_DONE Status: nothing to remove" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Remove Dropped Movies ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_TRASH Remove TMDb-Dropped Movies ━━━" + +START=$(date +%s) +REMOVED=() +FAILED=() +FILES_DELETED=0 +FILES_SKIPPED=0 + +while IFS=$'\t' read -r id title year tmdb_id has_file file_size; do + [[ -z "$id" ]] && continue + + SIZE_HUMAN="" + if [[ "$has_file" == "true" && "$file_size" -gt 0 ]]; then + SIZE_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $file_size / 1073741824}") + echo "$ICON_WARN $title ($year) [tmdbid $tmdb_id] — HAS FILE: $SIZE_HUMAN" + else + echo "$ICON_TRASH $title ($year) [tmdbid $tmdb_id] — no file" + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove: $title" + [[ "$DELETE_FILES" == true && "$has_file" == "true" ]] && \ + warn "DRY RUN — would delete file: $SIZE_HUMAN" + REMOVED+=("$title") + continue + fi + + DELETE_PARAM="false" + if [[ "$DELETE_FILES" == true && "$has_file" == "true" ]]; then + DELETE_PARAM="true" + fi + + RESP=$(curl -sf -X DELETE --connect-timeout 5 --max-time 15 \ + "$RADARR_URL/api/v3/movie/${id}?deleteFiles=${DELETE_PARAM}&addImportExclusion=${ADD_EXCLUSION}&apikey=$RADARR_API_KEY" \ + 2>/dev/null) + CURL_EXIT=$? + + if [[ "$CURL_EXIT" -eq 0 ]]; then + log " Removed from Radarr ✅" + REMOVED+=("$title") + if [[ "$DELETE_PARAM" == "true" ]]; then + (( FILES_DELETED++ )) + elif [[ "$has_file" == "true" ]]; then + (( FILES_SKIPPED++ )) + fi + else + warn " Failed to remove $title (curl exit $CURL_EXIT)" + FAILED+=("$title") + fi + +done < <(echo "$MOVIES" | jq -r ' + .[] | select(.status == "deleted") | + [ + (.id | tostring), + .title, + (.year | tostring), + (.tmdbId | tostring), + (if .hasFile then "true" else "false" end), + (if .movieFile.size? then (.movieFile.size | tostring) else "0" end) + ] | @tsv +') + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ $ICON_SUMMARY RADARR TMDB REMOVED SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_TRASH Removed: ${#REMOVED[@]} of $DROPPED" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ "$FILES_DELETED" -gt 0 ]] && echo "$ICON_TRASH Files deleted: $FILES_DELETED" +[[ "$FILES_SKIPPED" -gt 0 ]] && echo "$ICON_WARN Files kept: $FILES_SKIPPED (had files — use --delete-files to remove)" +[[ "$ADD_EXCLUSION" == "true" ]] && echo "$ICON_GEAR Import exclusion added for removed entries" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + log "$ICON_DONE Status: done ✅" +else + warn "Status: ${#FAILED[@]} removal(s) failed" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index 6c394c5..75194d8 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -164,8 +164,7 @@ if [[ ! -d "$SONARR_TV_ROOT" ]]; then exit 1 fi -log "Sonarr URL: $SONARR_URL" -log "TV root: $SONARR_TV_ROOT" +echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" [[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" @@ -298,7 +297,7 @@ fi # Safety Layer 3 — API version check check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1 -log "Querying Sonarr API: $SONARR_URL" +echo " Querying Sonarr API..." # Fetch all series SERIES_RESPONSE=$(sonarr_api "series") || { @@ -354,16 +353,14 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then exit 1 fi -warn "Sonarr tracks $TRACKED_COUNT episode files across $SERIES_COUNT series" +echo " $SERIES_COUNT series | $TRACKED_COUNT tracked episode files" # ============================================================================================== # ━━━ Scan TV Root ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning TV Root ━━━" -log "Root: $SONARR_TV_ROOT" -log "Orphan age: ${SONARR_ORPHAN_AGE} days" -log "Protected: ${SONARR_PROTECTED_PATTERNS[*]}" +echo " Root: $SONARR_TV_ROOT | Orphan age: ${SONARR_ORPHAN_AGE} days" echo "" START=$(date +%s) diff --git a/Media/sonarr_tvdb_removed.sh b/Media/sonarr_tvdb_removed.sh new file mode 100644 index 0000000..b369fbd --- /dev/null +++ b/Media/sonarr_tvdb_removed.sh @@ -0,0 +1,232 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Sonarr — TVDB Removed ========================================== +# ============================================================================================== +# Removes series from Sonarr that have been dropped from TVDB. +# Sonarr marks these with status="deleted" — they generate system health errors and +# can never be monitored or downloaded. +# +# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# Queries Sonarr API for series with status="deleted" (TVDB removal marker) +# Reports each found entry with file count and total size +# Removes the series record from Sonarr +# Optionally deletes associated files (disabled by default) +# Optionally adds to Sonarr's import exclusion list (default: true) +# +# ── SAFE DEFAULTS ───────────────────────────────────────────────────────────────────────────── +# Files are NOT deleted by default — use --delete-files to also remove from disk +# Import exclusion added by default — prevents Sonarr re-adding dropped series +# Per-deletion output always visible — deletions are never silently swallowed +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# Sonarr runs on HOST1 only. detect_hosts() sets SONARR_URL — if empty (HOST2) +# the script exits cleanly. +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# SONARR_DROPPED_ADD_EXCLUSION — add removed series to import exclusion (default: true) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# sonarr_tvdb_removed.sh — remove records, keep files, add exclusion +# sonarr_tvdb_removed.sh --delete-files — also delete files from disk +# sonarr_tvdb_removed.sh --dry-run — preview without removing anything +# sonarr_tvdb_removed.sh --log — verbose output +# sonarr_tvdb_removed.sh --status — show config and exit +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +# ── Parse --delete-files before standard parse_args ─────────────────────────────────────────── +DELETE_FILES=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--delete-files" ]]; then + DELETE_FILES=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found" + exit 1 +fi +if ! command -v jq >/dev/null 2>&1; then + error "jq not found" + exit 1 +fi + +acquire_lock + +detect_hosts + +if [[ -z "$SONARR_URL" ]]; then + log "Sonarr not configured for $MY_ID — nothing to do" + exit 0 +fi + +ADD_EXCLUSION="${SONARR_DROPPED_ADD_EXCLUSION:-true}" + +echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL" +[[ "$DELETE_FILES" == true ]] && warn "DELETE FILES MODE — files will be removed from disk" +[[ "$DELETE_FILES" == false ]] && echo " Files: records only (use --delete-files to also remove from disk)" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Sonarr URL: $SONARR_URL" + echo "$ICON_GEAR Add exclusion: $ADD_EXCLUSION" + echo "$ICON_GEAR Delete files: $DELETE_FILES" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Query Sonarr ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Querying Sonarr ━━━" + +if ! curl -sf --connect-timeout 5 --max-time 10 \ + "$SONARR_URL/api/v3/system/status?apikey=$SONARR_API_KEY" | jq -e '.version' >/dev/null 2>&1; then + error "Sonarr API unreachable at $SONARR_URL" + exit 1 +fi + +SERIES=$(curl -sf --connect-timeout 5 --max-time 30 \ + "$SONARR_URL/api/v3/series?apikey=$SONARR_API_KEY" 2>/dev/null) + +if [[ -z "$SERIES" || "$SERIES" == "null" ]]; then + error "Sonarr series API returned empty" + exit 1 +fi + +TOTAL=$(echo "$SERIES" | jq '. | length') +DROPPED=$(echo "$SERIES" | jq '[.[] | select(.status == "deleted")] | length') +echo " $TOTAL series total — $DROPPED dropped from TVDB" + +if [[ "$DROPPED" -eq 0 ]]; then + log "No TVDB-removed series found — nothing to do" + echo "" + echo "━━━━━ $ICON_SUMMARY SONARR TVDB REMOVED SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_DONE Status: nothing to remove" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Remove Dropped Series ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_TRASH Remove TVDB-Dropped Series ━━━" + +START=$(date +%s) +REMOVED=() +FAILED=() +FILES_DELETED=0 +FILES_SKIPPED=0 + +while IFS=$'\t' read -r id title year tvdb_id episode_file_count size_on_disk; do + [[ -z "$id" ]] && continue + + SIZE_HUMAN="" + if [[ "$size_on_disk" -gt 0 ]]; then + SIZE_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $size_on_disk / 1073741824}") + fi + + if [[ "$episode_file_count" -gt 0 ]]; then + echo "$ICON_WARN $title ($year) [tvdbid $tvdb_id] — $episode_file_count episode files${SIZE_HUMAN:+, $SIZE_HUMAN}" + else + echo "$ICON_TRASH $title ($year) [tvdbid $tvdb_id] — no files" + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove: $title" + [[ "$DELETE_FILES" == true && "$episode_file_count" -gt 0 ]] && \ + warn "DRY RUN — would delete $episode_file_count file(s)${SIZE_HUMAN:+, $SIZE_HUMAN}" + REMOVED+=("$title") + continue + fi + + DELETE_PARAM="false" + if [[ "$DELETE_FILES" == true && "$episode_file_count" -gt 0 ]]; then + DELETE_PARAM="true" + fi + + RESP=$(curl -sf -X DELETE --connect-timeout 5 --max-time 15 \ + "$SONARR_URL/api/v3/series/${id}?deleteFiles=${DELETE_PARAM}&addImportListExclusion=${ADD_EXCLUSION}&apikey=$SONARR_API_KEY" \ + 2>/dev/null) + CURL_EXIT=$? + + if [[ "$CURL_EXIT" -eq 0 ]]; then + log " Removed from Sonarr ✅" + REMOVED+=("$title") + if [[ "$DELETE_PARAM" == "true" ]]; then + (( FILES_DELETED++ )) + elif [[ "$episode_file_count" -gt 0 ]]; then + (( FILES_SKIPPED++ )) + fi + else + warn " Failed to remove $title (curl exit $CURL_EXIT)" + FAILED+=("$title") + fi + +done < <(echo "$SERIES" | jq -r ' + .[] | select(.status == "deleted") | + [ + (.id | tostring), + .title, + (.year | tostring), + (.tvdbId | tostring), + (if .episodeFileCount? then (.episodeFileCount | tostring) else "0" end), + (if .sizeOnDisk? then (.sizeOnDisk | tostring) else "0" end) + ] | @tsv +') + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ $ICON_SUMMARY SONARR TVDB REMOVED SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_TRASH Removed: ${#REMOVED[@]} of $DROPPED" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ "$FILES_DELETED" -gt 0 ]] && echo "$ICON_TRASH Files deleted: $FILES_DELETED series worth" +[[ "$FILES_SKIPPED" -gt 0 ]] && echo "$ICON_WARN Files kept: $FILES_SKIPPED series (had files — use --delete-files to remove)" +[[ "$ADD_EXCLUSION" == "true" ]] && echo "$ICON_GEAR Import exclusion added for removed entries" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + log "$ICON_DONE Status: done ✅" +else + warn "Status: ${#FAILED[@]} removal(s) failed" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 6fa3f08..69f4ebe 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -9,25 +9,32 @@ # Pre-sync: # git_pull_execute.sh — pull latest scripts first, always # +# Arr Sync (arr_sync.sh): +# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. +# All nodes agree on tracked library before any files are transferred. +# Remote nodes that don't have an arr running are skipped gracefully. +# # Rsync window (DAILY_SYNC_SHARES per host): -# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror +# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete) # HOST*_PERSONAL_SHARES — encrypted personal shares # # Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS): # media_shares_permissions.sh — fix ownership before arr cleanup # media_cleaner.sh anime — remove junk from anime shares # media_cleaner.sh media — remove junk from media shares -# lidarr_cleanup.sh — remove orphaned music files -# sonarr_cleanup.sh — remove orphaned TV files -# radarr_cleanup.sh — remove orphaned movie files +# lidarr_cleanup.sh — remove orphaned music files (local arr = truth) +# sonarr_cleanup.sh — remove orphaned TV files (local arr = truth) +# radarr_cleanup.sh — remove orphaned movie files (local arr = truth) # docker_daily_restart.sh — restart containers needing daily restart # # ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── -# git pull first — maintenance runs on latest code, not yesterday's -# Rsync before cleanup — cleanup sees the post-sync state -# Permissions before arr cleanup — arrs need correct ownership to delete/rename -# Arr cleanup after permissions — clean ownership = successful orphan deletion -# Docker restart last — containers already processed by cleanup +# git pull first — maintenance runs on latest code, not yesterday's +# arr sync before rsync — all nodes track the same library before files are spread; +# prevents remote arrs from searching for content already owned +# rsync before cleanup — cleanup sees fully spread state, rsync has no --delete +# permissions before arr cleanup — arrs need correct ownership to delete/rename +# arr cleanup after permissions — clean ownership = successful orphan deletion +# docker restart last — containers already processed by cleanup # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # Bidirectional — same script runs on both servers, correct direction automatic. @@ -193,6 +200,31 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then done fi +# ============================================================================================== +# ━━━ Arr Sync — library reconciliation before file spreading ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Arr Sync ━━━" + +ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" +if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then + log "ARR_SYNC_ENABLED=false — skipping" +elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then + warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping" + JOB_FAIL+=("arr_sync.sh") +else + _arr_sync_args=() + [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run") + if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then + log "Arr sync complete ✅" + JOB_PASS+=("arr_sync.sh") + else + warn "Arr sync completed with errors — continuing to rsync" + JOB_FAIL+=("arr_sync.sh") + fi + unset _arr_sync_args +fi + # ============================================================================================== # ━━━ Media Share Sync ━━━ # ============================================================================================== diff --git a/Tools/fallback_state_reset.sh b/Tools/fallback_state_reset.sh index 78c60b6..19b0f59 100644 --- a/Tools/fallback_state_reset.sh +++ b/Tools/fallback_state_reset.sh @@ -4,7 +4,7 @@ # ============================================================================================== # Resets the fallback state file to NORMAL and clears all tier flags. # Use when the fallback state file is stuck in a non-NORMAL state after: -# - Failover testing that left state as FAILOVER +# - Failover testing that left state as FALLBACK # - A failed handback that did not complete cleanly # - Manual intervention that left state inconsistent # - fallback.sh was killed mid-cycle and state is unknown diff --git a/common.sh b/common.sh index 3a01867..e024f4d 100644 --- a/common.sh +++ b/common.sh @@ -490,7 +490,7 @@ detect_hosts() { for _wd_var in "${_wd_checks[@]}"; do local _wd_src="${MY_ID}_${_wd_var}" # Only alias if the host-specific var is set — preserves master.conf defaults - [[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}="\${!_wd_src}"" + [[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\"" done TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD" @@ -782,6 +782,14 @@ check_remote_share() { # ── DISK TEMPERATURE CHECKS ─────────────────────────────────────────────────────────────────── # ============================================================================================== +# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD. +# Checks /sys/block//queue/rotational — 0=SSD, 1=HDD. +is_ssd() { + local base + base=$(basename "$1") + [[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]] +} + # Reads disk temperature thresholds from unRAID's dynamix.cfg. # Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX # Falls back to values in master.conf (SMART_TEMP_WARN/CRIT) if file not found. diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 0a0c896..d1ba5a8 100644 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -62,8 +62,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" - acquire_lock # detect_hosts() sets MY_ID — needed for sparse checkout configuration @@ -84,7 +82,7 @@ else log "Gitea not running locally — checking remote server" GITEA_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) if [[ -n "$GITEA_IP" ]]; then - info "$ICON_NET Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP" + echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP" elif [[ -n "${GITEA_DOMAIN:-}" ]]; then warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN" GITEA_IP="$GITEA_DOMAIN" @@ -172,7 +170,7 @@ configure_sparse_checkout() { done if [[ "$excluded" -gt 0 ]]; then - info "$ICON_LOCK Sparse checkout: excluding $excluded peer conf file(s) — credentials protected" + echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected" else log "Sparse checkout: no peer conf files to exclude (single server or first run)" fi @@ -201,7 +199,7 @@ else if [[ -d ".git" ]]; then # ── Existing repository ────────────────────────────────────────────── - info "$ICON_SYNC Existing repository detected — updating" + echo " Existing repository — updating" # Configure sparse checkout BEFORE pull # Uses conf files already present from last pull to determine exclusions @@ -213,9 +211,9 @@ else log "git clean -fd" git clean -fd - info "Pulling latest changes..." + echo " Pulling latest changes..." if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull; then - success "Git pull successful" + echo " Git pull successful" SYNC_SUCCESS=true else error "Git pull failed" @@ -225,20 +223,20 @@ else else # ── Fresh clone ────────────────────────────────────────────────────── - info "$ICON_SYNC No repository found — cloning" + echo " No repository found — cloning" # Clone first — need the repo to exist before configuring sparse checkout if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then - success "Clone successful" + echo " Clone successful" # Configure sparse checkout after clone # Now all master_host*.conf files are present — can detect exclusions configure_sparse_checkout "$TARGET_DIR" # Apply sparse checkout — removes excluded files from working tree - info "Applying sparse checkout..." + echo " Applying sparse checkout..." git read-tree -mu HEAD - success "Sparse checkout applied — peer credentials removed from working tree" + echo " Sparse checkout applied — peer credentials removed from working tree" SYNC_SUCCESS=true else @@ -253,7 +251,7 @@ else echo "━━━ $ICON_GEAR Permissions ━━━" log "Setting executable permissions on all .sh files..." find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \; - log "Permissions applied" + echo " Permissions set on .sh files" fi END=$(date +%s) diff --git a/load_config.sh b/load_config.sh index ad8d57f..2dfb53e 100644 --- a/load_config.sh +++ b/load_config.sh @@ -6,9 +6,9 @@ # Every script sources this file instead of sourcing master.conf files directly. # # ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── -# 1. Sources master.conf (shared config — thresholds, toggles, profiles, job lists) +# 1. Sources master.conf (shared config — hostnames, thresholds, toggles, profiles, job lists) # 2. Auto-discovers and sources all master_host*.conf files in the same directory -# Each host conf extends the shared profile arrays and adds host-specific identity +# Each host conf extends the shared profile arrays and adds host-specific credentials # 3. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.) # # ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── diff --git a/master.conf b/master.conf index 3c398b6..d1dff61 100644 --- a/master.conf +++ b/master.conf @@ -47,8 +47,8 @@ # REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort # RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs # -# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── -# FAILOVER Mutual container failover shared settings +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── +# FALLBACK Mutual container failover shared settings # FALLBACK TEST Simulated outage settings for fallback_test.sh # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── @@ -92,10 +92,20 @@ # # ============================================================================================== +# ============================================================================================== +# ── HOST IDENTITIES ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Hostnames for every server in the ecosystem — not credentials, safe for all machines. +# Must match the exact unRAID hostname AND Tailscale device name (case sensitive). +# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. +# Tailscale IP resolution uses these names — no hardcoded IPs needed. +# To add a new server: add HOST3="unRAID-NewServer" here + create master_host3.conf. + HOST1="unRAID-Gmer4Lfe" + HOST2="unRAID-Jayred365" + # ============================================================================================== # ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── # ============================================================================================== -# Hostnames defined in master_host*.conf — detect_hosts() reads them at runtime. # DATA_DIR is the same path on all servers — persistent script state and statistics. # Array share — survives reboots, no flash drive wear. # Created automatically if it doesn't exist. @@ -109,6 +119,19 @@ # "abort" — refuse to continue (strict — ensures both sides always match) UNRAID_VERSION_MISMATCH_ACTION="warn" +# ── Arr Sync ── +# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. +# Runs before rsync — all nodes agree on tracked library before files are transferred. +# Remote API keys are read live from each node's config.xml via SSH — never stored here. + ARR_SYNC_ENABLED=true + ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" + ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node + ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches + DOCKER_APPDATA_BASE="/mnt/user/appdata" + ARR_SYNC_LIDARR_PORT=8686 + ARR_SYNC_SONARR_PORT=8989 + ARR_SYNC_RADARR_PORT=7878 + # ── Remote Docker Daemon ── # Strike system for remote Docker daemon health checks. # Scripts that issue remote container commands check the remote daemon first. @@ -269,6 +292,8 @@ #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready #"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready + "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb + "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB "Docker_Essentials/docker_update.sh" # pull container image updates before restart "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last ) @@ -373,10 +398,11 @@ CONTAINER_DELAY=5 # seconds before starting delayed containers EXCLUDE_DIRS=() # directories excluded from transfer — profiles override -# --delete removes files on remote not on source (mirror behaviour) # --inplace writes directly to destination — better for large files # --no-whole-file forces delta transfer — sends only changed blocks - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) +# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. +# Media shares use this default; rsync spreads files only, never removes them. + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ # Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. @@ -502,13 +528,13 @@ # check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. # ============================================================================================== -# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers. # Each server runs Fallback/fallback.sh independently via array_start.sh. # All decisions based on two pings: remote reachable + internet reachable. # -# States: NORMAL | FAILOVER | NO_INTERNET | DARK +# States: NORMAL | FALLBACK | NO_INTERNET | DARK # # DDNS rules — absolute: # Internet loss → stop own DDNS immediately @@ -519,8 +545,8 @@ # Shared settings (intervals, state file, thresholds) live here. EXTERNAL_IP="8.8.8.8" - FALLBACK_CHECK_INTERVAL=120 # seconds between fallback state checks - FALLBACK_HANDBACK_STRIKES=2 # consecutive healthy checks before initiating handback + FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks + FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) FALLBACK_STATE_FILE="/boot/config/fallback_state.db" FALLBACK_ENABLED=false # HOST2 being rebuilt — set true when back online and tested # false = suppresses "not running" warnings in status scripts @@ -910,6 +936,10 @@ "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) +# Radarr/Sonarr TMDb/TVDB removed entry cleanup + RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list + SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list + # ━━━ Arr Failed/Stalled Recovery ━━━ # Auto blocklist + re-search failed imports and stalled downloads. # Runs every 6 hours — schedule: 0 */6 * * * diff --git a/master_host1.conf b/master_host1.conf index d028267..864293a 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -33,10 +33,10 @@ # DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list # DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh # -# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── # DDNS DDNS containers managed by HOST1 # INTERNET LOSS containers stopped when internet is lost -# FAILOVER TIERS what HOST1 runs for HOST2 per tier +# FALLBACK TIERS what HOST1 runs for HOST2 per tier # TIER DELAYS how long HOST1 must be down before each tier activates on HOST2 # RSYNC WRITEBACK HOST1 appdata synced back on handback # @@ -66,10 +66,7 @@ # ============================================================================================== # ━━━ Identity ━━━ -# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive. -# Used by detect_hosts() in common.sh to identify this server as HOST1. - HOST1="unRAID-Gmer4Lfe" - +# HOST1 hostname lives in master.conf (not a credential — safe for all servers). # SSH key used for all server-to-server operations — rsync, failover container commands. # Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys. HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation" @@ -278,7 +275,7 @@ ) # ============================================================================================== -# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ DDNS ━━━ diff --git a/master_host2.conf b/master_host2.conf index cb0272c..adb18f4 100644 --- a/master_host2.conf +++ b/master_host2.conf @@ -37,10 +37,10 @@ # DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list # DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh # -# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── # DDNS DDNS containers managed by HOST2 # INTERNET LOSS containers stopped when internet is lost -# FAILOVER TIERS what HOST2 runs for HOST1 per tier +# FALLBACK TIERS what HOST2 runs for HOST1 per tier # TIER DELAYS how long HOST2 must be down before each tier activates on HOST1 # RSYNC WRITEBACK HOST2 appdata synced back on handback # @@ -68,10 +68,7 @@ # ============================================================================================== # ━━━ Identity ━━━ -# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive. -# Used by detect_hosts() in common.sh to identify this server as HOST2. - HOST2="unRAID-Jayred365" - +# HOST2 hostname lives in master.conf (not a credential — safe for all servers). # SSH key used for all server-to-server operations — rsync, failover container commands. # Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys. HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" @@ -241,7 +238,7 @@ ) # ============================================================================================== -# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ DDNS ━━━ diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 7ce1761..e6760ff 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -203,7 +203,7 @@ # bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh -# ── RSYNC EMBY FAILOVER ─────────────────────────────────────────────────────────────────────── +# ── RSYNC EMBY FALLBACK ─────────────────────────────────────────────────────────────────────── # Schedule: */30 * * * * (every 30 minutes) # Background: YES # @@ -375,14 +375,14 @@ # ────────────────────────────────────────────────────────────────────────────────────────────── -# FAILOVER +# FALLBACK # ────────────────────────────────────────────────────────────────────────────────────────────── # fallback.sh — [continuous] mutual fallback state machine # Started by array_start.sh on both servers independently. # Every FALLBACK_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8 -# States: NORMAL / FAILOVER / NO_INTERNET / DARK -# FAILOVER: starts remote containers in tiers across 24 hours: +# States: NORMAL / FALLBACK / NO_INTERNET / DARK +# FALLBACK: starts remote containers in tiers across 24 hours: # Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV # Tier 2 at 4hr: NextCloud + Immich + Jellyseerr # Tier 3 at 12hr: AdGuard + Gitea + uptime monitoring @@ -399,7 +399,7 @@ # ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption. # iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes. # Remote is always reachable after the test completes regardless of what happened. -# 8 phases: pre-flight → block remote (iptables) → detect FAILOVER → verify Tier 1 started +# 8 phases: pre-flight → block remote (iptables) → detect FALLBACK → verify Tier 1 started # → restore remote → wait handback → verify NORMAL restored → full pass/fail report. # FALLBACK_TEST_BLOCK_WAIT must be > FALLBACK_CHECK_INTERVAL + buffer (default: 150s). # FALLBACK_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s). @@ -409,7 +409,7 @@ # fallback_state_reset.sh — reset fallback state file to NORMAL # Resets state file ONLY — does NOT start or stop any containers. -# Use when: state stuck in FAILOVER after testing, after killing fallback.sh mid-cycle, +# Use when: state stuck in FALLBACK after testing, after killing fallback.sh mid-cycle, # after manual intervention left state inconsistent, after a failed handback. # Shows current state file before asking for confirmation. # Verify before resetting: right containers on right server, DDNS correct,