diff --git a/Configurations/master.conf b/Configurations/master.conf index 943e481..968fe40 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -37,10 +37,14 @@ # # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── # ARRAY START Scripts launched at array start (array_started.sh) -# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) +# ARRAY STOP Scripts run at planned shutdown (array_stopping.sh) +# WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh) +# SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh +# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) +# INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh) # DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) # WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) -# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) +# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) # # ── RSYNC ────────────────────────────────────────────────────────────────────────────────── # RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator @@ -138,7 +142,6 @@ # Scripts that issue remote container commands check the remote daemon first. # Below limit → skip operation this run | At limit → notify critical + exit REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify - REMOTE_DOCKER_RETRY_WAIT=30 # seconds to wait before retrying after failure # ============================================================================================== # ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── @@ -259,18 +262,6 @@ # No changes to orchestrator scripts needed when adding or removing jobs. # Sections ordered by run frequency: array events first, then shortest interval to longest. -# ━━━ Array Stop ━━━ -# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. -# Run sequentially (foreground) — each must complete before the next starts. -# Order matters: user scripts first (prevents new ops), then data movement, then containers. - ARRAY_STOP_SCRIPTS=( - "unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops - "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) - "unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) - "unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) - "Docker_Essentials/docker_container_stop.sh" # stop all containers last - ) - # ━━━ Array Start ━━━ # Scripts launched by array_started.sh when the array comes online. # Launched in order — each as a background process. @@ -287,6 +278,18 @@ "Fallback/fallback.sh" # mutual failover — continuous ) +# ━━━ Array Stop ━━━ +# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. +# Run sequentially (foreground) — each must complete before the next starts. +# Order matters: user scripts first (prevents new ops), then data movement, then containers. + ARRAY_STOP_SCRIPTS=( + "unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops + "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) + "unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) + "unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) + "Docker_Essentials/docker_container_stop.sh" # stop all containers last + ) + # ━━━ Watchdog Orchestrator ━━━ # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # Schedule: * * * * * (every minute) @@ -300,9 +303,6 @@ "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense ) - WATCHDOG_ORCHESTRATOR_HEARTBEAT=true - WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1 - # ━━━ System Watchdog ━━━ # system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. # Called by watchdog_orchestrator.sh — not scheduled directly. @@ -312,23 +312,6 @@ "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy ) -# ━━━ Monthly Maintenance ━━━ -# monthly_maintenance.sh fires only when BOTH gates pass: -# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days -# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) -# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) -# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. -# Add scripts that require a long-stable settled system — scrubs, extended drive tests. -# State file on /boot/config — survives reboots (interval gate independent of uptime gate). - MONTHLY_MAINTENANCE_SCRIPTS=( - #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built - #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built - ) - - MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires - MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again - MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" - # ━━━ Critical Sync Maintenance ━━━ # critical_sync_maintenance.sh runs every 30 minutes. # Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check @@ -348,7 +331,6 @@ # and optional mid-day rsync for any shares that need sub-daily propagation. # Schedule: 0 */4 * * * # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. - INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares INTERMEDIATE_MAINTENANCE_SCRIPTS=( "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items @@ -423,6 +405,23 @@ WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH +# ━━━ Monthly Maintenance ━━━ +# monthly_maintenance.sh fires only when BOTH gates pass: +# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days +# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) +# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) +# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. +# Add scripts that require a long-stable settled system — scrubs, extended drive tests. +# State file on /boot/config — survives reboots (interval gate independent of uptime gate). + MONTHLY_MAINTENANCE_SCRIPTS=( + #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built + #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built + ) + + MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires + MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again + MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" + # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -439,18 +438,20 @@ # Use when: rebuilding secondary, testing, per-window bandwidth management # # Example — HOST2 data rebuild: -# RSYNC_ENABLED=true ← rsync works, individual scripts run fine -# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild -# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) -# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs -# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works +# RSYNC_ENABLED=true ← rsync works, individual scripts run fine +# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs +# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild +# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) +# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works # → Run individual: bash Rsync/rsync.sh /mnt/user/Movies -# → When ready: DAILY_RSYNC_ENABLED=true - RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below - DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready - WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section - CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready - FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback +# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true + RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below + CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready + INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section + DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready + WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. @@ -696,13 +697,6 @@ RESP_FAIL_LIMIT=2 # consecutive failed checks before restart CURL_TIMEOUT=5 # seconds per check before timeout -# Watchdog cycle interval - DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles (15 minutes) - -# Heartbeat — proof of life logged periodically even when everything is healthy - DOCKER_WATCHDOG_HEARTBEAT=true - DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 - # Tier 2 master toggle WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers @@ -1332,19 +1326,11 @@ # Single spikes (one bad reading) are ignored — sustained problems trigger reboot. SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger -# How often checks run — 300s = 5 minutes. -# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot. - SYSTEM_WATCHDOG_INTERVAL=300 - # Reboot loop protection — if system keeps rebooting something is seriously wrong. # After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. SYS_WATCHDOG_REBOOT_LIMIT=3 SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 -# Heartbeat — proof of life logged periodically even when everything is healthy. - SYSTEM_WATCHDOG_HEARTBEAT=true - SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 - # ━━━ RAM Reboot Threshold ━━━ # Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh # RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) diff --git a/Tools/emby_to_radarr_sync.sh b/Tools/emby_to_radarr_sync.sh index 38f845c..119f279 100644 --- a/Tools/emby_to_radarr_sync.sh +++ b/Tools/emby_to_radarr_sync.sh @@ -38,9 +38,10 @@ # without modification. Safe to run multiple times — the second run finds nothing to add. # # ============================================================================================== -# CONFIGURATION (host*.conf) +# CONFIGURATION (master.conf / host*.conf) # ============================================================================================== # +# RADARR_EMBY_LIBRARIES — Emby library names to scan (master.conf); empty = all libraries # HOST*_RADARR_URL / HOST*_RADARR_API_KEY — Radarr connection (aliased by detect_hosts) # HOST*_EMBY_URL / HOST*_EMBY_API_KEY — Emby connection (aliased by detect_hosts) # @@ -156,18 +157,39 @@ echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" echo "━━━ $ICON_SYNC Emby Movie Library ━━━" -EMBY_MOVIES_JSON=$(_emby_api "Items?IncludeItemTypes=Movie&Recursive=true&Fields=ProviderIds&Limit=10000") || { - error "Could not fetch Emby movies" - exit 1 -} - declare -A EMBY_MOVIES # name → tmdb_id (empty string if none) -while IFS='|' read -r name tmdb_id; do - [[ -z "$name" || "$name" == "null" ]] && continue - _is_episode_title "$name" && { log " $ICON_SKIP Skipping episode/garbage: $name"; continue; } - EMBY_MOVIES["$name"]="$tmdb_id" -done < <(echo "$EMBY_MOVIES_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tmdb // "")] | join("|")' 2>/dev/null) +if [[ "${#RADARR_EMBY_LIBRARIES[@]}" -gt 0 ]]; then + LIBRARIES_JSON=$(_emby_api "Library/VirtualFolders") || { error "Could not fetch Emby libraries"; exit 1; } + for lib_name in "${RADARR_EMBY_LIBRARIES[@]}"; do + lib_id=$(echo "$LIBRARIES_JSON" | jq -r --arg n "$lib_name" '.[] | select(.Name == $n) | .ItemId' 2>/dev/null) + if [[ -z "$lib_id" ]]; then + warn "Emby library not found: $lib_name" + continue + fi + log "Scanning library: $lib_name (ItemId: $lib_id)" + LIB_JSON=$(_emby_api "Items?ParentId=${lib_id}&IncludeItemTypes=Movie&Recursive=true&Fields=ProviderIds&Limit=5000") || { + warn "Could not fetch movies from library: $lib_name" + continue + } + while IFS='|' read -r name tmdb_id; do + [[ -z "$name" || "$name" == "null" ]] && continue + _is_episode_title "$name" && { log " $ICON_SKIP Skipping episode/garbage: $name"; continue; } + EMBY_MOVIES["$name"]="$tmdb_id" + done < <(echo "$LIB_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tmdb // "")] | join("|")' 2>/dev/null) + done +else + log "RADARR_EMBY_LIBRARIES empty — scanning all Emby libraries" + EMBY_MOVIES_JSON=$(_emby_api "Items?IncludeItemTypes=Movie&Recursive=true&Fields=ProviderIds&Limit=10000") || { + error "Could not fetch Emby movies" + exit 1 + } + while IFS='|' read -r name tmdb_id; do + [[ -z "$name" || "$name" == "null" ]] && continue + _is_episode_title "$name" && { log " $ICON_SKIP Skipping episode/garbage: $name"; continue; } + EMBY_MOVIES["$name"]="$tmdb_id" + done < <(echo "$EMBY_MOVIES_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tmdb // "")] | join("|")' 2>/dev/null) +fi EMBY_COUNT=${#EMBY_MOVIES[@]} log "$EMBY_COUNT movies in Emby library" diff --git a/Tools/emby_to_sonarr_sync.sh b/Tools/emby_to_sonarr_sync.sh index 0fef85a..163087a 100644 --- a/Tools/emby_to_sonarr_sync.sh +++ b/Tools/emby_to_sonarr_sync.sh @@ -41,7 +41,7 @@ # CONFIGURATION (master.conf / host*.conf) # ============================================================================================== # -# SONARR_EMBY_LIBRARIES — Emby library names to scan (master.conf) +# SONARR_EMBY_LIBRARIES — Emby library names to scan (master.conf); empty = all libraries # HOST*_SONARR_URL / HOST*_SONARR_API_KEY — Sonarr connection (aliased by detect_hosts) # HOST*_EMBY_URL / HOST*_EMBY_API_KEY — Emby connection (aliased by detect_hosts) # @@ -151,33 +151,39 @@ echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" echo "━━━ $ICON_SYNC Emby Series Library ━━━" -if [[ "${#SONARR_EMBY_LIBRARIES[@]}" -eq 0 ]]; then - error "SONARR_EMBY_LIBRARIES not configured — check master.conf" - exit 1 -fi - -# Fetch library list and resolve names → ItemIds -LIBRARIES_JSON=$(_emby_api "Library/VirtualFolders") || { error "Could not fetch Emby libraries"; exit 1; } - declare -A EMBY_SERIES # name → tvdb_id (empty string if none) -for lib_name in "${SONARR_EMBY_LIBRARIES[@]}"; do - lib_id=$(echo "$LIBRARIES_JSON" | jq -r --arg n "$lib_name" '.[] | select(.Name == $n) | .ItemId' 2>/dev/null) - if [[ -z "$lib_id" ]]; then - warn "Emby library not found: $lib_name" - continue - fi - log "Scanning library: $lib_name (ItemId: $lib_id)" - LIB_JSON=$(_emby_api "Items?ParentId=${lib_id}&IncludeItemTypes=Series&Recursive=true&Fields=ProviderIds&Limit=5000") || { - warn "Could not fetch series from library: $lib_name" - continue +if [[ "${#SONARR_EMBY_LIBRARIES[@]}" -gt 0 ]]; then + LIBRARIES_JSON=$(_emby_api "Library/VirtualFolders") || { error "Could not fetch Emby libraries"; exit 1; } + for lib_name in "${SONARR_EMBY_LIBRARIES[@]}"; do + lib_id=$(echo "$LIBRARIES_JSON" | jq -r --arg n "$lib_name" '.[] | select(.Name == $n) | .ItemId' 2>/dev/null) + if [[ -z "$lib_id" ]]; then + warn "Emby library not found: $lib_name" + continue + fi + log "Scanning library: $lib_name (ItemId: $lib_id)" + LIB_JSON=$(_emby_api "Items?ParentId=${lib_id}&IncludeItemTypes=Series&Recursive=true&Fields=ProviderIds&Limit=5000") || { + warn "Could not fetch series from library: $lib_name" + continue + } + while IFS='|' read -r name tvdb_id; do + [[ -z "$name" || "$name" == "null" ]] && continue + _is_garbage_title "$name" && { log " $ICON_SKIP Skipping garbage title: $name"; continue; } + EMBY_SERIES["$name"]="$tvdb_id" + done < <(echo "$LIB_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tvdb // "")] | join("|")' 2>/dev/null) + done +else + log "SONARR_EMBY_LIBRARIES empty — scanning all Emby libraries" + ALL_JSON=$(_emby_api "Items?IncludeItemTypes=Series&Recursive=true&Fields=ProviderIds&Limit=5000") || { + error "Could not fetch Emby series" + exit 1 } while IFS='|' read -r name tvdb_id; do [[ -z "$name" || "$name" == "null" ]] && continue _is_garbage_title "$name" && { log " $ICON_SKIP Skipping garbage title: $name"; continue; } EMBY_SERIES["$name"]="$tvdb_id" - done < <(echo "$LIB_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tvdb // "")] | join("|")' 2>/dev/null) -done + done < <(echo "$ALL_JSON" | jq -r '.Items[] | [.Name, (.ProviderIds.Tvdb // "")] | join("|")' 2>/dev/null) +fi EMBY_COUNT=${#EMBY_SERIES[@]} log "$EMBY_COUNT series in Emby library" diff --git a/common.sh b/common.sh index 360a864..582d486 100644 --- a/common.sh +++ b/common.sh @@ -186,16 +186,20 @@ ICON_SUCCESS="✅" # ============================================================================================== # Standardised output functions used across all scripts. # -# Two-tier model: -# echo — always visible — summaries, status conclusions, section headers -# warn() — always visible — state transitions, warnings, important events -# error() — always visible — something broke -# log() — only with --log flag — per-item detail, internal checks +# Output tiers: +# echo — always visible — summaries, status conclusions, section headers +# info() — always visible — status confirmations, reachability, check pass +# warn() — always visible — state transitions, warnings, important events +# error() — always visible — something broke +# success() — always visible — operation completed successfully +# log() — only with --log flag — per-item detail, internal checks # # All output goes to stdout — callers can redirect as needed. +info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } +success() { echo "$ICON_DONE [OK] $*"; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" @@ -673,11 +677,12 @@ ping_internet() { # Tier 2: orchestrator-specific flag — fine-grained control per window # # Usage: -# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED -# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED -# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED -# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED -# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) +# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED +# check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED +# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED +# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED +# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED +# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) # # Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly @@ -906,12 +911,12 @@ check_local_disk_temps() { if [[ ${#crit_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}" - error "$ICON_WARN Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}" + error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}" notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning" return 2 elif [[ ${#hot_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}" - warn "$ICON_WARN Drive temp WARNING — skipping this profile: ${hot_drives[*]}" + warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}" notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal" return 1 else @@ -1536,7 +1541,9 @@ notify_emby_scan() { # Get all scheduled tasks local tasks_response - tasks_response=$(curl -sf --max-time 15 -H "X-Api-Key: $EMBY_API_KEY" "${EMBY_URL}/ScheduledTasks" 2>/dev/null) + tasks_response=$(curl -sf --max-time 15 \ + -H "X-Api-Key: $EMBY_API_KEY" \ + "${EMBY_URL}/ScheduledTasks" 2>/dev/null) if [[ -z "$tasks_response" ]]; then warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan" @@ -1545,11 +1552,17 @@ notify_emby_scan() { # Find the "Clean Missing Files" task ID local task_id - task_id=$(echo "$tasks_response" | grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | grep -o '"Id":"[^"]*"' | sed 's/"Id":"//;s/"//' | head -1) + task_id=$(echo "$tasks_response" | \ + grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \ + grep -o '"Id":"[^"]*"' | \ + sed 's/"Id":"//;s/"//' | head -1) # Fallback — try "Scan Media Library" if Clean Missing not found if [[ -z "$task_id" ]]; then - task_id=$(echo "$tasks_response" | grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | grep -o '"Id":"[^"]*"' | sed 's/"Id":"//;s/"//' | head -1) + task_id=$(echo "$tasks_response" | \ + grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \ + grep -o '"Id":"[^"]*"' | \ + sed 's/"Id":"//;s/"//' | head -1) [[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library" fi @@ -1563,7 +1576,12 @@ notify_emby_scan() { # Trigger the task local http_code - http_code=$(curl -sf --max-time 15 -X POST -H "X-Api-Key: $EMBY_API_KEY" -w "%{http_code}" -o /dev/null "${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null) + http_code=$(curl -sf --max-time 15 \ + -X POST \ + -H "X-Api-Key: $EMBY_API_KEY" \ + -w "%{http_code}" \ + -o /dev/null \ + "${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null) if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed" @@ -1687,7 +1705,8 @@ REMOTE_DOCKER_STRIKES=0 check_remote_docker_daemon() { local timeout="${1:-10}" - if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" "timeout $timeout docker info" >/dev/null 2>&1; then + if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \ + "timeout $timeout docker info" >/dev/null 2>&1; then # Daemon healthy — clear strikes if we had issues if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes"