From 0ae31b5fa682df09f1c132dd6e0ca41267a71bce Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 10 May 2026 20:08:32 -0400 Subject: [PATCH] feat: Tailscale resolution hardening, partnership offboard completion, Emby provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit common.sh: - Add resolve_tailscale_ip() helper — tries `tailscale ip -4` first, falls back to parsing `tailscale status` output; handles hosts where MagicDNS short-name resolution is not active - Add PARTNERSHIP_OWN_CONTAINERS alias in detect_hosts() - Add aliasing for 4 Emby provisioning vars (PARTNERSHIP_PROVISION_EMBY_ADMIN, PARTNERSHIP_EMBY_ADMIN_USER, PARTNERSHIP_EMBY_ADMIN_PASS, PARTNERSHIP_EMBY_PORT) Partnership/partnership_manager.sh: - Replace 9 bare `tailscale ip -4` calls with resolve_tailscale_ip() - Add read_remote_conf_var() and read_remote_conf_array() — SSH to mirror, source its own load_config.sh + detect_hosts(), return aliased variable; solves sparse-checkout problem where HOST1 cannot read master_host2.conf directly - Add derive_short_name() — strips unraid- prefix, capitalises first char - Add cleanup_partner_containers() — removes partner containers via FolderView3 folder if enabled, else falls back to FALLBACK_*_COVERS_*_TIER* arrays - Add cleanup_owner_containers_on_mirror() — SSH to mirror, stops and removes containers matching *-${OWNER_SHORT} naming convention - Add start_own_stack() and start_mirror_own_stack() — restart own containers locally or on mirror via SSH using PARTNERSHIP_OWN_CONTAINERS - Add provision_emby_admin() — reads mirror credentials via read_remote_conf_var, checks for username collision, creates user + sets password + grants admin policy via Emby API - Add revoke_emby_admin() — looks up mirror username on local Emby, deletes via REST API - Wire offboard paths (both mirror-initiated and owner-initiated) to call container cleanup and stack restart; update --check finalisation paths accordingly - Fix write_state_file in --onboard not gated on DRY_RUN (was writing ACTIVE state on dry runs) master_host1.conf: - Add HOST1_PARTNERSHIP_OWN_CONTAINERS array - Add partnership Emby provisioning config (toggle + port + per-host credentials) master_host2.conf: - Add HOST2_PARTNERSHIP_OWN_CONTAINERS array - Add HOST2_PARTNERSHIP_EMBY_ADMIN_USER and HOST2_PARTNERSHIP_EMBY_ADMIN_PASS Tailscale fix applied to: - Initial_run/ssh_setup.sh (2 callsites) - unRAID_Essentials/rsync_stop.sh (1 callsite) Co-Authored-By: Claude Sonnet 4.6 --- Docker_Essentials/docker_container_stop.sh | 178 ++++++++ Docker_Essentials/docker_daily_restart.sh | 26 -- Docker_Essentials/docker_network_connect.sh | 0 Docker_Essentials/docker_update_remaining.sh | 145 ++++++- Docker_Essentials/docker_watchdog.sh | 2 +- Docker_Essentials/downloaders_reset.sh | 4 +- Fallback/fallback.sh | 0 Initial_run/ssh_setup.sh | 4 +- Media/lidarr_cleanup.sh | 56 +++ Media/lidarr_missing_art.sh | 61 ++- Media/radarr_cleanup.sh | 68 ++- Media/sonarr_cleanup.sh | 80 +++- Monitors/continuous_scripts_status.sh | 48 ++- Monitors/weekly_health_digest.sh | 4 +- Orchestrators/array_stop.sh | 178 ++++++++ Orchestrators/critical_sync_maintenance.sh | 4 +- .../intermediate_sync_maintenance.sh | 14 +- Partnership/partnership_manager.sh | 391 ++++++++++++++++-- Rsync/rsync.sh | 34 +- Tools/emby_database_repair.sh | 2 +- Transcodes/ramdisk_setup.sh | 0 common.sh | 26 +- git_pull_execute.sh | 0 master.conf | 23 +- master_host1.conf | 25 ++ master_host2.conf | 21 + unRAID_Essentials/docker_syslog_filter.sh | 0 unRAID_Essentials/inotify_tuning.sh | 0 unRAID_Essentials/mover_stop.sh | 0 unRAID_Essentials/php_fpm_max_children.sh | 0 unRAID_Essentials/rsync_stop.sh | 2 +- unRAID_Essentials/system_watchdog.sh | 0 unRAID_Essentials/user_scripts_stop.sh | 0 33 files changed, 1241 insertions(+), 155 deletions(-) create mode 100755 Docker_Essentials/docker_container_stop.sh mode change 100644 => 100755 Docker_Essentials/docker_network_connect.sh mode change 100644 => 100755 Docker_Essentials/docker_watchdog.sh mode change 100644 => 100755 Fallback/fallback.sh create mode 100755 Orchestrators/array_stop.sh mode change 100644 => 100755 Transcodes/ramdisk_setup.sh mode change 100644 => 100755 git_pull_execute.sh mode change 100644 => 100755 unRAID_Essentials/docker_syslog_filter.sh mode change 100644 => 100755 unRAID_Essentials/inotify_tuning.sh mode change 100644 => 100755 unRAID_Essentials/mover_stop.sh mode change 100644 => 100755 unRAID_Essentials/php_fpm_max_children.sh mode change 100644 => 100755 unRAID_Essentials/rsync_stop.sh mode change 100644 => 100755 unRAID_Essentials/system_watchdog.sh mode change 100644 => 100755 unRAID_Essentials/user_scripts_stop.sh diff --git a/Docker_Essentials/docker_container_stop.sh b/Docker_Essentials/docker_container_stop.sh new file mode 100755 index 0000000..b402489 --- /dev/null +++ b/Docker_Essentials/docker_container_stop.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Docker Container Stop ========================================== +# ============================================================================================== +# Stops all running Docker containers one at a time, verifying each is stopped before +# moving to the next. Called by array_stop.sh as part of a planned shutdown sequence. +# +# ── STOP SEQUENCE PER CONTAINER ────────────────────────────────────────────────────────────── +# 1. docker stop -t 30 (SIGTERM + 30s grace period — docker sends SIGKILL if needed) +# 2. Verify stopped — if still running, retry up to RETRY_COUNT times +# 3. docker kill (SIGKILL) if all retries exhausted +# 4. Final verify — error if still running after force-kill +# Never moves to the next container until the current one is confirmed stopped. +# +# ── WHY SEQUENTIAL ──────────────────────────────────────────────────────────────────────────── +# Containers may have dependencies — stopping one at a time avoids abruptly severing a +# service while its dependents are still running and trying to use it. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — docker requires root +# Per-container verify — confirmed stopped before proceeding to next +# Retry loop — RETRY_COUNT attempts before escalating to force-kill +# SIGTERM → SIGKILL — graceful then forced, never skips graceful +# DOCKER_TIMEOUT — all docker calls protected against hung daemon +# notify on failures — alert if any container cannot be stopped +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RETRY_COUNT — retry attempts before force-kill (default 3) +# SLEEP — seconds between retries +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_container_stop.sh — stop all running containers +# docker_container_stop.sh --dry-run — show which containers would be stopped +# docker_container_stop.sh --status — show running containers and exit +# docker_container_stop.sh --log — verbose output +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v docker &>/dev/null; then + error "Docker command not found" + exit 1 +fi + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped" + +DOCKER_TIMEOUT=30 +DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally +_RETRY_COUNT="${RETRY_COUNT:-3}" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort) + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_CONTAINERS Running: ${#RUNNING[@]} container(s)" + echo "$ICON_RETRY Retries: $_RETRY_COUNT" + for c in "${RUNNING[@]}"; do echo " $ICON_RUNNING $c"; done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Stop Containers ━━━ +# ============================================================================================== +mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort) + +echo "" +echo "━━━ $ICON_CONTAINERS Docker Container Stop — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +if [[ ${#RUNNING[@]} -eq 0 ]]; then + log "No running containers — nothing to do" + exit 0 +fi + +echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running" +echo "" + +START=$(date +%s) +STOPPED=() +FAILED=() + +for container in "${RUNNING[@]}"; do + [[ -z "$container" ]] && continue + echo "━━━ $ICON_CONTAINERS $container ━━━" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop $container" + STOPPED+=("$container") + echo "" + continue + fi + + attempt=1 + success=false + + while [[ "$attempt" -le "$_RETRY_COUNT" ]]; do + log "$ICON_RETRY Attempt $attempt of $_RETRY_COUNT — stopping $container..." + + timeout "$DOCKER_TIMEOUT" docker stop -t "$DOCKER_STOP_TIMEOUT" "$container" \ + >/dev/null 2>&1 + + STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \ + -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$STATE" != "true" ]]; then + echo "$ICON_DONE $container stopped ✅" + STOPPED+=("$container") + success=true + break + fi + + warn "Attempt $attempt failed — $container still running" + (( attempt++ )) + [[ "$attempt" -le "$_RETRY_COUNT" ]] && sleep "${SLEEP:-5}" + done + + if [[ "$success" == false ]]; then + warn "$container — retries exhausted, force-killing with SIGKILL..." + timeout "$DOCKER_TIMEOUT" docker kill "$container" >/dev/null 2>&1 || true + + STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \ + -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$STATE" != "true" ]]; then + warn "$container force-killed ✅" + STOPPED+=("$container") + else + error "$container still running after SIGKILL — manual intervention needed" + notify "$container could not be stopped on $(hostname) ($MY_ID)" \ + "Docker Container Stop" "warning" + FAILED+=("$container") + fi + fi + + echo "" +done + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "━━━━━ $ICON_SUMMARY DOCKER CONTAINER STOP SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_CONTAINERS Scope: ${#RUNNING[@]} running → ${#STOPPED[@]} stopped" +[[ ${#STOPPED[@]} -gt 0 ]] && echo "$ICON_DONE Stopped: ${STOPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no containers stopped" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + log "$ICON_DONE Status: done ✅ — ${#STOPPED[@]} container(s) stopped" +else + warn "Status: ${#FAILED[@]} container(s) could not be stopped" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 1673c17..4ca2723 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -107,32 +107,6 @@ fi # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. -# Usage: retry_docker docker restart ContainerName -retry_docker() { - local attempt=1 - - while [[ "$attempt" -le "$RETRY_COUNT" ]]; do - log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" - - if "$@"; then - log "Succeeded on attempt $attempt" - return 0 - else - warn "Attempt $attempt failed" - (( attempt++ )) - [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" - fi - done - - error "Command failed after $RETRY_COUNT attempts: $*" - return 1 -} - -# ============================================================================================== -# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - # Wraps docker commands with a 30 second timeout. # Prevents a hung Docker daemon from causing the script to hang indefinitely. # Usage: docker_cmd docker restart ContainerName diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh old mode 100644 new mode 100755 diff --git a/Docker_Essentials/docker_update_remaining.sh b/Docker_Essentials/docker_update_remaining.sh index becdbab..2f3f8ce 100644 --- a/Docker_Essentials/docker_update_remaining.sh +++ b/Docker_Essentials/docker_update_remaining.sh @@ -3,7 +3,8 @@ # ============================= Docker Update — Remaining ====================================== # ============================================================================================== # Pulls the latest image for every running container NOT already covered by the daily or -# weekly update/restart cycles. Runs at the end of the weekly maintenance window. +# weekly update/restart cycles. Restarts containers that received a new image, then prunes +# dangling images. Runs at the end of the weekly maintenance window. # # ── WHAT THIS COVERS ────────────────────────────────────────────────────────────────────────── # Daily update: DAILY_RESTART_CONTAINERS — auth stack, NPM, Dispatcharr, etc. @@ -14,6 +15,12 @@ # pull per week, with no container list to maintain here — it derives the remainder # automatically from `docker ps` minus the two managed lists. # +# ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── +# 1. Pull latest image for each remaining running container +# 2. Restart containers whose image ID changed (new update landed) +# 3. Prune dangling images left behind by the updates +# Containers already up to date are not restarted. +# # ── EXCLUSION LOGIC ─────────────────────────────────────────────────────────────────────────── # Exclusion set = DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS (aliased by detect_hosts) # Only running containers are targeted — stopped containers are intentionally excluded @@ -29,7 +36,7 @@ # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # docker_update_remaining.sh — normal run -# docker_update_remaining.sh --dry-run — show which containers would be pulled +# docker_update_remaining.sh --dry-run — show which containers would be pulled/restarted # docker_update_remaining.sh --log — verbose output # docker_update_remaining.sh --status — show config and exit # ============================================================================================== @@ -97,7 +104,48 @@ if [[ ${#REMAINING[@]} -eq 0 ]]; then exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted" + +# ============================================================================================== +# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +DOCKER_TIMEOUT=30 +docker_cmd() { + timeout "$DOCKER_TIMEOUT" "$@" + local exit_code=$? + if [[ "$exit_code" -eq 124 ]]; then + error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*" + return 1 + fi + return "$exit_code" +} + +retry_docker() { + local attempt=1 + while [[ "$attempt" -le "$RETRY_COUNT" ]]; do + log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + if docker_cmd "$@"; then + log "Succeeded on attempt $attempt" + return 0 + else + warn "Attempt $attempt failed" + (( attempt++ )) + [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" + fi + done + error "Command failed after $RETRY_COUNT attempts: $*" + return 1 +} + +RESTART_VERIFY_WAIT=5 +verify_running() { + local container="$1" + sleep "$RESTART_VERIFY_WAIT" + local state + state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + [[ "$state" == "true" ]] +} # ============================================================================================== # ━━━ Pull Updates ━━━ @@ -155,27 +203,102 @@ for container in "${REMAINING[@]}"; do echo "" done +# ============================================================================================== +# ━━━ Restart Updated Containers ━━━ +# ============================================================================================== +RESTARTED=() +RESTART_FAILED=() +SKIPPED_STOPPED=() + +if [[ ${#UPDATED[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}" + echo "" + + for container in "${UPDATED[@]}"; do + [[ -z "$container" ]] && continue + echo "━━━ $ICON_CONTAINERS $container ━━━" + + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + + if [[ "$STATUS" != "true" ]]; then + echo "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)" + SKIPPED_STOPPED+=("$container") + echo "" + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container" + RESTARTED+=("$container") + echo "" + continue + fi + + echo "$ICON_RUNNING $container is running — restarting on new image..." + if retry_docker docker restart "$container"; then + if verify_running "$container"; then + echo "$ICON_DONE $container restarted and running ✅" + RESTARTED+=("$container") + else + error "$container restarted but crashed immediately" + notify "$container crashed after update-restart on $(hostname)" "Docker Update Remaining" "warning" + RESTART_FAILED+=("$container") + fi + else + error "Failed to restart $container after $RETRY_COUNT attempts" + notify "$container failed to restart after update on $(hostname)" "Docker Update Remaining" "warning" + RESTART_FAILED+=("$container") + fi + echo "" + done +else + log "No containers received a new image — nothing to restart" +fi + +# ============================================================================================== +# ━━━ Prune Old Images ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would prune dangling images" + PRUNED_SUMMARY="(dry run)" +else + PRUNED_OUTPUT=$(docker image prune -f 2>&1) + echo "$PRUNED_OUTPUT" | sed 's/^/ /' + PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed") +fi + END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== +echo "" echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked" -[[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE Updated: ${UPDATED[*]}" -[[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE New image: ${UPDATED[*]}" +[[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}" +[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_DONE Restarted: ${RESTARTED[*]}" +[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && echo "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)" +[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}" +echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" +ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} )) if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no images pulled" -elif [[ ${#FAILED[@]} -eq 0 ]]; then - log "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current" + warn "DRY RUN — no changes made" +elif [[ "$ALL_FAILED" -eq 0 ]]; then + log "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current" else - warn "Status: ${#FAILED[@]} pull(s) failed — containers continue on existing images" + warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -# Pull failures are non-fatal +[[ "$ALL_FAILED" -gt 0 ]] && exit 1 exit 0 diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh old mode 100644 new mode 100755 index a77b451..2723a9d --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -787,7 +787,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue # Skip containers already monitored by required containers (Tier 1) - local already_required=false + already_required=false for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do [[ "$container" == "$req" ]] && already_required=true && break done diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index b2f24c2..ef1d54b 100644 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -286,7 +286,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then while IFS= read -r NZO_ID; do [[ -z "$NZO_ID" ]] && continue JOB_TIME=$(echo "$HISTORY" | grep -A5 "$NZO_ID" | \ - grep -o '"completed":[0-9]*' | grep -o '[0-9]*') + grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1) [[ -z "$JOB_TIME" ]] && continue [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue if [[ "$DRY_RUN" == true ]]; then @@ -332,7 +332,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then while IFS= read -r NZO_ID; do [[ -z "$NZO_ID" ]] && continue JOB_TIME=$(echo "$FAILED_HIST" | grep -A5 "$NZO_ID" | \ - grep -o '"completed":[0-9]*' | grep -o '[0-9]*') + grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1) [[ -z "$JOB_TIME" ]] && continue [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue if [[ "$DRY_RUN" == true ]]; then diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh old mode 100644 new mode 100755 diff --git a/Initial_run/ssh_setup.sh b/Initial_run/ssh_setup.sh index 54a4fca..f182592 100755 --- a/Initial_run/ssh_setup.sh +++ b/Initial_run/ssh_setup.sh @@ -189,7 +189,7 @@ if [[ "$MODE" == "status" ]]; then # Remote connectivity echo "" - REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) + REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -z "$REMOTE_IP" ]]; then echo " $ICON_ERROR Remote ($REMOTE_SERVER_NAME): Tailscale unreachable" elif [[ -f "$SSH_KEY_PATH" ]] && test_ssh_auth "$REMOTE_IP"; then @@ -218,7 +218,7 @@ if [[ "$MODE" == "validate" ]]; then MAX_STRIKES="${SSH_MAX_STRIKES:-5}" RESET_HRS="${SSH_STRIKE_RESET_HRS:-24}" - REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) + REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -z "$REMOTE_IP" ]]; then log "SSH validate — $REMOTE_SERVER_NAME Tailscale unreachable, not an SSH issue" exit 0 diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 691a783..a2e4516 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -61,6 +61,7 @@ # All aliased by detect_hosts() — script uses unprefixed names # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# LIDARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) # LIDARR_LOCK_WARN_AGE — override default lock warning age (large libraries) # LIDARR_ORPHAN_AGE — days before untracked file is eligible for deletion # LIDARR_MAX_DELETE_GB — require --i-know-what-im-doing above this @@ -293,6 +294,61 @@ is_protected_file() { return 1 } +# ============================================================================================== +# ━━━ Pre-flight: Lidarr Import Scan ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Pre-flight: Lidarr Import Scan ━━━" + +# Reverse-lookup container path from path map so Lidarr gets its own path, not the host path +LIDARR_CONTAINER_ROOT="" +for _cp in "${!ARR_PATH_MAP[@]}"; do + if [[ "${ARR_PATH_MAP[$_cp]}" == "$LIDARR_MUSIC_ROOT" ]]; then + LIDARR_CONTAINER_ROOT="$_cp" + break + fi +done +unset _cp + +if [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then + log "Triggering DownloadedAlbumsScan on: $LIDARR_CONTAINER_ROOT" + SCAN_PAYLOAD="{\"name\": \"DownloadedAlbumsScan\", \"path\": \"$LIDARR_CONTAINER_ROOT\"}" +else + log "No path map match — triggering DownloadedAlbumsScan (all root folders)" + SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}' +fi + +SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ + -H "X-Api-Key: $LIDARR_API_KEY" \ + -H "Content-Type: application/json" \ + -d "$SCAN_PAYLOAD" \ + "${LIDARR_URL}/api/v1/command" 2>/dev/null) + +SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) + +if [[ -z "$SCAN_CMD_ID" ]]; then + warn "Could not trigger import scan — proceeding without pre-flight" +else + echo " Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." + POLL_TIMEOUT=${LIDARR_IMPORT_SCAN_TIMEOUT:-600} + POLLED=0 + while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do + SCAN_STATUS=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $LIDARR_API_KEY" \ + "${LIDARR_URL}/api/v1/command/${SCAN_CMD_ID}" 2>/dev/null | \ + jq -r '.status // empty' 2>/dev/null) + case "$SCAN_STATUS" in + completed) log "Import scan complete ✅"; break ;; + failed) warn "Import scan reported failed — proceeding anyway"; break ;; + esac + sleep 10 + (( POLLED += 10 )) + [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" + done + [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ + warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" +fi + # ============================================================================================== # ━━━ Fetch Lidarr Tracked Files ━━━ # ============================================================================================== diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index 930fe5b..3b9cb6f 100644 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -88,6 +88,13 @@ acquire_lock detect_hosts +# Build path map for translate_path() — container path → host path +declare -A ARR_PATH_MAP +for _key in "${!HOST1_LIDARR_PATH_MAP[@]}"; do + ARR_PATH_MAP["$_key"]="${HOST1_LIDARR_PATH_MAP[$_key]}" +done +unset _key + # HOST guard — Lidarr runs on HOST1 only if [[ -z "$LIDARR_URL" ]]; then log "Lidarr not configured for $MY_ID — nothing to do" @@ -121,6 +128,8 @@ fi # ── Temp dir for subshell fetch/fail counters ───────────────────────────────────────────────── LIDARR_TMP=$(mktemp -d) trap 'rm -rf "$LIDARR_TMP"' EXIT +touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_fails" \ + "$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_fails" # ============================================================================================== # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── @@ -178,7 +187,7 @@ deezer_artist_image() { local query query=$(printf "%s" "$artist" | sed 's/ /+/g') curl_json "https://api.deezer.com/search/artist?q=$query" | - jq -r '.data[0].picture_xl // empty' + jq -r '.data[0].picture_xl // empty' 2>/dev/null } lastfm_artist_image() { @@ -186,7 +195,7 @@ lastfm_artist_image() { local encoded encoded=$(printf "%s" "$artist" | sed 's/ /%20/g') curl_json "https://ws.audioscrobbler.com/2.0/?method=artist.getinfo&artist=$encoded&api_key=$LASTFM_API_KEY&format=json" | - jq -r '.artist.image[-1]["#text"] // empty' + jq -r '.artist.image[-1]["#text"] // empty' 2>/dev/null } # ============================================================================================== @@ -210,6 +219,30 @@ ALBUMS_COMPLETE=0 ARTISTS_CHECKED=0 ARTISTS_COMPLETE=0 +# ============================================================================================== +# ━━━ Build Album Directory Map ━━━ +# ============================================================================================== +# Lidarr's album API never populates .path — derive album dirs from track file paths instead. +echo "" +echo "━━━ $ICON_SYNC Building Album Directory Map ━━━" + +declare -A ALBUM_DIR_MAP +_artist_list=$(curl_json "$LIDARR_URL/api/v1/artist?apikey=$LIDARR_API_KEY") +_map_artist_count=$(echo "$_artist_list" | jq '. | length') +echo " Fetching track files for $_map_artist_count artists..." + +while IFS= read -r _artist_id; do + [[ -z "$_artist_id" ]] && continue + while IFS=$'\t' read -r _album_id _track_path; do + [[ -z "$_album_id" || -z "$_track_path" || "$_track_path" == "null" ]] && continue + ALBUM_DIR_MAP["$_album_id"]=$(dirname "$_track_path") + done < <(curl_json "$LIDARR_URL/api/v1/trackFile?artistId=${_artist_id}&apikey=$LIDARR_API_KEY" | \ + jq -r '.[] | [(.albumId | tostring), .path] | @tsv' 2>/dev/null) +done < <(echo "$_artist_list" | jq -r '.[].id') +unset _artist_list _map_artist_count _artist_id _album_id _track_path + +echo " Mapped ${#ALBUM_DIR_MAP[@]} albums with local tracks" + # ============================================================================================== # ━━━ Albums ━━━ # ============================================================================================== @@ -227,7 +260,10 @@ fi total_albums=$(echo "$albums" | jq '. | length') echo " Processing $total_albums albums..." -while IFS=$'\t' read -r local_path mbid artist_name album_name; do +while IFS=$'\t' read -r mbid artist_name album_name album_id; do + raw_dir="${ALBUM_DIR_MAP[$album_id]:-}" + [[ -z "$raw_dir" ]] && continue # not downloaded, skip + local_path=$(translate_path "$raw_dir") (( ALBUMS_CHECKED++ )) [[ ! -d "$local_path" ]] && continue @@ -254,13 +290,13 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do fi if [[ ! -f "$local_path/cover.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.[].albumcover[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.[].albumcover[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/cover.jpg"; then (( _fetches++ )) else query=$(printf "%s %s" "$artist_name" "$album_name" | sed 's/ /+/g') itunes=$(curl_json "https://itunes.apple.com/search?term=$query&entity=album&limit=1" | - jq -r '.results[0].artworkUrl100 // empty' | sed 's/100x100/600x600/') + jq -r '.results[0].artworkUrl100 // empty' 2>/dev/null | sed 's/100x100/600x600/') if download_if_valid "$itunes" "$local_path/cover.jpg"; then (( _fetches++ )) else @@ -270,12 +306,12 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do fi if [[ ! -f "$local_path/cdart.png" ]]; then - IMG=$(echo "$JSON" | jq -r '.[].cdart[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.[].cdart[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/cdart.png"; then (( _fetches++ )); else (( _fails++ )); fi fi if [[ ! -f "$local_path/back.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.[].albumback[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.[].albumback[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/back.jpg"; then (( _fetches++ )); else (( _fails++ )); fi fi @@ -283,7 +319,7 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do (( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fails" ) & -done < <(echo "$albums" | jq -r '.[] | [.path, .foreignAlbumId, .artist.artistName, .title] | @tsv') +done < <(echo "$albums" | jq -r '.[] | [(.foreignAlbumId // ""), (.artist.artistName // ""), (.title // ""), (.id | tostring)] | @tsv') wait @@ -310,6 +346,7 @@ total_artists=$(echo "$artists" | jq '. | length') echo " Processing $total_artists artists..." while IFS=$'\t' read -r local_path mbid name; do + local_path=$(translate_path "$local_path") (( ARTISTS_CHECKED++ )) [[ ! -d "$local_path" ]] && continue @@ -335,7 +372,7 @@ while IFS=$'\t' read -r local_path mbid name; do sleep "$LIDARR_ART_SLEEP_BETWEEN" if [[ ! -f "$local_path/folder.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/folder.jpg"; then (( _fetches++ )) else @@ -354,7 +391,7 @@ while IFS=$'\t' read -r local_path mbid name; do fi if [[ ! -f "$local_path/fanart.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/fanart.jpg"; then (( _fetches++ )) else @@ -364,12 +401,12 @@ while IFS=$'\t' read -r local_path mbid name; do fi if [[ ! -f "$local_path/logo.png" ]]; then - IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/logo.png"; then (( _fetches++ )); else (( _fails++ )); fi fi if [[ ! -f "$local_path/banner.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty') + IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty' 2>/dev/null) if download_if_valid "$IMG" "$local_path/banner.jpg"; then (( _fetches++ )); else (( _fails++ )); fi fi diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 550a547..2ab1391 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -58,12 +58,13 @@ # All aliased by detect_hosts() — script uses unprefixed names # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion -# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this -# RADARR_EXTENSIONS — video file extensions considered for orphan classification -# RADARR_PROTECTED_PATTERNS — file patterns never deleted -# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check -# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion +# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# RADARR_EXTENSIONS — video file extensions considered for orphan classification +# RADARR_PROTECTED_PATTERNS — file patterns never deleted +# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check +# RADARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# ARR_CLEANUP_STATS — stats file path (read by coffee report) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # radarr_cleanup.sh — normal run @@ -282,6 +283,61 @@ format_bytes() { fi } +# ============================================================================================== +# ━━━ Pre-flight: Radarr Import Scan ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━" + +# Reverse-lookup container path from path map so Radarr gets its own path, not the host path +RADARR_CONTAINER_ROOT="" +for _cp in "${!ARR_PATH_MAP[@]}"; do + if [[ "${ARR_PATH_MAP[$_cp]}" == "$RADARR_MOVIES_ROOT" ]]; then + RADARR_CONTAINER_ROOT="$_cp" + break + fi +done +unset _cp + +if [[ -n "$RADARR_CONTAINER_ROOT" ]]; then + log "Triggering DownloadedMoviesScan on: $RADARR_CONTAINER_ROOT" + SCAN_PAYLOAD="{\"name\": \"DownloadedMoviesScan\", \"path\": \"$RADARR_CONTAINER_ROOT\"}" +else + log "No path map match — triggering DownloadedMoviesScan (all root folders)" + SCAN_PAYLOAD='{"name": "DownloadedMoviesScan"}' +fi + +SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ + -H "X-Api-Key: $RADARR_API_KEY" \ + -H "Content-Type: application/json" \ + -d "$SCAN_PAYLOAD" \ + "${RADARR_URL}/api/v3/command" 2>/dev/null) + +SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) + +if [[ -z "$SCAN_CMD_ID" ]]; then + warn "Could not trigger import scan — proceeding without pre-flight" +else + echo " Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." + POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600} + POLLED=0 + while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do + SCAN_STATUS=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $RADARR_API_KEY" \ + "${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \ + jq -r '.status // empty' 2>/dev/null) + case "$SCAN_STATUS" in + completed) log "Import scan complete ✅"; break ;; + failed) warn "Import scan reported failed — proceeding anyway"; break ;; + esac + sleep 10 + (( POLLED += 10 )) + [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" + done + [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ + warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" +fi + # ============================================================================================== # ━━━ Fetch Radarr Tracked Files ━━━ # ============================================================================================== diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index 75194d8..70f516c 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -58,12 +58,13 @@ # All aliased by detect_hosts() — script uses unprefixed names # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion -# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this -# SONARR_EXTENSIONS — video file extensions considered for orphan classification -# SONARR_PROTECTED_PATTERNS — file patterns never deleted -# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check -# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion +# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# SONARR_EXTENSIONS — video file extensions considered for orphan classification +# SONARR_PROTECTED_PATTERNS — file patterns never deleted +# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check +# SONARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# ARR_CLEANUP_STATS — stats file path (read by coffee report) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # sonarr_cleanup.sh — normal run @@ -282,6 +283,61 @@ format_bytes() { fi } +# ============================================================================================== +# ━━━ Pre-flight: Sonarr Import Scan ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━" + +# Reverse-lookup container path from path map so Sonarr gets its own path, not the host path +SONARR_CONTAINER_ROOT="" +for _cp in "${!ARR_PATH_MAP[@]}"; do + if [[ "${ARR_PATH_MAP[$_cp]}" == "$SONARR_TV_ROOT" ]]; then + SONARR_CONTAINER_ROOT="$_cp" + break + fi +done +unset _cp + +if [[ -n "$SONARR_CONTAINER_ROOT" ]]; then + log "Triggering DownloadedEpisodesScan on: $SONARR_CONTAINER_ROOT" + SCAN_PAYLOAD="{\"name\": \"DownloadedEpisodesScan\", \"path\": \"$SONARR_CONTAINER_ROOT\"}" +else + log "No path map match — triggering DownloadedEpisodesScan (all root folders)" + SCAN_PAYLOAD='{"name": "DownloadedEpisodesScan"}' +fi + +SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ + -H "X-Api-Key: $SONARR_API_KEY" \ + -H "Content-Type: application/json" \ + -d "$SCAN_PAYLOAD" \ + "${SONARR_URL}/api/v3/command" 2>/dev/null) + +SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) + +if [[ -z "$SCAN_CMD_ID" ]]; then + warn "Could not trigger import scan — proceeding without pre-flight" +else + echo " Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." + POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600} + POLLED=0 + while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do + SCAN_STATUS=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $SONARR_API_KEY" \ + "${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \ + jq -r '.status // empty' 2>/dev/null) + case "$SCAN_STATUS" in + completed) log "Import scan complete ✅"; break ;; + failed) warn "Import scan reported failed — proceeding anyway"; break ;; + esac + sleep 10 + (( POLLED += 10 )) + [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" + done + [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ + warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" +fi + # ============================================================================================== # ━━━ Fetch Sonarr Tracked Files ━━━ # ============================================================================================== @@ -343,6 +399,14 @@ while IFS= read -r series_id; do done <<< "$SERIES_IDS" sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" + +# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file +declare -A TRACKED_MAP +while IFS= read -r _tracked_path; do + [[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1 +done < "$TRACKED_FILE" +unset _tracked_path +log "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths" TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") # Safety Layer 5 — tracked count > 0 @@ -378,7 +442,7 @@ MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824 while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then log "TRACKED: $filepath" continue fi @@ -442,7 +506,7 @@ fi if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null && continue + [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue is_protected_file "$filepath" && continue FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) diff --git a/Monitors/continuous_scripts_status.sh b/Monitors/continuous_scripts_status.sh index 6d5af52..fcad8f7 100644 --- a/Monitors/continuous_scripts_status.sh +++ b/Monitors/continuous_scripts_status.sh @@ -21,7 +21,7 @@ # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. -# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly. +# Required containers and tier delays use MY_ID/REMOTE_ID correctly. # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # continuous_scripts_status.sh — show dashboard @@ -121,7 +121,6 @@ echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')" echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME" -echo " $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # ============================================================================================== @@ -303,28 +302,43 @@ if command -v docker >/dev/null 2>&1; then UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ --filter health=unhealthy -q 2>/dev/null | wc -l) - # Stopped containers — filter intentionally ignored ones - STOPPED_FILTERED=() + # Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely + CLEAN_STOPPED=() + UNEXPECTED_STOPPED=() while IFS= read -r name; do [[ -z "$name" ]] && continue SKIP=false for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do [[ "$name" == "$ignore" ]] && SKIP=true && break done - [[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name") + [[ "$SKIP" == true ]] && continue + exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null) + if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then + CLEAN_STOPPED+=("$name") + else + UNEXPECTED_STOPPED+=("$name") + fi done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ --format "{{.Names}}" 2>/dev/null) - STOPPED_COUNT="${#STOPPED_FILTERED[@]}" echo " Running: $RUNNING / $TOTAL total" [[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY" - if [[ "$STOPPED_COUNT" -gt 0 ]]; then + if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then echo " ⚠️ Stopped (unexpected):" - for name in "${STOPPED_FILTERED[@]}"; do + for name in "${UNEXPECTED_STOPPED[@]}"; do echo " → $name" done - else + fi + + if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then + echo " ⏸️ Stopped (clean):" + for name in "${CLEAN_STOPPED[@]}"; do + echo " → $name" + done + fi + + if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then echo " ✅ All containers running" fi @@ -458,22 +472,6 @@ case "$FALLBACK_STATE" in ;; esac -# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts() -echo "" -if command -v tailscale >/dev/null 2>&1; then - REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) - if [[ -n "$REMOTE_IP" ]]; then - if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then - echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅" - else - echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️" - fi - else - echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌" - fi -else - echo " 🌐 Tailscale: not available" -fi echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}" diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index 64a34c9..b882b1c 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -174,7 +174,7 @@ fi # ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────── if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0) + ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c ".") if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST") @@ -187,7 +187,7 @@ fi # ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────── if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0) + SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c ".") if [[ "$SYS_STRIKES" -gt 0 ]]; then SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST") diff --git a/Orchestrators/array_stop.sh b/Orchestrators/array_stop.sh new file mode 100755 index 0000000..f26cea2 --- /dev/null +++ b/Orchestrators/array_stop.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Array Stop Orchestrator ==================================== +# ============================================================================================== +# Planned shutdown orchestrator — stops all active processes cleanly before array maintenance. +# Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, each confirmed complete before next. +# +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# 1. user_scripts_stop.sh — kill background user scripts (prevents new operations) +# 2. rsync_stop.sh --rsync-only — kill rsync; skip container recovery (handled in step 4) +# 3. mover_stop.sh — stop mover after rsync (both write to same paths) +# 4. docker_container_stop.sh — stop all containers one-by-one with verification +# +# ── WHY THIS ORDER ──────────────────────────────────────────────────────────────────────────── +# User scripts stopped first — they can spawn new rsync/docker operations mid-shutdown. +# Rsync before mover — both write to the same paths; running together risks corruption. +# Containers last — apps should stay available as long as possible during shutdown prep. +# +# ── SEQUENTIAL vs BACKGROUND ───────────────────────────────────────────────────────────────── +# Unlike array_start.sh, all scripts run in the foreground. Each must complete (pass or fail) +# before the next starts — a failed stop is noted but does not prevent remaining steps. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — all stop scripts require root +# acquire_lock — prevents concurrent array stop runs +# detect_hosts() — MY_ID in notifications and logs +# validate_unraid_cmd — notify validated before use +# Non-fatal steps — a failed step is logged but remaining steps still run +# notify on failures — alert if any stop script fails +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# array_stop.sh — run full stop sequence +# array_stop.sh --dry-run — preview without stopping anything +# array_stop.sh --status — show configured scripts and exit +# array_stop.sh --log — verbose output +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$ECOSYSTEM_ROOT/load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Scripts: ${#ARRAY_STOP_SCRIPTS[@]} configured" + echo "" + + for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + extra_args=("${parts[@]:1}") + if [[ ! -f "$script_path" ]]; then + echo " $ICON_ERROR $script_name — FILE NOT FOUND" + else + echo " $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}" + fi + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Stop Sequence ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..." +echo "" + +START=$(date +%s) +PASSED=() +FAILED=() +STEP=0 + +for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + (( STEP++ )) + + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + extra_args=("${parts[@]:1}") + + echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + FAILED+=("$script_name") + echo "" + continue + fi + + if [[ ! -x "$script_path" ]]; then + warn "$script_name — not executable, fixing..." + chmod +x "$script_path" || { + error "$script_name — chmod +x failed" + FAILED+=("$script_name") + echo "" + continue + } + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would run: $script_name ${extra_args[*]}" + PASSED+=("$script_name") + echo "" + continue + fi + + if bash "$script_path" "${extra_args[@]}"; then + log "$script_name — done ✅" + PASSED+=("$script_name") + else + warn "$script_name — failed (exit $?) — continuing to next step" + FAILED+=("$script_name") + fi + + echo "" +done + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + log "$ICON_DONE Status: all $STEP step(s) complete ✅" + notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ + "Array Stop" "normal" +else + warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" + notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ + "Array Stop" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index 8818266..646c848 100644 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -197,10 +197,10 @@ if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run" if [[ "$RSYNC_OK" == true ]]; then - bash "$SCRIPT_DIR/partnership_manage.sh" \ + bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \ --check --remote-seen $PARTNER_DRY else - bash "$SCRIPT_DIR/partnership_manage.sh" \ + bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \ --check --remote-unseen $PARTNER_DRY fi else diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 63b5d54..67230e8 100644 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -23,8 +23,8 @@ # share sync stays in the daily window. # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID, DAILY_SYNC_SHARES, PERSONAL_SHARES from HOST*_ vars. -# INTERMEDIATE_SYNC_SHARES is a shared list in master.conf — same on all servers. +# detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES. +# Each server can have a different set of mid-day shares — configure in master_host*.conf. # Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic. # # ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── @@ -40,11 +40,11 @@ # Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch # Silent on success — runs 4x/day, only failures warrant notification # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) -# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) -# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync -# ARR_SYNC_ENABLED — toggle inside arr_sync.sh +# ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── +# master_host*.conf: HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) +# master.conf: INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) +# master.conf: INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync +# master.conf: ARR_SYNC_ENABLED — toggle inside arr_sync.sh # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # intermediate_sync_maintenance.sh — normal run diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index ecd6f68..7fb6c05 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -627,11 +627,317 @@ gather_partner_fallback_containers() { done } +# Read a scalar var from the mirror's own config via SSH. +# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works. +read_remote_conf_var() { + local mirror_ip="$1" var_name="$2" + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ + "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s' \"\${${var_name}:-}\"" 2>/dev/null +} + +# Read an array var from the mirror's own config via SSH — one element per line. +read_remote_conf_array() { + local mirror_ip="$1" var_name="$2" + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ + "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null +} + +# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe" +derive_short_name() { + local hostname="$1" + local short="${hostname,,}" + [[ "$short" == unraid-* ]] && short="${short:7}" + echo "${short^}" +} + +# Start this server's own parked containers after partnership ends. +start_own_stack() { + echo "" + echo "━━━ $ICON_START Restart Own Stack ━━━" + if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then + log "No PARTNERSHIP_OWN_CONTAINERS configured — skipping" + return 0 + fi + for container in "${PARTNERSHIP_OWN_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would start: $container" + continue + fi + if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then + log "$container started ✅" + else + warn "$container failed to start — check manually" + fi + done +} + +# Remove partnership containers on this server. +# Uses FolderView3 folder if enabled (precise list), else falls back to FALLBACK_*_COVERS_* config. +cleanup_partner_containers() { + local folder_name="$1" + if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then + folderview3_remove_partner_folder "$folder_name" + else + declare -a containers=() + gather_partner_fallback_containers containers + if [[ ${#containers[@]} -eq 0 ]]; then + log "No partner containers found to remove" + return 0 + fi + for container in "${containers[@]}"; do + [[ -z "$container" ]] && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop + rm: $container" + continue + fi + if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then + timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true + timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \ + log "$container removed ✅" || warn "$container rm failed" + else + log "$container not found — skipping" + fi + done + fi +} + +# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers). +cleanup_owner_containers_on_mirror() { + local mirror_ip="$1" + local owner_short + owner_short=$(derive_short_name "$OWNER") + + log "Removing owner-deployed containers from $MIRROR..." + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove *-${owner_short} containers from $MIRROR" + return 0 + fi + + local container_list + container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + "docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null) + + if [[ -z "$container_list" ]]; then + log "No *-${owner_short} containers found on $MIRROR" + return 0 + fi + + while IFS= read -r container; do + [[ -z "$container" ]] && continue + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + "docker stop '$container' >/dev/null 2>&1 + docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ + grep -q removed && \ + log "$container removed from $MIRROR ✅" || \ + warn "Failed to remove $container from $MIRROR" + done <<< "$container_list" +} + +# SSH to mirror — start mirror's own parked containers. +# Reads PARTNERSHIP_OWN_CONTAINERS from the mirror's own conf via SSH. +start_mirror_own_stack() { + local mirror_ip="$1" + + log "Reading own stack list from $MIRROR conf..." + local -a mirror_own=() + mapfile -t mirror_own < <(read_remote_conf_array "$mirror_ip" "PARTNERSHIP_OWN_CONTAINERS") + # Remove empty entries + local -a filtered=() + for c in "${mirror_own[@]}"; do [[ -n "$c" ]] && filtered+=("$c"); done + mirror_own=("${filtered[@]}") + + if [[ ${#mirror_own[@]} -eq 0 ]]; then + log "No PARTNERSHIP_OWN_CONTAINERS configured on $MIRROR — skipping remote stack restart" + return 0 + fi + + log "Restarting own stack on $MIRROR: ${mirror_own[*]}" + for container in "${mirror_own[@]}"; do + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would start $container on $MIRROR" + continue + fi + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + "docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \ + grep -q started && \ + log "$container started on $MIRROR ✅" || \ + warn "$container failed to start on $MIRROR — check manually" + done +} + +# Create the mirror's Emby admin account on the owner's deployed Emby. +# Credentials come from the MIRROR's conf (HOST*_PARTNERSHIP_EMBY_ADMIN_USER/PASS). +# Checks for username collision before creating — exits with guidance if taken. +provision_emby_admin() { + local mirror_ip="$1" + local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" + local emby_url="http://${mirror_ip}:${emby_port}" + + echo "" + echo "━━━ $ICON_EMBY Emby Admin Provisioning ━━━" + + if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then + log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" + return 0 + fi + + if [[ -z "$EMBY_API_KEY" ]]; then + warn "EMBY_API_KEY not set — skipping Emby admin provisioning" + return 1 + fi + + # Read mirror's desired credentials from their own conf via SSH + log "Reading Emby credentials from $MIRROR conf..." + local username password + username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER") + password=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_PASS") + [[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")" + + if [[ -z "$password" ]]; then + warn "PARTNERSHIP_EMBY_ADMIN_PASS is empty in $MIRROR conf" + warn "$MIRROR must set HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_PASS before onboard" + return 1 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would create Emby admin '$username' at $emby_url" + return 0 + fi + + # Collision check — username already exists? + local existing_users + existing_users=$(curl -sf --max-time 15 \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + "${emby_url}/Users" 2>/dev/null) + + if echo "$existing_users" | grep -q "\"Name\":\"${username}\""; then + error "Emby username '${username}' is already taken on the shared instance" + error "Options:" + error " 1. Sign in with that account — it may already be yours" + error " 2. Set a different name in ${user_var} and re-run --onboard" + return 1 + fi + + log "Creating Emby admin '$username' at $emby_url..." + + local create_response http_code body + create_response=$(curl -sf --max-time 15 -w "\n%{http_code}" \ + -X POST \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + -H "Content-Type: application/json" \ + -d "{\"Name\": \"$username\"}" \ + "${emby_url}/Users/New" 2>/dev/null) + http_code=$(echo "$create_response" | tail -1) + body=$(echo "$create_response" | head -n -1) + + if [[ "$http_code" != "200" ]] && [[ "$http_code" != "204" ]]; then + warn "Failed to create Emby user '$username' (HTTP $http_code)" + return 1 + fi + + local user_id + user_id=$(echo "$body" | grep -o '"Id":"[^"]*"' | head -1 | cut -d'"' -f4) + if [[ -z "$user_id" ]]; then + warn "Emby user created but could not parse user ID — set password manually" + return 1 + fi + + local pw_code + pw_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ + -X POST \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + -H "Content-Type: application/json" \ + -d "{\"NewPw\": \"$password\"}" \ + "${emby_url}/Users/${user_id}/Password" 2>/dev/null) + + if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then + log "Emby admin '$username' created (id: $user_id) ✅" + else + warn "User created but password set failed (HTTP $pw_code) — set password manually" + fi + + curl -sf --max-time 15 -o /dev/null \ + -X POST \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + -H "Content-Type: application/json" \ + -d '{"IsAdministrator": true, "IsDisabled": false}' \ + "${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \ + log "$username granted admin policy ✅" || \ + warn "Could not set admin policy — grant manually in Emby dashboard" +} + +# Delete the mirror's Emby admin account from the owner's deployed Emby. +revoke_emby_admin() { + local mirror_ip="$1" + local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" + local emby_url="http://${mirror_ip}:${emby_port}" + + echo "" + echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━" + + local username + username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER") + [[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")" + + if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then + log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" + return 0 + fi + + if [[ -z "$EMBY_API_KEY" ]]; then + warn "EMBY_API_KEY not set — skipping Emby admin revocation" + return 1 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete Emby admin '$username' at $emby_url" + return 0 + fi + + log "Looking up Emby user '$username' at $emby_url..." + + local users_json user_id + users_json=$(curl -sf --max-time 15 \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + "${emby_url}/Users" 2>/dev/null) + + user_id=$(echo "$users_json" | \ + grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \ + grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1) + + if [[ -z "$user_id" ]]; then + warn "Emby user '$username' not found at $emby_url — may already be removed" + return 0 + fi + + local del_code + del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ + -X DELETE \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + "${emby_url}/Users/${user_id}" 2>/dev/null) + + if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then + log "Emby admin '$username' removed ✅" + else + warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually" + fi +} + check_both_healthy() { mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; } local mirror_ip - mirror_ip=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) + mirror_ip=$(resolve_tailscale_ip "$MIRROR") [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ @@ -717,8 +1023,8 @@ fi if [[ "$MODE" == "status" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" - OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null || echo "unreachable") - MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null || echo "unreachable") + OWNER_IP=$(resolve_tailscale_ip "$OWNER" || echo "unreachable") + MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" || echo "unreachable") echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)" echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)" @@ -743,7 +1049,7 @@ if [[ "$MODE" == "status" ]]; then echo "" # Remote state - REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) + REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -n "$REMOTE_IP" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -n "$REMOTE_CONTENT" ]]; then @@ -851,7 +1157,7 @@ if [[ "$MODE" == "check" ]]; then fi # Read remote state file - REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) + REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -z "$REMOTE_IP" ]]; then log "Partnership check — remote unreachable, skipping state check" exit 0 @@ -884,6 +1190,10 @@ if [[ "$MODE" == "check" ]]; then warn "Owner finalising offboard request from mirror..." do_final_sync + # Remove owner's fallback containers, restart own stack + cleanup_partner_containers "$(derive_partner_folder_name "$MIRROR")" + start_own_stack + NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" @@ -915,6 +1225,8 @@ if [[ "$MODE" == "check" ]]; then # Mirror sees owner is INACTIVE — clean up own side warn "Owner has offboarded — cleaning up mirror side..." reconfigure_local_webuis "localhost" + cleanup_partner_containers "$(derive_partner_folder_name "$OWNER")" + start_own_stack NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" @@ -973,8 +1285,8 @@ if [[ "$MODE" == "onboard" ]]; then check_remote_array || exit 1 check_remote_docker_daemon || exit 1 - OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null) - MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) + OWNER_IP=$(resolve_tailscale_ip "$OWNER") + MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") [[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; } [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; } @@ -1016,13 +1328,15 @@ if [[ "$MODE" == "onboard" ]]; then echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') - write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" - log "Local state: ACTIVE ✅" - - [[ "$DRY_RUN" == false ]] && remove_from_blocklist "$MIRROR" - - push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" - echo "0" > "$OFFLINE_COUNTER" + if [[ "$DRY_RUN" == false ]]; then + write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" + log "Local state: ACTIVE ✅" + remove_from_blocklist "$MIRROR" + push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" + echo "0" > "$OFFLINE_COUNTER" + else + warn "DRY RUN — would write ACTIVE state and push to remote" + fi # FolderView3 — create partner folder with this server's failover containers for remote if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then @@ -1038,6 +1352,9 @@ if [[ "$MODE" == "onboard" ]]; then fi fi + # Emby admin provisioning — runs after container deployment (deploy step not yet built) + provision_emby_admin "$MIRROR_IP" + # Summary echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" @@ -1094,13 +1411,12 @@ if [[ "$MODE" == "offboard" ]]; then echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" - # FolderView3 — remove partner folder and clean containers - if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then - echo "" - echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━" - PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER") - folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME" - fi + # Remove owner's deployed containers + restart own stack + echo "" + echo "━━━ $ICON_CONTAINERS Cleanup Partner Containers ━━━" + PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER") + cleanup_partner_containers "$PARTNER_FOLDER_NAME" + start_own_stack NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ @@ -1109,7 +1425,7 @@ if [[ "$MODE" == "offboard" ]]; then [[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON" - OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null) + OWNER_IP=$(resolve_tailscale_ip "$OWNER") if [[ -n "$OWNER_IP" ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ @@ -1151,7 +1467,7 @@ if [[ "$MODE" == "offboard" ]]; then echo "━━━ $ICON_SYNC Final Sync ━━━" do_final_sync - MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) + MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") MIRROR_REACHABLE=false [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true @@ -1199,14 +1515,27 @@ if [[ "$MODE" == "offboard" ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" fi - # FolderView3 — remove partner folder and clean containers on this (owner) side - if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then - echo "" - echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━" - PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") - folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME" + # Local: remove fallback-coverage containers for mirror, restart own stack + echo "" + echo "━━━ $ICON_CONTAINERS Local Container Cleanup ━━━" + PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") + cleanup_partner_containers "$PARTNER_FOLDER_NAME" + start_own_stack + + # Remote: remove owner's deployed containers from mirror, restart mirror's own stack + echo "" + echo "━━━ $ICON_CONTAINERS Remote Container Cleanup ━━━" + if [[ "$MIRROR_REACHABLE" == true ]]; then + cleanup_owner_containers_on_mirror "$MIRROR_IP" + start_mirror_own_stack "$MIRROR_IP" + else + warn "$MIRROR unreachable — remote container cleanup skipped" + warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually" fi + # Emby admin revocation — before SSH key revocation while Emby still reachable + [[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" + # SSH key revocation — mutual, both directions # Must run before Tailscale removal (SSH needs network) and after state is pushed SSH_REVOKE_REMOTE_OK=false @@ -1354,8 +1683,8 @@ if [[ "$MODE" == "transfer" ]]; then NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}" NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}" - NEW_OWNER_IP=$(tailscale ip -4 "${NEW_OWNER,,}" 2>/dev/null) - NEW_MIRROR_IP=$(tailscale ip -4 "${NEW_MIRROR,,}" 2>/dev/null) + NEW_OWNER_IP=$(resolve_tailscale_ip "$NEW_OWNER") + NEW_MIRROR_IP=$(resolve_tailscale_ip "$NEW_MIRROR") [[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; } [[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; } diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index dcd6297..6fdd217 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -191,10 +191,19 @@ fi if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" - # Local first — flush local databases before pushing - stop_local_containers - # Remote next — prevent writes while receiving - stop_containers + if [[ "$DRY_RUN" == true ]]; then + for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do + [[ -n "$c" ]] && warn "DRY RUN — would stop local: $c" + done + for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do + [[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c" + done + else + # Local first — flush local databases before pushing + stop_local_containers + # Remote next — prevent writes while receiving + stop_containers + fi fi # ============================================================================================== @@ -260,10 +269,19 @@ done if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" - # Remote first — can be coming up while local restarts - start_containers - # Local next - start_local_containers + if [[ "$DRY_RUN" == true ]]; then + for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do + [[ -n "$c" ]] && warn "DRY RUN — would start remote: $c" + done + for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do + [[ -n "$c" ]] && warn "DRY RUN — would start local: $c" + done + else + # Remote first — can be coming up while local restarts + start_containers + # Local next + start_local_containers + fi fi # ============================================================================================== diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index b9ad60b..7d7494d 100644 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -218,7 +218,7 @@ for db_rel in "${DB_FILES[@]}"; do log "Checking $db_name ($DB_SIZE)..." # WAL file — different check (not a full SQLite database) - if [[ "$db_name" == "*.wal" || "$db_name" == "library.db-wal" ]]; then + if [[ "$db_name" == *.wal ]]; then if [[ -s "$db_path" ]]; then warn "$db_name exists and is non-empty (${DB_SIZE})" warn "Uncommitted WAL data — will be merged when Emby next starts cleanly" diff --git a/Transcodes/ramdisk_setup.sh b/Transcodes/ramdisk_setup.sh old mode 100644 new mode 100755 diff --git a/common.sh b/common.sh index e024f4d..05dbdc6 100644 --- a/common.sh +++ b/common.sh @@ -374,6 +374,7 @@ validate_int() { # # Also sets aliases for all host-specific arrays so scripts use unprefixed names: # DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES +# INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES # WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES # CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES # DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS @@ -473,6 +474,16 @@ detect_hosts() { EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY" EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}" + local _prov_var + for _prov_var in \ + PARTNERSHIP_PROVISION_EMBY_ADMIN \ + PARTNERSHIP_EMBY_ADMIN_USER \ + PARTNERSHIP_EMBY_ADMIN_PASS \ + PARTNERSHIP_EMBY_PORT; do + local _src="${MY_ID}_${_prov_var}" + printf -v "$_prov_var" '%s' "${!_src:-}" + done + # ── System watchdog check toggles — per-host ───────────────────────────── # Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh local _wd_checks=( @@ -540,6 +551,7 @@ detect_hosts() { _alias_array "DAILY_SYNC_SHARES" _alias_array "PERSONAL_SHARES" + _alias_array "INTERMEDIATE_SYNC_SHARES" _alias_array "WEEKLY_SYNC_SHARES" _alias_array "CRITICAL_SYNC_SHARES" _alias_array "BACKUP_VERIFY_SHARES" @@ -559,6 +571,7 @@ detect_hosts() { _alias_array "DDNS_CONTAINERS" _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" + _alias_array "PARTNERSHIP_OWN_CONTAINERS" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key @@ -610,6 +623,17 @@ resolve_remote_ip() { info "$ICON_NET Remote IP: $REMOTE_SERVER" } +# Resolve any hostname to a Tailscale IPv4 — tries direct lookup, falls back to status parse. +# Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure. +resolve_tailscale_ip() { + local hostname="${1,,}" + local ip + ip=$(tailscale ip -4 "$hostname" 2>/dev/null) + [[ -z "$ip" ]] && \ + ip=$(tailscale status 2>/dev/null | awk -v name="$hostname" '$2 ~ "^" name { print $1; exit }') + echo "$ip" +} + # ============================================================================================== # ── CONNECTIVITY CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -1455,7 +1479,7 @@ check_arr_version() { fi version=$(echo "$status_response" | \ - grep -o '"version":"[^"]*"' | \ + grep -o '"version": *"[^"]*"' | \ grep -o '[0-9][^"]*' | head -1) if [[ -z "$version" ]]; then diff --git a/git_pull_execute.sh b/git_pull_execute.sh old mode 100644 new mode 100755 diff --git a/master.conf b/master.conf index 832b95a..efb2d59 100644 --- a/master.conf +++ b/master.conf @@ -263,6 +263,17 @@ # All orchestrator job lists live here — edit arrays to add/remove scripts. # No changes to orchestrator scripts needed when adding or removing jobs. +# ━━━ Array Stop ━━━ +# Scripts run by array_stop.sh for a planned shutdown — stops everything cleanly in order. +# Run sequentially (foreground) — each must complete before the next starts. +# Order matters: user scripts first (prevents new ops), then data movement, then containers. + ARRAY_STOP_SCRIPTS=( + "unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops + "unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) + "unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) + "Docker_Essentials/docker_container_stop.sh" # stop all containers last + ) + # ━━━ Array Start ━━━ # Scripts launched by array_start.sh when the array comes online. # Launched in order — each as a background process. @@ -277,18 +288,14 @@ "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop -# "Fallback/fallback.sh" # mutual failover — enable when HOST2 ready + "Fallback/fallback.sh" # mutual failover — HOST2 back online ) # ━━━ Intermediate Sync Maintenance ━━━ # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, # and optional mid-day rsync for any shares that need sub-daily propagation. # Schedule: 0 */4 * * * - INTERMEDIATE_SYNC_SHARES=( - # Add shares here to enable mid-day rsync — empty = rsync section skipped entirely. - # Uses DEFAULT_RSYNC_OPTS (no --delete). Full media sync stays in the daily window. - # Example: "/mnt/user/Emby_Metadata" - ) + # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in master_host*.conf. INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares INTERMEDIATE_MAINTENANCE_SCRIPTS=( @@ -571,7 +578,7 @@ FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) FALLBACK_STATE_FILE="/boot/config/fallback_state.db" - FALLBACK_ENABLED=false # HOST2 being rebuilt — set true when back online and tested + FALLBACK_ENABLED=true # HOST2 back online # false = suppresses "not running" warnings in status scripts # ━━━ Failover Test ━━━ @@ -805,7 +812,6 @@ # ── Scene / download metadata ───────────────────────────────────────────────────────── '*.url' '*.lnk' # scene links - '*.nfo' # scene info files (arrs regenerate their own) '*.info' '*.diz' # scene description files '*.nzb' # usenet download files '*.torrent' # torrent files left by download clients @@ -842,7 +848,6 @@ # ── Scene / download metadata ───────────────────────────────────────────────────────── '*.url' '*.lnk' # scene links - '*.nfo' # scene info files (arrs regenerate their own) '*.info' '*.diz' # scene description files '*.nzb' # usenet download files '*.torrent' # torrent files left by download clients diff --git a/master_host1.conf b/master_host1.conf index a0e443b..5b156a1 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -105,6 +105,22 @@ # "/mnt/user/appdata-Fallback/Jayred365-Emby" ) +# Containers parked on this server when partnership is active. +# Stopped on onboard (owner deploys its stack instead), restarted on offboard. + HOST1_PARTNERSHIP_OWN_CONTAINERS=( + # "Emby" + # "NginxProxyManager" + ) + +# Emby admin provisioning — toggle is owner-only, credentials are per-host. +# Owner enables/disables the feature. Each host sets the account they want on the shared Emby. +# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account. +# On offboard: account is deleted. Username collision → onboard exits with error. + HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared + HOST1_PARTNERSHIP_EMBY_PORT=8096 + HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username + HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password + # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -153,6 +169,15 @@ "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack ) +# ━━━ Intermediate Sync Shares ━━━ +# Shares synced every 4 hours by intermediate_sync_maintenance.sh. +# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state. +# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely. + HOST1_INTERMEDIATE_SYNC_SHARES=( + # Add shares here to enable mid-day rsync + # Example: "/mnt/user/Emby_Metadata" + ) + # ━━━ Critical Sync Shares ━━━ # Appdata shares synced every 15 minutes by critical_sync_maintenance.sh. # Format: "/path/to/share" or "/path/to/share|profile-name" diff --git a/master_host2.conf b/master_host2.conf index 9bd8546..6203435 100644 --- a/master_host2.conf +++ b/master_host2.conf @@ -105,6 +105,18 @@ # fill in when HOST2 is back online ) +# Containers parked on this server when partnership is active. +# Stopped on onboard (owner deploys its stack instead), restarted on offboard. + HOST2_PARTNERSHIP_OWN_CONTAINERS=( + # "Emby" + # "NginxProxyManager" + ) + +# This server's desired Emby admin account on the shared Emby instance. +# Set these — owner reads them during --onboard to create the account. + HOST2_PARTNERSHIP_EMBY_ADMIN_USER="" # desired Emby username + HOST2_PARTNERSHIP_EMBY_ADMIN_PASS="" # desired Emby password + # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -152,6 +164,15 @@ # "/mnt/user/appdata-Fallback/Critical-Data" ) +# ━━━ Intermediate Sync Shares ━━━ +# Shares synced every 4 hours by intermediate_sync_maintenance.sh. +# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state. +# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely. + HOST2_INTERMEDIATE_SYNC_SHARES=( + # fill in when HOST2 is back online + # Example: "/mnt/user/Emby_Metadata" + ) + # ━━━ Critical Sync Shares ━━━ # Appdata shares synced every 15 minutes by critical_sync_maintenance.sh. # Format: "/path/to/share" or "/path/to/share|profile-name" diff --git a/unRAID_Essentials/docker_syslog_filter.sh b/unRAID_Essentials/docker_syslog_filter.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/inotify_tuning.sh b/unRAID_Essentials/inotify_tuning.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/mover_stop.sh b/unRAID_Essentials/mover_stop.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/php_fpm_max_children.sh b/unRAID_Essentials/php_fpm_max_children.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/rsync_stop.sh b/unRAID_Essentials/rsync_stop.sh old mode 100644 new mode 100755 index 14bad10..3032516 --- a/unRAID_Essentials/rsync_stop.sh +++ b/unRAID_Essentials/rsync_stop.sh @@ -87,7 +87,7 @@ detect_hosts # Soft IP resolution — rsync_stop continues local-only if remote unreachable REMOTE_REACHABLE=false -REMOTE_SERVER=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) +REMOTE_SERVER=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -z "$REMOTE_SERVER" ]]; then warn "$REMOTE_SERVER_NAME — cannot resolve Tailscale IP, remote operations will be skipped" elif timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/user_scripts_stop.sh b/unRAID_Essentials/user_scripts_stop.sh old mode 100644 new mode 100755