feat: arr-native distributed media sync + config architecture fixes
Arr sync (new): - Media/arr_sync.sh — full mesh bidirectional sync across all HOST* nodes - Lidarr (MusicBrainz), Sonarr (TVDB), Radarr (TMDB) all handled in one script - Remote API keys read live from config.xml via SSH — never stored in conf files - Shared blocklist (DATA_DIR/arr_sync_blocklist.tsv) merged from all nodes at runtime - Graceful skip if arr not configured locally or not reachable on a remote node - --blocklist-add / --blocklist-remove / --blocklist-list management flags - daily_sync_maintenance.sh — arr sync runs as explicit phase before rsync - partnership_onboard.sh — Step 3 bootstraps merged library on both sides at onboard - master.conf — ARR_SYNC_* config block, DOCKER_APPDATA_BASE Rsync / cleanup: - DEFAULT_RSYNC_OPTS — removed --delete; arr_cleanup.sh owns orphan enforcement - lidarr_cleanup.sh — removed HOST1-only guard; runs on any node with Lidarr configured Config architecture: - HOST1/HOST2 hostnames moved from master_host*.conf → master.conf (not credentials) - Sparse checkout now works correctly: each server only needs its own host conf - detect_hosts() still resolves MY_ID + REMOTE_ID via master.conf hostname values Bug fix: - common.sh line 493 — watchdog toggle eval had broken quoting; all SYS_WATCHDOG_CHECK_* globals were silently set to empty instead of their configured values Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
009820e981
commit
b65f572367
@@ -66,8 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
@@ -75,7 +73,6 @@ if ! command -v docker &>/dev/null; then
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
success "Docker found"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
@@ -86,6 +83,8 @@ if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} containers configured"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -114,10 +113,10 @@ retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
log "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
@@ -225,7 +224,7 @@ check_dependency_delay() {
|
||||
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||||
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||
echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
fi
|
||||
}
|
||||
@@ -240,7 +239,7 @@ retry_docker() {
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if docker_cmd "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
log "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
|
||||
@@ -66,8 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
@@ -75,7 +73,6 @@ if ! command -v docker &>/dev/null; then
|
||||
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
success "Docker found"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
@@ -86,6 +83,8 @@ if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#WEEKLY_RESTART_CONTAINERS[@]} containers configured"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -127,10 +126,10 @@ retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if docker_cmd "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
log "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
|
||||
+110
-123
@@ -21,7 +21,7 @@
|
||||
# NORMAL — remote up, internet up
|
||||
# Own containers only. DDNS ON. Silent operation.
|
||||
#
|
||||
# FAILOVER — remote down, internet up
|
||||
# FALLBACK — remote down, internet up
|
||||
# Start remote containers locally — tiered by outage duration.
|
||||
# Remote DDNS started immediately (Tier 1).
|
||||
# Own containers keep running — failover is additive.
|
||||
@@ -40,20 +40,21 @@
|
||||
# Script controls DDNS exclusively — network state NEVER auto-starts DDNS.
|
||||
# DDNS only starts after full handback sequence confirms containers are up.
|
||||
# One DDNS per domain active at all times — never two, never zero for long.
|
||||
# 1 minute TTL + 1 minute check interval = minimal user impact on failover.
|
||||
# 1 minute TTL + 90s strike window = minimal user impact on failover.
|
||||
#
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# When remote returns after FAILOVER:
|
||||
# When remote returns after FALLBACK:
|
||||
# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
|
||||
# 2. Pre-flight checks — version parity, remote array, remote Docker daemon
|
||||
# 3. Stop remote DDNS first — prevents split brain DNS during rsync
|
||||
# 4. Stop remote containers — clean state before rsync
|
||||
# 5. Tiered rsync writeback — skip if outage under threshold
|
||||
# 6. Start remote containers — in dependency order with verification
|
||||
# 7. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up
|
||||
# 8. Return to NORMAL
|
||||
# 3. Staged reverse-tier handback — Tier 4 → Tier 3 → Tier 2 → Tier 1
|
||||
# Each stage: stop local tier containers → rsync writeback → start on remote
|
||||
# Emby (Tier 1) stays on fallback serving users until all higher tiers are done
|
||||
# 4. DDNS handoff — immediately before Tier 1 goes down (not at start)
|
||||
# 5. Tier 1 handback — stop local → rsync → start remote
|
||||
# 6. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up
|
||||
# 7. Return to NORMAL
|
||||
#
|
||||
# ── TIERED FAILOVER ───────────────────────────────────────────────────────────────────────────
|
||||
# ── TIERED FALLBACK ───────────────────────────────────────────────────────────────────────────
|
||||
# Tier 1 — Immediate — vital services + Live TV — cannot wait
|
||||
# Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr)
|
||||
# Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr)
|
||||
@@ -72,16 +73,16 @@
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_DDNS_CONTAINERS — DDNS containers this host manages
|
||||
# FAILOVER_HOST*_STOP_ON_NO_NET — containers stopped on internet loss
|
||||
# FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
|
||||
# FALLBACK_HOST*_STOP_ON_NO_NET — containers stopped on internet loss
|
||||
# FALLBACK_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
|
||||
# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes
|
||||
# HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this
|
||||
# FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier
|
||||
# FALLBACK_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.)
|
||||
# FALLBACK_CHECK_INTERVAL — seconds between checks
|
||||
# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback
|
||||
# FALLBACK_CHECK_INTERVAL — seconds between checks (default 30)
|
||||
# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback (default 3 = 90s)
|
||||
# FALLBACK_STATE_FILE — /boot/config path — survives reboots
|
||||
# FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs
|
||||
# EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8)
|
||||
@@ -332,7 +333,7 @@ remote_ddns_stop() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TIERED FAILOVER HELPERS ───────────────────────────────────────────────────────────────────
|
||||
# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# All routing uses MY_ID — not hostname string comparison.
|
||||
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
|
||||
@@ -436,114 +437,133 @@ fi
|
||||
# ==============================================================================================
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Called when remote returns after FAILOVER state.
|
||||
# Called when remote returns after FALLBACK state.
|
||||
# CRITICAL sequencing — do not reorder without understanding the consequences.
|
||||
# Each step must succeed before proceeding — aborts and retries next cycle on failure.
|
||||
|
||||
run_handback() {
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME has returned — beginning handback"
|
||||
warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"
|
||||
|
||||
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Version parity — refuse handback if unRAID versions mismatch
|
||||
if ! check_unraid_version_parity; then
|
||||
warn "Version parity check failed — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote array must be mounted before rsync
|
||||
if ! check_remote_array; then
|
||||
warn "Remote array not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote Docker daemon must be responsive before issuing container commands
|
||||
if ! check_remote_docker_daemon; then
|
||||
warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Pre-flight checks passed ✅"
|
||||
echo " Pre-flight checks passed ✅"
|
||||
|
||||
# ── Step 2: Stop remote DDNS FIRST ──────────────────────────────────────────────────────
|
||||
# CRITICAL — prevents split-brain DNS while data is being synced
|
||||
local outage_minutes
|
||||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||||
local tier1_wb_delay t2_delay t3_delay t4_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
t2_delay=$(get_tier2_delay)
|
||||
t3_delay=$(get_tier3_delay)
|
||||
t4_delay=$(get_tier4_delay)
|
||||
|
||||
warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"
|
||||
|
||||
# ── Staged handback helper ────────────────────────────────────────────────────────────────
|
||||
# Each tier: stop local containers → rsync writeback → start on remote.
|
||||
# Skips tiers that were never activated. Emby (Tier 1) stays on fallback
|
||||
# serving users until all higher tiers complete.
|
||||
handback_tier() {
|
||||
local tier="$1" threshold="$2" label="$3"
|
||||
|
||||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"
|
||||
|
||||
local containers
|
||||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||||
|
||||
echo " Stopping local Tier $tier containers..."
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -n "$container" ]] && local_stop "$container"
|
||||
done
|
||||
|
||||
if ! check_rsync_enabled "FALLBACK"; then
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
|
||||
else
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
|
||||
echo " Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| warn "DRY RUN — would rsync: $job"
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " Starting Tier $tier on $REMOTE_SERVER_NAME..."
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
state_set "tier${tier}_started" "false"
|
||||
warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
|
||||
}
|
||||
|
||||
# ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
|
||||
handback_tier 4 "$t4_delay" "media shares + edge cases"
|
||||
handback_tier 3 "$t3_delay" "secondary services"
|
||||
handback_tier 2 "$t2_delay" "productivity services"
|
||||
|
||||
# ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
|
||||
# Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
|
||||
# Prevents split-brain DNS during Tier 1 rsync window.
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
||||
remote_ddns_stop
|
||||
sleep 5 # brief pause to ensure DDNS stop propagates before rsync
|
||||
sleep 5
|
||||
|
||||
# ── Step 3: Stop remote containers ──────────────────────────────────────────────────────
|
||||
# Clean state before rsync — no dirty writes during transfer window
|
||||
# ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Remote Containers ━━━"
|
||||
echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"
|
||||
|
||||
local ALL_FALLBACK_CONTAINERS=()
|
||||
|
||||
# Collect all started tiers in reverse order (highest tier stops first)
|
||||
if [[ "$(state_get tier4_started)" == "true" ]]; then
|
||||
read -r -a t4 <<< "$(get_tier_containers 4)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t4[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier3_started)" == "true" ]]; then
|
||||
read -r -a t3 <<< "$(get_tier_containers 3)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t3[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier2_started)" == "true" ]]; then
|
||||
read -r -a t2 <<< "$(get_tier_containers 2)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t2[@]}")
|
||||
fi
|
||||
|
||||
# Tier 1 always started — stop last (DDNS already handled above)
|
||||
read -r -a t1 <<< "$(get_tier_containers 1)"
|
||||
|
||||
echo " Stopping local Tier 1 containers..."
|
||||
for container in "${t1[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_ddns=false
|
||||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||||
done
|
||||
[[ "$is_ddns" == false ]] && ALL_FALLBACK_CONTAINERS+=("$container")
|
||||
[[ "$is_ddns" == false ]] && local_stop "$container"
|
||||
done
|
||||
|
||||
for container in "${ALL_FALLBACK_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local_stop "$container"
|
||||
done
|
||||
|
||||
log "All fallback containers stopped locally"
|
||||
|
||||
# ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
|
||||
# Tiered writeback with skip window — short outages do not benefit from writeback.
|
||||
# After a short outage primary's clean nightly state is more reliable than dirty
|
||||
# accumulation — skip writeback entirely for outages under the threshold.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Rsync Writeback ━━━"
|
||||
|
||||
local outage_minutes
|
||||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||||
local tier1_wb_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
|
||||
warn "Outage duration: ${outage_minutes}min"
|
||||
|
||||
if ! check_rsync_enabled "FALLBACK"; then
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping all writeback jobs"
|
||||
warn "Handback will complete without syncing state back to primary"
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
|
||||
else
|
||||
run_writeback_tier() {
|
||||
local tier="$1" threshold="$2" label="$3"
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
[[ ${#jobs[@]} -eq 0 ]] && return
|
||||
|
||||
if [[ "$outage_minutes" -ge "$threshold" ]]; then
|
||||
warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
local t1_jobs
|
||||
read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
|
||||
if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
|
||||
echo " Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
|
||||
for job in "${t1_jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
@@ -557,29 +577,12 @@ run_handback() {
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner"
|
||||
log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
|
||||
fi
|
||||
}
|
||||
|
||||
local t2_delay t3_delay t4_delay
|
||||
t2_delay=$(get_tier2_delay)
|
||||
t3_delay=$(get_tier3_delay)
|
||||
t4_delay=$(get_tier4_delay)
|
||||
|
||||
run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
|
||||
run_writeback_tier 2 "$t2_delay" "productivity services"
|
||||
run_writeback_tier 3 "$t3_delay" "secondary services"
|
||||
run_writeback_tier 4 "$t4_delay" "media shares + edge cases"
|
||||
fi
|
||||
|
||||
log "Writeback complete"
|
||||
|
||||
# ── Step 5: Start remote containers ─────────────────────────────────────────────────────
|
||||
# Start in tier order with dependency awareness — databases before apps
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Start Remote Containers ━━━"
|
||||
|
||||
# Tier 1 — excluding DDNS (handled separately as final step)
|
||||
echo " Starting Tier 1 on $REMOTE_SERVER_NAME..."
|
||||
sleep 10 # give databases time to initialise before apps
|
||||
for container in "${t1[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_ddns=false
|
||||
@@ -589,36 +592,17 @@ run_handback() {
|
||||
[[ "$is_ddns" == false ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
sleep 10 # give databases time to initialise before apps
|
||||
|
||||
[[ "$(state_get tier2_started)" == "true" ]] && \
|
||||
for container in "${t2[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier3_started)" == "true" ]] && \
|
||||
for container in "${t3[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier4_started)" == "true" ]] && \
|
||||
for container in "${t4[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
log "Remote containers started"
|
||||
|
||||
# ── Step 6: Start remote DDNS LAST ──────────────────────────────────────────────────────
|
||||
# DNS cuts over ONLY after containers confirmed up — this is the final step
|
||||
# ── Step 7: DNS Cutover — final step ────────────────────────────────────────────────────
|
||||
# DNS cuts over ONLY after Tier 1 containers confirmed up
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
||||
sleep 5 # brief pause to ensure containers are accepting connections
|
||||
sleep 5
|
||||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
remote_start "$container"
|
||||
done
|
||||
warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
||||
|
||||
# ── Step 7: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||||
# ── Step 8: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
state_set state "NORMAL"
|
||||
state_set fallback_start "0"
|
||||
@@ -627,6 +611,9 @@ run_handback() {
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
# Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
|
||||
local_ddns_start
|
||||
|
||||
warn "$ICON_DONE Handback complete — returned to NORMAL"
|
||||
notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
|
||||
"Fallback" "normal"
|
||||
@@ -671,7 +658,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
log "$ICON_SUCCESS NORMAL — all systems up"
|
||||
|
||||
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
||||
# Remote is down — enter FAILOVER
|
||||
# Remote is down — enter FALLBACK immediately
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
|
||||
@@ -713,7 +700,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
fi
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# FAILOVER STATE
|
||||
# FALLBACK STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
# Phase 1 — Pre-flight verify both servers reachable, daemons healthy,
|
||||
# version parity, fallback.sh exists, state is NORMAL
|
||||
# Phase 2 — Block Remote iptables rule drops all traffic to remote IP
|
||||
# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FAILOVER
|
||||
# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FALLBACK
|
||||
# Phase 4 — Container Start verify Tier 1 failover containers started locally
|
||||
# Phase 5 — Restore remove iptables rule, remote becomes reachable
|
||||
# Phase 6 — Handback wait for fallback.sh to complete handback to NORMAL
|
||||
|
||||
@@ -20,12 +20,17 @@
|
||||
# Step 2: partnership_manager.sh --onboard
|
||||
# — verifies both servers, reconfigures auth WebUIs → owner IP,
|
||||
# writes ACTIVE state, creates FolderView3 folder (if enabled)
|
||||
# Step 3: arr_sync.sh — bidirectional library bootstrap: partner gets our full
|
||||
# Lidarr/Sonarr/Radarr library, we get theirs. Both sides
|
||||
# start tracking the merged library from day one.
|
||||
# Non-fatal — partnership is valid even if arrs aren't live yet.
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# Initial_run/partnership_onboard.sh — full setup
|
||||
# Initial_run/partnership_onboard.sh --dry-run — preview without changes
|
||||
# Initial_run/partnership_onboard.sh --log — verbose output
|
||||
# Initial_run/partnership_onboard.sh --skip-ssh — skip ssh_setup.sh (key already set up)
|
||||
# Initial_run/partnership_onboard.sh --skip-arr-sync — skip arr sync (arrs not live yet)
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -35,10 +40,12 @@ source "$SCRIPTS_ROOT/load_config.sh"
|
||||
|
||||
# ── Parse --skip-ssh before parse_args ────────────────────────────────────────────────────────
|
||||
SKIP_SSH=false
|
||||
SKIP_ARR_SYNC=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--skip-ssh) SKIP_SSH=true ;;
|
||||
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
@@ -74,7 +81,7 @@ EXTRA_FLAGS=()
|
||||
# ━━━ Step 1: SSH Key Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ Step 1/2 — SSH Key Setup ━━━"
|
||||
echo "━━━ Step 1/3 — SSH Key Setup ━━━"
|
||||
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping SSH setup (--skip-ssh)"
|
||||
@@ -92,7 +99,7 @@ fi
|
||||
# ━━━ Step 2: Partnership Onboard ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ Step 2/2 — Partnership Onboard ━━━"
|
||||
echo "━━━ Step 2/3 — Partnership Onboard ━━━"
|
||||
|
||||
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
||||
log "Partnership onboard complete ✅"
|
||||
@@ -102,6 +109,37 @@ else
|
||||
ONBOARD_OK=false
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Step 3: Arr Library Bootstrap ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ Step 3/3 — Arr Library Bootstrap ━━━"
|
||||
|
||||
ARR_SYNC_OK=true
|
||||
if [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping arr sync — onboard did not complete"
|
||||
ARR_SYNC_OK=false
|
||||
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
||||
warn "Skipping arr sync (--skip-arr-sync)"
|
||||
ARR_SYNC_OK=false
|
||||
else
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||
if [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
warn "arr_sync.sh not found — skipping library bootstrap"
|
||||
warn "Run Media/arr_sync.sh manually once arrs are live"
|
||||
ARR_SYNC_OK=false
|
||||
else
|
||||
log "Syncing arr libraries with $REMOTE_SERVER_NAME..."
|
||||
if bash "$ARR_SYNC_SCRIPT" "${EXTRA_FLAGS[@]}"; then
|
||||
log "Arr library bootstrap complete ✅"
|
||||
else
|
||||
warn "Arr sync completed with errors — partnership is still valid"
|
||||
warn "Re-run Media/arr_sync.sh once all arr containers are live"
|
||||
ARR_SYNC_OK=false
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
@@ -113,20 +151,28 @@ echo "$ICON_NET Partner: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
echo " Step 1 — SSH key: ✅"
|
||||
echo " Step 2 — Onboard: ✅"
|
||||
_ssh_status() { [[ "$SKIP_SSH" == true ]] && echo "skipped" || echo "✅"; }
|
||||
_arr_status() {
|
||||
if [[ "$SKIP_ARR_SYNC" == true ]]; then echo "skipped (--skip-arr-sync)"
|
||||
elif [[ "$ONBOARD_OK" == false ]]; then echo "skipped (onboard failed)"
|
||||
elif [[ "$ARR_SYNC_OK" == true ]]; then echo "✅"
|
||||
else echo "⚠️ errors — re-run arr_sync.sh once arrs are live"
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — SSH key: $(_ssh_status)"
|
||||
echo " Step 2 — Onboard: $( [[ "$ONBOARD_OK" == true ]] && echo "✅" || echo "❌" )"
|
||||
echo " Step 3 — Arr bootstrap: $(_arr_status)"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
warn "$ICON_DONE DONE — partnership established ✅"
|
||||
warn "Next: verify with 'Partnership/partnership_manager.sh --status'"
|
||||
log "$ICON_DONE DONE — partnership established ✅"
|
||||
log "Next: verify with 'Partnership/partnership_manager.sh --status'"
|
||||
fi
|
||||
else
|
||||
echo " Step 1 — SSH key: $( [[ "$SKIP_SSH" == true ]] && echo "skipped" || echo "✅" )"
|
||||
echo " Step 2 — Onboard: ❌"
|
||||
echo ""
|
||||
error "Setup incomplete — resolve onboard errors above and re-run"
|
||||
fi
|
||||
|
||||
|
||||
Executable
+692
@@ -0,0 +1,692 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= ARR SYNC ===================================================
|
||||
# ==============================================================================================
|
||||
# Bidirectional arr library sync across all nodes in the ecosystem.
|
||||
# Syncs Lidarr, Sonarr, and Radarr libraries so every node tracks the same content.
|
||||
# Run before rsync — once arrs agree on library, rsync spreads the files.
|
||||
#
|
||||
# ── DESIGN ────────────────────────────────────────────────────────────────────────────────────
|
||||
# Full mesh: every node syncs with every other node — no primary, no hierarchy.
|
||||
# Union model: if any node tracks an item, all nodes get it (unless blocklisted).
|
||||
# Convergence: any node can add content; after one full cycle all nodes agree.
|
||||
# Upgrade-aware: server1 upgrades a file → arr tracks new path → rsync spreads it →
|
||||
# arr_cleanup removes old file on all nodes because arr no longer tracks it.
|
||||
#
|
||||
# ── NODE DISCOVERY ────────────────────────────────────────────────────────────────────────────
|
||||
# Reads HOST* vars from master.conf. Add HOST3= and it joins the sync automatically.
|
||||
# No scripts change when adding a new node.
|
||||
#
|
||||
# ── REMOTE API KEY ACCESS ─────────────────────────────────────────────────────────────────────
|
||||
# Remote API keys are not stored anywhere. Script SSHes to each remote node and reads
|
||||
# the key directly from that arr's config.xml in its appdata directory.
|
||||
# Only the API response (JSON) is returned — key never leaves the remote node.
|
||||
# Self-maintaining: remote key regeneration is picked up automatically.
|
||||
#
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added anywhere.
|
||||
# Read from ALL nodes via SSH at start of each run — immediate effect before rsync.
|
||||
# Propagates to all nodes via rsync on the next cycle.
|
||||
# Manage via --blocklist-add / --blocklist-remove / --blocklist-list.
|
||||
#
|
||||
# ── GRACEFUL SKIP ─────────────────────────────────────────────────────────────────────────────
|
||||
# Arr not configured locally → skip cleanly, no error.
|
||||
# Arr not reachable on a remote → skip that node for that arr type, continue with others.
|
||||
# Partial mesh works — nodes that share an arr type sync with each other.
|
||||
#
|
||||
# ── WHAT GETS SYNCED ──────────────────────────────────────────────────────────────────────────
|
||||
# Library items (tracked artists/series/movies) keyed on stable IDs:
|
||||
# Lidarr — MusicBrainz artist ID (foreignArtistId)
|
||||
# Sonarr — TVDB series ID (tvdbId)
|
||||
# Radarr — TMDB movie ID (tmdbId)
|
||||
# When adding to a remote node, that node's own quality profile, metadata profile,
|
||||
# and root folder path are used — settings are never copied from the source node.
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# arr_sync.sh — sync all arr types, all nodes
|
||||
# arr_sync.sh --dry-run — preview only, no changes
|
||||
# arr_sync.sh --log — verbose output
|
||||
# arr_sync.sh --status — show config and exit
|
||||
# arr_sync.sh --blocklist-add lidarr <id> "reason" — tombstone an ID on all nodes
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone an ID
|
||||
# arr_sync.sh --blocklist-list — show all blocklisted IDs
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# ARR_SYNC_ENABLED — global on/off toggle (default true)
|
||||
# ARR_SYNC_BLOCKLIST — path to TSV blocklist (default: DATA_DIR/arr_sync_blocklist.tsv)
|
||||
# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default 10)
|
||||
# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default 60)
|
||||
# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default /mnt/user/appdata)
|
||||
# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default 8686)
|
||||
# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default 8989)
|
||||
# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default 7878)
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts)
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Blocklist flag pre-processing ─────────────────────────────────────────────────────────────
|
||||
# Handle before parse_args — these are action flags, not standard options
|
||||
BLOCKLIST_ACTION=""
|
||||
BLOCKLIST_ARR=""
|
||||
BLOCKLIST_ID=""
|
||||
BLOCKLIST_REASON=""
|
||||
FILTERED_ARGS=()
|
||||
_skip_next=false
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_skip_next" == true ]]; then _skip_next=false; continue; fi
|
||||
case "$_arg" in
|
||||
--blocklist-add) BLOCKLIST_ACTION="add" ;;
|
||||
--blocklist-remove) BLOCKLIST_ACTION="remove" ;;
|
||||
--blocklist-list) BLOCKLIST_ACTION="list" ;;
|
||||
*) FILTERED_ARGS+=("$_arg") ;;
|
||||
esac
|
||||
done
|
||||
unset _arg _skip_next
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Consume blocklist positional args from remaining FILTERED_ARGS
|
||||
# --blocklist-add lidarr <id> "reason"
|
||||
# --blocklist-remove lidarr <id>
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]] && [[ "$BLOCKLIST_ACTION" != "list" ]]; then
|
||||
for _a in "${FILTERED_ARGS[@]}"; do
|
||||
[[ "$_a" == --* ]] && continue
|
||||
if [[ -z "$BLOCKLIST_ARR" ]]; then BLOCKLIST_ARR="$_a"
|
||||
elif [[ -z "$BLOCKLIST_ID" ]]; then BLOCKLIST_ID="$_a"
|
||||
elif [[ -z "$BLOCKLIST_REASON" ]]; then BLOCKLIST_REASON="$_a"
|
||||
fi
|
||||
done
|
||||
unset _a
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for _tool in curl jq; do
|
||||
if ! command -v "$_tool" >/dev/null 2>&1; then
|
||||
error "$_tool not found — required for arr API calls"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset _tool
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ── Runtime config with defaults ──────────────────────────────────────────────────────────────
|
||||
ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}"
|
||||
ARR_SYNC_BLOCKLIST="${ARR_SYNC_BLOCKLIST:-${DATA_DIR}/arr_sync_blocklist.tsv}"
|
||||
ARR_SYNC_CONNECT_TIMEOUT="${ARR_SYNC_CONNECT_TIMEOUT:-10}"
|
||||
ARR_SYNC_API_TIMEOUT="${ARR_SYNC_API_TIMEOUT:-60}"
|
||||
DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}"
|
||||
ARR_SYNC_LIDARR_PORT="${ARR_SYNC_LIDARR_PORT:-8686}"
|
||||
ARR_SYNC_SONARR_PORT="${ARR_SYNC_SONARR_PORT:-8989}"
|
||||
ARR_SYNC_RADARR_PORT="${ARR_SYNC_RADARR_PORT:-7878}"
|
||||
|
||||
if [[ "$ARR_SYNC_ENABLED" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Arr type definitions ───────────────────────────────────────────────────────────────────────
|
||||
# Each arr type maps to its port, API version, endpoint, stable ID field, and display name field
|
||||
declare -A _PORT=([lidarr]="$ARR_SYNC_LIDARR_PORT" [sonarr]="$ARR_SYNC_SONARR_PORT" [radarr]="$ARR_SYNC_RADARR_PORT")
|
||||
declare -A _VER=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3")
|
||||
declare -A _EP=( [lidarr]="artist" [sonarr]="series" [radarr]="movie")
|
||||
declare -A _ID=( [lidarr]="foreignArtistId" [sonarr]="tvdbId" [radarr]="tmdbId")
|
||||
declare -A _NAME=([lidarr]="artistName" [sonarr]="title" [radarr]="title")
|
||||
# ID type: "string" for MusicBrainz UUID, "int" for TVDB/TMDB numeric IDs
|
||||
declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
|
||||
ARR_TYPES=(lidarr sonarr radarr)
|
||||
|
||||
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
||||
REMOTE_NODES=()
|
||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||||
[[ -z "${!_hv:-}" ]] && continue
|
||||
REMOTE_NODES+=("$_hv")
|
||||
done
|
||||
unset _hv
|
||||
|
||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||
warn "No remote nodes defined in master.conf — nothing to sync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo " Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
declare -A BLOCKLIST_MAP # key: "arr_type:stable_id" → display name
|
||||
|
||||
_load_blocklist() {
|
||||
BLOCKLIST_MAP=()
|
||||
local count=0
|
||||
|
||||
_parse_blocklist_lines() {
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name rest; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
(( count++ ))
|
||||
done
|
||||
}
|
||||
|
||||
# Local blocklist
|
||||
[[ -f "$ARR_SYNC_BLOCKLIST" ]] && _parse_blocklist_lines < "$ARR_SYNC_BLOCKLIST"
|
||||
|
||||
# Remote blocklists — read via SSH so tombstones are effective immediately
|
||||
for _node_id in "${REMOTE_NODES[@]}"; do
|
||||
local _node_ip
|
||||
_node_ip=$(_resolve_node_ip "$_node_id") || continue
|
||||
local _remote_lines
|
||||
_remote_lines=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$_node_ip" "cat '$ARR_SYNC_BLOCKLIST' 2>/dev/null" 2>/dev/null) || continue
|
||||
_parse_blocklist_lines <<< "$_remote_lines"
|
||||
done
|
||||
unset _node_id _node_ip _remote_lines
|
||||
|
||||
log "Loaded blocklist: $count entries from $(( ${#REMOTE_NODES[@]} + 1 )) nodes"
|
||||
unset -f _parse_blocklist_lines
|
||||
}
|
||||
|
||||
_is_blocklisted() {
|
||||
[[ -n "${BLOCKLIST_MAP["${1}:${2}"]:-}" ]]
|
||||
}
|
||||
|
||||
_blocklist_add() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" reason="${4:-manually excluded}"
|
||||
mkdir -p "$(dirname "$ARR_SYNC_BLOCKLIST")"
|
||||
if ! grep -qP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" 2>/dev/null; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$arr_type" "$stable_id" "$display_name" \
|
||||
"$MY_ID" "$(date -Iseconds)" "$reason" \
|
||||
>> "$ARR_SYNC_BLOCKLIST"
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
log "Blocklisted: [$arr_type] $display_name ($stable_id)"
|
||||
else
|
||||
log "Already blocklisted: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
_blocklist_remove() {
|
||||
local arr_type="$1" stable_id="$2"
|
||||
if [[ -f "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
grep -vP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" > "$tmp" && \
|
||||
mv "$tmp" "$ARR_SYNC_BLOCKLIST" || rm -f "$tmp"
|
||||
unset "BLOCKLIST_MAP[${arr_type}:${stable_id}]"
|
||||
log "Removed from blocklist: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Blocklist management mode ──────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
case "$BLOCKLIST_ACTION" in
|
||||
list)
|
||||
echo ""
|
||||
echo "━━━━━ ARR SYNC BLOCKLIST ━━━━━"
|
||||
if [[ ! -f "$ARR_SYNC_BLOCKLIST" ]] || [[ ! -s "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
echo " (empty)"
|
||||
else
|
||||
echo ""
|
||||
printf '%-8s %-40s %-30s %s\n' "Arr" "Stable ID" "Name" "Reason"
|
||||
printf '%-8s %-40s %-30s %s\n' "---" "---------" "----" "------"
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name tombstoned_by ts reason; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
printf '%-8s %-40s %-30s %s\n' "$arr_type" "$stable_id" "$display_name" "$reason"
|
||||
done < "$ARR_SYNC_BLOCKLIST"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
;;
|
||||
add)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-add <arr_type> <stable_id> [reason]"
|
||||
error " arr_type: lidarr | sonarr | radarr"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "${BLOCKLIST_ID}" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $BLOCKLIST_ID — will propagate via rsync on next cycle"
|
||||
exit 0
|
||||
;;
|
||||
remove)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-remove <arr_type> <stable_id>"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_remove "$BLOCKLIST_ARR" "$BLOCKLIST_ID"
|
||||
echo " Removed [$BLOCKLIST_ARR] $BLOCKLIST_ID from blocklist"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_GEAR Appdata base: $DOCKER_APPDATA_BASE"
|
||||
echo "$ICON_GEAR Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
echo "$ICON_GEAR Connect timeout: ${ARR_SYNC_CONNECT_TIMEOUT}s"
|
||||
echo "$ICON_GEAR API timeout: ${ARR_SYNC_API_TIMEOUT}s"
|
||||
echo ""
|
||||
echo " Arr Port URL"
|
||||
for _arr in "${ARR_TYPES[@]}"; do
|
||||
local _url="" _key=""
|
||||
case "$_arr" in
|
||||
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
||||
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
||||
radarr) _url="${RADARR_URL:-not configured}" ;;
|
||||
esac
|
||||
printf ' %-8s %-6s %s\n' "$_arr" "${_PORT[$_arr]}" "$_url"
|
||||
done
|
||||
unset _arr _url
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── REMOTE HELPERS ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_resolve_node_ip() {
|
||||
local node_id="$1"
|
||||
local node_name="${!node_id}"
|
||||
local ts_name="${node_name,,}"
|
||||
local ip
|
||||
ip=$(tailscale ip -4 "$ts_name" 2>/dev/null)
|
||||
[[ -z "$ip" ]] && ip=$(tailscale status 2>/dev/null | \
|
||||
awk -v n="$ts_name" '$2 ~ "^" n { print $1; exit }')
|
||||
[[ -z "$ip" ]] && return 1
|
||||
echo "$ip"
|
||||
}
|
||||
|
||||
# Check if arr is reachable on remote node
|
||||
_remote_arr_up() {
|
||||
local node_ip="$1" port="$2" api_ver="$3" config_xml="$4"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time 5 -H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/system/status' >/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch full library from remote arr — returns raw JSON array
|
||||
_remote_library() {
|
||||
local node_ip="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/${endpoint}'" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch remote arr defaults: qualityProfileId, rootFolderPath, metadataProfileId (Lidarr)
|
||||
_remote_defaults() {
|
||||
local node_ip="$1" port="$2" api_ver="$3" arr_type="$4" config_xml="$5"
|
||||
local meta_field=""
|
||||
[[ "$arr_type" == "lidarr" ]] && \
|
||||
meta_field=', metadataProfileId: ($mp | map(select(.name == "Standard")) | .[0].id // .[0].id // 1)'
|
||||
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
QP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/qualityprofile")
|
||||
RF=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/rootfolder")
|
||||
MP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/metadataprofile" 2>/dev/null || echo '[]')
|
||||
jq -n --argjson qp "\$QP" --argjson rf "\$RF" --argjson mp "\$MP" \
|
||||
'{qualityProfileId: (\$qp | .[0].id // 1), rootFolderPath: (\$rf | .[0].path // "")${meta_field}}'
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# Add item to remote arr — payload is base64-encoded to avoid SSH quoting issues
|
||||
_remote_add() {
|
||||
local node_ip="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5"
|
||||
local encoded="$6" # base64-encoded JSON body
|
||||
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
BODY=\$(printf '%s' '${encoded}' | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "\$BODY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LOCAL HELPERS ─────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_local_library() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_local_defaults() {
|
||||
local url="$1" api_key="$2" api_ver="$3" arr_type="$4"
|
||||
local qp rf mp meta_field=""
|
||||
qp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/qualityprofile" | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/rootfolder" | jq -r '.[0].path // ""')
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/metadataprofile" | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
}
|
||||
|
||||
_local_add() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" payload="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PAYLOAD BUILDER ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Builds the minimal POST body to add an item to an arr.
|
||||
# Uses the TARGET node's own defaults — never copies source node settings.
|
||||
|
||||
_build_payload() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" monitored="$4" defaults_json="$5"
|
||||
case "$arr_type" in
|
||||
lidarr)
|
||||
jq -n \
|
||||
--arg id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
foreignArtistId: $id,
|
||||
artistName: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
metadataProfileId: ($df.metadataProfileId // 1),
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {monitor: "all", searchForMissingAlbums: false}
|
||||
}'
|
||||
;;
|
||||
sonarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tvdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
seasons: [],
|
||||
addOptions: {searchForMissingEpisodes: false, monitor: "all"}
|
||||
}'
|
||||
;;
|
||||
radarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tmdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {searchForMovie: false}
|
||||
}'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CORE SYNC — one arr type across all remote nodes ──────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_sync_arr() {
|
||||
local arr_type="$1"
|
||||
local port="${_PORT[$arr_type]}"
|
||||
local api_ver="${_VER[$arr_type]}"
|
||||
local endpoint="${_EP[$arr_type]}"
|
||||
local id_field="${_ID[$arr_type]}"
|
||||
local name_field="${_NAME[$arr_type]}"
|
||||
local id_type="${_ID_TYPE[$arr_type]}"
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/$(echo "${arr_type^}")/config.xml"
|
||||
|
||||
# Resolve local credentials
|
||||
local local_url local_key
|
||||
case "$arr_type" in
|
||||
lidarr) local_url="${LIDARR_URL:-}"; local_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) local_url="${SONARR_URL:-}"; local_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) local_url="${RADARR_URL:-}"; local_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
if [[ -z "$local_url" ]] || [[ -z "$local_key" ]]; then
|
||||
log "${arr_type^}: not configured on $MY_ID — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ ${arr_type^} ━━━"
|
||||
|
||||
# ── Fetch local library ────────────────────────────────────────────────────────────────────
|
||||
local local_json
|
||||
local_json=$(_local_library "$local_url" "$local_key" "$api_ver" "$endpoint")
|
||||
if [[ -z "$local_json" ]] || ! echo "$local_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch local library — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build local ID map: stable_id → "display_name|monitored"
|
||||
declare -A local_ids
|
||||
local local_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( local_count++ ))
|
||||
done < <(echo "$local_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $local_count items in local library"
|
||||
|
||||
local total_added_local=0 total_added_remote=0 total_skipped=0
|
||||
|
||||
# ── Sync with each remote node ─────────────────────────────────────────────────────────────
|
||||
for node_id in "${REMOTE_NODES[@]}"; do
|
||||
local node_name="${!node_id}"
|
||||
log "${arr_type^}: syncing with $node_name..."
|
||||
|
||||
local node_ip
|
||||
node_ip=$(_resolve_node_ip "$node_id") || {
|
||||
warn "${arr_type^}: cannot resolve Tailscale IP for $node_name — skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
if ! _remote_arr_up "$node_ip" "$port" "$api_ver" "$config_xml"; then
|
||||
log "${arr_type^}: not reachable on $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local remote_json
|
||||
remote_json=$(_remote_library "$node_ip" "$port" "$api_ver" "$endpoint" "$config_xml")
|
||||
if [[ -z "$remote_json" ]] || ! echo "$remote_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch library from $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build remote ID map
|
||||
declare -A remote_ids
|
||||
local remote_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
remote_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( remote_count++ ))
|
||||
done < <(echo "$remote_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $remote_count items on $node_name"
|
||||
|
||||
# ── Remote → Local: items on remote not in local ───────────────────────────────────────
|
||||
local to_add_local=()
|
||||
for stable_id in "${!remote_ids[@]}"; do
|
||||
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_local+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
|
||||
local local_defs
|
||||
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
|
||||
if [[ -z "$local_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
|
||||
else
|
||||
for stable_id in "${to_add_local[@]}"; do
|
||||
IFS='|' read -r display_name monitored <<< "${remote_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"$monitored" "$local_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
local http_code
|
||||
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
|
||||
"$endpoint" "$payload")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to local ${arr_type^}: $display_name"
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Local → Remote: items on local not on remote ───────────────────────────────────────
|
||||
local to_add_remote=()
|
||||
for stable_id in "${!local_ids[@]}"; do
|
||||
[[ -n "${remote_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_remote+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_remote[@]}" -gt 0 ]]; then
|
||||
local remote_defs
|
||||
remote_defs=$(_remote_defaults "$node_ip" "$port" "$api_ver" "$arr_type" "$config_xml")
|
||||
if [[ -z "$remote_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch defaults from $node_name — skipping remote adds"
|
||||
else
|
||||
for stable_id in "${to_add_remote[@]}"; do
|
||||
IFS='|' read -r display_name monitored <<< "${local_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"$monitored" "$remote_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to $node_name ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
local encoded http_code
|
||||
encoded=$(printf '%s' "$payload" | base64 -w0)
|
||||
http_code=$(_remote_add "$node_ip" "$port" "$api_ver" "$endpoint" \
|
||||
"$config_xml" "$encoded")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to $node_name ${arr_type^}: $display_name"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
warn "Failed to add to $node_name ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted"
|
||||
|
||||
unset remote_ids
|
||||
declare -A remote_ids
|
||||
done
|
||||
|
||||
echo " Total added to local: $total_added_local | to remotes: $total_added_remote | blocklisted: $total_skipped"
|
||||
unset local_ids
|
||||
declare -A local_ids
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Loading Blocklist ━━━"
|
||||
_load_blocklist
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
for arr_type in "${ARR_TYPES[@]}"; do
|
||||
_sync_arr "$arr_type"
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+9
-13
@@ -43,9 +43,8 @@
|
||||
# The user accepts full responsibility — this is 100% intentional by design.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Lidarr runs on HOST1 only — music library is HOST1's source of truth.
|
||||
# detect_hosts() sets MY_ID — if MY_ID != HOST1 script exits cleanly with no action.
|
||||
# LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT aliased by detect_hosts() automatically.
|
||||
# Runs on any node where Lidarr is configured — skips cleanly if LIDARR_URL/API_KEY not set.
|
||||
# detect_hosts() aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT from HOST*_LIDARR_*.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||
@@ -155,9 +154,9 @@ acquire_lock "wait"
|
||||
# detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT
|
||||
detect_hosts
|
||||
|
||||
# Lidarr is HOST1 only — exit cleanly on any other host
|
||||
if [[ "$MY_ID" != "HOST1" ]]; then
|
||||
log "Lidarr runs on HOST1 only — skipping on $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
# Skip if Lidarr is not configured on this host
|
||||
if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
|
||||
log "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -182,8 +181,7 @@ if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Lidarr URL: $LIDARR_URL"
|
||||
log "Music root: $LIDARR_MUSIC_ROOT"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $LIDARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
@@ -310,7 +308,7 @@ fi
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "Lidarr" || exit 1
|
||||
|
||||
log "Querying Lidarr API: $LIDARR_URL"
|
||||
echo " Querying Lidarr API..."
|
||||
|
||||
# Fetch all artists
|
||||
ARTIST_RESPONSE=$(lidarr_api "artist") || {
|
||||
@@ -371,7 +369,7 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
warn "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists"
|
||||
echo " $ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
|
||||
|
||||
# Safety Layer 6 — percentage drop vs last known count
|
||||
if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then
|
||||
@@ -399,9 +397,7 @@ echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE"
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning Music Root ━━━"
|
||||
log "Root: $LIDARR_MUSIC_ROOT"
|
||||
log "Orphan age: ${LIDARR_ORPHAN_AGE} days"
|
||||
log "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}"
|
||||
echo " Root: $LIDARR_MUSIC_ROOT | Orphan age: ${LIDARR_ORPHAN_AGE} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
+66
-27
@@ -24,6 +24,10 @@
|
||||
# Lidarr runs on HOST1 only. detect_hosts() sets LIDARR_URL — if empty (HOST2) the
|
||||
# script exits cleanly with no action rather than failing.
|
||||
#
|
||||
# ── OUTPUT ────────────────────────────────────────────────────────────────────────────────────
|
||||
# Minimal by default — section headers + per-section summary always visible.
|
||||
# --log shows per-item detail (each album, each artist, each file fetched).
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs during large library scans
|
||||
# curl + jq check — fail fast if tools missing
|
||||
@@ -50,7 +54,7 @@
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# lidarr_missing_art.sh — fetch all missing artwork
|
||||
# lidarr_missing_art.sh --dry-run — preview without downloading
|
||||
# lidarr_missing_art.sh --log — verbose output
|
||||
# lidarr_missing_art.sh --log — verbose per-item output
|
||||
# lidarr_missing_art.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -70,7 +74,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for API calls"
|
||||
@@ -80,7 +83,6 @@ if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
exit 1
|
||||
fi
|
||||
success "curl and jq found"
|
||||
|
||||
acquire_lock
|
||||
|
||||
@@ -91,7 +93,8 @@ if [[ -z "$LIDARR_URL" ]]; then
|
||||
log "Lidarr not configured for $MY_ID — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
success "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — tools OK"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
@@ -115,6 +118,10 @@ fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written"
|
||||
|
||||
# ── Temp dir for subshell fetch/fail counters ─────────────────────────────────────────────────
|
||||
LIDARR_TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$LIDARR_TMP"' EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -143,7 +150,7 @@ download_if_valid() {
|
||||
[[ -f "$dest" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
info "DRY RUN — would fetch: $(basename "$dest")"
|
||||
log "DRY RUN — would fetch: $(basename "$dest")"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -155,14 +162,14 @@ download_if_valid() {
|
||||
size=$(stat -c%s "$tmp" 2>/dev/null || echo 0)
|
||||
if (( size > LIDARR_ART_MIN_SIZE )); then
|
||||
mv "$tmp" "$dest"
|
||||
log "Fetched: $dest"
|
||||
log " Fetched: $(basename "$dest")"
|
||||
return 0
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
warn "Failed to fetch valid image: $(basename "$dest")"
|
||||
warn "Failed to fetch: $(basename "$dest")"
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -193,19 +200,15 @@ if ! curl_json "$LIDARR_URL/api/v1/system/status?apikey=$LIDARR_API_KEY" | jq -e
|
||||
notify "lidarr_missing_art failed — Lidarr API unreachable on $(hostname)" "Lidarr Missing Art" "warning"
|
||||
exit 1
|
||||
fi
|
||||
success "Lidarr API reachable"
|
||||
echo " Reachable — $LIDARR_URL"
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
ALBUMS_CHECKED=0
|
||||
ALBUMS_COMPLETE=0
|
||||
ALBUM_FETCHES=0
|
||||
ALBUM_FAILS=0
|
||||
|
||||
ARTISTS_CHECKED=0
|
||||
ARTISTS_COMPLETE=0
|
||||
ARTIST_FETCHES=0
|
||||
ARTIST_FAILS=0
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Albums ━━━
|
||||
@@ -222,7 +225,7 @@ if [[ -z "$albums" || "$albums" == "null" ]]; then
|
||||
fi
|
||||
|
||||
total_albums=$(echo "$albums" | jq '. | length')
|
||||
info "$total_albums albums to process"
|
||||
echo " Processing $total_albums albums..."
|
||||
|
||||
while IFS=$'\t' read -r local_path mbid artist_name album_name; do
|
||||
(( ALBUMS_CHECKED++ ))
|
||||
@@ -242,6 +245,8 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do
|
||||
wait_for_slot
|
||||
|
||||
(
|
||||
_fetches=0 _fails=0
|
||||
|
||||
JSON=""
|
||||
if [[ -n "$mbid" && "$mbid" != "null" ]]; then
|
||||
JSON=$(curl_json "http://webservice.fanart.tv/v3/music/albums/$mbid?api_key=$FANART_API_KEY")
|
||||
@@ -250,29 +255,43 @@ while IFS=$'\t' read -r local_path mbid artist_name album_name; do
|
||||
|
||||
if [[ ! -f "$local_path/cover.jpg" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.[].albumcover[0].url // empty')
|
||||
if ! download_if_valid "$IMG" "$local_path/cover.jpg"; then
|
||||
if download_if_valid "$IMG" "$local_path/cover.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
query=$(printf "%s %s" "$artist_name" "$album_name" | sed 's/ /+/g')
|
||||
itunes=$(curl_json "https://itunes.apple.com/search?term=$query&entity=album&limit=1" |
|
||||
jq -r '.results[0].artworkUrl100 // empty' | sed 's/100x100/600x600/')
|
||||
download_if_valid "$itunes" "$local_path/cover.jpg"
|
||||
if download_if_valid "$itunes" "$local_path/cover.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
(( _fails++ ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$local_path/cdart.png" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.[].cdart[0].url // empty')
|
||||
download_if_valid "$IMG" "$local_path/cdart.png"
|
||||
if download_if_valid "$IMG" "$local_path/cdart.png"; then (( _fetches++ )); else (( _fails++ )); fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$local_path/back.jpg" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.[].albumback[0].url // empty')
|
||||
download_if_valid "$IMG" "$local_path/back.jpg"
|
||||
if download_if_valid "$IMG" "$local_path/back.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
|
||||
fi
|
||||
|
||||
(( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fetches"
|
||||
(( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fails"
|
||||
) &
|
||||
|
||||
done < <(echo "$albums" | jq -r '.[] | [.path, .foreignAlbumId, .artist.artistName, .title] | @tsv')
|
||||
|
||||
wait
|
||||
|
||||
ALBUM_FETCHED=$(wc -l < "$LIDARR_TMP/album_fetches" 2>/dev/null || echo 0)
|
||||
ALBUM_FAILED=$(wc -l < "$LIDARR_TMP/album_fails" 2>/dev/null || echo 0)
|
||||
ALBUM_MISSING=$(( ALBUMS_CHECKED - ALBUMS_COMPLETE ))
|
||||
echo " Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Fetched: $ALBUM_FETCHED | Failed: $ALBUM_FAILED"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Artists ━━━
|
||||
# ==============================================================================================
|
||||
@@ -288,7 +307,7 @@ if [[ -z "$artists" || "$artists" == "null" ]]; then
|
||||
fi
|
||||
|
||||
total_artists=$(echo "$artists" | jq '. | length')
|
||||
info "$total_artists artists to process"
|
||||
echo " Processing $total_artists artists..."
|
||||
|
||||
while IFS=$'\t' read -r local_path mbid name; do
|
||||
(( ARTISTS_CHECKED++ ))
|
||||
@@ -310,43 +329,63 @@ while IFS=$'\t' read -r local_path mbid name; do
|
||||
wait_for_slot
|
||||
|
||||
(
|
||||
_fetches=0 _fails=0
|
||||
|
||||
JSON=$(curl_json "http://webservice.fanart.tv/v3/music/$mbid?api_key=$FANART_API_KEY")
|
||||
sleep "$LIDARR_ART_SLEEP_BETWEEN"
|
||||
|
||||
if [[ ! -f "$local_path/folder.jpg" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty')
|
||||
if ! download_if_valid "$IMG" "$local_path/folder.jpg"; then
|
||||
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
IMG=$(deezer_artist_image "$name")
|
||||
if ! download_if_valid "$IMG" "$local_path/folder.jpg"; then
|
||||
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
IMG=$(lastfm_artist_image "$name")
|
||||
download_if_valid "$IMG" "$local_path/folder.jpg"
|
||||
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
(( _fails++ ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$local_path/fanart.jpg" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty')
|
||||
if ! download_if_valid "$IMG" "$local_path/fanart.jpg"; then
|
||||
if download_if_valid "$IMG" "$local_path/fanart.jpg"; then
|
||||
(( _fetches++ ))
|
||||
else
|
||||
IMG=$(deezer_artist_image "$name")
|
||||
download_if_valid "$IMG" "$local_path/fanart.jpg"
|
||||
if download_if_valid "$IMG" "$local_path/fanart.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$local_path/logo.png" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty')
|
||||
download_if_valid "$IMG" "$local_path/logo.png"
|
||||
if download_if_valid "$IMG" "$local_path/logo.png"; then (( _fetches++ )); else (( _fails++ )); fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$local_path/banner.jpg" ]]; then
|
||||
IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty')
|
||||
download_if_valid "$IMG" "$local_path/banner.jpg"
|
||||
if download_if_valid "$IMG" "$local_path/banner.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
|
||||
fi
|
||||
|
||||
(( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fetches"
|
||||
(( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fails"
|
||||
) &
|
||||
|
||||
done < <(echo "$artists" | jq -r '.[] | [.path, .foreignArtistId, .artistName] | @tsv')
|
||||
|
||||
wait
|
||||
|
||||
ARTIST_FETCHED=$(wc -l < "$LIDARR_TMP/artist_fetches" 2>/dev/null || echo 0)
|
||||
ARTIST_FAILED=$(wc -l < "$LIDARR_TMP/artist_fails" 2>/dev/null || echo 0)
|
||||
ARTIST_MISSING=$(( ARTISTS_CHECKED - ARTISTS_COMPLETE ))
|
||||
echo " Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Fetched: $ARTIST_FETCHED | Failed: $ARTIST_FAILED"
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -356,8 +395,8 @@ echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY LIDARR MISSING ART SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked, $ALBUMS_COMPLETE already complete"
|
||||
echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked, $ARTISTS_COMPLETE already complete"
|
||||
echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched | $ALBUM_FAILED failed"
|
||||
echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched | $ARTIST_FAILED failed"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no files written"
|
||||
|
||||
@@ -164,8 +164,7 @@ if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Radarr URL: $RADARR_URL"
|
||||
log "Movies root: $RADARR_MOVIES_ROOT"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
@@ -298,7 +297,7 @@ fi
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1
|
||||
|
||||
log "Querying Radarr API: $RADARR_URL"
|
||||
echo " Querying Radarr API..."
|
||||
|
||||
# Fetch all movies
|
||||
MOVIES_RESPONSE=$(radarr_api "movie") || {
|
||||
@@ -363,16 +362,14 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
warn "Radarr tracks $TRACKED_COUNT movie files across $MOVIE_COUNT movies"
|
||||
echo " $MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan Movies Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
|
||||
log "Root: $RADARR_MOVIES_ROOT"
|
||||
log "Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
log "Protected: ${RADARR_PROTECTED_PATTERNS[*]}"
|
||||
echo " Root: $RADARR_MOVIES_ROOT | Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Radarr — TMDb Removed ==========================================
|
||||
# ==============================================================================================
|
||||
# Removes movies from Radarr that have been dropped from TMDb.
|
||||
# Radarr marks these with status="deleted" — they generate system health errors and
|
||||
# can never be monitored or downloaded. 99% are future/announced movies that were
|
||||
# delisted before release.
|
||||
#
|
||||
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
|
||||
# Queries Radarr API for movies with status="deleted" (TMDb removal marker)
|
||||
# Reports each found entry with file status and size
|
||||
# Removes the movie record from Radarr
|
||||
# Optionally deletes associated files (disabled by default — most have none)
|
||||
# Optionally adds to Radarr's import exclusion list (default: true)
|
||||
#
|
||||
# ── SAFE DEFAULTS ─────────────────────────────────────────────────────────────────────────────
|
||||
# Files are NOT deleted by default — use --delete-files to also remove from disk
|
||||
# Import exclusion added by default — prevents Radarr re-adding dropped movies
|
||||
# Per-deletion output always visible — deletions are never silently swallowed
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Radarr runs on HOST1 only. detect_hosts() sets RADARR_URL — if empty (HOST2)
|
||||
# the script exits cleanly.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RADARR_DROPPED_ADD_EXCLUSION — add removed movies to import exclusion (default: true)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# radarr_tmdb_removed.sh — remove records, keep files, add exclusion
|
||||
# radarr_tmdb_removed.sh --delete-files — also delete files from disk
|
||||
# radarr_tmdb_removed.sh --dry-run — preview without removing anything
|
||||
# radarr_tmdb_removed.sh --log — verbose output
|
||||
# radarr_tmdb_removed.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Parse --delete-files before standard parse_args ───────────────────────────────────────────
|
||||
DELETE_FILES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--delete-files" ]]; then
|
||||
DELETE_FILES=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "$RADARR_URL" ]]; then
|
||||
log "Radarr not configured for $MY_ID — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ADD_EXCLUSION="${RADARR_DROPPED_ADD_EXCLUSION:-true}"
|
||||
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL"
|
||||
[[ "$DELETE_FILES" == true ]] && warn "DELETE FILES MODE — files will be removed from disk"
|
||||
[[ "$DELETE_FILES" == false ]] && echo " Files: records only (use --delete-files to also remove from disk)"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
|
||||
echo "$ICON_GEAR Add exclusion: $ADD_EXCLUSION"
|
||||
echo "$ICON_GEAR Delete files: $DELETE_FILES"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Query Radarr ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Querying Radarr ━━━"
|
||||
|
||||
if ! curl -sf --connect-timeout 5 --max-time 10 \
|
||||
"$RADARR_URL/api/v3/system/status?apikey=$RADARR_API_KEY" | jq -e '.version' >/dev/null 2>&1; then
|
||||
error "Radarr API unreachable at $RADARR_URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MOVIES=$(curl -sf --connect-timeout 5 --max-time 30 \
|
||||
"$RADARR_URL/api/v3/movie?apikey=$RADARR_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$MOVIES" || "$MOVIES" == "null" ]]; then
|
||||
error "Radarr movie API returned empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL=$(echo "$MOVIES" | jq '. | length')
|
||||
DROPPED=$(echo "$MOVIES" | jq '[.[] | select(.status == "deleted")] | length')
|
||||
echo " $TOTAL movies total — $DROPPED dropped from TMDb"
|
||||
|
||||
if [[ "$DROPPED" -eq 0 ]]; then
|
||||
log "No TMDb-removed movies found — nothing to do"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RADARR TMDB REMOVED SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DONE Status: nothing to remove"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remove Dropped Movies ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Remove TMDb-Dropped Movies ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
REMOVED=()
|
||||
FAILED=()
|
||||
FILES_DELETED=0
|
||||
FILES_SKIPPED=0
|
||||
|
||||
while IFS=$'\t' read -r id title year tmdb_id has_file file_size; do
|
||||
[[ -z "$id" ]] && continue
|
||||
|
||||
SIZE_HUMAN=""
|
||||
if [[ "$has_file" == "true" && "$file_size" -gt 0 ]]; then
|
||||
SIZE_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $file_size / 1073741824}")
|
||||
echo "$ICON_WARN $title ($year) [tmdbid $tmdb_id] — HAS FILE: $SIZE_HUMAN"
|
||||
else
|
||||
echo "$ICON_TRASH $title ($year) [tmdbid $tmdb_id] — no file"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove: $title"
|
||||
[[ "$DELETE_FILES" == true && "$has_file" == "true" ]] && \
|
||||
warn "DRY RUN — would delete file: $SIZE_HUMAN"
|
||||
REMOVED+=("$title")
|
||||
continue
|
||||
fi
|
||||
|
||||
DELETE_PARAM="false"
|
||||
if [[ "$DELETE_FILES" == true && "$has_file" == "true" ]]; then
|
||||
DELETE_PARAM="true"
|
||||
fi
|
||||
|
||||
RESP=$(curl -sf -X DELETE --connect-timeout 5 --max-time 15 \
|
||||
"$RADARR_URL/api/v3/movie/${id}?deleteFiles=${DELETE_PARAM}&addImportExclusion=${ADD_EXCLUSION}&apikey=$RADARR_API_KEY" \
|
||||
2>/dev/null)
|
||||
CURL_EXIT=$?
|
||||
|
||||
if [[ "$CURL_EXIT" -eq 0 ]]; then
|
||||
log " Removed from Radarr ✅"
|
||||
REMOVED+=("$title")
|
||||
if [[ "$DELETE_PARAM" == "true" ]]; then
|
||||
(( FILES_DELETED++ ))
|
||||
elif [[ "$has_file" == "true" ]]; then
|
||||
(( FILES_SKIPPED++ ))
|
||||
fi
|
||||
else
|
||||
warn " Failed to remove $title (curl exit $CURL_EXIT)"
|
||||
FAILED+=("$title")
|
||||
fi
|
||||
|
||||
done < <(echo "$MOVIES" | jq -r '
|
||||
.[] | select(.status == "deleted") |
|
||||
[
|
||||
(.id | tostring),
|
||||
.title,
|
||||
(.year | tostring),
|
||||
(.tmdbId | tostring),
|
||||
(if .hasFile then "true" else "false" end),
|
||||
(if .movieFile.size? then (.movieFile.size | tostring) else "0" end)
|
||||
] | @tsv
|
||||
')
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RADARR TMDB REMOVED SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_TRASH Removed: ${#REMOVED[@]} of $DROPPED"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ "$FILES_DELETED" -gt 0 ]] && echo "$ICON_TRASH Files deleted: $FILES_DELETED"
|
||||
[[ "$FILES_SKIPPED" -gt 0 ]] && echo "$ICON_WARN Files kept: $FILES_SKIPPED (had files — use --delete-files to remove)"
|
||||
[[ "$ADD_EXCLUSION" == "true" ]] && echo "$ICON_GEAR Import exclusion added for removed entries"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
log "$ICON_DONE Status: done ✅"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} removal(s) failed"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -164,8 +164,7 @@ if [[ ! -d "$SONARR_TV_ROOT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Sonarr URL: $SONARR_URL"
|
||||
log "TV root: $SONARR_TV_ROOT"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
@@ -298,7 +297,7 @@ fi
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1
|
||||
|
||||
log "Querying Sonarr API: $SONARR_URL"
|
||||
echo " Querying Sonarr API..."
|
||||
|
||||
# Fetch all series
|
||||
SERIES_RESPONSE=$(sonarr_api "series") || {
|
||||
@@ -354,16 +353,14 @@ if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
warn "Sonarr tracks $TRACKED_COUNT episode files across $SERIES_COUNT series"
|
||||
echo " $SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan TV Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning TV Root ━━━"
|
||||
log "Root: $SONARR_TV_ROOT"
|
||||
log "Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
log "Protected: ${SONARR_PROTECTED_PATTERNS[*]}"
|
||||
echo " Root: $SONARR_TV_ROOT | Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Sonarr — TVDB Removed ==========================================
|
||||
# ==============================================================================================
|
||||
# Removes series from Sonarr that have been dropped from TVDB.
|
||||
# Sonarr marks these with status="deleted" — they generate system health errors and
|
||||
# can never be monitored or downloaded.
|
||||
#
|
||||
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
|
||||
# Queries Sonarr API for series with status="deleted" (TVDB removal marker)
|
||||
# Reports each found entry with file count and total size
|
||||
# Removes the series record from Sonarr
|
||||
# Optionally deletes associated files (disabled by default)
|
||||
# Optionally adds to Sonarr's import exclusion list (default: true)
|
||||
#
|
||||
# ── SAFE DEFAULTS ─────────────────────────────────────────────────────────────────────────────
|
||||
# Files are NOT deleted by default — use --delete-files to also remove from disk
|
||||
# Import exclusion added by default — prevents Sonarr re-adding dropped series
|
||||
# Per-deletion output always visible — deletions are never silently swallowed
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Sonarr runs on HOST1 only. detect_hosts() sets SONARR_URL — if empty (HOST2)
|
||||
# the script exits cleanly.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# SONARR_DROPPED_ADD_EXCLUSION — add removed series to import exclusion (default: true)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# sonarr_tvdb_removed.sh — remove records, keep files, add exclusion
|
||||
# sonarr_tvdb_removed.sh --delete-files — also delete files from disk
|
||||
# sonarr_tvdb_removed.sh --dry-run — preview without removing anything
|
||||
# sonarr_tvdb_removed.sh --log — verbose output
|
||||
# sonarr_tvdb_removed.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Parse --delete-files before standard parse_args ───────────────────────────────────────────
|
||||
DELETE_FILES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--delete-files" ]]; then
|
||||
DELETE_FILES=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "$SONARR_URL" ]]; then
|
||||
log "Sonarr not configured for $MY_ID — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ADD_EXCLUSION="${SONARR_DROPPED_ADD_EXCLUSION:-true}"
|
||||
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL"
|
||||
[[ "$DELETE_FILES" == true ]] && warn "DELETE FILES MODE — files will be removed from disk"
|
||||
[[ "$DELETE_FILES" == false ]] && echo " Files: records only (use --delete-files to also remove from disk)"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
|
||||
echo "$ICON_GEAR Add exclusion: $ADD_EXCLUSION"
|
||||
echo "$ICON_GEAR Delete files: $DELETE_FILES"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Query Sonarr ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Querying Sonarr ━━━"
|
||||
|
||||
if ! curl -sf --connect-timeout 5 --max-time 10 \
|
||||
"$SONARR_URL/api/v3/system/status?apikey=$SONARR_API_KEY" | jq -e '.version' >/dev/null 2>&1; then
|
||||
error "Sonarr API unreachable at $SONARR_URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERIES=$(curl -sf --connect-timeout 5 --max-time 30 \
|
||||
"$SONARR_URL/api/v3/series?apikey=$SONARR_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$SERIES" || "$SERIES" == "null" ]]; then
|
||||
error "Sonarr series API returned empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL=$(echo "$SERIES" | jq '. | length')
|
||||
DROPPED=$(echo "$SERIES" | jq '[.[] | select(.status == "deleted")] | length')
|
||||
echo " $TOTAL series total — $DROPPED dropped from TVDB"
|
||||
|
||||
if [[ "$DROPPED" -eq 0 ]]; then
|
||||
log "No TVDB-removed series found — nothing to do"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SONARR TVDB REMOVED SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DONE Status: nothing to remove"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remove Dropped Series ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Remove TVDB-Dropped Series ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
REMOVED=()
|
||||
FAILED=()
|
||||
FILES_DELETED=0
|
||||
FILES_SKIPPED=0
|
||||
|
||||
while IFS=$'\t' read -r id title year tvdb_id episode_file_count size_on_disk; do
|
||||
[[ -z "$id" ]] && continue
|
||||
|
||||
SIZE_HUMAN=""
|
||||
if [[ "$size_on_disk" -gt 0 ]]; then
|
||||
SIZE_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $size_on_disk / 1073741824}")
|
||||
fi
|
||||
|
||||
if [[ "$episode_file_count" -gt 0 ]]; then
|
||||
echo "$ICON_WARN $title ($year) [tvdbid $tvdb_id] — $episode_file_count episode files${SIZE_HUMAN:+, $SIZE_HUMAN}"
|
||||
else
|
||||
echo "$ICON_TRASH $title ($year) [tvdbid $tvdb_id] — no files"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove: $title"
|
||||
[[ "$DELETE_FILES" == true && "$episode_file_count" -gt 0 ]] && \
|
||||
warn "DRY RUN — would delete $episode_file_count file(s)${SIZE_HUMAN:+, $SIZE_HUMAN}"
|
||||
REMOVED+=("$title")
|
||||
continue
|
||||
fi
|
||||
|
||||
DELETE_PARAM="false"
|
||||
if [[ "$DELETE_FILES" == true && "$episode_file_count" -gt 0 ]]; then
|
||||
DELETE_PARAM="true"
|
||||
fi
|
||||
|
||||
RESP=$(curl -sf -X DELETE --connect-timeout 5 --max-time 15 \
|
||||
"$SONARR_URL/api/v3/series/${id}?deleteFiles=${DELETE_PARAM}&addImportListExclusion=${ADD_EXCLUSION}&apikey=$SONARR_API_KEY" \
|
||||
2>/dev/null)
|
||||
CURL_EXIT=$?
|
||||
|
||||
if [[ "$CURL_EXIT" -eq 0 ]]; then
|
||||
log " Removed from Sonarr ✅"
|
||||
REMOVED+=("$title")
|
||||
if [[ "$DELETE_PARAM" == "true" ]]; then
|
||||
(( FILES_DELETED++ ))
|
||||
elif [[ "$episode_file_count" -gt 0 ]]; then
|
||||
(( FILES_SKIPPED++ ))
|
||||
fi
|
||||
else
|
||||
warn " Failed to remove $title (curl exit $CURL_EXIT)"
|
||||
FAILED+=("$title")
|
||||
fi
|
||||
|
||||
done < <(echo "$SERIES" | jq -r '
|
||||
.[] | select(.status == "deleted") |
|
||||
[
|
||||
(.id | tostring),
|
||||
.title,
|
||||
(.year | tostring),
|
||||
(.tvdbId | tostring),
|
||||
(if .episodeFileCount? then (.episodeFileCount | tostring) else "0" end),
|
||||
(if .sizeOnDisk? then (.sizeOnDisk | tostring) else "0" end)
|
||||
] | @tsv
|
||||
')
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SONARR TVDB REMOVED SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_TRASH Removed: ${#REMOVED[@]} of $DROPPED"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ "$FILES_DELETED" -gt 0 ]] && echo "$ICON_TRASH Files deleted: $FILES_DELETED series worth"
|
||||
[[ "$FILES_SKIPPED" -gt 0 ]] && echo "$ICON_WARN Files kept: $FILES_SKIPPED series (had files — use --delete-files to remove)"
|
||||
[[ "$ADD_EXCLUSION" == "true" ]] && echo "$ICON_GEAR Import exclusion added for removed entries"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
log "$ICON_DONE Status: done ✅"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} removal(s) failed"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -9,25 +9,32 @@
|
||||
# Pre-sync:
|
||||
# git_pull_execute.sh — pull latest scripts first, always
|
||||
#
|
||||
# Arr Sync (arr_sync.sh):
|
||||
# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||||
# All nodes agree on tracked library before any files are transferred.
|
||||
# Remote nodes that don't have an arr running are skipped gracefully.
|
||||
#
|
||||
# Rsync window (DAILY_SYNC_SHARES per host):
|
||||
# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror
|
||||
# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete)
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
#
|
||||
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
|
||||
# media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# media_cleaner.sh anime — remove junk from anime shares
|
||||
# media_cleaner.sh media — remove junk from media shares
|
||||
# lidarr_cleanup.sh — remove orphaned music files
|
||||
# sonarr_cleanup.sh — remove orphaned TV files
|
||||
# radarr_cleanup.sh — remove orphaned movie files
|
||||
# lidarr_cleanup.sh — remove orphaned music files (local arr = truth)
|
||||
# sonarr_cleanup.sh — remove orphaned TV files (local arr = truth)
|
||||
# radarr_cleanup.sh — remove orphaned movie files (local arr = truth)
|
||||
# docker_daily_restart.sh — restart containers needing daily restart
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# git pull first — maintenance runs on latest code, not yesterday's
|
||||
# Rsync before cleanup — cleanup sees the post-sync state
|
||||
# Permissions before arr cleanup — arrs need correct ownership to delete/rename
|
||||
# Arr cleanup after permissions — clean ownership = successful orphan deletion
|
||||
# Docker restart last — containers already processed by cleanup
|
||||
# arr sync before rsync — all nodes track the same library before files are spread;
|
||||
# prevents remote arrs from searching for content already owned
|
||||
# rsync before cleanup — cleanup sees fully spread state, rsync has no --delete
|
||||
# permissions before arr cleanup — arrs need correct ownership to delete/rename
|
||||
# arr cleanup after permissions — clean ownership = successful orphan deletion
|
||||
# docker restart last — containers already processed by cleanup
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Bidirectional — same script runs on both servers, correct direction automatic.
|
||||
@@ -193,6 +200,31 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Arr Sync — library reconciliation before file spreading ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
else
|
||||
_arr_sync_args=()
|
||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||
log "Arr sync complete ✅"
|
||||
JOB_PASS+=("arr_sync.sh")
|
||||
else
|
||||
warn "Arr sync completed with errors — continuing to rsync"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
fi
|
||||
unset _arr_sync_args
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Media Share Sync ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# ==============================================================================================
|
||||
# Resets the fallback state file to NORMAL and clears all tier flags.
|
||||
# Use when the fallback state file is stuck in a non-NORMAL state after:
|
||||
# - Failover testing that left state as FAILOVER
|
||||
# - Failover testing that left state as FALLBACK
|
||||
# - A failed handback that did not complete cleanly
|
||||
# - Manual intervention that left state inconsistent
|
||||
# - fallback.sh was killed mid-cycle and state is unknown
|
||||
|
||||
@@ -490,7 +490,7 @@ detect_hosts() {
|
||||
for _wd_var in "${_wd_checks[@]}"; do
|
||||
local _wd_src="${MY_ID}_${_wd_var}"
|
||||
# Only alias if the host-specific var is set — preserves master.conf defaults
|
||||
[[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}="\${!_wd_src}""
|
||||
[[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\""
|
||||
done
|
||||
|
||||
TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD"
|
||||
@@ -782,6 +782,14 @@ check_remote_share() {
|
||||
# ── DISK TEMPERATURE CHECKS ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
||||
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
||||
is_ssd() {
|
||||
local base
|
||||
base=$(basename "$1")
|
||||
[[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]]
|
||||
}
|
||||
|
||||
# Reads disk temperature thresholds from unRAID's dynamix.cfg.
|
||||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||||
# Falls back to values in master.conf (SMART_TEMP_WARN/CRIT) if file not found.
|
||||
|
||||
+10
-12
@@ -62,8 +62,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
|
||||
@@ -84,7 +82,7 @@ else
|
||||
log "Gitea not running locally — checking remote server"
|
||||
GITEA_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null)
|
||||
if [[ -n "$GITEA_IP" ]]; then
|
||||
info "$ICON_NET Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
||||
echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
||||
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
|
||||
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
|
||||
GITEA_IP="$GITEA_DOMAIN"
|
||||
@@ -172,7 +170,7 @@ configure_sparse_checkout() {
|
||||
done
|
||||
|
||||
if [[ "$excluded" -gt 0 ]]; then
|
||||
info "$ICON_LOCK Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
||||
echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
||||
else
|
||||
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
|
||||
fi
|
||||
@@ -201,7 +199,7 @@ else
|
||||
|
||||
if [[ -d ".git" ]]; then
|
||||
# ── Existing repository ──────────────────────────────────────────────
|
||||
info "$ICON_SYNC Existing repository detected — updating"
|
||||
echo " Existing repository — updating"
|
||||
|
||||
# Configure sparse checkout BEFORE pull
|
||||
# Uses conf files already present from last pull to determine exclusions
|
||||
@@ -213,9 +211,9 @@ else
|
||||
log "git clean -fd"
|
||||
git clean -fd
|
||||
|
||||
info "Pulling latest changes..."
|
||||
echo " Pulling latest changes..."
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull; then
|
||||
success "Git pull successful"
|
||||
echo " Git pull successful"
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
error "Git pull failed"
|
||||
@@ -225,20 +223,20 @@ else
|
||||
|
||||
else
|
||||
# ── Fresh clone ──────────────────────────────────────────────────────
|
||||
info "$ICON_SYNC No repository found — cloning"
|
||||
echo " No repository found — cloning"
|
||||
|
||||
# Clone first — need the repo to exist before configuring sparse checkout
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
|
||||
success "Clone successful"
|
||||
echo " Clone successful"
|
||||
|
||||
# Configure sparse checkout after clone
|
||||
# Now all master_host*.conf files are present — can detect exclusions
|
||||
configure_sparse_checkout "$TARGET_DIR"
|
||||
|
||||
# Apply sparse checkout — removes excluded files from working tree
|
||||
info "Applying sparse checkout..."
|
||||
echo " Applying sparse checkout..."
|
||||
git read-tree -mu HEAD
|
||||
success "Sparse checkout applied — peer credentials removed from working tree"
|
||||
echo " Sparse checkout applied — peer credentials removed from working tree"
|
||||
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
@@ -253,7 +251,7 @@ else
|
||||
echo "━━━ $ICON_GEAR Permissions ━━━"
|
||||
log "Setting executable permissions on all .sh files..."
|
||||
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
|
||||
log "Permissions applied"
|
||||
echo " Permissions set on .sh files"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
+2
-2
@@ -6,9 +6,9 @@
|
||||
# Every script sources this file instead of sourcing master.conf files directly.
|
||||
#
|
||||
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Sources master.conf (shared config — thresholds, toggles, profiles, job lists)
|
||||
# 1. Sources master.conf (shared config — hostnames, thresholds, toggles, profiles, job lists)
|
||||
# 2. Auto-discovers and sources all master_host*.conf files in the same directory
|
||||
# Each host conf extends the shared profile arrays and adds host-specific identity
|
||||
# Each host conf extends the shared profile arrays and adds host-specific credentials
|
||||
# 3. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.)
|
||||
#
|
||||
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
+39
-9
@@ -47,8 +47,8 @@
|
||||
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||||
# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs
|
||||
#
|
||||
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
||||
# FAILOVER Mutual container failover shared settings
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# FALLBACK Mutual container failover shared settings
|
||||
# FALLBACK TEST Simulated outage settings for fallback_test.sh
|
||||
#
|
||||
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||||
@@ -92,10 +92,20 @@
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HOST IDENTITIES ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Hostnames for every server in the ecosystem — not credentials, safe for all machines.
|
||||
# Must match the exact unRAID hostname AND Tailscale device name (case sensitive).
|
||||
# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID.
|
||||
# Tailscale IP resolution uses these names — no hardcoded IPs needed.
|
||||
# To add a new server: add HOST3="unRAID-NewServer" here + create master_host3.conf.
|
||||
HOST1="unRAID-Gmer4Lfe"
|
||||
HOST2="unRAID-Jayred365"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Hostnames defined in master_host*.conf — detect_hosts() reads them at runtime.
|
||||
# DATA_DIR is the same path on all servers — persistent script state and statistics.
|
||||
# Array share — survives reboots, no flash drive wear.
|
||||
# Created automatically if it doesn't exist.
|
||||
@@ -109,6 +119,19 @@
|
||||
# "abort" — refuse to continue (strict — ensures both sides always match)
|
||||
UNRAID_VERSION_MISMATCH_ACTION="warn"
|
||||
|
||||
# ── Arr Sync ──
|
||||
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||||
# Runs before rsync — all nodes agree on tracked library before files are transferred.
|
||||
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
|
||||
ARR_SYNC_ENABLED=true
|
||||
ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
|
||||
ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
|
||||
ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
|
||||
DOCKER_APPDATA_BASE="/mnt/user/appdata"
|
||||
ARR_SYNC_LIDARR_PORT=8686
|
||||
ARR_SYNC_SONARR_PORT=8989
|
||||
ARR_SYNC_RADARR_PORT=7878
|
||||
|
||||
# ── Remote Docker Daemon ──
|
||||
# Strike system for remote Docker daemon health checks.
|
||||
# Scripts that issue remote container commands check the remote daemon first.
|
||||
@@ -269,6 +292,8 @@
|
||||
#"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready
|
||||
#"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready
|
||||
#"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready
|
||||
"Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||||
)
|
||||
@@ -373,10 +398,11 @@
|
||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||
EXCLUDE_DIRS=() # directories excluded from transfer — profiles override
|
||||
|
||||
# --delete removes files on remote not on source (mirror behaviour)
|
||||
# --inplace writes directly to destination — better for large files
|
||||
# --no-whole-file forces delta transfer — sends only changed blocks
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||||
# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
|
||||
# Media shares use this default; rsync spreads files only, never removes them.
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --no-whole-file)
|
||||
|
||||
# ━━━ Remote Health Checks ━━━
|
||||
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
|
||||
@@ -502,13 +528,13 @@
|
||||
# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Mutual container failover between two unRAID servers.
|
||||
# Each server runs Fallback/fallback.sh independently via array_start.sh.
|
||||
# All decisions based on two pings: remote reachable + internet reachable.
|
||||
#
|
||||
# States: NORMAL | FAILOVER | NO_INTERNET | DARK
|
||||
# States: NORMAL | FALLBACK | NO_INTERNET | DARK
|
||||
#
|
||||
# DDNS rules — absolute:
|
||||
# Internet loss → stop own DDNS immediately
|
||||
@@ -519,8 +545,8 @@
|
||||
# Shared settings (intervals, state file, thresholds) live here.
|
||||
|
||||
EXTERNAL_IP="8.8.8.8"
|
||||
FALLBACK_CHECK_INTERVAL=120 # seconds between fallback state checks
|
||||
FALLBACK_HANDBACK_STRIKES=2 # consecutive healthy checks before initiating handback
|
||||
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
||||
FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
|
||||
FALLBACK_STATE_FILE="/boot/config/fallback_state.db"
|
||||
FALLBACK_ENABLED=false # HOST2 being rebuilt — set true when back online and tested
|
||||
# false = suppresses "not running" warnings in status scripts
|
||||
@@ -910,6 +936,10 @@
|
||||
"theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||||
)
|
||||
|
||||
# Radarr/Sonarr TMDb/TVDB removed entry cleanup
|
||||
RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list
|
||||
SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list
|
||||
|
||||
# ━━━ Arr Failed/Stalled Recovery ━━━
|
||||
# Auto blocklist + re-search failed imports and stalled downloads.
|
||||
# Runs every 6 hours — schedule: 0 */6 * * *
|
||||
|
||||
+4
-7
@@ -33,10 +33,10 @@
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FAILOVER TIERS what HOST1 runs for HOST2 per tier
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
@@ -66,10 +66,7 @@
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive.
|
||||
# Used by detect_hosts() in common.sh to identify this server as HOST1.
|
||||
HOST1="unRAID-Gmer4Lfe"
|
||||
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
@@ -278,7 +275,7 @@
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
|
||||
+4
-7
@@ -37,10 +37,10 @@
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST2
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FAILOVER TIERS what HOST2 runs for HOST1 per tier
|
||||
# FALLBACK TIERS what HOST2 runs for HOST1 per tier
|
||||
# TIER DELAYS how long HOST2 must be down before each tier activates on HOST1
|
||||
# RSYNC WRITEBACK HOST2 appdata synced back on handback
|
||||
#
|
||||
@@ -68,10 +68,7 @@
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive.
|
||||
# Used by detect_hosts() in common.sh to identify this server as HOST2.
|
||||
HOST2="unRAID-Jayred365"
|
||||
|
||||
# HOST2 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys.
|
||||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||||
@@ -241,7 +238,7 @@
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
|
||||
@@ -203,7 +203,7 @@
|
||||
# bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
|
||||
|
||||
# ── RSYNC EMBY FAILOVER ───────────────────────────────────────────────────────────────────────
|
||||
# ── RSYNC EMBY FALLBACK ───────────────────────────────────────────────────────────────────────
|
||||
# Schedule: */30 * * * * (every 30 minutes)
|
||||
# Background: YES
|
||||
#
|
||||
@@ -375,14 +375,14 @@
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
# FAILOVER
|
||||
# FALLBACK
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# fallback.sh — [continuous] mutual fallback state machine
|
||||
# Started by array_start.sh on both servers independently.
|
||||
# Every FALLBACK_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8
|
||||
# States: NORMAL / FAILOVER / NO_INTERNET / DARK
|
||||
# FAILOVER: starts remote containers in tiers across 24 hours:
|
||||
# States: NORMAL / FALLBACK / NO_INTERNET / DARK
|
||||
# FALLBACK: starts remote containers in tiers across 24 hours:
|
||||
# Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV
|
||||
# Tier 2 at 4hr: NextCloud + Immich + Jellyseerr
|
||||
# Tier 3 at 12hr: AdGuard + Gitea + uptime monitoring
|
||||
@@ -399,7 +399,7 @@
|
||||
# ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption.
|
||||
# iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes.
|
||||
# Remote is always reachable after the test completes regardless of what happened.
|
||||
# 8 phases: pre-flight → block remote (iptables) → detect FAILOVER → verify Tier 1 started
|
||||
# 8 phases: pre-flight → block remote (iptables) → detect FALLBACK → verify Tier 1 started
|
||||
# → restore remote → wait handback → verify NORMAL restored → full pass/fail report.
|
||||
# FALLBACK_TEST_BLOCK_WAIT must be > FALLBACK_CHECK_INTERVAL + buffer (default: 150s).
|
||||
# FALLBACK_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s).
|
||||
@@ -409,7 +409,7 @@
|
||||
|
||||
# fallback_state_reset.sh — reset fallback state file to NORMAL
|
||||
# Resets state file ONLY — does NOT start or stop any containers.
|
||||
# Use when: state stuck in FAILOVER after testing, after killing fallback.sh mid-cycle,
|
||||
# Use when: state stuck in FALLBACK after testing, after killing fallback.sh mid-cycle,
|
||||
# after manual intervention left state inconsistent, after a failed handback.
|
||||
# Shows current state file before asking for confirmation.
|
||||
# Verify before resetting: right containers on right server, DDNS correct,
|
||||
|
||||
Reference in New Issue
Block a user