diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh index df8aa3c..61a5f90 100755 --- a/Media/arr_sync.sh +++ b/Media/arr_sync.sh @@ -25,10 +25,16 @@ # # ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── # ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added anywhere. -# Read from ALL nodes via SSH at start of each run — immediate effect before rsync. -# Propagates to all nodes via rsync on the next cycle. +# Read from ALL nodes via SSH at start of each run — immediate effect across all nodes. +# (Each node reads every other node's blocklist file via SSH — no rsync delay.) # Manage via --blocklist-add / --blocklist-remove / --blocklist-list. # +# --blocklist-add does three things atomically: +# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync) +# 2. Deletes the item from the local arr API (deleteFiles=false) +# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false) +# Files become orphans on all nodes — arr_cleanup.sh removes them on next run. +# # ── GRACEFUL SKIP ───────────────────────────────────────────────────────────────────────────── # Arr not configured locally → skip cleanly, no error. # Arr not reachable on a remote → skip that node for that arr type, continue with others. @@ -47,8 +53,10 @@ # arr_sync.sh --dry-run — preview only, no changes # arr_sync.sh --log — verbose output # arr_sync.sh --status — show config and exit -# arr_sync.sh --blocklist-add lidarr "reason" — tombstone an ID on all nodes -# arr_sync.sh --blocklist-remove lidarr — un-tombstone an ID +# arr_sync.sh --blocklist-add lidarr "reason" — remove from all arrs + tombstone +# arr_sync.sh --blocklist-add sonarr "reason" — remove from all arrs + tombstone +# arr_sync.sh --blocklist-add radarr "reason" — remove from all arrs + tombstone +# arr_sync.sh --blocklist-remove lidarr — un-tombstone (does NOT re-add) # arr_sync.sh --blocklist-list — show all blocklisted IDs # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── @@ -238,6 +246,68 @@ _blocklist_remove() { fi } +# Look up item in local arr by stable_id — returns "internal_id\tdisplay_name" or empty +_lookup_local_item() { + local url="$1" api_key="$2" api_ver="$3" endpoint="$4" + local id_field="$5" id_type="$6" name_field="$7" stable_id="$8" + local raw select_expr + raw=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_ver}/${endpoint}" 2>/dev/null) + [[ -z "$raw" ]] && return 1 + if [[ "$id_type" == "string" ]]; then + select_expr=".[] | select(.${id_field} == \"${stable_id}\")" + else + select_expr=".[] | select(.${id_field} == ${stable_id})" + fi + echo "$raw" | jq -r "${select_expr} | [(.id | tostring), .${name_field}] | @tsv" 2>/dev/null | head -1 +} + +# Delete item from local arr by internal integer id — returns HTTP status code +_delete_local_item() { + local url="$1" api_key="$2" api_ver="$3" endpoint="$4" internal_id="$5" + curl -sf -o /dev/null -w '%{http_code}' -X DELETE \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null +} + +# Delete item from remote arr by stable_id via SSH. +# Outputs: HTTP code on success | "not_found" if item absent | empty on SSH/API failure. +# deleteFiles=false — files become orphans for arr_cleanup to handle with its safety checks. +_delete_remote_item() { + local node_id="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5" + local id_field="$6" id_type="$7" stable_id="$8" + local node_name="${!node_id}" + local ts_name="${node_name,,}" + local node_ip + node_ip=$(tailscale ip -4 "$ts_name" 2>/dev/null) + [[ -z "$node_ip" ]] && node_ip=$(tailscale status 2>/dev/null | \ + awk -v n="$ts_name" '$2 ~ "^" n { print $1; exit }') + [[ -z "$node_ip" ]] && return 1 + + local select_expr + if [[ "$id_type" == "string" ]]; then + select_expr=".[] | select(.${id_field} == \"${stable_id}\") | .id" + else + select_expr=".[] | select(.${id_field} == ${stable_id}) | .id" + fi + + ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" bash </dev/null +KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) +[[ -z "\$KEY" ]] && exit 1 +LIBRARY=\$(curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \ + -H "X-Api-Key: \$KEY" \ + "http://localhost:${port}/api/${api_ver}/${endpoint}" 2>/dev/null) +[[ -z "\$LIBRARY" ]] && exit 1 +INTERNAL_ID=\$(echo "\$LIBRARY" | jq -r '${select_expr}' 2>/dev/null | head -1) +[[ -z "\$INTERNAL_ID" ]] && echo "not_found" && exit 0 +curl -sf -o /dev/null -w '%{http_code}' -X DELETE \ + -H "X-Api-Key: \$KEY" \ + "http://localhost:${port}/api/${api_ver}/${endpoint}/\${INTERNAL_ID}?deleteFiles=false" +REMOTE +} + # ── Blocklist management mode ────────────────────────────────────────────────────────────────── if [[ -n "$BLOCKLIST_ACTION" ]]; then case "$BLOCKLIST_ACTION" in @@ -264,8 +334,65 @@ if [[ -n "$BLOCKLIST_ACTION" ]]; then error " arr_type: lidarr | sonarr | radarr" exit 1 fi - _blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "${BLOCKLIST_ID}" "${BLOCKLIST_REASON:-manually excluded}" - echo " Blocklisted [$BLOCKLIST_ARR] $BLOCKLIST_ID — will propagate via rsync on next cycle" + if [[ -z "${_PORT[$BLOCKLIST_ARR]:-}" ]]; then + error "Unknown arr type: $BLOCKLIST_ARR — use lidarr, sonarr, or radarr" + exit 1 + fi + + _bl_port="${_PORT[$BLOCKLIST_ARR]}" + _bl_ver="${_VER[$BLOCKLIST_ARR]}" + _bl_ep="${_EP[$BLOCKLIST_ARR]}" + _bl_id_field="${_ID[$BLOCKLIST_ARR]}" + _bl_id_type="${_ID_TYPE[$BLOCKLIST_ARR]}" + _bl_name_field="${_NAME[$BLOCKLIST_ARR]}" + _bl_config_xml="${DOCKER_APPDATA_BASE}/${BLOCKLIST_ARR^}/config.xml" + _bl_url="" _bl_key="" + case "$BLOCKLIST_ARR" in + lidarr) _bl_url="${LIDARR_URL:-}"; _bl_key="${LIDARR_API_KEY:-}" ;; + sonarr) _bl_url="${SONARR_URL:-}"; _bl_key="${SONARR_API_KEY:-}" ;; + radarr) _bl_url="${RADARR_URL:-}"; _bl_key="${RADARR_API_KEY:-}" ;; + esac + + # Look up display name and internal id from local arr + _bl_display_name="$BLOCKLIST_ID" + _bl_internal_id="" + if [[ -n "$_bl_url" ]] && [[ -n "$_bl_key" ]]; then + _bl_lookup=$(_lookup_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" \ + "$_bl_id_field" "$_bl_id_type" "$_bl_name_field" "$BLOCKLIST_ID") + if [[ -n "$_bl_lookup" ]]; then + IFS=$'\t' read -r _bl_internal_id _bl_display_name <<< "$_bl_lookup" + fi + fi + + _blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "$_bl_display_name" "${BLOCKLIST_REASON:-manually excluded}" + echo " Blocklisted [$BLOCKLIST_ARR] $_bl_display_name ($BLOCKLIST_ID)" + + # Remove from local arr (deleteFiles=false — arr_cleanup handles file removal) + if [[ -n "$_bl_internal_id" ]]; then + _bl_http=$(_delete_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" "$_bl_internal_id") + if [[ "$_bl_http" == "200" ]]; then + log "Removed from local ${BLOCKLIST_ARR^}: $_bl_display_name" + else + warn "Failed to remove from local ${BLOCKLIST_ARR^} (HTTP ${_bl_http:-no response}) — remove manually via UI" + fi + else + log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally" + fi + + # Remove from all remote arrs + for _bl_node_id in "${REMOTE_NODES[@]}"; do + _bl_node_name="${!_bl_node_id}" + _bl_result=$(_delete_remote_item "$_bl_node_id" "$_bl_port" "$_bl_ver" "$_bl_ep" \ + "$_bl_config_xml" "$_bl_id_field" "$_bl_id_type" "$BLOCKLIST_ID") + case "$_bl_result" in + 200) log "Removed from $_bl_node_name ${BLOCKLIST_ARR^}: $_bl_display_name" ;; + not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;; + *) warn "Failed to remove from $_bl_node_name ${BLOCKLIST_ARR^} (${_bl_result:-SSH error}) — remove manually via UI" ;; + esac + done + + echo "" + echo " Files are now orphans on all nodes — arr_cleanup.sh will remove them on next run" exit 0 ;; remove) diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh new file mode 100644 index 0000000..63b5d54 --- /dev/null +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -0,0 +1,312 @@ +#!/bin/bash +# ============================================================================================== +# =========================== Intermediate Sync Maintenance ==================================== +# ============================================================================================== +# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync. +# Schedule: 0 */4 * * * (every 4 hours) +# +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# 1. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes +# 2. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured +# 3. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs +# +# ── WHY A SEPARATE ORCHESTRATOR ─────────────────────────────────────────────────────────────── +# arr libraries need to converge more frequently than once a day. If a remote node adds +# something at 2am, the next daily window is 23 hours away — remote arrs search for content +# they don't know is already owned. Running every 4 hours closes that gap. +# +# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill. +# Pairing it here means artwork catches up within 4 hours of a new album landing. +# +# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config +# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media +# share sync stays in the daily window. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID, DAILY_SYNC_SHARES, PERSONAL_SHARES from HOST*_ vars. +# INTERMEDIATE_SYNC_SHARES is a shared list in master.conf — same on all servers. +# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic. +# +# ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── +# Same as daily_sync_maintenance.sh: +# exit 1 = temp WARN — skip this share, continue to next +# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — scripts called here require root +# acquire_lock — prevents concurrent intermediate windows +# check_connectivity — verified before any rsync (skipped if no shares) +# check_remote_rootfs — aborts rsync if remote rootfs nearly full +# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch +# Silent on success — runs 4x/day, only failures warrant notification +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) +# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) +# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync +# ARR_SYNC_ENABLED — toggle inside arr_sync.sh +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# intermediate_sync_maintenance.sh — normal run +# intermediate_sync_maintenance.sh --dry-run — preview without changes +# intermediate_sync_maintenance.sh --log — verbose per-job output +# intermediate_sync_maintenance.sh --status — show configured shares/jobs and exit +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" +SCRIPTS_ROOT="$SCRIPT_DIR/.." + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +detect_hosts +resolve_remote_ip + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ── Helper — run a maintenance job, track pass/fail ─────────────────────────────────────────── +run_job() { + local script_entry="$1" + local extra_dry="" + [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" + + read -r -a script_args <<< "$script_entry" + local script_path="$SCRIPTS_ROOT/${script_args[0]}" + local script_name + script_name=$(basename "${script_args[0]}") + local extra_args=("${script_args[@]:1}") + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + return 1 + fi + + log "Running: $script_name ${extra_args[*]}" + # shellcheck disable=SC2086 + if bash "$script_path" "${extra_args[@]}" $extra_dry; then + log "$script_name — done ✅" + JOB_PASS+=("$script_name ${extra_args[*]}") + else + warn "$script_name — failed (exit $?)" + JOB_FAIL+=("$script_name ${extra_args[*]}") + fi +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" + echo "$ICON_SYNC Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}" + echo "$ICON_GEAR Arr sync: ${ARR_SYNC_ENABLED:-true}" + echo "" + echo "━━━ Intermediate Sync Shares ━━━" + if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then + echo " None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable" + else + for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do + [[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)" + done + fi + echo "" + echo "━━━ Intermediate Maintenance Scripts ━━━" + if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then + echo " None configured" + else + for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do + [[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}" + done + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +WINDOW_START=$(date +%s) +JOB_PASS=() +JOB_FAIL=() +PASS=() +FAIL=() +SHARE_TIMES=() + +echo "" +echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +# ============================================================================================== +# ━━━ Arr Sync ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Arr Sync ━━━" + +ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" +if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then + log "ARR_SYNC_ENABLED=false — skipping" +elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then + warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping" + JOB_FAIL+=("arr_sync.sh") +else + _arr_sync_args=() + [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run") + if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then + log "Arr sync complete ✅" + JOB_PASS+=("arr_sync.sh") + else + warn "Arr sync completed with errors — continuing" + JOB_FAIL+=("arr_sync.sh") + fi + unset _arr_sync_args +fi + +# ============================================================================================== +# ━━━ Rsync (optional) ━━━ +# ============================================================================================== +SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]} + +echo "" +echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━" + +TOTAL_START=$(date +%s) +SHARE_INDEX=0 +ABORT_ALL_SYNCS=false + +if [[ "$SHARE_COUNT" -eq 0 ]]; then + log "No INTERMEDIATE_SYNC_SHARES configured — skipping" + log "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync" +elif ! check_rsync_enabled "INTERMEDIATE"; then + warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)" +else + check_connectivity + check_remote_rootfs + + RSYNC_DRY="" + [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" + + for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do + (( SHARE_INDEX++ )) + SHARE_NAME=$(basename "$SHARE") + SHARE_START=$(date +%s) + + echo "" + echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━" + + if [[ "$ABORT_ALL_SYNCS" == true ]]; then + warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)" + FAIL+=("$SHARE_NAME:temp-critical") + continue + fi + + bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY + RSYNC_EXIT=$? + + SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))") + + case "$RSYNC_EXIT" in + 0) + PASS+=("$SHARE_NAME") + log "$SHARE_NAME — done ✅" + ;; + 1) + FAIL+=("$SHARE_NAME:temp-warn") + warn "$SHARE_NAME skipped — drive temps too high" + ;; + 2) + FAIL+=("$SHARE_NAME:temp-critical") + ABORT_ALL_SYNCS=true + error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs" + notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \ + "Intermediate Sync" "warning" + ;; + *) + FAIL+=("$SHARE_NAME") + error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share" + ;; + esac + done +fi + +TOTAL_END=$(date +%s) + +# ============================================================================================== +# ━━━ Maintenance Jobs ━━━ +# ============================================================================================== +if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_GEAR Maintenance Jobs ━━━" + for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + echo "" + run_job "$script_entry" + done +fi + +WINDOW_END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" +echo "" + +if [[ "$SHARE_COUNT" -gt 0 ]]; then + echo "$ICON_SYNC Shares ($SHARE_COUNT):" + for entry in "${SHARE_TIMES[@]}"; do + sname="${entry%%:*}" + sdur="${entry##*:}" + if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then + echo " $ICON_ERROR $sname — $(format_duration "$sdur")" + else + echo " $ICON_DONE $sname — $(format_duration "$sdur")" + fi + done + echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" + echo "" +fi + +if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then + echo "$ICON_GEAR Jobs:" + for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done + for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done + echo "" +fi + +TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ "$TOTAL_FAIL" -eq 0 ]]; then + log "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" +else + warn "Status: $TOTAL_FAIL failure(s)" + notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ + "Intermediate Sync" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 +exit 0 diff --git a/master.conf b/master.conf index d1dff61..832b95a 100644 --- a/master.conf +++ b/master.conf @@ -280,6 +280,23 @@ # "Fallback/fallback.sh" # mutual failover — enable when HOST2 ready ) +# ━━━ Intermediate Sync Maintenance ━━━ +# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, +# and optional mid-day rsync for any shares that need sub-daily propagation. +# Schedule: 0 */4 * * * + INTERMEDIATE_SYNC_SHARES=( + # Add shares here to enable mid-day rsync — empty = rsync section skipped entirely. + # Uses DEFAULT_RSYNC_OPTS (no --delete). Full media sync stays in the daily window. + # Example: "/mnt/user/Emby_Metadata" + ) + INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares + + INTERMEDIATE_MAINTENANCE_SCRIPTS=( + "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) + ) + # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. + # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). + # ━━━ Daily Sync Maintenance ━━━ # daily_sync_maintenance.sh runs media share sync first, then iterates # DAILY_MAINTENANCE_SCRIPTS for all jobs. @@ -305,9 +322,10 @@ # Media shares synced daily by daily_sync_maintenance.sh. # Defined per-host in master_host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. -# Each server syncs only the shares it owns — direction is automatic. -# HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1. -# Never both pushing the same share — one server is always the truth holder. +# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. +# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track. # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. # For shares needing custom options or container stops — create a profile in RSYNC section. @@ -398,11 +416,16 @@ CONTAINER_DELAY=5 # seconds before starting delayed containers EXCLUDE_DIRS=() # directories excluded from transfer — profiles override -# --inplace writes directly to destination — better for large files -# --no-whole-file forces delta transfer — sends only changed blocks +# --inplace writes directly to destination — delta against existing file, better for large media +# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers +# --timeout kill stalled transfers instead of hanging indefinitely +# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers # --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. # Media shares use this default; rsync spreads files only, never removes them. - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --no-whole-file) +# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default). +# Note: arr_cleanup only catches true orphans under the union model — intentional removals +# require arr_sync.sh --blocklist-add first, then manual file deletion. + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids) # ━━━ Remote Health Checks ━━━ # Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. diff --git a/master_host1.conf b/master_host1.conf index e5e0c27..a0e443b 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -110,9 +110,12 @@ # ============================================================================================== # ━━━ Daily Sync Shares ━━━ -# Shares HOST1 owns and pushes to HOST2 every night (1am via daily_sync_maintenance.sh). -# HOST1 is the source of truth — HOST2 is the mirror. -# Never push a share both directions — one server always owns it. +# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh). +# Mesh model: every node pushes every media share — no ownership, no mirrors. +# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete). +# arr_cleanup removes true orphans based on local arr state. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed). # Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. # For shares needing container stops or custom options — add a profile in master.conf. HOST1_DAILY_SYNC_SHARES=( @@ -129,6 +132,8 @@ /mnt/user/Tv_Shows /mnt/user/Anime_Shows-Old /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows ) # Personal encrypted shares — synced for offsite backup, independent of media shares. diff --git a/master_host2.conf b/master_host2.conf index adb18f4..9bd8546 100644 --- a/master_host2.conf +++ b/master_host2.conf @@ -110,14 +110,29 @@ # ============================================================================================== # ━━━ Daily Sync Shares ━━━ -# Shares HOST2 owns and pushes to HOST1 every night (1am via daily_sync_maintenance.sh). -# HOST2 is the source of truth — HOST1 is the mirror. -# Never push a share both directions — one server always owns it. +# Shares HOST2 pushes to all other nodes every night (1am via daily_sync_maintenance.sh). +# Mesh model: every node pushes every media share — no ownership, no mirrors. +# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete). +# arr_cleanup removes true orphans based on local arr state. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Nextcloud excluded — personal data, not arr-managed, synced HOST1→HOST2 only as offsite backup. # Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. # For shares needing container stops or custom options — add a profile in master.conf. HOST2_DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Shows + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old ) # Personal encrypted shares — synced for offsite backup, independent of media shares.