feat: intermediate orchestrator + arr_sync blocklist end-to-end + mesh sync
Add intermediate_sync_maintenance.sh (0 */4 * * *) — arr library sync as fixed first step, optional mid-day rsync (INTERMEDIATE_SYNC_SHARES, empty by default), then INTERMEDIATE_MAINTENANCE_SCRIPTS. Wired with full rsync infrastructure (temperature abort, check_rsync_enabled, INTERMEDIATE_RSYNC_ENABLED toggle) matching daily/weekly pattern. lidarr_missing_art.sh scheduled here. arr_sync.sh: blocklist --add now end-to-end — looks up real display name from local arr API, writes TSV, deletes from local arr API (deleteFiles=false), SSHes each remote node and deletes from their arr API. Files become orphans for arr_cleanup safety pass. master.conf: DEFAULT_RSYNC_OPTS updated (--partial, --timeout=60, --numeric-ids; --no-whole-file removed). Media share comments updated to mesh model. master_host1.conf + master_host2.conf: both hosts now push all media shares bidirectionally (true mesh — no ownership per share, arr_cleanup enforces truth). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
6674dd4c17
commit
f277ce92d5
+133
-6
@@ -25,10 +25,16 @@
|
||||
#
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added anywhere.
|
||||
# Read from ALL nodes via SSH at start of each run — immediate effect before rsync.
|
||||
# Propagates to all nodes via rsync on the next cycle.
|
||||
# Read from ALL nodes via SSH at start of each run — immediate effect across all nodes.
|
||||
# (Each node reads every other node's blocklist file via SSH — no rsync delay.)
|
||||
# Manage via --blocklist-add / --blocklist-remove / --blocklist-list.
|
||||
#
|
||||
# --blocklist-add does three things atomically:
|
||||
# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync)
|
||||
# 2. Deletes the item from the local arr API (deleteFiles=false)
|
||||
# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false)
|
||||
# Files become orphans on all nodes — arr_cleanup.sh removes them on next run.
|
||||
#
|
||||
# ── GRACEFUL SKIP ─────────────────────────────────────────────────────────────────────────────
|
||||
# Arr not configured locally → skip cleanly, no error.
|
||||
# Arr not reachable on a remote → skip that node for that arr type, continue with others.
|
||||
@@ -47,8 +53,10 @@
|
||||
# arr_sync.sh --dry-run — preview only, no changes
|
||||
# arr_sync.sh --log — verbose output
|
||||
# arr_sync.sh --status — show config and exit
|
||||
# arr_sync.sh --blocklist-add lidarr <id> "reason" — tombstone an ID on all nodes
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone an ID
|
||||
# arr_sync.sh --blocklist-add lidarr <mbid> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add sonarr <tvdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add radarr <tmdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone (does NOT re-add)
|
||||
# arr_sync.sh --blocklist-list — show all blocklisted IDs
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
@@ -238,6 +246,68 @@ _blocklist_remove() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Look up item in local arr by stable_id — returns "internal_id\tdisplay_name" or empty
|
||||
_lookup_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
local id_field="$5" id_type="$6" name_field="$7" stable_id="$8"
|
||||
local raw select_expr
|
||||
raw=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$raw" ]] && return 1
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\")"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id})"
|
||||
fi
|
||||
echo "$raw" | jq -r "${select_expr} | [(.id | tostring), .${name_field}] | @tsv" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# Delete item from local arr by internal integer id — returns HTTP status code
|
||||
_delete_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" internal_id="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
}
|
||||
|
||||
# Delete item from remote arr by stable_id via SSH.
|
||||
# Outputs: HTTP code on success | "not_found" if item absent | empty on SSH/API failure.
|
||||
# deleteFiles=false — files become orphans for arr_cleanup to handle with its safety checks.
|
||||
_delete_remote_item() {
|
||||
local node_id="$1" port="$2" api_ver="$3" endpoint="$4" config_xml="$5"
|
||||
local id_field="$6" id_type="$7" stable_id="$8"
|
||||
local node_name="${!node_id}"
|
||||
local ts_name="${node_name,,}"
|
||||
local node_ip
|
||||
node_ip=$(tailscale ip -4 "$ts_name" 2>/dev/null)
|
||||
[[ -z "$node_ip" ]] && node_ip=$(tailscale status 2>/dev/null | \
|
||||
awk -v n="$ts_name" '$2 ~ "^" n { print $1; exit }')
|
||||
[[ -z "$node_ip" ]] && return 1
|
||||
|
||||
local select_expr
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\") | .id"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id}) | .id"
|
||||
fi
|
||||
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
LIBRARY=\$(curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "\$LIBRARY" ]] && exit 1
|
||||
INTERNAL_ID=\$(echo "\$LIBRARY" | jq -r '${select_expr}' 2>/dev/null | head -1)
|
||||
[[ -z "\$INTERNAL_ID" ]] && echo "not_found" && exit 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}/\${INTERNAL_ID}?deleteFiles=false"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ── Blocklist management mode ──────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
case "$BLOCKLIST_ACTION" in
|
||||
@@ -264,8 +334,65 @@ if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
error " arr_type: lidarr | sonarr | radarr"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "${BLOCKLIST_ID}" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $BLOCKLIST_ID — will propagate via rsync on next cycle"
|
||||
if [[ -z "${_PORT[$BLOCKLIST_ARR]:-}" ]]; then
|
||||
error "Unknown arr type: $BLOCKLIST_ARR — use lidarr, sonarr, or radarr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_bl_port="${_PORT[$BLOCKLIST_ARR]}"
|
||||
_bl_ver="${_VER[$BLOCKLIST_ARR]}"
|
||||
_bl_ep="${_EP[$BLOCKLIST_ARR]}"
|
||||
_bl_id_field="${_ID[$BLOCKLIST_ARR]}"
|
||||
_bl_id_type="${_ID_TYPE[$BLOCKLIST_ARR]}"
|
||||
_bl_name_field="${_NAME[$BLOCKLIST_ARR]}"
|
||||
_bl_config_xml="${DOCKER_APPDATA_BASE}/${BLOCKLIST_ARR^}/config.xml"
|
||||
_bl_url="" _bl_key=""
|
||||
case "$BLOCKLIST_ARR" in
|
||||
lidarr) _bl_url="${LIDARR_URL:-}"; _bl_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) _bl_url="${SONARR_URL:-}"; _bl_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) _bl_url="${RADARR_URL:-}"; _bl_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
# Look up display name and internal id from local arr
|
||||
_bl_display_name="$BLOCKLIST_ID"
|
||||
_bl_internal_id=""
|
||||
if [[ -n "$_bl_url" ]] && [[ -n "$_bl_key" ]]; then
|
||||
_bl_lookup=$(_lookup_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" \
|
||||
"$_bl_id_field" "$_bl_id_type" "$_bl_name_field" "$BLOCKLIST_ID")
|
||||
if [[ -n "$_bl_lookup" ]]; then
|
||||
IFS=$'\t' read -r _bl_internal_id _bl_display_name <<< "$_bl_lookup"
|
||||
fi
|
||||
fi
|
||||
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "$_bl_display_name" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $_bl_display_name ($BLOCKLIST_ID)"
|
||||
|
||||
# Remove from local arr (deleteFiles=false — arr_cleanup handles file removal)
|
||||
if [[ -n "$_bl_internal_id" ]]; then
|
||||
_bl_http=$(_delete_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" "$_bl_internal_id")
|
||||
if [[ "$_bl_http" == "200" ]]; then
|
||||
log "Removed from local ${BLOCKLIST_ARR^}: $_bl_display_name"
|
||||
else
|
||||
warn "Failed to remove from local ${BLOCKLIST_ARR^} (HTTP ${_bl_http:-no response}) — remove manually via UI"
|
||||
fi
|
||||
else
|
||||
log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally"
|
||||
fi
|
||||
|
||||
# Remove from all remote arrs
|
||||
for _bl_node_id in "${REMOTE_NODES[@]}"; do
|
||||
_bl_node_name="${!_bl_node_id}"
|
||||
_bl_result=$(_delete_remote_item "$_bl_node_id" "$_bl_port" "$_bl_ver" "$_bl_ep" \
|
||||
"$_bl_config_xml" "$_bl_id_field" "$_bl_id_type" "$BLOCKLIST_ID")
|
||||
case "$_bl_result" in
|
||||
200) log "Removed from $_bl_node_name ${BLOCKLIST_ARR^}: $_bl_display_name" ;;
|
||||
not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;;
|
||||
*) warn "Failed to remove from $_bl_node_name ${BLOCKLIST_ARR^} (${_bl_result:-SSH error}) — remove manually via UI" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Files are now orphans on all nodes — arr_cleanup.sh will remove them on next run"
|
||||
exit 0
|
||||
;;
|
||||
remove)
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# =========================== Intermediate Sync Maintenance ====================================
|
||||
# ==============================================================================================
|
||||
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync.
|
||||
# Schedule: 0 */4 * * * (every 4 hours)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
||||
# 2. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
||||
# 3. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
||||
#
|
||||
# ── WHY A SEPARATE ORCHESTRATOR ───────────────────────────────────────────────────────────────
|
||||
# arr libraries need to converge more frequently than once a day. If a remote node adds
|
||||
# something at 2am, the next daily window is 23 hours away — remote arrs search for content
|
||||
# they don't know is already owned. Running every 4 hours closes that gap.
|
||||
#
|
||||
# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill.
|
||||
# Pairing it here means artwork catches up within 4 hours of a new album landing.
|
||||
#
|
||||
# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config
|
||||
# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media
|
||||
# share sync stays in the daily window.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID, DAILY_SYNC_SHARES, PERSONAL_SHARES from HOST*_ vars.
|
||||
# INTERMEDIATE_SYNC_SHARES is a shared list in master.conf — same on all servers.
|
||||
# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic.
|
||||
#
|
||||
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
|
||||
# Same as daily_sync_maintenance.sh:
|
||||
# exit 1 = temp WARN — skip this share, continue to next
|
||||
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — scripts called here require root
|
||||
# acquire_lock — prevents concurrent intermediate windows
|
||||
# check_connectivity — verified before any rsync (skipped if no shares)
|
||||
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
||||
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
||||
# Silent on success — runs 4x/day, only failures warrant notification
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
|
||||
# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true)
|
||||
# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
|
||||
# ARR_SYNC_ENABLED — toggle inside arr_sync.sh
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# intermediate_sync_maintenance.sh — normal run
|
||||
# intermediate_sync_maintenance.sh --dry-run — preview without changes
|
||||
# intermediate_sync_maintenance.sh --log — verbose per-job output
|
||||
# intermediate_sync_maintenance.sh --status — show configured shares/jobs and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
warn "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_GEAR Arr sync: ${ARR_SYNC_ENABLED:-true}"
|
||||
echo ""
|
||||
echo "━━━ Intermediate Sync Shares ━━━"
|
||||
if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
echo " None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
|
||||
else
|
||||
for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Intermediate Maintenance Scripts ━━━"
|
||||
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Arr Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
else
|
||||
_arr_sync_args=()
|
||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||
log "Arr sync complete ✅"
|
||||
JOB_PASS+=("arr_sync.sh")
|
||||
else
|
||||
warn "Arr sync completed with errors — continuing"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
fi
|
||||
unset _arr_sync_args
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Rsync (optional) ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
TOTAL_START=$(date +%s)
|
||||
SHARE_INDEX=0
|
||||
ABORT_ALL_SYNCS=false
|
||||
|
||||
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
log "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
||||
log "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
||||
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
||||
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
||||
else
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
||||
(( SHARE_INDEX++ ))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
||||
|
||||
case "$RSYNC_EXIT" in
|
||||
0)
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done ✅"
|
||||
;;
|
||||
1)
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
;;
|
||||
2)
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
||||
"Intermediate Sync" "warning"
|
||||
;;
|
||||
*)
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Maintenance Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
|
||||
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
sname="${entry%%:*}"
|
||||
sdur="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||
else
|
||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
log "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+29
-6
@@ -280,6 +280,23 @@
|
||||
# "Fallback/fallback.sh" # mutual failover — enable when HOST2 ready
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Maintenance ━━━
|
||||
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
|
||||
# and optional mid-day rsync for any shares that need sub-daily propagation.
|
||||
# Schedule: 0 */4 * * *
|
||||
INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync — empty = rsync section skipped entirely.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete). Full media sync stays in the daily window.
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares
|
||||
|
||||
INTERMEDIATE_MAINTENANCE_SCRIPTS=(
|
||||
"Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
)
|
||||
# arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
||||
# It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
||||
|
||||
# ━━━ Daily Sync Maintenance ━━━
|
||||
# daily_sync_maintenance.sh runs media share sync first, then iterates
|
||||
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
|
||||
@@ -305,9 +322,10 @@
|
||||
|
||||
# Media shares synced daily by daily_sync_maintenance.sh.
|
||||
# Defined per-host in master_host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
|
||||
# Each server syncs only the shares it owns — direction is automatic.
|
||||
# HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1.
|
||||
# Never both pushing the same share — one server is always the truth holder.
|
||||
# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
|
||||
# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
|
||||
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
|
||||
# For shares needing custom options or container stops — create a profile in RSYNC section.
|
||||
|
||||
@@ -398,11 +416,16 @@
|
||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||
EXCLUDE_DIRS=() # directories excluded from transfer — profiles override
|
||||
|
||||
# --inplace writes directly to destination — better for large files
|
||||
# --no-whole-file forces delta transfer — sends only changed blocks
|
||||
# --inplace writes directly to destination — delta against existing file, better for large media
|
||||
# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers
|
||||
# --timeout kill stalled transfers instead of hanging indefinitely
|
||||
# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers
|
||||
# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
|
||||
# Media shares use this default; rsync spreads files only, never removes them.
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --no-whole-file)
|
||||
# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default).
|
||||
# Note: arr_cleanup only catches true orphans under the union model — intentional removals
|
||||
# require arr_sync.sh --blocklist-add first, then manual file deletion.
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids)
|
||||
|
||||
# ━━━ Remote Health Checks ━━━
|
||||
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
|
||||
|
||||
+8
-3
@@ -110,9 +110,12 @@
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 owns and pushes to HOST2 every night (1am via daily_sync_maintenance.sh).
|
||||
# HOST1 is the source of truth — HOST2 is the mirror.
|
||||
# Never push a share both directions — one server always owns it.
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
@@ -129,6 +132,8 @@
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
|
||||
+18
-3
@@ -110,14 +110,29 @@
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST2 owns and pushes to HOST1 every night (1am via daily_sync_maintenance.sh).
|
||||
# HOST2 is the source of truth — HOST1 is the mirror.
|
||||
# Never push a share both directions — one server always owns it.
|
||||
# Shares HOST2 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud excluded — personal data, not arr-managed, synced HOST1→HOST2 only as offsite backup.
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST2_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
|
||||
Reference in New Issue
Block a user