diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh index 10e0162..441d488 100755 --- a/Media/arr_sync.sh +++ b/Media/arr_sync.sh @@ -124,6 +124,7 @@ BLOCKLIST_ACTION="" BLOCKLIST_ARR="" BLOCKLIST_ID="" BLOCKLIST_REASON="" +FORCE_MODE=false FILTERED_ARGS=() _skip_next=false for _arg in "$@"; do @@ -132,6 +133,7 @@ for _arg in "$@"; do --blocklist-add) BLOCKLIST_ACTION="add" ;; --blocklist-remove) BLOCKLIST_ACTION="remove" ;; --blocklist-list) BLOCKLIST_ACTION="list" ;; + --force) FORCE_MODE=true ;; *) FILTERED_ARGS+=("$_arg") ;; esac done @@ -606,6 +608,56 @@ curl -sf -o /dev/null -w '%{http_code}' -X POST \ REMOTE } +# Trigger a full library rescan on a remote arr — used after merge-run to force the arr +# to accept what is now on disk as ground truth rather than chasing stale file versions. +# Args: arr_type node_id node_ip port api_ver +_trigger_rescan() { + local arr_type="$1" node_id="$2" node_ip="$3" port="$4" api_ver="$5" + local command_name + case "$arr_type" in + sonarr) command_name="RefreshSeries" ;; + radarr) command_name="RefreshMovie" ;; + lidarr) command_name="RefreshArtist" ;; + *) return 0 ;; + esac + + local encoded + encoded=$(printf '{"name":"%s"}' "$command_name" | base64 -w0) + local _kvar="${node_id}_${arr_type^^}_API_KEY" + local cached_key="${!_kvar:-}" + + local http_code + if [[ -n "$cached_key" ]]; then + local body + body=$(printf '%s' "$encoded" | base64 -d) + http_code=$(curl -sf -o /dev/null -w '%{http_code}' -X POST \ + -H "X-Api-Key: $cached_key" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "http://${node_ip}:${port}/api/${api_ver}/command" 2>/dev/null) + else + local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml" + http_code=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \ + root@"$node_ip" bash </dev/null +KEY=\$(grep -oP '(?<=)[^<]+' '${config_xml}' 2>/dev/null) +[[ -z "\$KEY" ]] && exit 1 +BODY=\$(printf '%s' '${encoded}' | base64 -d) +curl -sf -o /dev/null -w '%{http_code}' -X POST \ + -H "X-Api-Key: \$KEY" \ + -H 'Content-Type: application/json' \ + -d "\$BODY" \ + "http://localhost:${port}/api/${api_ver}/command" +REMOTE +) + fi + + if [[ "$http_code" == "201" || "$http_code" == "200" ]]; then + log "${arr_type^}: triggered ${command_name} on ${node_id}" + else + warn "${arr_type^}: failed to trigger ${command_name} on ${node_id} (HTTP ${http_code:-timeout})" + fi +} + # ============================================================================================== # ── LOCAL HELPERS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -841,45 +893,50 @@ _sync_arr() { log "${arr_type^}: $remote_count items on $node_name" # ── Remote → Local: items on remote not in local ─────────────────────────────────────── - local to_add_local=() - for stable_id in "${!remote_ids[@]}"; do - [[ -n "${local_ids[$stable_id]:-}" ]] && continue - if _is_blocklisted "$arr_type" "$stable_id"; then - log "BLOCKLISTED [$arr_type] $stable_id — skipping" - (( total_skipped++ )) - continue - fi - to_add_local+=("$stable_id") - done + # Skipped in force mode — local arr is authoritative; remote tracking does not propagate back. + if [[ "$FORCE_MODE" == false ]]; then + local to_add_local=() + for stable_id in "${!remote_ids[@]}"; do + [[ -n "${local_ids[$stable_id]:-}" ]] && continue + if _is_blocklisted "$arr_type" "$stable_id"; then + log "BLOCKLISTED [$arr_type] $stable_id — skipping" + (( total_skipped++ )) + continue + fi + to_add_local+=("$stable_id") + done - if [[ "${#to_add_local[@]}" -gt 0 ]]; then - local local_defs - local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type") - if [[ -z "$local_defs" ]]; then - warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name" - else - for stable_id in "${to_add_local[@]}"; do - IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}" - local payload - payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \ - "true" "$local_defs") - if [[ "$DRY_RUN" == true ]]; then - log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)" - (( total_added_local++ )) - else - local http_code - http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \ - "$endpoint" "$payload") - if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then - log "Added to local ${arr_type^}: $display_name" - local_ids["$stable_id"]="${display_name}|true" + if [[ "${#to_add_local[@]}" -gt 0 ]]; then + local local_defs + local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type") + if [[ -z "$local_defs" ]]; then + warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name" + else + for stable_id in "${to_add_local[@]}"; do + IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}" + local payload + payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \ + "true" "$local_defs") + if [[ "$DRY_RUN" == true ]]; then + log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)" (( total_added_local++ )) else - warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)" + local http_code + http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \ + "$endpoint" "$payload") + if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then + log "Added to local ${arr_type^}: $display_name" + local_ids["$stable_id"]="${display_name}|true" + (( total_added_local++ )) + else + warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)" + fi fi - fi - done + done + fi fi + else + log "${arr_type^}: force mode — skipping remote→local (local is authoritative)" fi # ── Local → Remote: items on local not on remote ─────────────────────────────────────── @@ -923,9 +980,22 @@ _sync_arr() { fi fi - log " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted" + # ── Force mode: trigger library rescan on remote ────────────────────────────────────────── + # After files are settled by merge-run and tracking pushed by force arr-sync, the remote + # arr rescans its library so it accepts the authoritative file versions as ground truth. + if [[ "$FORCE_MODE" == true ]]; then + if [[ "$DRY_RUN" == true ]]; then + log "DRY RUN: would trigger ${arr_type^} rescan on $node_name" + else + _trigger_rescan "$arr_type" "$node_id" "$node_ip" "$port" "$api_ver" + fi + fi - unset remote_ids + local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0} + local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0} + log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted" + + unset remote_ids to_add_local to_add_remote _n_local _n_remote declare -A remote_ids done @@ -954,7 +1024,8 @@ echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━" echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made" +[[ "$FORCE_MODE" == true ]] && echo " Mode: force (local authoritative — remote→local skipped, rescan triggered)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 8574730..64d8a36 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -131,6 +131,13 @@ # rsync.sh /path/to/share --seed # Skip the empty-remote-share guard. Use for first-time seeding of a new share. # +# rsync.sh /path/to/share --merge-run +# Bidirectional merge: pull remote-unique content to local first (--ignore-existing), +# then push local → remote with --delete so the remote matches local exactly. +# Local is always authoritative — remote loses divergent file versions but keeps +# any content the local did not have (pulled in pass 1 before the delete push). +# Also auto-triggered when pre-scan detects ≥75% directory overlap with remote. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -141,12 +148,14 @@ source "$SCRIPT_DIR/../load_config.sh" DIRECTORY="" PROFILE_OVERRIDE="" SEED=false +MERGE_RUN=false RAW_ARGS=() for ARG in "$@"; do case "$ARG" in --profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;; --seed) SEED=true ;; + --merge-run) MERGE_RUN=true ;; --*|*=*) RAW_ARGS+=("$ARG") ;; *) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;; esac @@ -260,6 +269,31 @@ check_remote_rootfs [[ "$SEED" == false ]] && check_remote_share "$DIRECTORY" check_remote_disks "$DIRECTORY" +# ── Merge-run pre-scan ──────────────────────────────────────────────────────────────────────── +# Auto-promote to merge mode when ≥75% of remote's top-level entries exist locally. +# Skipped when --merge-run is already set or --seed is active (remote was just created). +if [[ "$MERGE_RUN" == false && "$SEED" == false ]]; then + _remote_top=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \ + root@"$REMOTE_SERVER" "ls -1A '$DIRECTORY' 2>/dev/null | sort" 2>/dev/null) + _remote_count=$(echo "$_remote_top" | grep -c . 2>/dev/null || echo 0) + + if [[ "$_remote_count" -gt 0 ]]; then + _local_top=$(ls -1A "$DIRECTORY" 2>/dev/null | sort) + _overlap=$(comm -12 \ + <(echo "$_local_top") \ + <(echo "$_remote_top") | grep -c . 2>/dev/null || echo 0) + _overlap_pct=$(( _overlap * 100 / _remote_count )) + + if [[ "$_overlap_pct" -ge 75 ]]; then + info "Overlap ${_overlap_pct}% (${_overlap}/${_remote_count} entries) — auto-promoting to merge mode" + MERGE_RUN=true + else + log "Overlap ${_overlap_pct}% (${_overlap}/${_remote_count} entries) — normal sync" + fi + fi + unset _remote_top _remote_count _local_top _overlap _overlap_pct +fi + # Remote Docker daemon — check before attempting container operations if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then check_remote_docker_daemon || { @@ -315,6 +349,29 @@ RSYNC_OPTS+=(--stats) [[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") +# ── Merge pass 1: pull remote-unique content to local (--ignore-existing) ──────────────────── +# Runs before the push so anything the remote has that we don't is preserved locally. +# After this pass, local is the superset — the delete push in pass 2 is then safe. +if [[ "$MERGE_RUN" == true ]]; then + echo "$ICON_SYNC Merge pass 1 — pulling ${REMOTE_SERVER_NAME}-unique content to local..." + _pull_opts=(-av --ignore-existing --stats) + [[ "$DRY_RUN" == true ]] && _pull_opts+=(--dry-run) + _pull_output=$(rsync "${_pull_opts[@]}" \ + -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ + "root@${REMOTE_SERVER}:${DIRECTORY}/" "${DIRECTORY}/" 2>&1) + _pull_exit=$? + _pull_bytes=$(echo "$_pull_output" | \ + awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}') + if [[ "$_pull_exit" -eq 0 ]]; then + echo "$ICON_DONE Merge pass 1 complete — ${_pull_bytes:-0} bytes pulled" + else + warn "Merge pass 1 failed (exit $_pull_exit) — continuing with push" + fi + unset _pull_opts _pull_output _pull_exit _pull_bytes + # Pass 2: push with --delete — local is now the authoritative superset + RSYNC_OPTS+=(--delete) +fi + START=$(date +%s) RSYNC_SUCCESS=false BYTES_TRANSFERRED=0