FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add --merge-run to rsync and --force to arr_sync for authoritative node compliance

Browse Source
This commit is contained in:
Gmer4Lfe
2026-06-14 15:51:28 -04:00
parent 5a7d5b5005
commit 3c303210a3
2 changed files with 165 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Media/arr_sync.sh
+108 -37
View File
@@ -124,6 +124,7 @@ BLOCKLIST_ACTION=""
BLOCKLIST_ARR=""
BLOCKLIST_ID=""
BLOCKLIST_REASON=""
FORCE_MODE=false
FILTERED_ARGS=()
_skip_next=false
for _arg in "$@"; do
@@ -132,6 +133,7 @@ for _arg in "$@"; do
--blocklist-add) BLOCKLIST_ACTION="add" ;;
--blocklist-remove) BLOCKLIST_ACTION="remove" ;;
--blocklist-list) BLOCKLIST_ACTION="list" ;;
--force) FORCE_MODE=true ;;
*) FILTERED_ARGS+=("$_arg") ;;
esac
done
@@ -606,6 +608,56 @@ curl -sf -o /dev/null -w '%{http_code}' -X POST \
REMOTE
}
# Trigger a full library rescan on a remote arr — used after merge-run to force the arr
# to accept what is now on disk as ground truth rather than chasing stale file versions.
# Args: arr_type node_id node_ip port api_ver
_trigger_rescan() {
local arr_type="$1" node_id="$2" node_ip="$3" port="$4" api_ver="$5"
local command_name
case "$arr_type" in
sonarr) command_name="RefreshSeries" ;;
radarr) command_name="RefreshMovie" ;;
lidarr) command_name="RefreshArtist" ;;
*) return 0 ;;
esac
local encoded
encoded=$(printf '{"name":"%s"}' "$command_name" | base64 -w0)
local _kvar="${node_id}_${arr_type^^}_API_KEY"
local cached_key="${!_kvar:-}"
local http_code
if [[ -n "$cached_key" ]]; then
local body
body=$(printf '%s' "$encoded" | base64 -d)
http_code=$(curl -sf -o /dev/null -w '%{http_code}' -X POST \
-H "X-Api-Key: $cached_key" \
-H "Content-Type: application/json" \
-d "$body" \
"http://${node_ip}:${port}/api/${api_ver}/command" 2>/dev/null)
else
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
http_code=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
root@"$node_ip" bash <<REMOTE 2>/dev/null
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
[[ -z "\$KEY" ]] && exit 1
BODY=\$(printf '%s' '${encoded}' | base64 -d)
curl -sf -o /dev/null -w '%{http_code}' -X POST \
-H "X-Api-Key: \$KEY" \
-H 'Content-Type: application/json' \
-d "\$BODY" \
"http://localhost:${port}/api/${api_ver}/command"
REMOTE
)
fi
if [[ "$http_code" == "201" || "$http_code" == "200" ]]; then
log "${arr_type^}: triggered ${command_name} on ${node_id}"
else
warn "${arr_type^}: failed to trigger ${command_name} on ${node_id} (HTTP ${http_code:-timeout})"
fi
}
# ==============================================================================================
# ── LOCAL HELPERS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -841,45 +893,50 @@ _sync_arr() {
log "${arr_type^}: $remote_count items on $node_name"
# ── Remote → Local: items on remote not in local ───────────────────────────────────────
local to_add_local=()
for stable_id in "${!remote_ids[@]}"; do
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
if _is_blocklisted "$arr_type" "$stable_id"; then
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
(( total_skipped++ ))
continue
fi
to_add_local+=("$stable_id")
done
# Skipped in force mode — local arr is authoritative; remote tracking does not propagate back.
if [[ "$FORCE_MODE" == false ]]; then
local to_add_local=()
for stable_id in "${!remote_ids[@]}"; do
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
if _is_blocklisted "$arr_type" "$stable_id"; then
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
(( total_skipped++ ))
continue
fi
to_add_local+=("$stable_id")
done
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
local local_defs
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
if [[ -z "$local_defs" ]]; then
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
else
for stable_id in "${to_add_local[@]}"; do
IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}"
local payload
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
"true" "$local_defs")
if [[ "$DRY_RUN" == true ]]; then
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
(( total_added_local++ ))
else
local http_code
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
"$endpoint" "$payload")
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
log "Added to local ${arr_type^}: $display_name"
local_ids["$stable_id"]="${display_name}|true"
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
local local_defs
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
if [[ -z "$local_defs" ]]; then
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
else
for stable_id in "${to_add_local[@]}"; do
IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}"
local payload
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
"true" "$local_defs")
if [[ "$DRY_RUN" == true ]]; then
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
(( total_added_local++ ))
else
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
local http_code
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
"$endpoint" "$payload")
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
log "Added to local ${arr_type^}: $display_name"
local_ids["$stable_id"]="${display_name}|true"
(( total_added_local++ ))
else
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
fi
fi
fi
done
done
fi
fi
else
log "${arr_type^}: force mode — skipping remote→local (local is authoritative)"
fi
# ── Local → Remote: items on local not on remote ───────────────────────────────────────
@@ -923,9 +980,22 @@ _sync_arr() {
fi
fi
log " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted"
# ── Force mode: trigger library rescan on remote ──────────────────────────────────────────
# After files are settled by merge-run and tracking pushed by force arr-sync, the remote
# arr rescans its library so it accepts the authoritative file versions as ground truth.
if [[ "$FORCE_MODE" == true ]]; then
if [[ "$DRY_RUN" == true ]]; then
log "DRY RUN: would trigger ${arr_type^} rescan on $node_name"
else
_trigger_rescan "$arr_type" "$node_id" "$node_ip" "$port" "$api_ver"
fi
fi
unset remote_ids
local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0}
local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0}
log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted"
unset remote_ids to_add_local to_add_remote _n_local _n_remote
declare -A remote_ids
done
@@ -954,7 +1024,8 @@ echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━"
echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
[[ "$FORCE_MODE" == true ]] && echo " Mode: force (local authoritative — remote→local skipped, rescan triggered)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Rsync/rsync.sh
+57
View File
@@ -131,6 +131,13 @@
# rsync.sh /path/to/share --seed
# Skip the empty-remote-share guard. Use for first-time seeding of a new share.
#
# rsync.sh /path/to/share --merge-run
# Bidirectional merge: pull remote-unique content to local first (--ignore-existing),
# then push local → remote with --delete so the remote matches local exactly.
# Local is always authoritative — remote loses divergent file versions but keeps
# any content the local did not have (pulled in pass 1 before the delete push).
# Also auto-triggered when pre-scan detects ≥75% directory overlap with remote.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -141,12 +148,14 @@ source "$SCRIPT_DIR/../load_config.sh"
DIRECTORY=""
PROFILE_OVERRIDE=""
SEED=false
MERGE_RUN=false
RAW_ARGS=()
for ARG in "$@"; do
case "$ARG" in
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
--seed) SEED=true ;;
--merge-run) MERGE_RUN=true ;;
--*|*=*) RAW_ARGS+=("$ARG") ;;
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
esac
@@ -260,6 +269,31 @@ check_remote_rootfs
[[ "$SEED" == false ]] && check_remote_share "$DIRECTORY"
check_remote_disks "$DIRECTORY"
# ── Merge-run pre-scan ────────────────────────────────────────────────────────────────────────
# Auto-promote to merge mode when ≥75% of remote's top-level entries exist locally.
# Skipped when --merge-run is already set or --seed is active (remote was just created).
if [[ "$MERGE_RUN" == false && "$SEED" == false ]]; then
_remote_top=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
root@"$REMOTE_SERVER" "ls -1A '$DIRECTORY' 2>/dev/null | sort" 2>/dev/null)
_remote_count=$(echo "$_remote_top" | grep -c . 2>/dev/null || echo 0)
if [[ "$_remote_count" -gt 0 ]]; then
_local_top=$(ls -1A "$DIRECTORY" 2>/dev/null | sort)
_overlap=$(comm -12 \
<(echo "$_local_top") \
<(echo "$_remote_top") | grep -c . 2>/dev/null || echo 0)
_overlap_pct=$(( _overlap * 100 / _remote_count ))
if [[ "$_overlap_pct" -ge 75 ]]; then
info "Overlap ${_overlap_pct}% (${_overlap}/${_remote_count} entries) — auto-promoting to merge mode"
MERGE_RUN=true
else
log "Overlap ${_overlap_pct}% (${_overlap}/${_remote_count} entries) — normal sync"
fi
fi
unset _remote_top _remote_count _local_top _overlap _overlap_pct
fi
# Remote Docker daemon — check before attempting container operations
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
check_remote_docker_daemon || {
@@ -315,6 +349,29 @@ RSYNC_OPTS+=(--stats)
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
# ── Merge pass 1: pull remote-unique content to local (--ignore-existing) ────────────────────
# Runs before the push so anything the remote has that we don't is preserved locally.
# After this pass, local is the superset — the delete push in pass 2 is then safe.
if [[ "$MERGE_RUN" == true ]]; then
echo "$ICON_SYNC Merge pass 1 — pulling ${REMOTE_SERVER_NAME}-unique content to local..."
_pull_opts=(-av --ignore-existing --stats)
[[ "$DRY_RUN" == true ]] && _pull_opts+=(--dry-run)
_pull_output=$(rsync "${_pull_opts[@]}" \
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
"root@${REMOTE_SERVER}:${DIRECTORY}/" "${DIRECTORY}/" 2>&1)
_pull_exit=$?
_pull_bytes=$(echo "$_pull_output" | \
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
if [[ "$_pull_exit" -eq 0 ]]; then
echo "$ICON_DONE Merge pass 1 complete — ${_pull_bytes:-0} bytes pulled"
else
warn "Merge pass 1 failed (exit $_pull_exit) — continuing with push"
fi
unset _pull_opts _pull_output _pull_exit _pull_bytes
# Pass 2: push with --delete — local is now the authoritative superset
RSYNC_OPTS+=(--delete)
fi
START=$(date +%s)
RSYNC_SUCCESS=false
BYTES_TRANSFERRED=0