1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Weekly Sync Maintenance ========================================
4	# ==============================================================================================
5	# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
6	# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   1. Stop local containers         — Emby + auth stack stopped locally
10	#   2. Stop remote containers        — Emby + auth stack stopped remotely via SSH
11	#   3. Pull updates locally          — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
12	#   4. Pull updates remotely         — if WEEKLY_SYNC_UPDATES_REMOTE=true
13	#   5. rsync WEEKLY_SYNC_SHARES      — full clean mirror, containers stopped both sides
14	#   6. Start remote containers       — correct order, delayed start respected
15	#   7. Start local containers        — correct order, delayed start respected
16	#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
17	#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window
18	#
19	# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
20	# Emby builds a warm image cache on HOST2 throughout the week.
21	# Syncing nightly resets cache — cold loads every morning for users.
22	# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
23	# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
24	#
25	# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
26	# Containers already stopped for sync — updates pull at zero extra downtime.
27	# Both servers start on identical image versions after the window completes.
28	# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
29	#
30	# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
31	# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
32	# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
33	# Same script runs correctly on both servers.
34	#
35	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
36	#   Root check           — stop/start containers, rsync require root
37	#   acquire_lock         — prevents concurrent weekly windows
38	#   check_connectivity   — verifies remote before any remote operations
39	#   check_remote_rootfs  — aborts if remote rootfs nearly full
40	#   DOCKER_TIMEOUT       — all docker calls protected
41	#   SSH_TIMEOUT          — all SSH calls protected
42	#   platform_require_cmd  — notify validated before use
43	#   Silent on success    — runs weekly, only failures warrant notification
44	#
45	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
46	#   WEEKLY_SYNC_SHARES             — shares synced during window
47	#   WEEKLY_MAINTENANCE_SCRIPTS     — scripts run after sync
48	#   WEEKLY_SYNC_UPDATES            — toggle local container updates
49	#   WEEKLY_SYNC_UPDATES_REMOTE     — toggle remote container updates
50	#   WEEKLY_RSYNC_ENABLED           — enable/disable rsync section
51	#
52	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
53	#   weekly_sync_maintenance.sh              — normal run
54	#   weekly_sync_maintenance.sh --dry-run    — preview without stopping containers or syncing
55	#   weekly_sync_maintenance.sh --log        — verbose per-share/per-job output
56	#   weekly_sync_maintenance.sh --status     — show configuration and exit
57	# ==============================================================================================
58
59	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
60
61	source "$SCRIPT_DIR/../load_config.sh"
62
63	RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
64	SCRIPTS_ROOT="$SCRIPT_DIR/.."
65
66	parse_args "$@"
67
68	DOCKER_TIMEOUT=30   # container stop/start needs longer than normal
69	SSH_TIMEOUT=30      # remote pulls can be slow
70
71	# ==============================================================================================
72	# ━━━ Setup ━━━
73	# ==============================================================================================
74	if [[ "$EUID" -ne 0 ]]; then
75	    error "Must be run as root"
76	    exit 1
77	fi
78
79
80	acquire_lock
81
82	if ! command -v docker &>/dev/null; then
83	    error "Docker command not found"
84	    exit 1
85	fi
86
87	detect_hosts
88	resolve_remote_ip
89
90	WINDOW_START=$(date +%s)
91	PASS=()
92	FAIL=()
93	JOB_PASS=()
94	JOB_FAIL=()
95
96	# Load container lists from profile config
97	read -r -a MAINTENANCE_CONTAINERS <<< \
98	    "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
99
100	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
101
102	# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
103	run_job() {
104	    local script_entry="$1"
105	    local extra_dry=""
106	    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
107
108	    read -r -a script_args <<< "$script_entry"
109	    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
110	    local script_name
111	    script_name=$(basename "${script_args[0]}")
112	    local extra_args=("${script_args[@]:1}")
113
114	    if [[ ! -f "$script_path" ]]; then
115	        error "$script_name — not found at $script_path"
116	        JOB_FAIL+=("$script_name")
117	        return 1
118	    fi
119
120	    log "Running: $script_name ${extra_args[*]}"
121	    # shellcheck disable=SC2086
122	    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
123	        log "$script_name — done ✅"
124	        JOB_PASS+=("$script_name ${extra_args[*]}")
125	    else
126	        error "$script_name — failed (exit $?)"
127	        JOB_FAIL+=("$script_name ${extra_args[*]}")
128	    fi
129	}
130
131	# ==============================================================================================
132	# ━━━ Status ━━━
133	# ==============================================================================================
134	if [[ "$SHOW_STATUS" == true ]]; then
135	    echo ""
136	    echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
137	    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
138	    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
139	    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
140	    echo "$ICON_SYNC   Weekly enabled:  ${WEEKLY_RSYNC_ENABLED:-false}"
141	    echo "$ICON_GEAR   Local updates:   ${WEEKLY_SYNC_UPDATES:-false}"
142	    echo "$ICON_GEAR   Remote updates:  ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
143	    echo ""
144	    echo "━━━ Weekly Sync Shares ━━━"
145	    if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
146	        warn "  No WEEKLY_SYNC_SHARES configured"
147	    else
148	        for share in "${WEEKLY_SYNC_SHARES[@]}"; do
149	            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
150	        done
151	    fi
152	    echo ""
153	    echo "━━━ Weekly Maintenance Scripts ━━━"
154	    if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
155	        echo "  None configured"
156	    else
157	        for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
158	            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
159	        done
160	    fi
161	    echo ""
162	    echo "━━━ Containers (from profile config) ━━━"
163	    for c in "${MAINTENANCE_CONTAINERS[@]}"; do
164	        [[ -n "$c" ]] && echo "  $ICON_CONTAINERS $c"
165	    done
166	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
167	    exit 0
168	fi
169
170	# ==============================================================================================
171	# ━━━ Pre-flight Checks ━━━
172	# ==============================================================================================
173	echo ""
174	echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
175	echo ""
176	echo "━━━ $ICON_SHIELD Pre-flight ━━━"
177
178	check_connectivity
179	check_remote_rootfs
180
181	# ==============================================================================================
182	# ━━━ Stop Containers ━━━
183	# ==============================================================================================
184	echo ""
185	echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
186
187	if [[ "$DRY_RUN" == true ]]; then
188	    warn "DRY RUN — containers will not be stopped"
189	else
190	    # Load container lists for stop functions
191	    read -r -a CRITICAL_CONTAINER_NAMES <<< \
192	        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
193	    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
194	    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
195
196	    stop_local_containers
197	    stop_containers
198	fi
199
200	# ==============================================================================================
201	# ━━━ Container Updates ━━━
202	# ==============================================================================================
203	echo ""
204	echo "━━━ $ICON_GEAR Container Updates ━━━"
205
206	if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
207	    if [[ "$DRY_RUN" == true ]]; then
208	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
209	            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
210	        done
211	    else
212	        log "Pulling local container updates..."
213	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
214	            [[ -z "$c" ]] && continue
215	            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
216	                "$c" --format '{{.Config.Image}}' 2>/dev/null)
217	            if [[ -z "$IMAGE" ]]; then
218	                log "$c — not found locally, skipping update"
219	                continue
220	            fi
221	            log "Pulling $IMAGE for $c..."
222	            if docker pull "$IMAGE" >/dev/null 2>&1; then
223	                log "$c — image updated ✅"
224	            else
225	                warn "$c — pull failed, will start on existing image"
226	            fi
227	        done
228	    fi
229	else
230	    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
231	fi
232
233	if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
234	    if [[ "$DRY_RUN" == true ]]; then
235	        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
236	    else
237	        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
238	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
239	            [[ -z "$c" ]] && continue
240	            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
241	                -o ConnectTimeout="$SSH_TIMEOUT" \
242	                root@"$REMOTE_SERVER" \
243	                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
244	            if [[ -z "$IMAGE" ]]; then
245	                log "$c — not found on remote, skipping update"
246	                continue
247	            fi
248	            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
249	            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
250	                -o ConnectTimeout="$SSH_TIMEOUT" \
251	                root@"$REMOTE_SERVER" \
252	                "docker pull $IMAGE" >/dev/null 2>&1; then
253	                log "$c — remote image updated ✅"
254	            else
255	                warn "$c — remote pull failed, will start on existing image"
256	            fi
257	        done
258	    fi
259	else
260	    echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
261	fi
262
263	# ==============================================================================================
264	# ━━━ Critical Shares Sync ━━━
265	# ==============================================================================================
266	SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
267
268	echo ""
269	echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
270
271	SYNC_START=$(date +%s)
272	JOB_NUM=0
273
274	if ! check_rsync_enabled "WEEKLY"; then
275	    warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
276	    warn "Proceeding to container start and maintenance scripts..."
277	elif [[ "$SHARE_COUNT" -eq 0 ]]; then
278	    warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
279	    warn "Check WEEKLY_SYNC_SHARES in master.conf"
280	else
281	    RSYNC_DRY=""
282	    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
283
284	    for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
285	        (( JOB_NUM++ ))
286	        JOB_NAME=$(basename "$JOB")
287
288	        echo ""
289	        echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
290
291	        JOB_START=$(date +%s)
292	        bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
293	        EXIT_CODE=$?
294	        JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
295
296	        if [[ "$EXIT_CODE" -eq 0 ]]; then
297	            PASS+=("$JOB_NAME")
298	            log "$JOB_NAME — done in $JOB_DUR ✅"
299	        else
300	            FAIL+=("$JOB_NAME")
301	            error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
302	        fi
303	        echo ""
304	    done
305	fi
306
307	SYNC_END=$(date +%s)
308
309	# ==============================================================================================
310	# ━━━ Start Containers ━━━
311	# ==============================================================================================
312	echo ""
313	echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
314
315	if [[ "$DRY_RUN" == true ]]; then
316	    warn "DRY RUN — containers will not be started"
317	else
318	    start_containers
319	    start_local_containers
320	fi
321
322	# ==============================================================================================
323	# ━━━ Post-sync Jobs ━━━
324	# ==============================================================================================
325	if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
326	    echo ""
327	    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
328	    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
329	        [[ -z "$script_entry" ]] && continue
330	        echo ""
331	        run_job "$script_entry"
332	    done
333	fi
334
335	# ==============================================================================================
336	# ━━━ Remainder Container Updates ━━━
337	# ==============================================================================================
338	# Updates all running containers not already covered by daily or the weekly sync window.
339	# Runs last — weekly sync-window containers are back up before this pulls their peers.
340	echo ""
341	echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"
342
343	DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
344	if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
345	    warn "docker_update.sh not found — skipping remainder updates"
346	    JOB_FAIL+=("docker_update.sh --remainder")
347	else
348	    _remainder_args=("--remainder")
349	    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
350	    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
351	        echo "Remainder updates complete ✅"
352	        JOB_PASS+=("docker_update.sh --remainder")
353	    else
354	        warn "Remainder updates completed with errors"
355	        JOB_FAIL+=("docker_update.sh --remainder")
356	    fi
357	    unset _remainder_args
358	fi
359
360	WINDOW_END=$(date +%s)
361
362	# ==============================================================================================
363	# ━━━ Summary ━━━
364	# ==============================================================================================
365	echo ""
366	echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
367	echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
368	echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
369	echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
370	echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
371	echo ""
372
373	echo "$ICON_SYNC  Sync jobs ($SHARE_COUNT):"
374	for job in "${PASS[@]}"; do echo "  $ICON_DONE  $job"; done
375	for job in "${FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
376	echo "  Passed: ${#PASS[@]}  Failed: ${#FAIL[@]}"
377
378	if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
379	    echo ""
380	    echo "$ICON_GEAR  Post-sync jobs:"
381	    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
382	    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
383	fi
384
385	TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
386
387	echo ""
388	if [[ "$DRY_RUN" == true ]]; then
389	    warn "DRY RUN — no changes made"
390	elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
391	    echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
392	else
393	    warn "Status: $TOTAL_FAIL failure(s)"
394	    notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
395	        "Weekly Maintenance" "warning"
396	fi
397	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
398
399	[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
400	exit 0

#!/bin/bash
# ==============================================================================================
# ============================= Weekly Sync Maintenance ========================================
# ==============================================================================================
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. Stop local containers         — Emby + auth stack stopped locally
#   2. Stop remote containers        — Emby + auth stack stopped remotely via SSH
#   3. Pull updates locally          — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
#   4. Pull updates remotely         — if WEEKLY_SYNC_UPDATES_REMOTE=true
#   5. rsync WEEKLY_SYNC_SHARES      — full clean mirror, containers stopped both sides
#   6. Start remote containers       — correct order, delayed start respected
#   7. Start local containers        — correct order, delayed start respected
#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window
#
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
# Emby builds a warm image cache on HOST2 throughout the week.
# Syncing nightly resets cache — cold loads every morning for users.
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
#
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
# Containers already stopped for sync — updates pull at zero extra downtime.
# Both servers start on identical image versions after the window completes.
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
# Same script runs correctly on both servers.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — stop/start containers, rsync require root
#   acquire_lock         — prevents concurrent weekly windows
#   check_connectivity   — verifies remote before any remote operations
#   check_remote_rootfs  — aborts if remote rootfs nearly full
#   DOCKER_TIMEOUT       — all docker calls protected
#   SSH_TIMEOUT          — all SSH calls protected
#   platform_require_cmd  — notify validated before use
#   Silent on success    — runs weekly, only failures warrant notification
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   WEEKLY_SYNC_SHARES             — shares synced during window
#   WEEKLY_MAINTENANCE_SCRIPTS     — scripts run after sync
#   WEEKLY_SYNC_UPDATES            — toggle local container updates
#   WEEKLY_SYNC_UPDATES_REMOTE     — toggle remote container updates
#   WEEKLY_RSYNC_ENABLED           — enable/disable rsync section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   weekly_sync_maintenance.sh              — normal run
#   weekly_sync_maintenance.sh --dry-run    — preview without stopping containers or syncing
#   weekly_sync_maintenance.sh --log        — verbose per-share/per-job output
#   weekly_sync_maintenance.sh --status     — show configuration and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."

parse_args "$@"

DOCKER_TIMEOUT=30   # container stop/start needs longer than normal
SSH_TIMEOUT=30      # remote pulls can be slow

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts
resolve_remote_ip

WINDOW_START=$(date +%s)
PASS=()
FAIL=()
JOB_PASS=()
JOB_FAIL=()

# Load container lists from profile config
read -r -a MAINTENANCE_CONTAINERS <<< \
    "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"

# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
run_job() {
    local script_entry="$1"
    local extra_dry=""
    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"

    read -r -a script_args <<< "$script_entry"
    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
    local script_name
    script_name=$(basename "${script_args[0]}")
    local extra_args=("${script_args[@]:1}")

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        JOB_FAIL+=("$script_name")
        return 1
    fi

    log "Running: $script_name ${extra_args[*]}"
    # shellcheck disable=SC2086
    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
        log "$script_name — done ✅"
        JOB_PASS+=("$script_name ${extra_args[*]}")
    else
        error "$script_name — failed (exit $?)"
        JOB_FAIL+=("$script_name ${extra_args[*]}")
    fi
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Weekly enabled:  ${WEEKLY_RSYNC_ENABLED:-false}"
    echo "$ICON_GEAR   Local updates:   ${WEEKLY_SYNC_UPDATES:-false}"
    echo "$ICON_GEAR   Remote updates:  ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
    echo ""
    echo "━━━ Weekly Sync Shares ━━━"
    if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
        warn "  No WEEKLY_SYNC_SHARES configured"
    else
        for share in "${WEEKLY_SYNC_SHARES[@]}"; do
            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
        done
    fi
    echo ""
    echo "━━━ Weekly Maintenance Scripts ━━━"
    if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
        done
    fi
    echo ""
    echo "━━━ Containers (from profile config) ━━━"
    for c in "${MAINTENANCE_CONTAINERS[@]}"; do
        [[ -n "$c" ]] && echo "  $ICON_CONTAINERS $c"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

check_connectivity
check_remote_rootfs

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be stopped"
else
    # Load container lists for stop functions
    read -r -a CRITICAL_CONTAINER_NAMES <<< \
        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"

    stop_local_containers
    stop_containers
fi

# ==============================================================================================
# ━━━ Container Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                log "$c — image updated ✅"
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
    else
        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found on remote, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker pull $IMAGE" >/dev/null 2>&1; then
                log "$c — remote image updated ✅"
            else
                warn "$c — remote pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi

# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}

echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"

SYNC_START=$(date +%s)
JOB_NUM=0

if ! check_rsync_enabled "WEEKLY"; then
    warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
    warn "Proceeding to container start and maintenance scripts..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
    warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
    warn "Check WEEKLY_SYNC_SHARES in master.conf"
else
    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
        (( JOB_NUM++ ))
        JOB_NAME=$(basename "$JOB")

        echo ""
        echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"

        JOB_START=$(date +%s)
        bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
        EXIT_CODE=$?
        JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))

        if [[ "$EXIT_CODE" -eq 0 ]]; then
            PASS+=("$JOB_NAME")
            log "$JOB_NAME — done in $JOB_DUR ✅"
        else
            FAIL+=("$JOB_NAME")
            error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
        fi
        echo ""
    done
fi

SYNC_END=$(date +%s)

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers
    start_local_containers
fi

# ==============================================================================================
# ━━━ Post-sync Jobs ━━━
# ==============================================================================================
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_job "$script_entry"
    done
fi

# ==============================================================================================
# ━━━ Remainder Container Updates ━━━
# ==============================================================================================
# Updates all running containers not already covered by daily or the weekly sync window.
# Runs last — weekly sync-window containers are back up before this pulls their peers.
echo ""
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"

DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
    warn "docker_update.sh not found — skipping remainder updates"
    JOB_FAIL+=("docker_update.sh --remainder")
else
    _remainder_args=("--remainder")
    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
        echo "Remainder updates complete ✅"
        JOB_PASS+=("docker_update.sh --remainder")
    else
        warn "Remainder updates completed with errors"
        JOB_FAIL+=("docker_update.sh --remainder")
    fi
    unset _remainder_args
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""

echo "$ICON_SYNC  Sync jobs ($SHARE_COUNT):"
for job in "${PASS[@]}"; do echo "  $ICON_DONE  $job"; done
for job in "${FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
echo "  Passed: ${#PASS[@]}  Failed: ${#FAIL[@]}"

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo ""
    echo "$ICON_GEAR  Post-sync jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

echo ""
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
    echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
else
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Weekly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0


ok now daily and weekly ran. no containers were updated


Run docker network script during onboarding


I spot two issues. The weekly script hardcodes `docker_update.sh --remainder` at the end, but `WEEKLY_MAINTENANCE_SCRIPTS` in master.conf already includes `docker_update_remaining.sh` — that's duplicate work with different exclusion logic. Let me also check what's in the actual logs and look for the rebuild function.


Bash

ls /boot/config/plugins/varaverk/data/ | sort

List all data/log files


arr_cleanup_stats.db
arr_recovery_stats.db
bandwidth_history.db
container_restart_history.db
lidarr_discovery_history.db
lidarr_tracked.count
radarr_discovery_history.db
sonarr_discovery_history.db
transcode_daily.db
zfs-weekly-health.log


grep -n "platform_rebuild_container\|rebuild_container" /boot/config/plugins/varaverk/Plugin/unraid/adapter.sh | head -20

Find platform_rebuild_container implementation


38:#   platform_rebuild_container    — rebuild a container from its stored XML template
302:# platform_rebuild_container <container_name>
306:platform_rebuild_container() {
308:    local rebuild_script="/usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container"


302	# platform_rebuild_container <container_name>
303	#   Rebuilds a container from its stored Unraid CA XML template (stops old, recreates on new
304	#   image digest, prunes old image). Returns 1 if the rebuild script is absent or fails.
305	# ──────────────────────────────────────────────────────────────────────────────────────────────
306	platform_rebuild_container() {
307	    local container="$1"
308	    local rebuild_script="/usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container"
309	    [[ -x "$rebuild_script" ]] || return 1
310	    "$rebuild_script" "$container" >/dev/null 2>&1
311	}
312
313	# ──────────────────────────────────────────────────────────────────────────────────────────────
314	# platform_push_conf
315	#   Pushes master.conf to all listed hosts via the WebGUI PHP API.
316	#   Writes per-host push results to stdout. Returns 1 if any host failed.
317	#   Returns 0 silently if php is unavailable (caller should warn manually).
318	# ──────────────────────────────────────────────────────────────────────────────────────────────
319	platform_push_conf() {
320	    command -v php &>/dev/null || return 0
321	    local rc=0
322	    php -r "
323	require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
324	\$results = vv_push_master_conf();
325	if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
326	\$failed = 0;
327	foreach (\$results as \$r) {
328	    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
329	    if (!\$r['ok']) \$failed++;
330	}
331	exit(\$failed > 0 ? 1 : 0);
332	" 2>/dev/null
333	    rc=$?
334	    return $rc
335	}
336
337	# ──────────────────────────────────────────────────────────────────────────────────────────────
338	# platform_push_setup_state
339	#   Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
340	#   No-op if php is unavailable. Always returns 0.
341	# ──────────────────────────────────────────────────────────────────────────────────────────────
342	platform_push_setup_state() {
343	    command -v php &>/dev/null || return 0
344	    php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null
345	    return 0
346	}
347

# platform_rebuild_container <container_name>
#   Rebuilds a container from its stored Unraid CA XML template (stops old, recreates on new
#   image digest, prunes old image). Returns 1 if the rebuild script is absent or fails.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_rebuild_container() {
    local container="$1"
    local rebuild_script="/usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container"
    [[ -x "$rebuild_script" ]] || return 1
    "$rebuild_script" "$container" >/dev/null 2>&1
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_push_conf
#   Pushes master.conf to all listed hosts via the WebGUI PHP API.
#   Writes per-host push results to stdout. Returns 1 if any host failed.
#   Returns 0 silently if php is unavailable (caller should warn manually).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_push_conf() {
    command -v php &>/dev/null || return 0
    local rc=0
    php -r "
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
\$results = vv_push_master_conf();
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
\$failed = 0;
foreach (\$results as \$r) {
    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
    if (!\$r['ok']) \$failed++;
}
exit(\$failed > 0 ? 1 : 0);
" 2>/dev/null
    rc=$?
    return $rc
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_push_setup_state
#   Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
#   No-op if php is unavailable. Always returns 0.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_push_setup_state() {
    command -v php &>/dev/null || return 0
    php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null
    return 0
}


grep -n "DAILY_RESTART_CONTAINERS\|WEEKLY_RESTART_CONTAINERS" /boot/config/plugins/varaverk/Configurations/host1.conf 2>/dev/null | head -30

Check host1.conf for restart container lists


329:    HOST1_DAILY_RESTART_CONTAINERS=(
342:    HOST1_WEEKLY_RESTART_CONTAINERS=(


325	# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
326	# ==============================================================================================
327
328	# ━━━ Docker Daily Restart ━━━
329	    HOST1_DAILY_RESTART_CONTAINERS=(
330	        "NginxProxyManager"
331	        "Lldap-Gmer4Lfe"
332	        "Authelia"
333	        "Authelia-Secondary"
334	        "Dispatcharr-Iptv-Users"
335	        "Dispatcharr"               # Live TV scheduler — degrades without daily restart
336	        "Dispatcharr-Basic"
337	        "ErsatzTV-Emby"
338	        "slskd"                     # Soulseek connection drops after extended uptime; restart refreshes share index
339	    )
340
341	# ━━━ Docker Weekly Restart ━━━
342	    HOST1_WEEKLY_RESTART_CONTAINERS=(
343	        "NextCloud"
344	        "Organizrv2-Gmer4Lfe"
345	        "AdGuard-Home"
346	        "Immich-Gmer4Lfe"
347	    )
348
349	# ━━━ Docker Watchdog ━━━
350
351	# Memory hard limits in MB — immediate restart if exceeded.
352	# 20GB=20480  16GB=16384  8GB=8192  4GB=4096  2GB=2048  1GB=1024
353	    declare -A HOST1_WATCHDOG_CONTAINERS=(
354	        ["Emby"]=20480              # 20GB — large library + active transcodes

# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Docker Daily Restart ━━━
    HOST1_DAILY_RESTART_CONTAINERS=(
        "NginxProxyManager"
        "Lldap-Gmer4Lfe"
        "Authelia"
        "Authelia-Secondary"
        "Dispatcharr-Iptv-Users"
        "Dispatcharr"               # Live TV scheduler — degrades without daily restart
        "Dispatcharr-Basic"
        "ErsatzTV-Emby"
        "slskd"                     # Soulseek connection drops after extended uptime; restart refreshes share index
    )

# ━━━ Docker Weekly Restart ━━━
    HOST1_WEEKLY_RESTART_CONTAINERS=(
        "NextCloud"
        "Organizrv2-Gmer4Lfe"
        "AdGuard-Home"
        "Immich-Gmer4Lfe"
    )

# ━━━ Docker Watchdog ━━━

# Memory hard limits in MB — immediate restart if exceeded.
# 20GB=20480  16GB=16384  8GB=8192  4GB=4096  2GB=2048  1GB=1024
    declare -A HOST1_WATCHDOG_CONTAINERS=(
        ["Emby"]=20480              # 20GB — large library + active transcodes


Let me check if there are logs from the recent runs and whether the rebuild script exists on this system.


ls -la /usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container 2>/dev/null && echo "EXISTS" || echo "NOT FOUND"

Check if rebuild_container script exists


ls /var/log/varaverk/ 2>/dev/null || ls /tmp/varaverk*.log 2>/dev/null || find /var/log -name "*varaverk*" -o -name "*daily*maint*" -o -name "*weekly*maint*" 2>/dev/null | head -10

Find maintenance run logs


-rwxr-xr-x 1 root root 1460 Nov 15  2023 /usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container*
EXISTS


Orchestrators/
Plugin/
Tools/
git_pull_execute.json
git_pull_execute.log
git_pull_execute.manual_ts
rebuild_cron.log


grep -n "DOCKER_UPDATE\|docker_update\|DAILY_CONTAINER\|WEEKLY_REMAINING" /boot/config/plugins/varaverk/Configurations/master.conf | head -20

Check all docker update toggles in master.conf


377:        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
384:    DAILY_CONTAINER_UPDATES=true
405:        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
415:    WEEKLY_REMAINING_UPDATES=true


docker ps --format '{{.Names}}' 2>/dev/null | head -20

List currently running containers


Emby
NextCloud
AdGuard-Home
degoog
Unraid-progressive-web-app
Trailarr
Sonarr
SeerrFin
SABnzbd-Gmer4Lfe
Redis-Immich
Redis-Authelia-Secondary
Redis-Authelia
Radarr
Qbittorrent-Gmer4Lfe
Prowlarr
Mariadb-Authelia-Secondary
Mariadb-Authelia
Lidarr
Koffan
Healarr


docker ps --format '{{.Names}}' 2>/dev/null | tail -30

Get remaining running containers


Dispatcharr-Basic
Dispatcharr
Authelia-Secondary
Authelia
Lldap-Gmer4Lfe
NginxProxyManager
Immich-Gmer4Lfe
Seerr
musicseerr
VaultWarden-Gmer4Lfe
EmyCredits-Ocr
Fetcharr
slskd
Soularr
Readarr
Profilarr
Pinchflat
Organizrv2-Gmer4Lfe
UptimeKuma-Gmer4Lfe
honeygain
DHCP-Netboot
speedtest-tracker
LocalAI
Krusader
ErsatzTV-Emby
Postgres-NextCloud
PostgreSQL_Immich
Gmer4Lfe.com
7DaysToDie
Playerr


grep -n "PROFILE_CRITICAL_CONTAINER_NAMES\|WEEKLY_RESTART_CONTAINERS" /boot/config/plugins/varaverk/Configurations/master.conf | head -20

Check profile container names and weekly restart list in master.conf


585:    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
697:#   HOST1_WEEKLY_RESTART_CONTAINERS
698:#   HOST2_WEEKLY_RESTART_CONTAINERS


583	# Only running containers get restarted — stopped containers stay stopped.
584	# SPACE-SEPARATED STRINGS — converted to array at runtime
585	    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
586	        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
587	        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
588	        [critical-fallback]=""      # dirty sync — auth stays running both sides
589	        [important-data]="Postgres-NextCloud NextCloud"
590	        [emby]="Emby"
591	        [emby-fallback]=""          # dirty sync — Emby stays running both sides
592	    )
593
594	# Containers needing a delay after rsync before starting.
595	# SPACE-SEPARATED STRINGS — converted to array at runtime
596	    declare -A PROFILE_DELAYED_CONTAINERS=(
597	        [arrs_stack]=""

# Only running containers get restarted — stopped containers stay stopped.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
        [critical-fallback]=""      # dirty sync — auth stays running both sides
        [important-data]="Postgres-NextCloud NextCloud"
        [emby]="Emby"
        [emby-fallback]=""          # dirty sync — Emby stays running both sides
    )

# Containers needing a delay after rsync before starting.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_DELAYED_CONTAINERS=(
        [arrs_stack]=""


Found it. There's a duplication: `WEEKLY_MAINTENANCE_SCRIPTS` includes `docker_update_remaining.sh`, but `weekly_sync_maintenance.sh` also has a hardcoded `docker_update.sh --remainder` at the end. They do essentially the same thing with slightly different exclusion logic — whichever runs first takes all the updates and the second always finds nothing.

Additionally, `docker_update.sh --remainder` doesn't exclude `WEEKLY_RESTART_CONTAINERS` (NextCloud, AdGuard, etc.), while `docker_update_remaining.sh` does but misses the Emby/critical-data exclusion that the remainder mode handles correctly.

Fix: remove `docker_update_remaining.sh` from `WEEKLY_MAINTENANCE_SCRIPTS`, add `WEEKLY_RESTART_CONTAINERS` to the remainder exclusion list in `docker_update.sh`.


Edit

    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority

    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority


The file /boot/config/plugins/varaverk/Configurations/master.conf has been updated successfully. (file state is current in your context — no need to Read it back)

    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority

    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority

-        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers

"Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history

"Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations


Now add `WEEKLY_RESTART_CONTAINERS` to the exclusion logic in `docker_update.sh --remainder`:


    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Fallback coverage containers — owned by the remote server's update cycle.

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Weekly restart containers — restarted by docker_weekly_restart.sh after sync
    for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done

    # Fallback coverage containers — owned by the remote server's update cycle.


The file /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh has been updated successfully. (file state is current in your context — no need to Read it back)

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Fallback coverage containers — owned by the remote server's update cycle.

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Weekly restart containers — restarted by docker_weekly_restart.sh after sync
    for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done

    # Fallback coverage containers — owned by the remote server's update cycle.

+    # Weekly restart containers — restarted by docker_weekly_restart.sh after sync

+    for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do

# This server runs them during fallback but should never update them independently.

# Updating them here risks version divergence: if remote's writeback after handback


Also update the status output so it reflects the weekly restart exclusion:


echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"

echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"


echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"

echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"

if [[ "$REMAINDER_MODE" == true ]]; then

-        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"

+        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"

for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done

echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"


Also update the docblock at the top of `docker_update.sh` to document the weekly restart exclusion:


# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.

# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     WEEKLY_RESTART_CONTAINERS      — restarted by docker_weekly_restart.sh after sync
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.


# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.

# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     WEEKLY_RESTART_CONTAINERS      — restarted by docker_weekly_restart.sh after sync
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.

+#     WEEKLY_RESTART_CONTAINERS      — restarted by docker_weekly_restart.sh after sync


Two things fixed:

1. **Duplication removed** — `docker_update_remaining.sh` was in `WEEKLY_MAINTENANCE_SCRIPTS` AND the weekly had a hardcoded `docker_update.sh --remainder` call at the end. Whichever ran first would take all available updates; the second always found nothing. `docker_update_remaining.sh` is removed from the weekly list — the hardcoded remainder step is the right one to keep (it handles fallback container exclusion that `docker_update_remaining.sh` missed).

2. **Missing exclusion added** — `WEEKLY_RESTART_CONTAINERS` (NextCloud, AdGuard, Immich, Organizrr) weren't excluded from `docker_update.sh --remainder`. The weekly restart script handles those; the remainder step shouldn't touch them.

On "no containers were updated" — if this is a recently set up system with freshly pulled images, that's expected. The update scripts only rebuild containers when a new image digest lands. Next weekly run should catch anything that's actually shipped an update.


no half the containers need updates and were not touched, this system has been running unraid for 5 or 6 years


That's a real bug then. Let me check `acquire_lock` — the orchestrators hold the lock, then call scripts that also try to acquire it. That would deadlock.


Bash

grep -n "acquire_lock\|LOCK_FILE\|flock" /boot/config/plugins/varaverk/common.sh | head -30

Find acquire_lock implementation in common.sh


19:#   Locking          — acquire_lock, acquire_rsync_lock — concurrent execution protection
47:#            Script locking system added — acquire_lock(), acquire_rsync_lock()
1356:# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
1358:declare -ga _LOCK_FILES=()
1361:    for lf in "${_LOCK_FILES[@]}"; do
1366:    _LOCK_FILES+=("$1")
1387:# Usage: acquire_lock [strict|wait|continuous]
1388:acquire_lock() {


1356	# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
1357	# call used to overwrite the EXIT trap, orphaning the previous lock file.
1358	declare -ga _LOCK_FILES=()
1359	_release_all_locks() {
1360	    local lf
1361	    for lf in "${_LOCK_FILES[@]}"; do
1362	        [[ -f "$lf" ]] && rm -f "$lf"
1363	    done
1364	}
1365	_register_lock() {
1366	    _LOCK_FILES+=("$1")
1367	    trap '_release_all_locks' EXIT
1368	}
1369
1370	# Internal — script name used as lock identifier
1371	_lock_name() {
1372	    basename "${BASH_SOURCE[1]:-$0}" .sh
1373	}
1374
1375	# Internal — lock file path for this script
1376	_lock_file() {
1377	    echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
1378	}
1379
1380	# Internal — release lock on exit (registered via EXIT trap)
1381	_release_on_exit() {
1382	    local lockfile="$1"
1383	    [[ -f "$lockfile" ]] && rm -f "$lockfile"
1384	}
1385
1386	# Acquire exclusive lock for this script.
1387	# Usage: acquire_lock [strict|wait|continuous]
1388	acquire_lock() {
1389	    local mode="${1:-strict}"
1390	    local script_name
1391	    script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh)
1392	    local lockfile
1393	    lockfile="$(_lock_file "$script_name")"
1394
1395	    mkdir -p "$LOCK_DIR"
1396
1397	    if [[ -f "$lockfile" ]]; then
1398	        local lock_content existing_pid locked_name
1399	        lock_content=$(cat "$lockfile" 2>/dev/null)
1400	        existing_pid="${lock_content%%:*}"
1401	        locked_name="${lock_content##*:}"
1402
1403	        # Stale lock — PID dead
1404	        if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
1405	            warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
1406	            rm -f "$lockfile"
1407
1408	        # PID reuse — PID alive but belongs to different process
1409	        elif [[ "$locked_name" != "$script_name" ]]; then
1410	            warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing"
1411	            rm -f "$lockfile"
1412
1413	        else
1414	            # Lock is genuinely active — check age
1415	            local lock_age
1416	            lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
1417	            if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
1418	                warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
1419	            fi
1420
1421	            if [[ "$mode" == "continuous" ]]; then
1422	                log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
1423	                exit 0
1424	            elif [[ "$mode" == "wait" ]]; then
1425	                info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
1426	                local waited=0
1427	                while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
1428	                    sleep 1
1429	                    ((waited++))
1430	                    lock_content=$(cat "$lockfile" 2>/dev/null)
1431	                    existing_pid="${lock_content%%:*}"
1432	                    locked_name="${lock_content##*:}"
1433	                    if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
1434	                        warn "Lock became stale while waiting — clearing"
1435	                        rm -f "$lockfile"

# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
# call used to overwrite the EXIT trap, orphaning the previous lock file.
declare -ga _LOCK_FILES=()
_release_all_locks() {
    local lf
    for lf in "${_LOCK_FILES[@]}"; do
        [[ -f "$lf" ]] && rm -f "$lf"
    done
}
_register_lock() {
    _LOCK_FILES+=("$1")
    trap '_release_all_locks' EXIT
}

# Internal — script name used as lock identifier
_lock_name() {
    basename "${BASH_SOURCE[1]:-$0}" .sh
}

# Internal — lock file path for this script
_lock_file() {
    echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
}

# Internal — release lock on exit (registered via EXIT trap)
_release_on_exit() {
    local lockfile="$1"
    [[ -f "$lockfile" ]] && rm -f "$lockfile"
}

# Acquire exclusive lock for this script.
# Usage: acquire_lock [strict|wait|continuous]
acquire_lock() {
    local mode="${1:-strict}"
    local script_name
    script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh)
    local lockfile
    lockfile="$(_lock_file "$script_name")"

    mkdir -p "$LOCK_DIR"

    if [[ -f "$lockfile" ]]; then
        local lock_content existing_pid locked_name
        lock_content=$(cat "$lockfile" 2>/dev/null)
        existing_pid="${lock_content%%:*}"
        locked_name="${lock_content##*:}"

        # Stale lock — PID dead
        if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
            warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
            rm -f "$lockfile"

        # PID reuse — PID alive but belongs to different process
        elif [[ "$locked_name" != "$script_name" ]]; then
            warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing"
            rm -f "$lockfile"

        else
            # Lock is genuinely active — check age
            local lock_age
            lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
            if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
                warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
            fi

            if [[ "$mode" == "continuous" ]]; then
                log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
                exit 0
            elif [[ "$mode" == "wait" ]]; then
                info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
                local waited=0
                while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
                    sleep 1
                    ((waited++))
                    lock_content=$(cat "$lockfile" 2>/dev/null)
                    existing_pid="${lock_content%%:*}"
                    locked_name="${lock_content##*:}"
                    if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
                        warn "Lock became stale while waiting — clearing"
                        rm -f "$lockfile"


1434	                        warn "Lock became stale while waiting — clearing"
1435	                        rm -f "$lockfile"
1436	                        break
1437	                    elif [[ "$locked_name" != "$script_name" ]]; then
1438	                        warn "Lock PID reused while waiting — clearing"
1439	                        rm -f "$lockfile"
1440	                        break
1441	                    fi
1442	                done
1443	                if [[ -f "$lockfile" ]]; then
1444	                    error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting"
1445	                    exit 1
1446	                fi
1447	            else
1448	                error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
1449	                case "$script_name" in
1450	                    fallback|transcode_management|daily_sync_maintenance|stability_watchdog)
1451	                        notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
1452	                        ;;
1453	                esac
1454	                exit 1
1455	            fi
1456	        fi
1457	    fi
1458
1459	    # Acquire lock — store PID:scriptname to prevent PID reuse false positives
1460	    echo "$$:$script_name" > "$lockfile"
1461	    _register_lock "$lockfile"
1462	    log "$ICON_LOCK Lock acquired: $script_name (PID $$)"
1463	}

                        warn "Lock became stale while waiting — clearing"
                        rm -f "$lockfile"
                        break
                    elif [[ "$locked_name" != "$script_name" ]]; then
                        warn "Lock PID reused while waiting — clearing"
                        rm -f "$lockfile"
                        break
                    fi
                done
                if [[ -f "$lockfile" ]]; then
                    error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting"
                    exit 1
                fi
            else
                error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
                case "$script_name" in
                    fallback|transcode_management|daily_sync_maintenance|stability_watchdog)
                        notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
                        ;;
                esac
                exit 1
            fi
        fi
    fi

    # Acquire lock — store PID:scriptname to prevent PID reuse false positives
    echo "$$:$script_name" > "$lockfile"
    _register_lock "$lockfile"
    log "$ICON_LOCK Lock acquired: $script_name (PID $$)"
}


The lock is per-script name so no deadlock. Let me read the `rebuild_container` script itself and check if there's a log showing what actually happened:


Read the Unraid rebuild_container script


#!/usr/bin/php -q
<?PHP
/* Copyright 2005-2023, Lime Technology
 * Copyright 2012-2023, Bergware International.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2,
 * as published by the Free Software Foundation.
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 */
?>
<?
$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
require_once "$docroot/plugins/dynamix.docker.manager/include/DockerClient.php";

$DockerClient = new DockerClient();
$DockerUpdate = new DockerUpdate();
$DockerTemplates = new DockerTemplates();

$start  = array_map(function($row){return strtok($row,' ');},@file('/var/lib/docker/unraid-autostart',FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES) ?: []);
$custom = DockerUtil::custom();
$subnet = DockerUtil::network($custom);
$cpus   = DockerUtil::cpus();
$tmpl   = $DockerTemplates->getUserTemplate($argv[1]);
$xml    = file_get_contents($tmpl);

[$cmd,$name,$repository] = xmlToCommand($tmpl);
$old = $DockerClient->getImageID($repository);

removeContainer($name,false,false);
execCommand(str_replace('/docker create ','/docker run -d ',$cmd),false);
if (!in_array($name,$start)) stopContainer($name,false,false);
$DockerClient->flushCaches();
$new = $DockerClient->getImageID($repository);
if ($old && $old != $new) removeImage($old,false);
?>


ls /var/log/varaverk/Orchestrators/ 2>/dev/null || ls /tmp/*.log 2>/dev/null | head -20

Find orchestrator logs


array_started.json
array_started.log
critical_sync_maintenance.json
critical_sync_maintenance.log
daily_sync_maintenance.json
daily_sync_maintenance.log
intermediate_sync_maintenance.json
intermediate_sync_maintenance.log
sunday_morning_coffee_report.json
sunday_morning_coffee_report.log
transcode_management.json
transcode_management.log
watchdog_orchestrator.json
watchdog_orchestrator.log
weekly_sync_maintenance.json
weekly_sync_maintenance.log


grep -i "docker_update\|updated\|rebuild\|pulled\|DAILY_CONTAINER\|RESTART_CONTAINERS\|nothing to update\|No containers\|up to date\|new image" /var/log/varaverk/Orchestrators/daily_sync_maintenance.log | tail -40

Check daily maintenance log for docker update output


🏁 Status: done — 20 shares updated
🏁 Status: done ✅ — 0 updated, 9 current
  🏁  docker_update.sh
Already up to date.
  Already up to date — no changes needed.
    + HOSTN_DAILY_RESTART_CONTAINERS
    + HOSTN_WEEKLY_RESTART_CONTAINERS
    - HOST1_WEEKLY_RESTART_CONTAINERS
    - HOST1_DAILY_RESTART_CONTAINERS
Updated: /boot/config/plugins/varaverk/Configurations/host1.conf
❌ [ERROR] docker_update.sh — failed (exit 1)
  ❌ docker_update.sh
Already up to date.
  Already up to date — no changes needed.
Updated: /boot/config/plugins/varaverk/Configurations/host1.conf
🏁 Status: done — 20 shares updated
🏁 Status: done ✅ — 0 updated, 9 current
  🏁  docker_update.sh


Found it — the log shows `docker_update.sh — failed (exit 1)` and the diff shows `HOSTN_` replacing `HOST1_`. The git pull is bringing in template-style variable names. Let me see the full context:


Check the tail of the daily maintenance log for the most recent run


━━━ 🔄 Pre-flight: Radarr Import Scan ━━━
ℹ  [INFO]  Scan targets (5): /mnt/user/Movies /mnt/user/Kids_Movies /mnt/user/Anime_Movies-Old /ext-ufc-ppv /mnt/user/stand-up_comedy/specials
ℹ  [INFO]  Triggering DownloadedMoviesScan (all root folders)
ℹ  [INFO]  Import scan queued (command ID: 4386089) — waiting for completion...
⚠  [WARN]  Import scan reported failed — proceeding anyway

━━━ 🔄 Fetching Radarr Tracked Files ━━━
🏁  [OK]    Radarr version: 6.1.1.10360 (major 6 — tested ✅)
ℹ  [INFO]  Querying Radarr API...
ℹ  [INFO]  Found 2869 movies — fetching movie files...
ℹ  [INFO]  Built in-memory lookup map: 2730 tracked paths
ℹ  [INFO]  2869 movies | 2730 tracked movie files

━━━ 🧹 Scanning Movies Root ━━━
ℹ  [INFO]  Root: /mnt/user/Movies | Orphan age: 7 days
ℹ  [INFO]  Cleaning up empty folders...
ℹ  [INFO]  Empty folders removed

━━━━━ 📋 RADARR CLEANUP SUMMARY ━━━━━
🖥    Identity:        HOST1 (unRAID-Gmer4Lfe)
🔄    Tracked:         2730 files (2869 movies)
🛡  Protected:       28074 files (artwork, subtitles, metadata)
🗑   Orphans:         0 files (0B)
🗑   Junk:            0 files (0B)
⏭    Recent skipped:  0 files (under 7 days)
⏱    Duration:        3m48s

🏁 Clean — nothing to remove
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

ℹ  [INFO]  HOST1 (unRAID-Gmer4Lfe) — tools OK

━━━ 🌐 Lidarr API ━━━
ℹ  [INFO]  Lidarr reachable — http://localhost:8686

━━━ 🔄 Building Album Directory Map ━━━
ℹ  [INFO]  Fetching track files for 1347 artists...
ℹ  [INFO]  Mapped 10553 albums with local tracks

━━━ 🎬 Albums ━━━
ℹ  [INFO]  Processing 11789 albums...
⚠  [WARN]  Failed to fetch: cover.jpg
ℹ  [INFO]  Checked: 10553 | Complete: 0 | Needed art: 10553 | Fetched: 81 | Failed: 10553

━━━ 🎬 Artists ━━━
ℹ  [INFO]  Processing 1347 artists...
⚠  [WARN]  Failed to fetch: folder.jpg
ℹ  [INFO]  Checked: 1347 | Complete: 0 | Needed art: 1347 | Fetched: 0 | Failed: 1332

━━━━━ 📋 LIDARR MISSING ART SUMMARY ━━━━━
🖥    Identity:  HOST1 (unRAID-Gmer4Lfe)
⏱    Duration:  3h 0m
🎬   Albums:    10553 checked | 0 complete | 81 fetched | 10553 failed
🎬   Artists:   1347 checked | 0 complete | 0 fetched | 1332 failed
🏁    Status:    ✅ ALL DONE
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

  HOST1 (unRAID-Gmer4Lfe) — http://localhost:7878
  Files: records only (use --delete-files to also remove from disk)

━━━ 🔄 Querying Radarr ━━━
  2871 movies total — 0 dropped from TMDb

━━━━━ 📋 RADARR TMDB REMOVED SUMMARY ━━━━━
🖥   Identity: HOST1 (unRAID-Gmer4Lfe)
🏁   Status:   nothing to remove
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

  HOST1 (unRAID-Gmer4Lfe) — http://localhost:8989
  Files: records only (use --delete-files to also remove from disk)

━━━ 🔄 Querying Sonarr ━━━
  1165 series total — 0 dropped from TVDB

━━━━━ 📋 SONARR TVDB REMOVED SUMMARY ━━━━━
🖥   Identity: HOST1 (unRAID-Gmer4Lfe)
🏁   Status:   nothing to remove
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

━━━ 📦 Docker Update — 2026-06-14 05:46:07 ━━━

━━━━━ 📋 DOCKER UPDATE SUMMARY ━━━━━
🖥       Identity:    HOST1 (unRAID-Gmer4Lfe)
⏱       Duration:    2m20s
🔄       Pruned:      Total reclaimed space: 0B
🏁 Status: done ✅ — 0 updated, 9 current
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

━━━ 📦 Daily Restart — 2026-06-14 05:48:27 ━━━
🖥 HOST1 (unRAID-Gmer4Lfe) — 9 container(s)
Authelia
  Waiting 5s — Authelia-Secondary depends on Authelia...
Authelia-Secondary
NginxProxyManager
Lldap-Gmer4Lfe
Dispatcharr-Iptv-Users
Dispatcharr
Dispatcharr-Basic
ErsatzTV-Emby
slskd

━━━ 🔄 Pruning Dangling Images — 2026-06-14 05:49:50 ━━━
━━━━━ 📋 DAILY RESTART SUMMARY ━━━━━
🖥     Identity:  HOST1 (unRAID-Gmer4Lfe)
⏱     Duration:  1m23s
📦 Scope:   9 restarted, 0 skipped, 0 failed
🔄     Pruned:    Total reclaimed space: 0B
🏁     Status:    ALL DONE ✅
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

━━━━━ 📋 DAILY MAINTENANCE SUMMARY ━━━━━
🖥   Identity: HOST1 (unRAID-Gmer4Lfe)
⏱   Window:   2026-06-14 01:00:06 → 05:49:51
⏱   Duration: 4h 49m

🔄  Shares (14):
  ❌ Books — 0s
  ❌ Intros — 0s
  ❌ Kids_Movies — 0s
  ❌ Kids_Tv_Shows — 0s
  ❌ Movies — 0s
  ❌ Music — 0s
  ❌ Music_Videos — 0s
  ❌ Nextcloud — 0s
  ❌ stand-up_comedy — 0s
  ❌ Sports — 1s
  ❌ Anime_Shows-Old — 0s
  ❌ Anime_Movies-Old — 0s
  ❌ Anime_Movies — 0s
  ❌ Anime_Shows — 0s
  Passed: 0/14  Failed: 14/14

⚙  Jobs:
  🏁  git_pull_execute.sh
  🏁  arr_sync.sh
  🏁  media_shares_permissions.sh
  🏁  media_cleaner.sh anime
  🏁  media_cleaner.sh media
  🏁  radarr_cleanup.sh
  🏁  lidarr_missing_art.sh
  🏁  radarr_tmdb_removed.sh
  🏁  sonarr_tvdb_removed.sh
  🏁  docker_update.sh
  🏁  docker_daily_restart.sh

⚠  [WARN]  Status: 14 failure(s)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━


Docker update ran but found 0 updated, 9 current for `DAILY_RESTART_CONTAINERS`. Now check the weekly log — the non-daily containers that need updates should be caught by the remainder step:


grep -A5 -B2 "Remainder\|docker_update.*remainder\|UPDATED\|updated\|rebuild\|0 updated\|No containers" /var/log/varaverk/Orchestrators/weekly_sync_maintenance.log | tail -60

Check weekly log for remainder update results


━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

━━━ 📦 Remainder Container Updates ━━━

━━━ 📦 Docker Update (remainder) — 2026-06-07 02:59:29 ━━━
📦 Updating 47 container(s) (not in daily or weekly sync)

━━━━━ 📋 DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━
--
🔄       Rebuilt:     1
🔄       Pruned:      Total reclaimed space: 6.303GB
🏁 Status: done ✅ — 1 updated, 46 current
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Remainder updates complete ✅

━━━━━ 📋 WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━
🖥   Identity:  HOST1 (unRAID-Gmer4Lfe)
⏱   Window:    2026-06-07 02:30:06 → 03:14:55
⏱   Duration:  44m49s
--
  🏁  playback_aware_radarr_discovery.sh
  🏁  playback_aware_sonarr_discovery.sh
  🏁  docker_update.sh --remainder

⚠  [WARN]  Status: 2 failure(s)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

── 2026-06-14 02:30:01 ────────────────────────────────────────────────
--
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

━━━ 📦 Remainder Container Updates ━━━

━━━ 📦 Docker Update (remainder) — 2026-06-14 03:44:01 ━━━
📦 Updating 44 container(s) (not in daily or weekly sync)

━━━━━ 📋 DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━
--
🔄       Rebuilt:     2
🔄       Pruned:      Total reclaimed space: 0B
🏁 Status: done ✅ — 2 updated, 42 current
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Remainder updates complete ✅

━━━━━ 📋 WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━
🖥   Identity:  HOST1 (unRAID-Gmer4Lfe)
⏱   Window:    2026-06-14 02:30:07 → 03:57:09
⏱   Duration:  1h 27m
--
  🏁  playback_aware_radarr_discovery.sh
  🏁  playback_aware_sonarr_discovery.sh
  🏁  docker_update.sh --remainder

🏁 Status: all complete ✅ — 2 share(s) synced, 7 job(s) run
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━


Weekly found 2 updated, 42 current. But something's off — let me check how `start_containers` works during the weekly sync window, because `docker pull` + `docker start` doesn't use the new image:


grep -n "WEEKLY_SYNC_UPDATES\|start_containers\|stop_containers" /boot/config/plugins/varaverk/Configurations/master.conf | head -20

Check weekly sync update toggles and container functions in master.conf


425:    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
426:    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH


grep -n "^start_containers\|^stop_containers\|^start_local\|^stop_local" /boot/config/plugins/varaverk/common.sh

Find start/stop container functions in common.sh


1185:stop_containers() {
1221:stop_local_containers() {
1252:start_containers() {
1280:start_local_containers() {


1252	start_containers() {
1253	    if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
1254	        log "No remote containers to restart."
1255	        return
1256	    fi
1257	    info "Starting remote containers..."
1258	    for c in "${RUNNING_CONTAINERS[@]}"; do
1259	        [[ -z "$c" ]] && continue
1260	        local needs_delay=false
1261	        for d in "${DELAYED_CONTAINERS[@]}"; do
1262	            [[ "$c" == "$d" ]] && needs_delay=true && break
1263	        done
1264	        if [[ "$needs_delay" == true ]]; then
1265	            info "Waiting ${CONTAINER_DELAY}s before starting $c..."
1266	            sleep "$CONTAINER_DELAY"
1267	        fi
1268	        echo "$ICON_START Starting remote $c..."
1269	        if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then
1270	            echo "$ICON_STARTED $c started"
1271	        else
1272	            error "Failed to start remote $c — start manually if needed"
1273	        fi
1274	    done
1275	}
1276
1277	# Restart containers on the LOCAL server that were running before sync.
1278	# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers().
1279	# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote.
1280	start_local_containers() {
1281	    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
1282	        log "No local containers to restart."
1283	        return
1284	    fi
1285	    info "Starting local containers..."
1286	    for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
1287	        [[ -z "$c" ]] && continue
1288	        local needs_delay=false
1289	        for d in "${DELAYED_CONTAINERS[@]}"; do
1290	            [[ "$c" == "$d" ]] && needs_delay=true && break
1291	        done
1292	        if [[ "$needs_delay" == true ]]; then
1293	            info "Waiting ${CONTAINER_DELAY}s before starting local $c..."
1294	            sleep "$CONTAINER_DELAY"
1295	        fi
1296	        echo "$ICON_START Starting local $c..."
1297	        if docker start "$c" >/dev/null 2>&1; then
1298	            echo "$ICON_STARTED $c started"
1299	        else
1300	            error "Failed to start local $c — start manually if needed"
1301	        fi

start_containers() {
    if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No remote containers to restart."
        return
    fi
    info "Starting remote containers..."
    for c in "${RUNNING_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue
        local needs_delay=false
        for d in "${DELAYED_CONTAINERS[@]}"; do
            [[ "$c" == "$d" ]] && needs_delay=true && break
        done
        if [[ "$needs_delay" == true ]]; then
            info "Waiting ${CONTAINER_DELAY}s before starting $c..."
            sleep "$CONTAINER_DELAY"
        fi
        echo "$ICON_START Starting remote $c..."
        if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then
            echo "$ICON_STARTED $c started"
        else
            error "Failed to start remote $c — start manually if needed"
        fi
    done
}

# Restart containers on the LOCAL server that were running before sync.
# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers().
# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote.
start_local_containers() {
    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No local containers to restart."
        return
    fi
    info "Starting local containers..."
    for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue
        local needs_delay=false
        for d in "${DELAYED_CONTAINERS[@]}"; do
            [[ "$c" == "$d" ]] && needs_delay=true && break
        done
        if [[ "$needs_delay" == true ]]; then
            info "Waiting ${CONTAINER_DELAY}s before starting local $c..."
            sleep "$CONTAINER_DELAY"
        fi
        echo "$ICON_START Starting local $c..."
        if docker start "$c" >/dev/null 2>&1; then
            echo "$ICON_STARTED $c started"
        else
            error "Failed to start local $c — start manually if needed"
        fi


Confirmed — the weekly sync window calls `docker pull` for the maintenance containers, but `start_local_containers()` uses `docker start`, which re-uses the image ID baked in at creation time. New images are pulled but containers never actually switch to them.

Emby, Mariadb-Authelia, Redis-Authelia (and their secondaries) are the affected containers — they're excluded from remainder mode and the daily only `docker start`s them after sync, not rebuilds. They never pick up pulled images.

The fix: track digest changes during the pull phase, then use `platform_rebuild_container` instead of `docker start` for any container that got a new image.


183	# ==============================================================================================
184	echo ""
185	echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
186
187	if [[ "$DRY_RUN" == true ]]; then
188	    warn "DRY RUN — containers will not be stopped"
189	else
190	    # Load container lists for stop functions
191	    read -r -a CRITICAL_CONTAINER_NAMES <<< \
192	        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
193	    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
194	    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
195
196	    stop_local_containers
197	    stop_containers
198	fi
199
200	# ==============================================================================================
201	# ━━━ Container Updates ━━━
202	# ==============================================================================================
203	echo ""
204	echo "━━━ $ICON_GEAR Container Updates ━━━"
205
206	if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
207	    if [[ "$DRY_RUN" == true ]]; then
208	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
209	            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
210	        done
211	    else
212	        log "Pulling local container updates..."
213	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
214	            [[ -z "$c" ]] && continue
215	            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
216	                "$c" --format '{{.Config.Image}}' 2>/dev/null)
217	            if [[ -z "$IMAGE" ]]; then
218	                log "$c — not found locally, skipping update"
219	                continue
220	            fi
221	            log "Pulling $IMAGE for $c..."
222	            if docker pull "$IMAGE" >/dev/null 2>&1; then
223	                log "$c — image updated ✅"
224	            else
225	                warn "$c — pull failed, will start on existing image"
226	            fi
227	        done
228	    fi
229	else
230	    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
231	fi
232
233	if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
234	    if [[ "$DRY_RUN" == true ]]; then
235	        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
236	    else
237	        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
238	        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
239	            [[ -z "$c" ]] && continue
240	            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
241	                -o ConnectTimeout="$SSH_TIMEOUT" \
242	                root@"$REMOTE_SERVER" \
243	                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
244	            if [[ -z "$IMAGE" ]]; then
245	                log "$c — not found on remote, skipping update"
246	                continue
247	            fi
248	            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
249	            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
250	                -o ConnectTimeout="$SSH_TIMEOUT" \
251	                root@"$REMOTE_SERVER" \
252	                "docker pull $IMAGE" >/dev/null 2>&1; then
253	                log "$c — remote image updated ✅"
254	            else
255	                warn "$c — remote pull failed, will start on existing image"
256	            fi
257	        done
258	    fi
259	else
260	    echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
261	fi
262

# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be stopped"
else
    # Load container lists for stop functions
    read -r -a CRITICAL_CONTAINER_NAMES <<< \
        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"

    stop_local_containers
    stop_containers
fi

# ==============================================================================================
# ━━━ Container Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                log "$c — image updated ✅"
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
    else
        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found on remote, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker pull $IMAGE" >/dev/null 2>&1; then
                log "$c — remote image updated ✅"
            else
                warn "$c — remote pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi


308
309	# ==============================================================================================
310	# ━━━ Start Containers ━━━
311	# ==============================================================================================
312	echo ""
313	echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
314
315	if [[ "$DRY_RUN" == true ]]; then
316	    warn "DRY RUN — containers will not be started"
317	else
318	    start_containers
319	    start_local_containers
320	fi
321
322	# ==============================================================================================
323	# ━━━ Post-sync Jobs ━━━
324	# ==============================================================================================
325	if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
326	    echo ""
327	    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
328	    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
329	        [[ -z "$script_entry" ]] && continue
330	        echo ""
331	        run_job "$script_entry"
332	    done

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers
    start_local_containers
fi

# ==============================================================================================
# ━━━ Post-sync Jobs ━━━
# ==============================================================================================
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_job "$script_entry"
    done


Now I have the full picture. Making both changes:


Edit

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                log "$c — image updated ✅"
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

declare -A _weekly_needs_rebuild=()

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            _old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                _new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
                if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
                    log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
                    _weekly_needs_rebuild["$c"]=1
                else
                    log "$c — already current"
                fi
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
        unset _old_id _new_id
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi


The file /boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh has been updated successfully. (file state is current in your context — no need to Read it back)

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                log "$c — image updated ✅"
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

declare -A _weekly_needs_rebuild=()

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            _old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                _new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
                if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
                    log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
                    _weekly_needs_rebuild["$c"]=1
                else
                    log "$c — already current"
                fi
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
        unset _old_id _new_id
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

echo "━━━ $ICON_GEAR Container Updates ━━━"

+            _old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

-                log "$c — image updated ✅"

+                _new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

+                    log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"

+                    _weekly_needs_rebuild["$c"]=1

+                    log "$c — already current"


Now replace `start_local_containers` with rebuild-aware logic:


if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers
    start_local_containers
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers

    # Local start — rebuild containers that received a new image, docker start the rest
    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No local containers to restart."
    else
        for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
            [[ -z "$_c" ]] && continue
            _needs_delay=false
            for _d in "${DELAYED_CONTAINERS[@]}"; do
                [[ "$_c" == "$_d" ]] && _needs_delay=true && break
            done
            [[ "$_needs_delay" == true ]] && {
                info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
                sleep "$CONTAINER_DELAY"
            }
            if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
                log "Rebuilding $_c on new image..."
                if platform_rebuild_container "$_c"; then
                    log "$_c rebuilt on new image ✅"
                else
                    warn "$_c rebuild failed — falling back to docker start"
                    docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
                fi
            else
                docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"
            fi
        done
        unset _c _d _needs_delay
    fi
fi


if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers
    start_local_containers
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers

    # Local start — rebuild containers that received a new image, docker start the rest
    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No local containers to restart."
    else
        for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
            [[ -z "$_c" ]] && continue
            _needs_delay=false
            for _d in "${DELAYED_CONTAINERS[@]}"; do
                [[ "$_c" == "$_d" ]] && _needs_delay=true && break
            done
            [[ "$_needs_delay" == true ]] && {
                info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
                sleep "$CONTAINER_DELAY"
            }
            if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
                log "Rebuilding $_c on new image..."
                if platform_rebuild_container "$_c"; then
                    log "$_c rebuilt on new image ✅"
                else
                    warn "$_c rebuild failed — falling back to docker start"
                    docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
                fi
            else
                docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"
            fi
        done
        unset _c _d _needs_delay
    fi
fi

+    # Local start — rebuild containers that received a new image, docker start the rest

+    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then

+        log "No local containers to restart."

+        for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do

+            for _d in "${DELAYED_CONTAINERS[@]}"; do

+                info "Waiting ${CONTAINER_DELAY}s before starting $_c..."

+                sleep "$CONTAINER_DELAY"

+            if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then

+                log "Rebuilding $_c on new image..."

+                if platform_rebuild_container "$_c"; then

+                    log "$_c rebuilt on new image ✅"

+                    warn "$_c rebuild failed — falling back to docker start"

+                    docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"

+                docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"


Now update the header comment to reflect the correct flow: