1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Docker Update ==============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Pulls the latest images for configured containers. Two modes: normal (daily)
9	# and remainder (weekly).
10	#
11	# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
12	# Containers stay running during the pull — no extra downtime beyond what the
13	# nightly restart already causes.
14	#
15	# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
16	# It catches everything that normal mode and the weekly sync window did not already
17	# update — derived automatically from docker ps, nothing to configure.
18	#
19	# ==============================================================================================
20	# OPERATIONAL MODEL
21	# ==============================================================================================
22	#
23	# Normal mode (daily):
24	#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
25	#   Pull → compare old vs new image ID → mark updated or already current.
26	#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
27	#
28	# Remainder mode (weekly):
29	#   Targets all currently running containers NOT in:
30	#     DAILY_RESTART_CONTAINERS       — already updated daily
31	#     emby + critical-data profiles  — updated inline by the weekly sync window
32	#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
33	#   Pull → compare → prune dangling images.
34	#
35	# ==============================================================================================
36	# DESIGN PRINCIPLES
37	# ==============================================================================================
38	#
39	# Single List
40	#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
41	#   separate update list. Adding or removing a container from the restart list
42	#   automatically updates the image pull list — one change, both places.
43	#
44	# Version Ownership
45	#   Fallback containers are excluded from remainder mode. This server only runs
46	#   them during a fallback. The remote server owns their version — if remainder
47	#   updates them independently and a handback occurs, the remote's older image
48	#   may not handle data written by the newer version.
49	#
50	# State Respect
51	#   Stopped containers are never targeted. Pulling while stopped adds no value
52	#   and a stopped container was likely halted intentionally.
53	#
54	# ==============================================================================================
55	# OPERATIONAL SAFEGUARDS
56	# ==============================================================================================
57	#
58	# Lock Acquisition
59	#   Prevents concurrent execution via acquire_lock(). Safe to call from
60	#   maintenance scripts without risk of overlap.
61	#
62	# Host Detection
63	#   detect_hosts() identifies which server is running the script and aliases
64	#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
65	#
66	# Root Enforcement
67	#   Docker operations require root privileges.
68	#
69	# DAILY_CONTAINER_UPDATES Toggle
70	#   Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
71	#   regardless — update and restart are independent operations.
72	#
73	# Fallback Exclusion
74	#   Remainder mode excludes containers owned by the remote server's update cycle
75	#   to prevent version divergence across the fallback boundary.
76	#
77	# Running-Only Filter
78	#   Stopped containers excluded from remainder mode — intentionally down.
79	#
80	# Image ID Comparison
81	#   Containers not restarted unless their image actually changed. Pulls that
82	#   result in "already up to date" produce no restart.
83	#
84	# ==============================================================================================
85	# CONFIGURATION
86	# ==============================================================================================
87	#
88	# master.conf
89	#
90	#   DAILY_CONTAINER_UPDATES
91	#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
92	#       (default: true)
93	#
94	#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
95	#       Container names for emby and critical-data profiles — excluded from
96	#       remainder mode (already updated by the weekly sync window)
97	#
98	# host*.conf
99	#
100	#   HOST*_DAILY_RESTART_CONTAINERS
101	#       Containers updated in normal mode. Aliased by detect_hosts() →
102	#       DAILY_RESTART_CONTAINERS
103	#
104	# ==============================================================================================
105	# RUNTIME MODES
106	# ==============================================================================================
107	#
108	# docker_update.sh
109	#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
110	#
111	# docker_update.sh --remainder
112	#     Remainder mode — pull all running containers not in managed lists,
113	#     restart those that received updates, prune dangling images
114	#
115	# docker_update.sh --dry-run
116	#     Preview which containers would be pulled without making changes
117	#
118	# docker_update.sh --status
119	#     Show configuration and container list for current mode
120	#
121	# docker_update.sh --log
122	#     Verbose per-container pull and comparison output
123	#
124	# ==============================================================================================
125
126	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
127
128	source "$SCRIPT_DIR/../load_config.sh"
129
130	# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
131	REMAINDER_MODE=false
132	_filtered_args=()
133	for _arg in "$@"; do
134	    if [[ "$_arg" == "--remainder" ]]; then
135	        REMAINDER_MODE=true
136	    else
137	        _filtered_args+=("$_arg")
138	    fi
139	done
140	unset _arg
141
142	parse_args "${_filtered_args[@]}"
143	unset _filtered_args
144
145	# ==============================================================================================
146	# ━━━ Setup ━━━
147	# ==============================================================================================
148	if [[ "$EUID" -ne 0 ]]; then
149	    error "Must be run as root"
150	    exit 1
151	fi
152
153	acquire_lock
154
155	if ! command -v docker &>/dev/null; then
156	    error "Docker command not found"
157	    exit 1
158	fi
159
160	detect_hosts
161
162	# ==============================================================================================
163	# ━━━ Container Discovery ━━━
164	# ==============================================================================================
165	if [[ "$REMAINDER_MODE" == true ]]; then
166	    declare -A _exclude=()
167
168	    # Daily containers — updated by docker_update.sh normal mode
169	    for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
170	        [[ -n "$_c" ]] && _exclude["$_c"]=1
171	    done
172
173	    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
174	    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
175	    read -r -a _weekly_arr <<< "$_weekly_str"
176	    for _c in "${_weekly_arr[@]}"; do
177	        [[ -n "$_c" ]] && _exclude["$_c"]=1
178	    done
179	    unset _weekly_str _weekly_arr
180
181	    # Fallback coverage containers — owned by the remote server's update cycle.
182	    # This server runs them during fallback but should never update them independently.
183	    # Updating them here risks version divergence: if remote's writeback after handback
184	    # encounters data written by a newer version, it may not handle it correctly.
185	    for _tier in 1 2 3 4; do
186	        _tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
187	        eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
188	        for _c in "${_tier_arr[@]}"; do
189	            [[ -n "$_c" ]] && _exclude["$_c"]=1
190	        done
191	    done
192	    unset _tier _tier_var _tier_arr _c
193
194	    mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
195	    TARGET_CONTAINERS=()
196	    for _c in "${_all_running[@]}"; do
197	        [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
198	    done
199	    unset _all_running _exclude _c
200	else
201	    if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
202	        echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
203	        exit 0
204	    fi
205
206	    if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
207	        warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
208	        warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
209	        exit 0
210	    fi
211
212	    TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
213	fi
214
215	# ==============================================================================================
216	# ━━━ Status ━━━
217	# ==============================================================================================
218	if [[ "$SHOW_STATUS" == true ]]; then
219	    echo ""
220	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
221	    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
222	    echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
223	    if [[ "$REMAINDER_MODE" == true ]]; then
224	        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
225	        for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done
226	    else
227	        echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
228	        echo "$ICON_GEAR       Enabled:    ${DAILY_CONTAINER_UPDATES:-true}"
229	    fi
230	    echo "$ICON_GEAR       Dry Run:    $DRY_RUN"
231	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
232	    exit 0
233	fi
234
235	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
236
237	if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
238	    echo "No containers to update"
239	    exit 0
240	fi
241
242	# ==============================================================================================
243	# ━━━ Pull Updates ━━━
244	# ==============================================================================================
245	echo ""
246	if [[ "$REMAINDER_MODE" == true ]]; then
247	    echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
248	    echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
249	    log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
250	else
251	    echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
252	    log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
253	fi
254	echo ""
255
256	START=$(date +%s)
257	UPDATED=()
258	UP_TO_DATE=()
259	FAILED=()
260	OLD_IMAGE_IDS=()   # old image IDs to explicitly remove after rebuilds
261	SKIPPED=()
262
263	for container in "${TARGET_CONTAINERS[@]}"; do
264	    [[ -z "$container" ]] && continue
265	    log "━━━ $ICON_CONTAINERS $container ━━━"
266
267	    if ! docker inspect "$container" &>/dev/null; then
268	        warn "$container — not found, skipping"
269	        SKIPPED+=("$container")
270	        continue
271	    fi
272
273	    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
274	    if [[ -z "$IMAGE" ]]; then
275	        warn "$container — could not determine image, skipping"
276	        SKIPPED+=("$container")
277	        continue
278	    fi
279
280	    log "$container — image: $IMAGE"
281
282	    if [[ "$DRY_RUN" == true ]]; then
283	        warn "DRY RUN — would pull: $IMAGE"
284	        UPDATED+=("$container")
285	        continue
286	    fi
287
288	    # Capture the image ID the container is currently running on, and the
289	    # image ID :latest points to before the pull. After pulling, we rebuild if
290	    # either a new digest landed OR the container is behind what :latest is now.
291	    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
292	    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
293
294	    log "$ICON_SYNC  Pulling $IMAGE..."
295	    if [[ "$ENABLE_LOGGING" == "true" ]]; then
296	        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
297	        _pull_rc=${PIPESTATUS[0]}
298	    else
299	        docker pull "$IMAGE" >/dev/null 2>&1
300	        _pull_rc=$?
301	    fi
302	    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
303
304	    if [[ $_pull_rc -eq 0 ]]; then
305	        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
306	        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
307
308	        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
309	            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
310	            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
311	            UPDATED+=("$container")
312	            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
313	        else
314	            log "$container — up to date (${NEW_ID:7:12})"
315	            UP_TO_DATE+=("$container")
316	        fi
317	    else
318	        warn "$container — pull failed ($IMAGE)"
319	        FAILED+=("$container")
320	    fi
321
322	done
323
324	# ── Recreate containers that received a new image ────────────────────────────
325	# docker restart uses the image ID baked in at creation time — it never picks
326	# up the new digest. rebuild_container reads the stored XML template, stops the
327	# old container, recreates it (new image, same config), then prunes the old image.
328	REBUILT=()
329	REBUILD_FAILED=()
330	if [[ ${#UPDATED[@]} -gt 0 ]]; then
331	    for container in "${UPDATED[@]}"; do
332	        [[ -z "$container" ]] && continue
333	        if [[ "$DRY_RUN" == true ]]; then
334	            warn "DRY RUN — would rebuild $container from template"
335	            REBUILT+=("$container")
336	            continue
337	        fi
338	        log "$ICON_SYNC Rebuilding $container from template on new image..."
339	        if platform_rebuild_container "$container"; then
340	            log "$ICON_DONE $container rebuilt ✅"
341	            REBUILT+=("$container")
342	        else
343	            error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
344	            notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
345	            REBUILD_FAILED+=("$container")
346	        fi
347	    done
348	fi
349
350	# ── Remove old images ────────────────────────────────────────────────────────
351	# Explicitly rmi by the IDs captured before each pull. Tagged images are never
352	# caught by dangling-only prune, so this is the only reliable cleanup path.
353	# Fall through to dangling prune to catch any leftovers from other update paths.
354	if [[ "$DRY_RUN" == true ]]; then
355	    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
356	    PRUNED_SUMMARY="(dry run)"
357	else
358	    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
359	        docker rmi "$_old_id" >/dev/null 2>&1 || true
360	    done
361	    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
362	    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
363	    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
364	fi
365
366	END=$(date +%s)
367
368	# ==============================================================================================
369	# ━━━ Summary ━━━
370	# ==============================================================================================
371	if [[ "$REMAINDER_MODE" == true ]]; then
372	    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
373	else
374	    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
375	fi
376	echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
377	echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
378	if [[ ${#UPDATED[@]} -gt 0 ]]; then
379	    echo "$ICON_DONE       Updated:     ${#UPDATED[@]}"
380	    log "  ${UPDATED[*]}"
381	fi
382	[[ ${#REBUILT[@]} -gt 0 ]]       && echo "$ICON_SYNC       Rebuilt:     ${#REBUILT[@]}"
383	[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR      Rebuild fail:${REBUILD_FAILED[*]}"
384	[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
385	[[ ${#SKIPPED[@]} -gt 0 ]]       && log "$ICON_WARN       Skipped:     ${SKIPPED[*]}"
386	[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Failed:      ${FAILED[*]}"
387	echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"
388
389	if [[ "$DRY_RUN" == true ]]; then
390	    warn "DRY RUN — no images pulled"
391	elif [[ ${#FAILED[@]} -eq 0 ]]; then
392	    echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
393	else
394	    warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
395	fi
396	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
397
398	# Pull failures are non-fatal — restart proceeds regardless
399	exit 0
400

#!/bin/bash
# ==============================================================================================
# ================================= Docker Update ==============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pulls the latest images for configured containers. Two modes: normal (daily)
# and remainder (weekly).
#
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
# Containers stay running during the pull — no extra downtime beyond what the
# nightly restart already causes.
#
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
# It catches everything that normal mode and the weekly sync window did not already
# update — derived automatically from docker ps, nothing to configure.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Normal mode (daily):
#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
#   Pull → compare old vs new image ID → mark updated or already current.
#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
#
# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Single List
#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
#   separate update list. Adding or removing a container from the restart list
#   automatically updates the image pull list — one change, both places.
#
# Version Ownership
#   Fallback containers are excluded from remainder mode. This server only runs
#   them during a fallback. The remote server owns their version — if remainder
#   updates them independently and a handback occurs, the remote's older image
#   may not handle data written by the newer version.
#
# State Respect
#   Stopped containers are never targeted. Pulling while stopped adds no value
#   and a stopped container was likely halted intentionally.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# DAILY_CONTAINER_UPDATES Toggle
#   Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
#   regardless — update and restart are independent operations.
#
# Fallback Exclusion
#   Remainder mode excludes containers owned by the remote server's update cycle
#   to prevent version divergence across the fallback boundary.
#
# Running-Only Filter
#   Stopped containers excluded from remainder mode — intentionally down.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed. Pulls that
#   result in "already up to date" produce no restart.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   DAILY_CONTAINER_UPDATES
#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
#       (default: true)
#
#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
#       Container names for emby and critical-data profiles — excluded from
#       remainder mode (already updated by the weekly sync window)
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Containers updated in normal mode. Aliased by detect_hosts() →
#       DAILY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update.sh
#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
#
# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#
# docker_update.sh --dry-run
#     Preview which containers would be pulled without making changes
#
# docker_update.sh --status
#     Show configuration and container list for current mode
#
# docker_update.sh --log
#     Verbose per-container pull and comparison output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
REMAINDER_MODE=false
_filtered_args=()
for _arg in "$@"; do
    if [[ "$_arg" == "--remainder" ]]; then
        REMAINDER_MODE=true
    else
        _filtered_args+=("$_arg")
    fi
done
unset _arg

parse_args "${_filtered_args[@]}"
unset _filtered_args

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

# ==============================================================================================
# ━━━ Container Discovery ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    declare -A _exclude=()

    # Daily containers — updated by docker_update.sh normal mode
    for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Fallback coverage containers — owned by the remote server's update cycle.
    # This server runs them during fallback but should never update them independently.
    # Updating them here risks version divergence: if remote's writeback after handback
    # encounters data written by a newer version, it may not handle it correctly.
    for _tier in 1 2 3 4; do
        _tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
        eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
        for _c in "${_tier_arr[@]}"; do
            [[ -n "$_c" ]] && _exclude["$_c"]=1
        done
    done
    unset _tier _tier_var _tier_arr _c

    mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
    TARGET_CONTAINERS=()
    for _c in "${_all_running[@]}"; do
        [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
    done
    unset _all_running _exclude _c
else
    if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
        echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
        exit 0
    fi

    if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
        warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
        warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
        exit 0
    fi

    TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
fi

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
    if [[ "$REMAINDER_MODE" == true ]]; then
        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
        for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done
    else
        echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
        echo "$ICON_GEAR       Enabled:    ${DAILY_CONTAINER_UPDATES:-true}"
    fi
    echo "$ICON_GEAR       Dry Run:    $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"

if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
    echo "No containers to update"
    exit 0
fi

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
    log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
else
    echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
fi
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()
OLD_IMAGE_IDS=()   # old image IDs to explicitly remove after rebuilds
SKIPPED=()

for container in "${TARGET_CONTAINERS[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    if ! docker inspect "$container" &>/dev/null; then
        warn "$container — not found, skipping"
        SKIPPED+=("$container")
        continue
    fi

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        SKIPPED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        continue
    fi

    # Capture the image ID the container is currently running on, and the
    # image ID :latest points to before the pull. After pulling, we rebuild if
    # either a new digest landed OR the container is behind what :latest is now.
    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    log "$ICON_SYNC  Pulling $IMAGE..."
    if [[ "$ENABLE_LOGGING" == "true" ]]; then
        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
        _pull_rc=${PIPESTATUS[0]}
    else
        docker pull "$IMAGE" >/dev/null 2>&1
        _pull_rc=$?
    fi
    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    if [[ $_pull_rc -eq 0 ]]; then
        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)

        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
            UPDATED+=("$container")
            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
        else
            log "$container — up to date (${NEW_ID:7:12})"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

done

# ── Recreate containers that received a new image ────────────────────────────
# docker restart uses the image ID baked in at creation time — it never picks
# up the new digest. rebuild_container reads the stored XML template, stops the
# old container, recreates it (new image, same config), then prunes the old image.
REBUILT=()
REBUILD_FAILED=()
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would rebuild $container from template"
            REBUILT+=("$container")
            continue
        fi
        log "$ICON_SYNC Rebuilding $container from template on new image..."
        if platform_rebuild_container "$container"; then
            log "$ICON_DONE $container rebuilt ✅"
            REBUILT+=("$container")
        else
            error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
            notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
            REBUILD_FAILED+=("$container")
        fi
    done
fi

# ── Remove old images ────────────────────────────────────────────────────────
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
# caught by dangling-only prune, so this is the only reliable cleanup path.
# Fall through to dangling prune to catch any leftovers from other update paths.
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
    PRUNED_SUMMARY="(dry run)"
else
    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
        docker rmi "$_old_id" >/dev/null 2>&1 || true
    done
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
else
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
fi
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Updated:     ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#REBUILT[@]} -gt 0 ]]       && echo "$ICON_SYNC       Rebuilt:     ${#REBUILT[@]}"
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR      Rebuild fail:${REBUILD_FAILED[*]}"
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]]       && log "$ICON_WARN       Skipped:     ${SKIPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Failed:      ${FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no images pulled"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
else
    warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Pull failures are non-fatal — restart proceeds regardless
exit 0


ok now daily and weekly ran. no containers were updated


Run docker network script during onboarding


Error: Exit code 2
ugrep: warning: /boot/config/plugins/varaverk/master.conf: No such file or directory


Bash

find /boot/config/plugins/varaverk -name "master.conf" -o -name "Configurations" -type d | grep -v ".git" | grep -v "claude-data"

Find master.conf location


/boot/config/plugins/varaverk/Configurations
/boot/config/plugins/varaverk/Configurations/master.conf


Check master.conf for maintenance script lists and docker update config


364:# DAILY_MAINTENANCE_SCRIPTS for all jobs.
366:    DAILY_MAINTENANCE_SCRIPTS=(
377:        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
384:    DAILY_CONTAINER_UPDATES=true
401:# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
403:    WEEKLY_MAINTENANCE_SCRIPTS=(
405:        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
687:# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
694:# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
859:# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
896:# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
922:# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media


360	    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
361
362	# ━━━ Daily Sync Maintenance ━━━
363	# daily_sync_maintenance.sh runs media share sync first, then iterates
364	# DAILY_MAINTENANCE_SCRIPTS for all jobs.
365	# Schedule: 0 1 * * * (1am daily)
366	    DAILY_MAINTENANCE_SCRIPTS=(
367	        "git_pull_execute.sh"                         # pull latest scripts — always runs first
368	        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
369	        "Media/media_cleaner.sh anime"                # remove junk from anime shares
370	        "Media/media_cleaner.sh media"                # remove junk from media shares
371	        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
372	        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
373	        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
374	        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
375	        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
376	        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
377	        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
378	        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
379	    )
380
381	# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
382	# Containers keep running during pull — no extra downtime.
383	# Set false to skip updates while still running the daily restart.
384	    DAILY_CONTAINER_UPDATES=true
385
386	# Media shares synced daily by daily_sync_maintenance.sh.
387	# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
388	# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
389	# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
390	# Any node can download content to any share — it propagates to all nodes on the next cycle.
391	# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
392	# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
393	# For shares needing custom options or container stops — create a profile in RSYNC section.
394
395	# Personal encrypted shares defined per-host in host*.conf.
396	# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
397	# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
398
399	# ━━━ Weekly Sync Maintenance ━━━
400	# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
401	# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
402	# Schedule: 30 2 * * 0 (Sunday 2:30am)
403	    WEEKLY_MAINTENANCE_SCRIPTS=(
404	        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
405	        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
406	        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
407	        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
408	        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
409	        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
410	    )
411
412	# Pull updates for all running containers NOT in daily/weekly restart lists.
413	# Ensures every deployed container receives at least one image pull per week.
414	# Set false to skip — docker_weekly_restart.sh still runs regardless.
415	    WEEKLY_REMAINING_UPDATES=true
416
417	# Shares synced during the weekly maintenance window — defined per host in host*.conf.
418	# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
419	# Containers stopped both sides before sync — full clean state guaranteed.

    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
# Schedule: 0 1 * * * (1am daily)
    DAILY_MAINTENANCE_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts — always runs first
        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
        "Media/media_cleaner.sh anime"                # remove junk from anime shares
        "Media/media_cleaner.sh media"                # remove junk from media shares
        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
    )

# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
# Containers keep running during pull — no extra downtime.
# Set false to skip updates while still running the daily restart.
    DAILY_CONTAINER_UPDATES=true

# Media shares synced daily by daily_sync_maintenance.sh.
# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
# Any node can download content to any share — it propagates to all nodes on the next cycle.
# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
# For shares needing custom options or container stops — create a profile in RSYNC section.

# Personal encrypted shares defined per-host in host*.conf.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
# Schedule: 30 2 * * 0 (Sunday 2:30am)
    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
    )

# Pull updates for all running containers NOT in daily/weekly restart lists.
# Ensures every deployed container receives at least one image pull per week.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_REMAINING_UPDATES=true

# Shares synced during the weekly maintenance window — defined per host in host*.conf.
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
# Containers stopped both sides before sync — full clean state guaranteed.


`docker_update.sh` IS in the daily list. Let me check `docker_update_remaining.sh` and the weekly script, then look for logs.


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Docker Update — Remaining ======================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Weekly sweep that pulls the latest image for every running container not
9	# already covered by the daily or weekly managed update cycles. Restarts
10	# containers that received a new image, then prunes dangling images.
11	#
12	# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
13	# Derives its target list automatically from docker ps minus the two managed
14	# lists — there is nothing to configure for this script.
15	#
16	# Together with docker_update.sh (normal + remainder modes), every deployed
17	# container receives at least one image pull per week without any per-container
18	# configuration required here.
19	#
20	# ==============================================================================================
21	# OPERATIONAL SAFEGUARDS
22	# ==============================================================================================
23	#
24	# Lock Acquisition
25	#   Prevents concurrent execution via acquire_lock(). Safe to call from
26	#   weekly maintenance scripts without risk of overlap.
27	#
28	# Host Detection
29	#   detect_hosts() identifies which server is running the script and aliases
30	#   HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to
31	#   the correct host's values for exclusion.
32	#
33	# Root Enforcement
34	#   Docker operations require root privileges.
35	#
36	# WEEKLY_REMAINING_UPDATES Toggle
37	#   Exits cleanly when disabled via master.conf.
38	#
39	# Running-Only Filter
40	#   Stopped containers excluded — intentionally down, pulling adds no value.
41	#
42	# Image ID Comparison
43	#   Containers not restarted unless their image actually changed.
44	#
45	# ==============================================================================================
46	# CONFIGURATION
47	# ==============================================================================================
48	#
49	# master.conf
50	#
51	#   WEEKLY_REMAINING_UPDATES
52	#       Enable or disable this script. (default: true)
53	#       To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
54	#
55	# host*.conf
56	#
57	#   HOST*_DAILY_RESTART_CONTAINERS
58	#       Excluded from this script — already updated daily. Aliased by
59	#       detect_hosts() → DAILY_RESTART_CONTAINERS
60	#
61	#   HOST*_WEEKLY_RESTART_CONTAINERS
62	#       Excluded from this script — already updated by weekly sync window.
63	#       Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
64	#
65	# ==============================================================================================
66	# RUNTIME MODES
67	# ==============================================================================================
68	#
69	# docker_update_remaining.sh
70	#     Pull all remaining running containers, restart those updated, prune images
71	#
72	# docker_update_remaining.sh --dry-run
73	#     Preview which containers would be pulled and restarted
74	#
75	# docker_update_remaining.sh --status
76	#     Show exclusion lists and current remaining container count
77	#
78	# docker_update_remaining.sh --log
79	#     Verbose per-container pull and restart output
80	#
81	# ==============================================================================================
82
83	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
84
85	source "$SCRIPT_DIR/../load_config.sh"
86
87	parse_args "$@"
88
89	# ==============================================================================================
90	# ━━━ Setup ━━━
91	# ==============================================================================================
92	if [[ "$EUID" -ne 0 ]]; then
93	    error "Must be run as root"
94	    exit 1
95	fi
96
97	acquire_lock
98
99	if ! command -v docker &>/dev/null; then
100	    error "Docker command not found"
101	    exit 1
102	fi
103
104	detect_hosts
105
106	if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
107	    echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
108	    exit 0
109	fi
110
111	# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
112	declare -A EXCLUDED
113	for c in "${DAILY_RESTART_CONTAINERS[@]}"  "${WEEKLY_RESTART_CONTAINERS[@]}"; do
114	    [[ -n "$c" ]] && EXCLUDED["$c"]=1
115	done
116
117	# ── Get all running containers ────────────────────────────────────────────────────────────────
118	mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
119
120	# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
121	REMAINING=()
122	for c in "${ALL_RUNNING[@]}"; do
123	    [[ -z "$c" ]] && continue
124	    [[ -n "${EXCLUDED[$c]:-}" ]] && continue
125	    REMAINING+=("$c")
126	done
127
128	# ==============================================================================================
129	# ━━━ Status ━━━
130	# ==============================================================================================
131	if [[ "$SHOW_STATUS" == true ]]; then
132	    echo ""
133	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
134	    echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
135	    echo "$ICON_GEAR       Enabled:     ${WEEKLY_REMAINING_UPDATES:-true}"
136	    echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
137	    echo "$ICON_CONTAINERS Excluded:    ${!EXCLUDED[*]}"
138	    echo "$ICON_CONTAINERS Remaining:   ${REMAINING[*]:-none}"
139	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
140	    exit 0
141	fi
142
143	if [[ ${#REMAINING[@]} -eq 0 ]]; then
144	    echo "No remaining containers to update — all running containers are covered by daily/weekly lists"
145	    exit 0
146	fi
147
148	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"
149
150	# ==============================================================================================
151	# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
152	# ==============================================================================================
153
154	# docker_cmd, retry_docker, verify_running — defined in common.sh
155
156	# ==============================================================================================
157	# ━━━ Pull Updates ━━━
158	# ==============================================================================================
159	echo ""
160	echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
161	log "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
162	log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
163	echo ""
164
165	START=$(date +%s)
166	UPDATED=()
167	UP_TO_DATE=()
168	FAILED=()
169	OLD_IMAGE_IDS=()
170
171	for container in "${REMAINING[@]}"; do
172	    [[ -z "$container" ]] && continue
173	    log "━━━ $ICON_CONTAINERS $container ━━━"
174
175	    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
176	    if [[ -z "$IMAGE" ]]; then
177	        warn "$container — could not determine image, skipping"
178	        FAILED+=("$container")
179	        continue
180	    fi
181
182	    log "$container — image: $IMAGE"
183
184	    if [[ "$DRY_RUN" == true ]]; then
185	        warn "DRY RUN — would pull: $IMAGE"
186	        UPDATED+=("$container")
187	        continue
188	    fi
189
190	    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
191	    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
192
193	    log "$ICON_SYNC  Pulling $IMAGE..."
194	    if [[ "$ENABLE_LOGGING" == "true" ]]; then
195	        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
196	        _pull_rc=${PIPESTATUS[0]}
197	    else
198	        docker pull "$IMAGE" >/dev/null 2>&1
199	        _pull_rc=$?
200	    fi
201	    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
202
203	    if [[ $_pull_rc -eq 0 ]]; then
204	        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
205	        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
206
207	        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
208	            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
209	            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
210	            UPDATED+=("$container")
211	            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
212	        else
213	            log "$container — up to date (${NEW_ID:7:12})"
214	            UP_TO_DATE+=("$container")
215	        fi
216	    else
217	        warn "$container — pull failed ($IMAGE)"
218	        FAILED+=("$container")
219	    fi
220
221	done
222
223	# ==============================================================================================
224	# ━━━ Restart Updated Containers ━━━
225	# ==============================================================================================
226	RESTARTED=()
227	RESTART_FAILED=()
228	SKIPPED_STOPPED=()
229
230	if [[ ${#UPDATED[@]} -gt 0 ]]; then
231	    echo ""
232	    echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
233	    log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"
234
235	    for container in "${UPDATED[@]}"; do
236	        [[ -z "$container" ]] && continue
237	        log "━━━ $ICON_CONTAINERS $container ━━━"
238
239	        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
240
241	        if [[ "$STATUS" != "true" ]]; then
242	            log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
243	            SKIPPED_STOPPED+=("$container")
244	            continue
245	        fi
246
247	        if [[ "$DRY_RUN" == true ]]; then
248	            warn "DRY RUN — would restart $container"
249	            RESTARTED+=("$container")
250	            continue
251	        fi
252
253	        log "$ICON_RUNNING $container — recreating from template on new image..."
254	        if platform_rebuild_container "$container"; then
255	            if verify_running "$container"; then
256	                log "$ICON_DONE  $container recreated and running ✅"
257	                RESTARTED+=("$container")
258	            else
259	                error "$container recreated but not running — may be intentionally stopped"
260	                RESTARTED+=("$container")
261	            fi
262	        else
263	            error "Failed to rebuild $container from template"
264	            notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning"
265	            RESTART_FAILED+=("$container")
266	        fi
267	    done
268	else
269	    log "No containers received a new image — nothing to restart"
270	fi
271
272	# ==============================================================================================
273	# ━━━ Prune Old Images ━━━
274	# ==============================================================================================
275	echo ""
276	echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
277
278	if [[ "$DRY_RUN" == true ]]; then
279	    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
280	    PRUNED_SUMMARY="(dry run)"
281	else
282	    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
283	        docker rmi "$_old_id" >/dev/null 2>&1 || true
284	    done
285	    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
286	    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
287	    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
288	fi
289
290	END=$(date +%s)
291
292	# ==============================================================================================
293	# ━━━ Summary ━━━
294	# ==============================================================================================
295	echo ""
296	echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
297	echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
298	echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
299	echo "$ICON_CONTAINERS Scope:       ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
300	if [[ ${#UPDATED[@]} -gt 0 ]]; then
301	    echo "$ICON_DONE       New image:   ${#UPDATED[@]}"
302	    log "  ${UPDATED[*]}"
303	fi
304	[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
305	[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Pull failed: ${FAILED[*]}"
306	if [[ ${#RESTARTED[@]} -gt 0 ]]; then
307	    echo "$ICON_DONE       Restarted:   ${#RESTARTED[@]}"
308	    log "  ${RESTARTED[*]}"
309	fi
310	[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN     Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
311	[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR     Restart fail:${RESTART_FAILED[*]}"
312	echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"
313
314	ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
315	if [[ "$DRY_RUN" == true ]]; then
316	    warn "DRY RUN — no changes made"
317	elif [[ "$ALL_FAILED" -eq 0 ]]; then
318	    echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
319	else
320	    warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
321	fi
322	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
323
324	[[ "$ALL_FAILED" -gt 0 ]] && exit 1
325	exit 0
326

#!/bin/bash
# ==============================================================================================
# ============================= Docker Update — Remaining ======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly sweep that pulls the latest image for every running container not
# already covered by the daily or weekly managed update cycles. Restarts
# containers that received a new image, then prunes dangling images.
#
# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
# Derives its target list automatically from docker ps minus the two managed
# lists — there is nothing to configure for this script.
#
# Together with docker_update.sh (normal + remainder modes), every deployed
# container receives at least one image pull per week without any per-container
# configuration required here.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   weekly maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to
#   the correct host's values for exclusion.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# WEEKLY_REMAINING_UPDATES Toggle
#   Exits cleanly when disabled via master.conf.
#
# Running-Only Filter
#   Stopped containers excluded — intentionally down, pulling adds no value.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WEEKLY_REMAINING_UPDATES
#       Enable or disable this script. (default: true)
#       To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Excluded from this script — already updated daily. Aliased by
#       detect_hosts() → DAILY_RESTART_CONTAINERS
#
#   HOST*_WEEKLY_RESTART_CONTAINERS
#       Excluded from this script — already updated by weekly sync window.
#       Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update_remaining.sh
#     Pull all remaining running containers, restart those updated, prune images
#
# docker_update_remaining.sh --dry-run
#     Preview which containers would be pulled and restarted
#
# docker_update_remaining.sh --status
#     Show exclusion lists and current remaining container count
#
# docker_update_remaining.sh --log
#     Verbose per-container pull and restart output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
    echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
    exit 0
fi

# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
declare -A EXCLUDED
for c in "${DAILY_RESTART_CONTAINERS[@]}"  "${WEEKLY_RESTART_CONTAINERS[@]}"; do
    [[ -n "$c" ]] && EXCLUDED["$c"]=1
done

# ── Get all running containers ────────────────────────────────────────────────────────────────
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)

# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
REMAINING=()
for c in "${ALL_RUNNING[@]}"; do
    [[ -z "$c" ]] && continue
    [[ -n "${EXCLUDED[$c]:-}" ]] && continue
    REMAINING+=("$c")
done

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Enabled:     ${WEEKLY_REMAINING_UPDATES:-true}"
    echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
    echo "$ICON_CONTAINERS Excluded:    ${!EXCLUDED[*]}"
    echo "$ICON_CONTAINERS Remaining:   ${REMAINING[*]:-none}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ ${#REMAINING[@]} -eq 0 ]]; then
    echo "No remaining containers to update — all running containers are covered by daily/weekly lists"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"

# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# docker_cmd, retry_docker, verify_running — defined in common.sh

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()
OLD_IMAGE_IDS=()

for container in "${REMAINING[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        FAILED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        continue
    fi

    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    log "$ICON_SYNC  Pulling $IMAGE..."
    if [[ "$ENABLE_LOGGING" == "true" ]]; then
        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
        _pull_rc=${PIPESTATUS[0]}
    else
        docker pull "$IMAGE" >/dev/null 2>&1
        _pull_rc=$?
    fi
    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    if [[ $_pull_rc -eq 0 ]]; then
        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)

        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
            UPDATED+=("$container")
            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
        else
            log "$container — up to date (${NEW_ID:7:12})"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

done

# ==============================================================================================
# ━━━ Restart Updated Containers ━━━
# ==============================================================================================
RESTARTED=()
RESTART_FAILED=()
SKIPPED_STOPPED=()

if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"

    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        log "━━━ $ICON_CONTAINERS $container ━━━"

        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

        if [[ "$STATUS" != "true" ]]; then
            log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
            SKIPPED_STOPPED+=("$container")
            continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container"
            RESTARTED+=("$container")
            continue
        fi

        log "$ICON_RUNNING $container — recreating from template on new image..."
        if platform_rebuild_container "$container"; then
            if verify_running "$container"; then
                log "$ICON_DONE  $container recreated and running ✅"
                RESTARTED+=("$container")
            else
                error "$container recreated but not running — may be intentionally stopped"
                RESTARTED+=("$container")
            fi
        else
            error "Failed to rebuild $container from template"
            notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning"
            RESTART_FAILED+=("$container")
        fi
    done
else
    log "No containers received a new image — nothing to restart"
fi

# ==============================================================================================
# ━━━ Prune Old Images ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
    PRUNED_SUMMARY="(dry run)"
else
    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
        docker rmi "$_old_id" >/dev/null 2>&1 || true
    done
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
echo "$ICON_CONTAINERS Scope:       ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       New image:   ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Pull failed: ${FAILED[*]}"
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Restarted:   ${#RESTARTED[@]}"
    log "  ${RESTARTED[*]}"
fi
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN     Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR     Restart fail:${RESTART_FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$ALL_FAILED" -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
else
    warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$ALL_FAILED" -gt 0 ]] && exit 1
exit 0