1	#!/bin/bash
2	# ==============================================================================================
3	# ========================= Monthly Maintenance Orchestrator ===================================
4	# ==============================================================================================
5	# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system.
6	# Fires only when BOTH gates pass:
7	#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
8	#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
9	#
10	# ── WHY UPTIME-GATED ─────────────────────────────────────────────────────────────────────────
11	#   A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a
12	#   stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance
13	#   only runs after the server has been healthy for a full month, never immediately post-boot.
14	#   If uptime or interval gate is not met on the 15th, the run is skipped until next month.
15	#
16	# ── HOW TO CALL ──────────────────────────────────────────────────────────────────────────────
17	# Schedule: 0 0 15 * *  (15th of each month at midnight)
18	#   Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires.
19	#
20	# ── STATE FILE ────────────────────────────────────────────────────────────────────────────────
21	#   MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts.
22	#   Written after each run (pass or partial fail). Format: Unix timestamp.
23	#   A reboot does NOT reset the last-run state — the interval gate survives independently
24	#   of the uptime gate. Both must pass before maintenance fires again.
25	#
26	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
27	#   MONTHLY_MAINTENANCE_SCRIPTS    — ordered list of scripts to run
28	#   MONTHLY_UPTIME_THRESHOLD_DAYS  — minimum uptime in days before maintenance fires
29	#   MONTHLY_RUN_INTERVAL_DAYS      — minimum days since last run before running again
30	#   MONTHLY_LAST_RUN_FILE          — state file path — /boot/config, survives reboots
31	#
32	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
33	#   monthly_maintenance.sh           — normal run (uptime + interval gates enforced)
34	#   monthly_maintenance.sh --dry-run — preview gate state and scripts without running
35	#   monthly_maintenance.sh --status  — show gate state, last run, and configured scripts
36	#   monthly_maintenance.sh --force   — bypass uptime + interval gates (manual override)
37	#   monthly_maintenance.sh --log     — verbose output
38	# ==============================================================================================
39
40	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
41	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
42
43	source "$ECOSYSTEM_ROOT/load_config.sh"
44
45	# ── Parse --force before common parse_args ────────────────────────────────────────────────────
46	FORCE_RUN=false
47	FILTERED_ARGS=()
48	for _arg in "$@"; do
49	    if [[ "$_arg" == "--force" ]]; then
50	        FORCE_RUN=true
51	    else
52	        FILTERED_ARGS+=("$_arg")
53	    fi
54	done
55	parse_args "${FILTERED_ARGS[@]}"
56
57	# ==============================================================================================
58	# ━━━ Setup ━━━
59	# ==============================================================================================
60	if [[ "$EUID" -ne 0 ]]; then
61	    error "Must be run as root"
62	    exit 1
63	fi
64
65	acquire_lock
66
67	detect_hosts
68
69	[[ "$DRY_RUN" == true ]]   && warn "DRY RUN — no scripts will be executed"
70	[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"
71
72	# Defaults — overridden by master.conf values
73	MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
74	MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
75	MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}"
76
77	UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
78	INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
79	UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
80	NOW=$(date +%s)
81
82	# ==============================================================================================
83	# ━━━ Gate Evaluation ━━━
84	# ==============================================================================================
85	_uptime_days=$(( UPTIME_SECS / 86400 ))
86	_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))
87
88	UPTIME_GATE_PASS=false
89	if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
90	    UPTIME_GATE_PASS=true
91	fi
92
93	INTERVAL_GATE_PASS=false
94	LAST_RUN=0
95	DAYS_SINCE_LAST_RUN="never"
96	if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
97	    LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
98	    ELAPSED=$(( NOW - LAST_RUN ))
99	    DAYS_SINCE=$(( ELAPSED / 86400 ))
100	    DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
101	    if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
102	        INTERVAL_GATE_PASS=true
103	    fi
104	else
105	    INTERVAL_GATE_PASS=true  # never run
106	fi
107
108	# ==============================================================================================
109	# ━━━ Status ━━━
110	# ==============================================================================================
111	if [[ "$SHOW_STATUS" == true ]]; then
112	    echo ""
113	    echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
114	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
115	    echo ""
116	    echo "── Gates ──"
117
118	    if [[ "$UPTIME_GATE_PASS" == true ]]; then
119	        echo "  $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
120	    else
121	        echo "  $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
122	    fi
123
124	    if [[ "$INTERVAL_GATE_PASS" == true ]]; then
125	        echo "  $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
126	    else
127	        echo "  $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
128	    fi
129
130	    if [[ "$LAST_RUN" -gt 0 ]]; then
131	        echo "  Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
132	    else
133	        echo "  Last run: never"
134	    fi
135
136	    echo ""
137	    echo "── Scripts ──"
138	    if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
139	        echo "  (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
140	    else
141	        for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
142	            [[ -z "$entry" ]] && continue
143	            read -r -a parts <<< "$entry"
144	            script_path="$ECOSYSTEM_ROOT/${parts[0]}"
145	            script_name=$(basename "${parts[0]}")
146	            if [[ -f "$script_path" ]]; then
147	                [[ -x "$script_path" ]] && echo "  $ICON_DONE $script_name" || echo "  $ICON_WARN $script_name (not executable)"
148	            else
149	                echo "  $ICON_ERROR $script_name — NOT FOUND: $script_path"
150	            fi
151	        done
152	    fi
153
154	    echo ""
155	    echo "  Schedule:   0 0 15 * * (15th of each month at midnight)"
156	    echo "  Force flag: --force bypasses both gates"
157	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
158	    exit 0
159	fi
160
161	# ==============================================================================================
162	# ━━━ Gate Check — Silent Exit When Not Due ━━━
163	# ==============================================================================================
164	if [[ "$FORCE_RUN" != true ]]; then
165	    if [[ "$UPTIME_GATE_PASS" != true ]]; then
166	        echo "Uptime gate not met — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — no-op"
167	        exit 0
168	    fi
169	    if [[ "$INTERVAL_GATE_PASS" != true ]]; then
170	        echo "Interval gate not met — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — no-op"
171	        exit 0
172	    fi
173	fi
174
175	# ==============================================================================================
176	# ━━━ Pre-flight ━━━
177	# ==============================================================================================
178	if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
179	    warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
180	    exit 0
181	fi
182
183	echo ""
184	echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
185	echo "$ICON_HOST  Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
186	echo "$ICON_GEAR  Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
187	echo ""
188
189	START=$(date +%s)
190	PASSED=()
191	FAILED=()
192	STEP=0
193
194	# ==============================================================================================
195	# ━━━ Run Sequence ━━━
196	# ==============================================================================================
197	for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
198	    [[ -z "$entry" ]] && continue
199	    (( STEP++ ))
200
201	    read -r -a parts <<< "$entry"
202	    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
203	    script_name=$(basename "${parts[0]}")
204	    extra_args=("${parts[@]:1}")
205
206	    echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
207
208	    if [[ ! -f "$script_path" ]]; then
209	        error "$script_name — not found at $script_path"
210	        FAILED+=("$script_name")
211	        echo ""
212	        continue
213	    fi
214
215	    if [[ ! -x "$script_path" ]]; then
216	        warn "$script_name — not executable, fixing..."
217	        chmod +x "$script_path" || {
218	            error "$script_name — chmod +x failed"
219	            FAILED+=("$script_name")
220	            echo ""
221	            continue
222	        }
223	    fi
224
225	    if [[ "$DRY_RUN" == true ]]; then
226	        warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
227	        PASSED+=("$script_name")
228	        echo ""
229	        continue
230	    fi
231
232	    local_args=()
233	    [[ "$VERBOSE" == true ]] && local_args+=("--log")
234
235	    if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
236	        log "$script_name — done ✅"
237	        PASSED+=("$script_name")
238	    else
239	        warn "$script_name — failed (exit $?) — continuing to next step"
240	        FAILED+=("$script_name")
241	    fi
242
243	    echo ""
244	done
245
246	END=$(date +%s)
247
248	# ==============================================================================================
249	# ━━━ Record Last Run ━━━
250	# ==============================================================================================
251	# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
252	if [[ "$DRY_RUN" != true ]]; then
253	    echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
254	    log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
255	fi
256
257	# ==============================================================================================
258	# ━━━ Summary ━━━
259	# ==============================================================================================
260	echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
261	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
262	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
263	[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE    Passed:   ${PASSED[*]}"
264	[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR   Failed:   ${FAILED[*]}"
265	echo ""
266
267	if [[ "$DRY_RUN" == true ]]; then
268	    warn "DRY RUN — no changes made"
269	elif [[ ${#FAILED[@]} -eq 0 ]]; then
270	    echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
271	    notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
272	        "Monthly Maintenance" "normal"
273	else
274	    warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
275	    notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
276	        "Monthly Maintenance" "warning"
277	fi
278	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
279
280	[[ ${#FAILED[@]} -gt 0 ]] && exit 1
281	exit 0
282

#!/bin/bash
# ==============================================================================================
# ========================= Monthly Maintenance Orchestrator ===================================
# ==============================================================================================
# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system.
# Fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
#
# ── WHY UPTIME-GATED ─────────────────────────────────────────────────────────────────────────
#   A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a
#   stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance
#   only runs after the server has been healthy for a full month, never immediately post-boot.
#   If uptime or interval gate is not met on the 15th, the run is skipped until next month.
#
# ── HOW TO CALL ──────────────────────────────────────────────────────────────────────────────
# Schedule: 0 0 15 * *  (15th of each month at midnight)
#   Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires.
#
# ── STATE FILE ────────────────────────────────────────────────────────────────────────────────
#   MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts.
#   Written after each run (pass or partial fail). Format: Unix timestamp.
#   A reboot does NOT reset the last-run state — the interval gate survives independently
#   of the uptime gate. Both must pass before maintenance fires again.
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   MONTHLY_MAINTENANCE_SCRIPTS    — ordered list of scripts to run
#   MONTHLY_UPTIME_THRESHOLD_DAYS  — minimum uptime in days before maintenance fires
#   MONTHLY_RUN_INTERVAL_DAYS      — minimum days since last run before running again
#   MONTHLY_LAST_RUN_FILE          — state file path — /boot/config, survives reboots
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   monthly_maintenance.sh           — normal run (uptime + interval gates enforced)
#   monthly_maintenance.sh --dry-run — preview gate state and scripts without running
#   monthly_maintenance.sh --status  — show gate state, last run, and configured scripts
#   monthly_maintenance.sh --force   — bypass uptime + interval gates (manual override)
#   monthly_maintenance.sh --log     — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

# ── Parse --force before common parse_args ────────────────────────────────────────────────────
FORCE_RUN=false
FILTERED_ARGS=()
for _arg in "$@"; do
    if [[ "$_arg" == "--force" ]]; then
        FORCE_RUN=true
    else
        FILTERED_ARGS+=("$_arg")
    fi
done
parse_args "${FILTERED_ARGS[@]}"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]]   && warn "DRY RUN — no scripts will be executed"
[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"

# Defaults — overridden by master.conf values
MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}"

UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
NOW=$(date +%s)

# ==============================================================================================
# ━━━ Gate Evaluation ━━━
# ==============================================================================================
_uptime_days=$(( UPTIME_SECS / 86400 ))
_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))

UPTIME_GATE_PASS=false
if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
    UPTIME_GATE_PASS=true
fi

INTERVAL_GATE_PASS=false
LAST_RUN=0
DAYS_SINCE_LAST_RUN="never"
if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
    LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
    ELAPSED=$(( NOW - LAST_RUN ))
    DAYS_SINCE=$(( ELAPSED / 86400 ))
    DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
    if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
        INTERVAL_GATE_PASS=true
    fi
else
    INTERVAL_GATE_PASS=true  # never run
fi

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""
    echo "── Gates ──"

    if [[ "$UPTIME_GATE_PASS" == true ]]; then
        echo "  $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
    else
        echo "  $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
    fi

    if [[ "$INTERVAL_GATE_PASS" == true ]]; then
        echo "  $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
    else
        echo "  $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
    fi

    if [[ "$LAST_RUN" -gt 0 ]]; then
        echo "  Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
    else
        echo "  Last run: never"
    fi

    echo ""
    echo "── Scripts ──"
    if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
        echo "  (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
    else
        for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -z "$entry" ]] && continue
            read -r -a parts <<< "$entry"
            script_path="$ECOSYSTEM_ROOT/${parts[0]}"
            script_name=$(basename "${parts[0]}")
            if [[ -f "$script_path" ]]; then
                [[ -x "$script_path" ]] && echo "  $ICON_DONE $script_name" || echo "  $ICON_WARN $script_name (not executable)"
            else
                echo "  $ICON_ERROR $script_name — NOT FOUND: $script_path"
            fi
        done
    fi

    echo ""
    echo "  Schedule:   0 0 15 * * (15th of each month at midnight)"
    echo "  Force flag: --force bypasses both gates"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Gate Check — Silent Exit When Not Due ━━━
# ==============================================================================================
if [[ "$FORCE_RUN" != true ]]; then
    if [[ "$UPTIME_GATE_PASS" != true ]]; then
        echo "Uptime gate not met — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — no-op"
        exit 0
    fi
    if [[ "$INTERVAL_GATE_PASS" != true ]]; then
        echo "Interval gate not met — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — no-op"
        exit 0
    fi
fi

# ==============================================================================================
# ━━━ Pre-flight ━━━
# ==============================================================================================
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
    warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
    exit 0
fi

echo ""
echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST  Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
echo "$ICON_GEAR  Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
echo ""

START=$(date +%s)
PASSED=()
FAILED=()
STEP=0

# ==============================================================================================
# ━━━ Run Sequence ━━━
# ==============================================================================================
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
    [[ -z "$entry" ]] && continue
    (( STEP++ ))

    read -r -a parts <<< "$entry"
    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
    script_name=$(basename "${parts[0]}")
    extra_args=("${parts[@]:1}")

    echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        FAILED+=("$script_name")
        echo ""
        continue
    fi

    if [[ ! -x "$script_path" ]]; then
        warn "$script_name — not executable, fixing..."
        chmod +x "$script_path" || {
            error "$script_name — chmod +x failed"
            FAILED+=("$script_name")
            echo ""
            continue
        }
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
        PASSED+=("$script_name")
        echo ""
        continue
    fi

    local_args=()
    [[ "$VERBOSE" == true ]] && local_args+=("--log")

    if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
        log "$script_name — done ✅"
        PASSED+=("$script_name")
    else
        warn "$script_name — failed (exit $?) — continuing to next step"
        FAILED+=("$script_name")
    fi

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Record Last Run ━━━
# ==============================================================================================
# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
if [[ "$DRY_RUN" != true ]]; then
    echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
    log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE    Passed:   ${PASSED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR   Failed:   ${FAILED[*]}"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
    notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
        "Monthly Maintenance" "normal"
else
    warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
    notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
        "Monthly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0


370	    done
371	fi
372
373	# ==============================================================================================
374	# ━━━ Remainder Container Updates ━━━
375	# ==============================================================================================
376	# Updates all running containers not already covered by daily or the weekly sync window.
377	# Runs last — weekly sync-window containers are back up before this pulls their peers.
378	echo ""
379	echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"
380
381	DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
382	if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
383	    warn "docker_update.sh not found — skipping remainder updates"
384	    JOB_FAIL+=("docker_update.sh --remainder")
385	else
386	    _remainder_args=("--remainder")
387	    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
388	    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
389	        echo "Remainder updates complete ✅"
390	        JOB_PASS+=("docker_update.sh --remainder")
391	    else
392	        warn "Remainder updates completed with errors"
393	        JOB_FAIL+=("docker_update.sh --remainder")
394	    fi
395	    unset _remainder_args
396	fi
397
398	WINDOW_END=$(date +%s)
399
400	# ==============================================================================================
401	# ━━━ Summary ━━━
402	# ==============================================================================================
403	echo ""
404	echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
405	echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
406	echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
407	echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
408	echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
409	echo ""

    done
fi

# ==============================================================================================
# ━━━ Remainder Container Updates ━━━
# ==============================================================================================
# Updates all running containers not already covered by daily or the weekly sync window.
# Runs last — weekly sync-window containers are back up before this pulls their peers.
echo ""
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"

DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
    warn "docker_update.sh not found — skipping remainder updates"
    JOB_FAIL+=("docker_update.sh --remainder")
else
    _remainder_args=("--remainder")
    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
        echo "Remainder updates complete ✅"
        JOB_PASS+=("docker_update.sh --remainder")
    else
        warn "Remainder updates completed with errors"
        JOB_FAIL+=("docker_update.sh --remainder")
    fi
    unset _remainder_args
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""


ok, so at the time i didnt have a monthly orch... daily needs to update its containers, weekly its containers and we need to move remainder to montly


Run docker network script during onboarding


Now I have the full picture. Four changes:

1. Remove the remainder block from `weekly_sync_maintenance.sh`
2. Add `docker_update.sh --remainder` to `MONTHLY_MAINTENANCE_SCRIPTS` in `master.conf`
3. Rename toggle `WEEKLY_REMAINING_UPDATES` → `MONTHLY_REMAINING_UPDATES` everywhere
4. Update the docblock in `docker_update.sh`

Let me check the weekly header and master.conf first.


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Weekly Sync Maintenance ========================================
4	# ==============================================================================================
5	# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
6	# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   1. Stop local containers         — Emby + auth stack stopped locally
10	#   2. Stop remote containers        — Emby + auth stack stopped remotely via SSH
11	#   3. Pull updates locally          — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
12	#   4. Pull updates remotely         — if WEEKLY_SYNC_UPDATES_REMOTE=true
13	#   5. rsync WEEKLY_SYNC_SHARES      — full clean mirror, containers stopped both sides
14	#   6. Start remote containers       — correct order, delayed start respected
15	#   7. Start local containers        — rebuild if new image pulled, docker start otherwise
16	#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
17	#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window
18	#
19	# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
20	# Emby builds a warm image cache on HOST2 throughout the week.
21	# Syncing nightly resets cache — cold loads every morning for users.
22	# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
23	# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
24	#
25	# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
26	# Containers already stopped for sync — updates pull at zero extra downtime.
27	# Both servers start on identical image versions after the window completes.
28	# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
29	#
30	# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────

#!/bin/bash
# ==============================================================================================
# ============================= Weekly Sync Maintenance ========================================
# ==============================================================================================
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. Stop local containers         — Emby + auth stack stopped locally
#   2. Stop remote containers        — Emby + auth stack stopped remotely via SSH
#   3. Pull updates locally          — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
#   4. Pull updates remotely         — if WEEKLY_SYNC_UPDATES_REMOTE=true
#   5. rsync WEEKLY_SYNC_SHARES      — full clean mirror, containers stopped both sides
#   6. Start remote containers       — correct order, delayed start respected
#   7. Start local containers        — rebuild if new image pulled, docker start otherwise
#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window
#
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
# Emby builds a warm image cache on HOST2 throughout the week.
# Syncing nightly resets cache — cold loads every morning for users.
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
#
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
# Containers already stopped for sync — updates pull at zero extra downtime.
# Both servers start on identical image versions after the window completes.
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────


Bash

Find monthly scripts list and existing toggle in master.conf


414:    WEEKLY_REMAINING_UPDATES=true
436:    MONTHLY_MAINTENANCE_SCRIPTS=(


grep -n "MONTHLY_MAINTENANCE_SCRIPTS\|WEEKLY_REMAINING" /boot/config/plugins/varaverk/Configurations/master.conf | head -20

Get line numbers for monthly scripts block


410
411	# Pull updates for all running containers NOT in daily/weekly restart lists.
412	# Ensures every deployed container receives at least one image pull per week.
413	# Set false to skip — docker_weekly_restart.sh still runs regardless.
414	    WEEKLY_REMAINING_UPDATES=true
415
416	# Shares synced during the weekly maintenance window — defined per host in host*.conf.
417	# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
418	# Containers stopped both sides before sync — full clean state guaranteed.
419	# Profiles drive container stops, excludes, and options — configure in RSYNC section.
420	# Order matters — Emby first (larger), then Critical-Data (auth stack).
421
422	# Container update toggles for the weekly sync window.
423	# Containers already stopped for sync — updates pull at no extra downtime.
424	# Both false → sync only, no updates.
425	    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
426	    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH
427
428	# ━━━ Monthly Maintenance ━━━
429	# monthly_maintenance.sh fires only when BOTH gates pass:

# Pull updates for all running containers NOT in daily/weekly restart lists.
# Ensures every deployed container receives at least one image pull per week.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_REMAINING_UPDATES=true

# Shares synced during the weekly maintenance window — defined per host in host*.conf.
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
# Order matters — Emby first (larger), then Critical-Data (auth stack).

# Container update toggles for the weekly sync window.
# Containers already stopped for sync — updates pull at no extra downtime.
# Both false → sync only, no updates.
    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH

# ━━━ Monthly Maintenance ━━━
# monthly_maintenance.sh fires only when BOTH gates pass:


428	# ━━━ Monthly Maintenance ━━━
429	# monthly_maintenance.sh fires only when BOTH gates pass:
430	#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
431	#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
432	# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
433	# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
434	# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
435	# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
436	    MONTHLY_MAINTENANCE_SCRIPTS=(
437	        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
438	        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
439	    )
440
441	    MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires
442	    MONTHLY_RUN_INTERVAL_DAYS=30        # minimum days since last run before running again
443	    MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"
444
445	# ━━━ Sunday Morning Coffee Report ━━━
446	# Orchestrator that runs all Sunday monitor scripts in sequence.
447	# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
448	# Each script runs independently and notifies on its own findings.
449	    COFFEE_REPORT_SCRIPTS=(
450	        "Monitors/zfs_memory_snapshot.sh"       # ZFS pool health + ARC + Docker memory snapshot
451	        "Monitors/smart_health.sh"              # drive SMART attributes — reallocated, pending, temp
452	        "Monitors/cert_monitor.sh"              # SSL certificate expiry for all configured domains

# ━━━ Monthly Maintenance ━━━
# monthly_maintenance.sh fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
    )

    MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires
    MONTHLY_RUN_INTERVAL_DAYS=30        # minimum days since last run before running again
    MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"

# ━━━ Sunday Morning Coffee Report ━━━
# Orchestrator that runs all Sunday monitor scripts in sequence.
# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
# Each script runs independently and notifies on its own findings.
    COFFEE_REPORT_SCRIPTS=(
        "Monitors/zfs_memory_snapshot.sh"       # ZFS pool health + ARC + Docker memory snapshot
        "Monitors/smart_health.sh"              # drive SMART attributes — reallocated, pending, temp
        "Monitors/cert_monitor.sh"              # SSL certificate expiry for all configured domains


Clear picture. Making all four changes now.


Edit

# Pull updates for all running containers NOT in daily/weekly restart lists.
# Ensures every deployed container receives at least one image pull per week.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_REMAINING_UPDATES=true

# Pull updates for all running containers NOT in daily/weekly managed lists.
# Ensures every deployed container receives at least one image pull per month.
# Set false to skip.
    MONTHLY_REMAINING_UPDATES=true


The file /boot/config/plugins/varaverk/Configurations/master.conf has been updated successfully. (file state is current in your context — no need to Read it back)

# Pull updates for all running containers NOT in daily/weekly restart lists.
# Ensures every deployed container receives at least one image pull per week.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_REMAINING_UPDATES=true

# Pull updates for all running containers NOT in daily/weekly managed lists.
# Ensures every deployed container receives at least one image pull per month.
# Set false to skip.
    MONTHLY_REMAINING_UPDATES=true

"Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations

-# Pull updates for all running containers NOT in daily/weekly restart lists.

-# Ensures every deployed container receives at least one image pull per week.

-# Set false to skip — docker_weekly_restart.sh still runs regardless.

+# Pull updates for all running containers NOT in daily/weekly managed lists.

+# Ensures every deployed container receives at least one image pull per month.

# Shares synced during the weekly maintenance window — defined per host in host*.conf.

# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES


    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
    )

    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists
        "Tools/zfs_pool_scrub.sh"                         # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"                        # SMART extended self-test — all drives, sequential
    )


    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
    )

    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists
        "Tools/zfs_pool_scrub.sh"                         # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"                        # SMART extended self-test — all drives, sequential
    )

# Add scripts that require a long-stable settled system — scrubs, extended drive tests.

# State file on /boot/config — survives reboots (interval gate independent of uptime gate).

-        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS

-        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential

+        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists

+        "Tools/zfs_pool_scrub.sh"                         # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS

+        "Tools/smart_long_test.sh"                        # SMART extended self-test — all drives, sequential

MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires


Now remove the remainder block and step 9 from weekly:


#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window

#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)


The file /boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh has been updated successfully. (file state is current in your context — no need to Read it back)

#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window

#   6. Start remote containers       — correct order, delayed start respected

#   7. Start local containers        — rebuild if new image pulled, docker start otherwise

-#   9. docker_update.sh --remainder  — update all containers not in daily or weekly sync window

# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────

# Emby builds a warm image cache on HOST2 throughout the week.


# ==============================================================================================
# ━━━ Remainder Container Updates ━━━
# ==============================================================================================
# Updates all running containers not already covered by daily or the weekly sync window.
# Runs last — weekly sync-window containers are back up before this pulls their peers.
echo ""
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"

DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
    warn "docker_update.sh not found — skipping remainder updates"
    JOB_FAIL+=("docker_update.sh --remainder")
else
    _remainder_args=("--remainder")
    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
        echo "Remainder updates complete ✅"
        JOB_PASS+=("docker_update.sh --remainder")
    else
        warn "Remainder updates completed with errors"
        JOB_FAIL+=("docker_update.sh --remainder")
    fi
    unset _remainder_args
fi

WINDOW_END=$(date +%s)


# ==============================================================================================
# ━━━ Remainder Container Updates ━━━
# ==============================================================================================
# Updates all running containers not already covered by daily or the weekly sync window.
# Runs last — weekly sync-window containers are back up before this pulls their peers.
echo ""
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"

DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
    warn "docker_update.sh not found — skipping remainder updates"
    JOB_FAIL+=("docker_update.sh --remainder")
else
    _remainder_args=("--remainder")
    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
        echo "Remainder updates complete ✅"
        JOB_PASS+=("docker_update.sh --remainder")
    else
        warn "Remainder updates completed with errors"
        JOB_FAIL+=("docker_update.sh --remainder")
    fi
    unset _remainder_args
fi

WINDOW_END=$(date +%s)

-# Updates all running containers not already covered by daily or the weekly sync window.

-# Runs last — weekly sync-window containers are back up before this pulls their peers.

-echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"

-if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then

-    warn "docker_update.sh not found — skipping remainder updates"

-    JOB_FAIL+=("docker_update.sh --remainder")

-    [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")

-    if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then

-        echo "Remainder updates complete ✅"

-        JOB_PASS+=("docker_update.sh --remainder")

-        warn "Remainder updates completed with errors"


Now update `docker_update.sh` — rename the toggle and fix the caller reference:


# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by weekly_sync_maintenance.sh at the end of the Sunday window

# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by monthly_maintenance.sh


The file /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh has been updated successfully. (file state is current in your context — no need to Read it back)

# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by weekly_sync_maintenance.sh at the end of the Sunday window

# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by monthly_maintenance.sh

-#     Called by weekly_sync_maintenance.sh at the end of the Sunday window

#     Preview which containers would be pulled without making changes


#   WEEKLY_REMAINING_UPDATES
#       Enable or disable remainder mode. (default: true)

#   MONTHLY_REMAINING_UPDATES
#       Enable or disable remainder mode. (default: true)


#   WEEKLY_REMAINING_UPDATES
#       Enable or disable remainder mode. (default: true)

#   MONTHLY_REMAINING_UPDATES
#       Enable or disable remainder mode. (default: true)

#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.

#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]


    if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
        echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"

    if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
        echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"


    if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
        echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"

    if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
        echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"

if [[ "$REMAINDER_MODE" == true ]]; then

-    if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then

-        echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"

+    if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then

+        echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"


Bash

grep -rn "WEEKLY_REMAINING_UPDATES\|docker_update.*remainder\|remainder.*docker_update" /boot/config/plugins/varaverk --include="*.sh" --include="*.conf" --include="*.php" 2>/dev/null

Verify no stale references remain


/boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh:74:#           docker_update.sh (normal + --remainder): daily/weekly image pulls.
/boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh:333:# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists
/boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh:666:#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.
/boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh:671:# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder
/boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh:672:# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder --dry-run
/boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh:117:# docker_update.sh --remainder
/boot/config/plugins/varaverk/Configurations/master.conf:437:        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists


660	#     Containers stay running during the pull. Rebuilds containers that received a new image.
661	#     docker_daily_restart.sh restarts the rest. Toggle: DAILY_CONTAINER_UPDATES in master.conf.
662	#
663	#   Remainder (weekly) — targets all running containers not in managed lists.
664	#     Excludes daily list, weekly restart list, emby+critical-data sync-window containers,
665	#     and fallback containers owned by the remote server's update cycle.
666	#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.
667	#
668	# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --dry-run
669	# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --status
670	# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh
671	# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder
672	# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder --dry-run
673
674	# downloaders_reset.sh — maintenance reset for all download clients on this server
675	# Called every 30 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS.
676	# Host-aware: sections skip gracefully if a downloader URL is empty for this server.
677	#
678	# slskd:
679	#   Stuck searches    — clears Completed/Errored searches left by Soularr crashes

#     Containers stay running during the pull. Rebuilds containers that received a new image.
#     docker_daily_restart.sh restarts the rest. Toggle: DAILY_CONTAINER_UPDATES in master.conf.
#
#   Remainder (weekly) — targets all running containers not in managed lists.
#     Excludes daily list, weekly restart list, emby+critical-data sync-window containers,
#     and fallback containers owned by the remote server's update cycle.
#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder --dry-run

# downloaders_reset.sh — maintenance reset for all download clients on this server
# Called every 30 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS.
# Host-aware: sections skip gracefully if a downloader URL is empty for this server.
#
# slskd:
#   Stuck searches    — clears Completed/Errored searches left by Soularr crashes


70	#           individually below for standalone use. Inline descriptions on every entry.
71	#           Correct schedules verified against script headers. sunday_morning_coffee_report
72	#           and rsync_emby_fallback added as separate scheduled entries.
73	#   v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork).
74	#           docker_update.sh (normal + --remainder): daily/weekly image pulls.
75	#           downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance.
76	#           arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh.
77	#           lidarr_missing_art.sh: fetch missing album/artist artwork.
78	#           radarr_tmdb_removed.sh + sonarr_tvdb_removed.sh: dropped-entry cleanup.
79	#           Initial_run/ section: ssh_setup.sh + partnership_onboard.sh.

#           individually below for standalone use. Inline descriptions on every entry.
#           Correct schedules verified against script headers. sunday_morning_coffee_report
#           and rsync_emby_fallback added as separate scheduled entries.
#   v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork).
#           docker_update.sh (normal + --remainder): daily/weekly image pulls.
#           downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance.
#           arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh.
#           lidarr_missing_art.sh: fetch missing album/artist artwork.
#           radarr_tmdb_removed.sh + sonarr_tvdb_removed.sh: dropped-entry cleanup.
#           Initial_run/ section: ssh_setup.sh + partnership_onboard.sh.


329	#        Critical-Data full clean auth stack mirror
330	#   7. Start remote containers  dependency order, new image, verify each container up
331	#   8. Start local containers   dependency order, new image, verify each container up
332	#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
333	# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists
334	#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.
335	#                                playback_aware_lidarr_discovery.sh: 0–5 music adds/week
336	#                                  based on your Emby play history + Last.fm similar artists.
337	#                                playback_aware_radarr_discovery.sh: 0–5 movie adds/week
338	#                                  based on recently watched + TMDB recommendations.

#        Critical-Data full clean auth stack mirror
#   7. Start remote containers  dependency order, new image, verify each container up
#   8. Start local containers   dependency order, new image, verify each container up
#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists
#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.
#                                playback_aware_lidarr_discovery.sh: 0–5 music adds/week
#                                  based on your Emby play history + Last.fm similar artists.
#                                playback_aware_radarr_discovery.sh: 0–5 movie adds/week
#                                  based on recently watched + TMDB recommendations.


Edit

#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists
#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.

#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
#  10. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.


The file /boot/config/plugins/varaverk/Plugin/unraid/user_script_plug-in.sh has been updated successfully. (file state is current in your context — no need to Read it back)

#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists
#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.

#   9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich
#  10. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.

#   7. Start remote containers  dependency order, new image, verify each container up

#   8. Start local containers   dependency order, new image, verify each container up

-# 10. docker_update.sh --remainder  pull latest images for all containers not in daily/weekly lists

-#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.

+#  10. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.

#                                playback_aware_lidarr_discovery.sh: 0–5 music adds/week

#                                  based on your Emby play history + Last.fm similar artists.

#                                playback_aware_radarr_discovery.sh: 0–5 movie adds/week


  Remainder (weekly) — targets all running containers not in managed lists.
#     Excludes daily list, weekly restart list, emby+critical-data sync-window containers,
#     and fallback containers owned by the remote server's update cycle.
#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.

  Remainder (monthly) — targets all running containers not in managed lists.
#     Excludes daily list, emby+critical-data sync-window containers,
#     and fallback containers owned by the remote server's update cycle.
#     Rebuilds containers that received a new image. Toggle: MONTHLY_REMAINING_UPDATES in master.conf.


  Remainder (weekly) — targets all running containers not in managed lists.
#     Excludes daily list, weekly restart list, emby+critical-data sync-window containers,
#     and fallback containers owned by the remote server's update cycle.
#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.

  Remainder (monthly) — targets all running containers not in managed lists.
#     Excludes daily list, emby+critical-data sync-window containers,
#     and fallback containers owned by the remote server's update cycle.
#     Rebuilds containers that received a new image. Toggle: MONTHLY_REMAINING_UPDATES in master.conf.

#     Containers stay running during the pull. Rebuilds containers that received a new image.

#     docker_daily_restart.sh restarts the rest. Toggle: DAILY_CONTAINER_UPDATES in master.conf.

-#   Remainder (weekly) — targets all running containers not in managed lists.

-#     Excludes daily list, weekly restart list, emby+critical-data sync-window containers,

+#   Remainder (monthly) — targets all running containers not in managed lists.

+#     Excludes daily list, emby+critical-data sync-window containers,

-#     Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf.

+#     Rebuilds containers that received a new image. Toggle: MONTHLY_REMAINING_UPDATES in master.conf.

# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --dry-run

# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --status


docker_update.sh (normal + --remainder): daily/weekly image pulls.

docker_update.sh (normal + --remainder): daily/monthly image pulls.


#           Correct schedules verified against script headers. sunday_morning_coffee_report

#           and rsync_emby_fallback added as separate scheduled entries.

#   v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork).

-#           docker_update.sh (normal + --remainder): daily/weekly image pulls.

+#           docker_update.sh (normal + --remainder): daily/monthly image pulls.

#           downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance.

#           arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh.

#           lidarr_missing_art.sh: fetch missing album/artist artwork.


Bash

grep -rn "WEEKLY_REMAINING_UPDATES" /boot/config/plugins/varaverk --include="*.sh" --include="*.conf" --include="*.php" 2>/dev/null

Confirm no stale WEEKLY_REMAINING_UPDATES references remain


(Bash completed with no output)


Clean. Commit and push:


cd /boot/config/plugins/varaverk && git add Configurations/master.conf Orchestrators/weekly_sync_maintenance.sh Docker_Essentials/docker_update.sh Plugin/unraid/user_script_plug-in.sh && git commit -m "$(cat <<'EOF'
Move remainder container updates from weekly to monthly

Daily updates daily containers, weekly updates its sync-window containers,
monthly now handles everything else via MONTHLY_MAINTENANCE_SCRIPTS.
EOF
)" && git push

Commit and push remainder move to monthly


Error: Exit code 1
The following paths are ignored by one of your .gitignore files:
Configurations/master.conf
hint: Use -f if you really want to add them.
hint: Disable this message with "git config set advice.addIgnoredFile false"


cd /boot/config/plugins/varaverk && git add -f Configurations/master.conf && git add Orchestrators/weekly_sync_maintenance.sh Docker_Essentials/docker_update.sh Plugin/unraid/user_script_plug-in.sh && git commit -m "$(cat <<'EOF'
Move remainder container updates from weekly to monthly

Daily updates daily containers, weekly updates its sync-window containers,
monthly now handles everything else via MONTHLY_MAINTENANCE_SCRIPTS.
EOF
)" && git push

Force-add gitignored master.conf and commit


[main 55f8842] Move remainder container updates from weekly to monthly
 4 files changed, 1461 insertions(+), 36 deletions(-)
 create mode 100644 Configurations/master.conf
remote: . Processing 1 references
remote: Processed 1 references in total
To ssh://192.168.50.2:221/FailedProxy/Varaverk.git
   bfc3cbe..55f8842  main -> main
Shell cwd was reset to /root


Done. Update cadence is now:

- **Daily** — pulls + rebuilds `DAILY_RESTART_CONTAINERS`
- **Weekly** — pulls + rebuilds emby/critical-data in the sync window; `docker_weekly_restart.sh` restarts `WEEKLY_RESTART_CONTAINERS`
- **Monthly** — `docker_update.sh --remainder` catches everything else (including `WEEKLY_RESTART_CONTAINERS` whose images were never pulled by weekly); toggle is now `MONTHLY_REMAINING_UPDATES`


# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update.sh
#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
#     Called by daily_sync_maintenance.sh before docker_daily_restart.sh
#
# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by monthly_maintenance.sh
#
# docker_update.sh --dry-run
#     Preview which containers would be pulled without making changes
#
# docker_update.sh --status
#     Show configuration and container list for current mode
#
# docker_update.sh --log
#     Verbose per-container pull and comparison output
#
# ==============================================================================================
, again what does weekly call? does it get its own arg to know to read the weekly array?


Weekly doesn't call it at all — the sync-window pulls (emby, mariadb-authelia, redis-authelia) happen inline in `weekly_sync_maintenance.sh` with direct `docker pull` calls. `WEEKLY_RESTART_CONTAINERS` (NextCloud, AdGuard, Immich, Organizrr) only get restarted by `docker_weekly_restart.sh` with no image pull beforehand.

You're pointing at the gap: the pattern is update → restart, and daily has both halves, but weekly's restart containers have no update step. They need a `--weekly` mode that pulls `WEEKLY_RESTART_CONTAINERS` before `docker_weekly_restart.sh` runs — then remainder goes back to excluding them since weekly owns their update cycle.

Shall I add `--weekly` to `docker_update.sh`, wire it into `WEEKLY_MAINTENANCE_SCRIPTS` before `docker_weekly_restart.sh`, and remove `WEEKLY_RESTART_CONTAINERS` from remainder's exclusion?


yes


Let me read the current state of the key files before touching anything.

Find weekly scripts list in master.conf


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Docker Update ==============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Pulls the latest images for configured containers. Two modes: normal (daily)
9	# and remainder (weekly).
10	#
11	# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
12	# Containers stay running during the pull — no extra downtime beyond what the
13	# nightly restart already causes.
14	#
15	# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
16	# It catches everything that normal mode and the weekly sync window did not already
17	# update — derived automatically from docker ps, nothing to configure.
18	#
19	# ==============================================================================================
20	# OPERATIONAL MODEL
21	# ==============================================================================================
22	#
23	# Normal mode (daily):
24	#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
25	#   Pull → compare old vs new image ID → mark updated or already current.
26	#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
27	#
28	# Remainder mode (weekly):
29	#   Targets all currently running containers NOT in:
30	#     DAILY_RESTART_CONTAINERS       — already updated daily
31	#     emby + critical-data profiles  — updated inline by the weekly sync window
32	#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
33	#   WEEKLY_RESTART_CONTAINERS are included — docker_weekly_restart.sh restarts
34	#   them but doesn't pull images; remainder is the only place they get updated.
35	#   Pull → compare → prune dangling images.
36	#
37	# ==============================================================================================
38	# DESIGN PRINCIPLES
39	# ==============================================================================================
40	#
41	# Single List
42	#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
43	#   separate update list. Adding or removing a container from the restart list
44	#   automatically updates the image pull list — one change, both places.
45	#
46	# Version Ownership
47	#   Fallback containers are excluded from remainder mode. This server only runs
48	#   them during a fallback. The remote server owns their version — if remainder
49	#   updates them independently and a handback occurs, the remote's older image
50	#   may not handle data written by the newer version.
51	#
52	# State Respect
53	#   Stopped containers are never targeted. Pulling while stopped adds no value
54	#   and a stopped container was likely halted intentionally.
55	#
56	# ==============================================================================================
57	# OPERATIONAL SAFEGUARDS
58	# ==============================================================================================
59	#
60	# Lock Acquisition
61	#   Prevents concurrent execution via acquire_lock(). Safe to call from
62	#   maintenance scripts without risk of overlap.
63	#
64	# Host Detection
65	#   detect_hosts() identifies which server is running the script and aliases
66	#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
67	#
68	# Root Enforcement
69	#   Docker operations require root privileges.
70	#
71	# DAILY_CONTAINER_UPDATES Toggle
72	#   Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
73	#   regardless — update and restart are independent operations.
74	#
75	# Fallback Exclusion
76	#   Remainder mode excludes containers owned by the remote server's update cycle
77	#   to prevent version divergence across the fallback boundary.
78	#
79	# Running-Only Filter
80	#   Stopped containers excluded from remainder mode — intentionally down.
81	#
82	# Image ID Comparison
83	#   Containers not restarted unless their image actually changed. Pulls that
84	#   result in "already up to date" produce no restart.
85	#
86	# ==============================================================================================
87	# CONFIGURATION
88	# ==============================================================================================
89	#
90	# master.conf
91	#
92	#   DAILY_CONTAINER_UPDATES
93	#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
94	#       (default: true)
95	#
96	#   MONTHLY_REMAINING_UPDATES
97	#       Enable or disable remainder mode. (default: true)
98	#
99	#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
100	#       Container names for emby and critical-data profiles — excluded from
101	#       remainder mode (already updated by the weekly sync window)
102	#
103	# host*.conf
104	#
105	#   HOST*_DAILY_RESTART_CONTAINERS
106	#       Containers updated in normal mode. Aliased by detect_hosts() →
107	#       DAILY_RESTART_CONTAINERS
108	#
109	# ==============================================================================================
110	# RUNTIME MODES
111	# ==============================================================================================
112	#
113	# docker_update.sh
114	#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
115	#     Called by daily_sync_maintenance.sh before docker_daily_restart.sh
116	#
117	# docker_update.sh --remainder
118	#     Remainder mode — pull all running containers not in managed lists,
119	#     restart those that received updates, prune dangling images
120	#     Called by monthly_maintenance.sh
121	#
122	# docker_update.sh --dry-run
123	#     Preview which containers would be pulled without making changes
124	#
125	# docker_update.sh --status
126	#     Show configuration and container list for current mode
127	#
128	# docker_update.sh --log
129	#     Verbose per-container pull and comparison output
130	#
131	# ==============================================================================================
132
133	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
134
135	source "$SCRIPT_DIR/../load_config.sh"

#!/bin/bash
# ==============================================================================================
# ================================= Docker Update ==============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pulls the latest images for configured containers. Two modes: normal (daily)
# and remainder (weekly).
#
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
# Containers stay running during the pull — no extra downtime beyond what the
# nightly restart already causes.
#
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
# It catches everything that normal mode and the weekly sync window did not already
# update — derived automatically from docker ps, nothing to configure.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Normal mode (daily):
#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
#   Pull → compare old vs new image ID → mark updated or already current.
#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
#
# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   WEEKLY_RESTART_CONTAINERS are included — docker_weekly_restart.sh restarts
#   them but doesn't pull images; remainder is the only place they get updated.
#   Pull → compare → prune dangling images.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Single List
#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
#   separate update list. Adding or removing a container from the restart list
#   automatically updates the image pull list — one change, both places.
#
# Version Ownership
#   Fallback containers are excluded from remainder mode. This server only runs
#   them during a fallback. The remote server owns their version — if remainder
#   updates them independently and a handback occurs, the remote's older image
#   may not handle data written by the newer version.
#
# State Respect
#   Stopped containers are never targeted. Pulling while stopped adds no value
#   and a stopped container was likely halted intentionally.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# DAILY_CONTAINER_UPDATES Toggle
#   Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
#   regardless — update and restart are independent operations.
#
# Fallback Exclusion
#   Remainder mode excludes containers owned by the remote server's update cycle
#   to prevent version divergence across the fallback boundary.
#
# Running-Only Filter
#   Stopped containers excluded from remainder mode — intentionally down.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed. Pulls that
#   result in "already up to date" produce no restart.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   DAILY_CONTAINER_UPDATES
#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
#       (default: true)
#
#   MONTHLY_REMAINING_UPDATES
#       Enable or disable remainder mode. (default: true)
#
#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
#       Container names for emby and critical-data profiles — excluded from
#       remainder mode (already updated by the weekly sync window)
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Containers updated in normal mode. Aliased by detect_hosts() →
#       DAILY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update.sh
#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
#     Called by daily_sync_maintenance.sh before docker_daily_restart.sh
#
# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#     Called by monthly_maintenance.sh
#
# docker_update.sh --dry-run
#     Preview which containers would be pulled without making changes
#
# docker_update.sh --status
#     Show configuration and container list for current mode
#
# docker_update.sh --log
#     Verbose per-container pull and comparison output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"


377:        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
401:# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
403:    WEEKLY_MAINTENANCE_SCRIPTS=(
404:        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
437:        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists
694:# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
859:# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.


395	# Personal encrypted shares defined per-host in host*.conf.
396	# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
397	# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
398
399	# ━━━ Weekly Sync Maintenance ━━━
400	# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
401	# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
402	# Schedule: 30 2 * * 0 (Sunday 2:30am)
403	    WEEKLY_MAINTENANCE_SCRIPTS=(
404	        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
405	        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
406	        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
407	        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
408	        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
409	    )
410
411	# Pull updates for all running containers NOT in daily/weekly managed lists.
412	# Ensures every deployed container receives at least one image pull per month.
413	# Set false to skip.
414	    MONTHLY_REMAINING_UPDATES=true

# Personal encrypted shares defined per-host in host*.conf.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
# Schedule: 30 2 * * 0 (Sunday 2:30am)
    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
    )

# Pull updates for all running containers NOT in daily/weekly managed lists.
# Ensures every deployed container receives at least one image pull per month.
# Set false to skip.
    MONTHLY_REMAINING_UPDATES=true


135	source "$SCRIPT_DIR/../load_config.sh"
136
137	# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
138	REMAINDER_MODE=false
139	_filtered_args=()
140	for _arg in "$@"; do
141	    if [[ "$_arg" == "--remainder" ]]; then
142	        REMAINDER_MODE=true
143	    else
144	        _filtered_args+=("$_arg")
145	    fi
146	done
147	unset _arg
148
149	parse_args "${_filtered_args[@]}"
150	unset _filtered_args
151
152	# ==============================================================================================
153	# ━━━ Setup ━━━
154	# ==============================================================================================
155	if [[ "$EUID" -ne 0 ]]; then
156	    error "Must be run as root"
157	    exit 1
158	fi
159
160	acquire_lock
161
162	if ! command -v docker &>/dev/null; then
163	    error "Docker command not found"
164	    exit 1
165	fi
166
167	detect_hosts
168
169	# ==============================================================================================
170	# ━━━ Container Discovery ━━━
171	# ==============================================================================================
172	if [[ "$REMAINDER_MODE" == true ]]; then
173	    if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
174	        echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"
175	        exit 0
176	    fi
177
178	    declare -A _exclude=()
179
180	    # Daily containers — updated by docker_update.sh normal mode
181	    for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
182	        [[ -n "$_c" ]] && _exclude["$_c"]=1
183	    done
184
185	    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
186	    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
187	    read -r -a _weekly_arr <<< "$_weekly_str"
188	    for _c in "${_weekly_arr[@]}"; do
189	        [[ -n "$_c" ]] && _exclude["$_c"]=1
190	    done
191	    unset _weekly_str _weekly_arr
192
193	    # Fallback coverage containers — owned by the remote server's update cycle.
194	    # This server runs them during fallback but should never update them independently.
195	    # Updating them here risks version divergence: if remote's writeback after handback
196	    # encounters data written by a newer version, it may not handle it correctly.
197	    for _tier in 1 2 3 4; do
198	        _tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
199	        eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
200	        for _c in "${_tier_arr[@]}"; do
201	            [[ -n "$_c" ]] && _exclude["$_c"]=1
202	        done
203	    done
204	    unset _tier _tier_var _tier_arr _c
205
206	    mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
207	    TARGET_CONTAINERS=()
208	    for _c in "${_all_running[@]}"; do
209	        [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
210	    done
211	    unset _all_running _exclude _c
212	else
213	    if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
214	        echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
215	        exit 0
216	    fi
217
218	    if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
219	        warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
220	        warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
221	        exit 0
222	    fi
223
224	    TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")

source "$SCRIPT_DIR/../load_config.sh"

# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
REMAINDER_MODE=false
_filtered_args=()
for _arg in "$@"; do
    if [[ "$_arg" == "--remainder" ]]; then
        REMAINDER_MODE=true
    else
        _filtered_args+=("$_arg")
    fi
done
unset _arg

parse_args "${_filtered_args[@]}"
unset _filtered_args

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

# ==============================================================================================
# ━━━ Container Discovery ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
        echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"
        exit 0
    fi

    declare -A _exclude=()

    # Daily containers — updated by docker_update.sh normal mode
    for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Fallback coverage containers — owned by the remote server's update cycle.
    # This server runs them during fallback but should never update them independently.
    # Updating them here risks version divergence: if remote's writeback after handback
    # encounters data written by a newer version, it may not handle it correctly.
    for _tier in 1 2 3 4; do
        _tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
        eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
        for _c in "${_tier_arr[@]}"; do
            [[ -n "$_c" ]] && _exclude["$_c"]=1
        done
    done
    unset _tier _tier_var _tier_arr _c

    mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
    TARGET_CONTAINERS=()
    for _c in "${_all_running[@]}"; do
        [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
    done
    unset _all_running _exclude _c
else
    if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
        echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
        exit 0
    fi

    if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
        warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
        warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
        exit 0
    fi

    TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")


grep -n "WEEKLY_RESTART_CONTAINERS" /boot/config/plugins/varaverk/Configurations/host1.conf | head -5

Confirm WEEKLY_RESTART_CONTAINERS alias name


225	fi
226
227	# ==============================================================================================
228	# ━━━ Status ━━━
229	# ==============================================================================================
230	if [[ "$SHOW_STATUS" == true ]]; then
231	    echo ""
232	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
233	    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
234	    echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
235	    if [[ "$REMAINDER_MODE" == true ]]; then
236	        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
237	        for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done
238	    else
239	        echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
240	        echo "$ICON_GEAR       Enabled:    ${DAILY_CONTAINER_UPDATES:-true}"
241	    fi
242	    echo "$ICON_GEAR       Dry Run:    $DRY_RUN"
243	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
244	    exit 0
245	fi
246
247	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
248
249	if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
250	    echo "No containers to update"
251	    exit 0
252	fi
253
254	# ==============================================================================================
255	# ━━━ Pull Updates ━━━
256	# ==============================================================================================
257	echo ""
258	if [[ "$REMAINDER_MODE" == true ]]; then
259	    echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
260	    echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
261	    log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
262	else
263	    echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
264	    log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
265	fi
266	echo ""
267
268	START=$(date +%s)
269	UPDATED=()
270	UP_TO_DATE=()
271	FAILED=()
272	OLD_IMAGE_IDS=()   # old image IDs to explicitly remove after rebuilds
273	SKIPPED=()
274
275	for container in "${TARGET_CONTAINERS[@]}"; do
276	    [[ -z "$container" ]] && continue
277	    log "━━━ $ICON_CONTAINERS $container ━━━"
278
279	    if ! docker inspect "$container" &>/dev/null; then
280	        warn "$container — not found, skipping"
281	        SKIPPED+=("$container")
282	        continue
283	    fi
284
285	    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
286	    if [[ -z "$IMAGE" ]]; then
287	        warn "$container — could not determine image, skipping"
288	        SKIPPED+=("$container")
289	        continue
290	    fi
291
292	    log "$container — image: $IMAGE"
293
294	    if [[ "$DRY_RUN" == true ]]; then

fi

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
    if [[ "$REMAINDER_MODE" == true ]]; then
        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
        for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done
    else
        echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
        echo "$ICON_GEAR       Enabled:    ${DAILY_CONTAINER_UPDATES:-true}"
    fi
    echo "$ICON_GEAR       Dry Run:    $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"

if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
    echo "No containers to update"
    exit 0
fi

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
    log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
else
    echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
fi
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()
OLD_IMAGE_IDS=()   # old image IDs to explicitly remove after rebuilds
SKIPPED=()

for container in "${TARGET_CONTAINERS[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    if ! docker inspect "$container" &>/dev/null; then
        warn "$container — not found, skipping"
        SKIPPED+=("$container")
        continue
    fi

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        SKIPPED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then


342:    HOST1_WEEKLY_RESTART_CONTAINERS=(

342:    HOST1_WEEKLY_RESTART_CONTAINERS=(


375	    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
376	fi
377
378	END=$(date +%s)
379
380	# ==============================================================================================
381	# ━━━ Summary ━━━
382	# ==============================================================================================
383	if [[ "$REMAINDER_MODE" == true ]]; then
384	    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
385	else
386	    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
387	fi
388	echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
389	echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
390	if [[ ${#UPDATED[@]} -gt 0 ]]; then
391	    echo "$ICON_DONE       Updated:     ${#UPDATED[@]}"
392	    log "  ${UPDATED[*]}"
393	fi
394	[[ ${#REBUILT[@]} -gt 0 ]]       && echo "$ICON_SYNC       Rebuilt:     ${#REBUILT[@]}"
395	[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR      Rebuild fail:${REBUILD_FAILED[*]}"
396	[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
397	[[ ${#SKIPPED[@]} -gt 0 ]]       && log "$ICON_WARN       Skipped:     ${SKIPPED[*]}"
398	[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Failed:      ${FAILED[*]}"
399	echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"
400
401	if [[ "$DRY_RUN" == true ]]; then
402	    warn "DRY RUN — no images pulled"
403	elif [[ ${#FAILED[@]} -eq 0 ]]; then
404	    echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
405	else
406	    warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
407	fi
408	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
409
410	# Pull failures are non-fatal — restart proceeds regardless
411	exit 0
412

    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
else
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
fi
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Updated:     ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#REBUILT[@]} -gt 0 ]]       && echo "$ICON_SYNC       Rebuilt:     ${#REBUILT[@]}"
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR      Rebuild fail:${REBUILD_FAILED[*]}"
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]]       && log "$ICON_WARN       Skipped:     ${SKIPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Failed:      ${FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no images pulled"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
else
    warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Pull failures are non-fatal — restart proceeds regardless
exit 0


Got everything I need. Making all the changes now.


Edit

# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pulls the latest images for configured containers. Two modes: normal (daily)
# and remainder (weekly).
#
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
# Containers stay running during the pull — no extra downtime beyond what the
# nightly restart already causes.
#
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
# It catches everything that normal mode and the weekly sync window did not already
# update — derived automatically from docker ps, nothing to configure.

# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pulls the latest images for configured containers. Three modes:
#
# Normal mode — called by daily_sync_maintenance.sh before docker_daily_restart.sh.
# Containers stay running during the pull — no extra downtime beyond what the
# nightly restart already causes.
#
# Weekly mode — called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS
# before docker_weekly_restart.sh. Pulls WEEKLY_RESTART_CONTAINERS so the
# subsequent restart lands on the fresh image.
#
# Remainder mode — called by monthly_maintenance.sh. Catches everything not
# already owned by daily or weekly — derived from docker ps, nothing to configure.