1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Downloaders Reset ==========================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Maintenance reset for all download clients on this server. Clears accumulated
9	# state that download clients generate but never clean up themselves — stuck
10	# searches, dead transfers, failed imports, stale queue entries, completed history.
11	#
12	# Called every 30 minutes by critical_sync_maintenance.sh via
13	# CRITICAL_MAINTENANCE_SCRIPTS. Can also be run manually for ad hoc cleanup.
14	# If a downloader is not configured for this host, that section skips cleanly.
15	#
16	# ==============================================================================================
17	# OPERATIONAL MODEL
18	# ==============================================================================================
19	#
20	# slskd
21	#   Stuck searches    — clears Completed/Errored searches left by Soularr crashes
22	#                       prevents 409 Conflict on next Soularr startup
23	#   Dead transfers    — removes completed/errored/aborted transfer records per user
24	#                       prevents Soularr 404 loop when polling a user whose transfer is gone
25	#                       NEVER removes InProgress or Queued transfers
26	#   Failed imports    — purges albums Soularr downloaded but Lidarr rejected
27	#                       Soularr moves these to failed_imports/ and never cleans them up
28	#
29	# SABnzbd
30	#   Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS
31	#   Failed history    — removes failed download records older than DOWNLOADER_RETENTION_DAYS
32	#   Stalled queue     — removes Paused or Stuck queue items no longer progressing
33	#                       active downloading items are never touched
34	#
35	# qBittorrent
36	#   Age failsafe      — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
37	#                       deleteFiles=false — removes from qBit, leaves files for arrs to manage
38	#                       optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO
39	#
40	# ==============================================================================================
41	# DESIGN PRINCIPLES
42	# ==============================================================================================
43	#
44	# Never Interrupt Active Downloads
45	#   Each downloader section checks for active state before any removal. slskd
46	#   skips users with InProgress or Queued transfers. SABnzbd only removes items
47	#   past the retention threshold. qBittorrent applies minimum age and optional
48	#   ratio requirements. In-progress work is never touched.
49	#
50	# Graceful Skip on Unavailability
51	#   If a downloader's URL is empty or the service is unreachable, that section
52	#   skips cleanly with a log message. The script never exits fatally on a single
53	#   unreachable downloader — the others still run.
54	#
55	# Host-Aware Configuration
56	#   detect_hosts() aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
57	#   to their unprefixed names. Downloaders not configured for this host are absent
58	#   from the aliased vars and skip automatically.
59	#
60	# ==============================================================================================
61	# OPERATIONAL SAFEGUARDS
62	# ==============================================================================================
63	#
64	# Active Transfer Protection
65	#   slskd: skips users with InProgress or Queued transfers before any removal.
66	#   SABnzbd: age threshold enforced before deletion.
67	#   qBittorrent: minimum age plus optional ratio gate before failsafe removal.
68	#
69	# Reachability Check
70	#   Each section validates its downloader URL before API calls. Missing or
71	#   unreachable downloaders skip without affecting other sections.
72	#
73	# Host Detection
74	#   detect_hosts() identifies which server is running the script and aliases
75	#   all HOST*_SLSKD_*, HOST*_SABNZBD_*, and HOST*_QBIT_* vars to the correct
76	#   host's values. Downloaders not configured on this host skip automatically.
77	#
78	# Lock Acquisition
79	#   acquire_lock "wait" — waits for previous run to finish since this runs every
80	#   30 minutes and prior execution may still be completing.
81	#
82	# ==============================================================================================
83	# CONFIGURATION
84	# ==============================================================================================
85	#
86	# host*.conf
87	#
88	#   HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR
89	#       slskd connection and failed imports path. Aliased by detect_hosts()
90	#
91	#   HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY
92	#       SABnzbd connection details. Aliased by detect_hosts()
93	#
94	#   HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD
95	#       qBittorrent connection details. Aliased by detect_hosts()
96	#
97	# master.conf
98	#
99	#   DOWNLOADER_RETENTION_DAYS
100	#       Days before SABnzbd history entries (completed or failed) are removed
101	#
102	#   QBIT_FAILSAFE_MIN_DAYS
103	#       Minimum torrent age in days before failsafe removal is considered
104	#
105	#   QBIT_FAILSAFE_MIN_RATIO
106	#       Minimum seeding ratio required alongside age gate (0 = age only)
107	#
108	# ==============================================================================================
109	# RUNTIME MODES
110	# ==============================================================================================
111	#
112	# downloaders_reset.sh
113	#     Run maintenance reset for all configured download clients
114	#
115	# downloaders_reset.sh --dry-run
116	#     Preview what would be removed without making any changes
117	#
118	# downloaders_reset.sh --status
119	#     Show configured downloaders, current queue depths, and retention settings
120	#
121	# downloaders_reset.sh --log
122	#     Verbose per-client per-item output
123	#
124	# ==============================================================================================
125
126	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
127
128	source "$SCRIPT_DIR/../load_config.sh"
129
130	parse_args "$@"
131
132	# ==============================================================================================
133	# ━━━ Setup ━━━
134	# ==============================================================================================
135	if [[ "$EUID" -ne 0 ]]; then
136	    error "Must be run as root"
137	    exit 1
138	fi
139
140	# Lock first — wait mode since this runs every 30min and previous may still be finishing
141	acquire_lock "wait"
142
143	# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
144	detect_hosts
145
146	START_TIME=$(date +%s)
147	CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
148	TOTAL_PASS=0
149	TOTAL_FAIL=0
150
151	log "$ICON_GEAR Config: retention=${DOWNLOADER_RETENTION_DAYS}d qbit-age=${QBIT_FAILSAFE_MIN_DAYS}d qbit-ratio=${QBIT_FAILSAFE_MIN_RATIO}"
152
153	# ==============================================================================================
154	# ━━━ Status ━━━
155	# ==============================================================================================
156	if [[ "$SHOW_STATUS" == true ]]; then
157	    echo ""
158	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
159	    echo "$ICON_HOST    Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
160	    echo "$ICON_GEAR    slskd:       ${SLSKD_URL:-not configured}"
161	    echo "$ICON_GEAR    SABnzbd:     ${SABNZBD_URL:-not configured}"
162	    echo "$ICON_GEAR    qBittorrent: ${QBIT_URL:-not configured}"
163	    echo "$ICON_TIME    Retention:   ${DOWNLOADER_RETENTION_DAYS} days"
164	    echo "$ICON_GEAR    qBit age:    ${QBIT_FAILSAFE_MIN_DAYS} days"

... [486 lines truncated] ...


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Docker Update ==============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Pulls the latest images for configured containers. Three modes:
9	#
10	# Normal mode — called by daily_sync_maintenance.sh before docker_daily_restart.sh.
11	# Containers stay running during the pull — no extra downtime beyond what the
12	# nightly restart already causes.
13	#
14	# Weekly mode — called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS
15	# before docker_weekly_restart.sh. Pulls WEEKLY_RESTART_CONTAINERS so the
16	# subsequent restart lands on the fresh image.
17	#
18	# Remainder mode — called by monthly_maintenance.sh. Catches everything not
19	# already owned by daily or weekly — derived from docker ps, nothing to configure.
20	#
21	# ==============================================================================================
22	# OPERATIONAL MODEL
23	# ==============================================================================================
24	#
25	# Normal mode (daily):
26	#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
27	#   Pull → compare old vs new image ID → mark updated or already current.
28	#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
29	#
30	# Weekly mode:
31	#   Targets WEEKLY_RESTART_CONTAINERS — same list used by docker_weekly_restart.sh.
32	#   Pull → compare → rebuild if changed.
33	#   docker_weekly_restart.sh runs after — containers restart onto the fresh image.
34	#
35	# Remainder mode (monthly):
36	#   Targets all currently running containers NOT in:
37	#     DAILY_RESTART_CONTAINERS       — already updated daily
38	#     WEEKLY_RESTART_CONTAINERS      — already updated weekly
39	#     emby + critical-data profiles  — updated inline by the weekly sync window
40	#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
41	#   Pull → compare → prune dangling images.
42	#
43	# ==============================================================================================
44	# DESIGN PRINCIPLES
45	# ==============================================================================================
46	#
47	# Single List
48	#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
49	#   separate update list. Adding or removing a container from the restart list
50	#   automatically updates the image pull list — one change, both places.
51	#
52	# Version Ownership
53	#   Fallback containers are excluded from remainder mode. This server only runs
54	#   them during a fallback. The remote server owns their version — if remainder
55	#   updates them independently and a handback occurs, the remote's older image
56	#   may not handle data written by the newer version.
57	#
58	# State Respect
59	#   Stopped containers are never targeted. Pulling while stopped adds no value
60	#   and a stopped container was likely halted intentionally.
61	#
62	# ==============================================================================================
63	# OPERATIONAL SAFEGUARDS
64	# ==============================================================================================
65	#
66	# Lock Acquisition
67	#   Prevents concurrent execution via acquire_lock(). Safe to call from
68	#   maintenance scripts without risk of overlap.
69	#
70	# Host Detection
71	#   detect_hosts() identifies which server is running the script and aliases
72	#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
73	#
74	# Root Enforcement
75	#   Docker operations require root privileges.
76	#
77	# DAILY_CONTAINER_UPDATES / WEEKLY_CONTAINER_UPDATES Toggles
78	#   Each mode exits cleanly when disabled. Restart scripts run regardless —
79	#   update and restart are independent operations.
80	#
81	# Fallback Exclusion
82	#   Remainder mode excludes containers owned by the remote server's update cycle
83	#   to prevent version divergence across the fallback boundary.
84	#
85	# Running-Only Filter
86	#   Stopped containers excluded from remainder mode — intentionally down.
87	#
88	# Image ID Comparison
89	#   Containers not restarted unless their image actually changed. Pulls that
90	#   result in "already up to date" produce no restart.
91	#
92	# ==============================================================================================
93	# CONFIGURATION
94	# ==============================================================================================
95	#
96	# master.conf
97	#
98	#   DAILY_CONTAINER_UPDATES
99	#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
100	#       (default: true)
101	#
102	#   WEEKLY_CONTAINER_UPDATES
103	#       Enable or disable weekly mode. docker_weekly_restart.sh runs regardless.
104	#       (default: true)
105	#
106	#   MONTHLY_REMAINING_UPDATES
107	#       Enable or disable remainder mode. (default: true)
108	#
109	#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
110	#       Container names for emby and critical-data profiles — excluded from
111	#       remainder mode (already updated by the weekly sync window)
112	#
113	# host*.conf
114	#
115	#   HOST*_DAILY_RESTART_CONTAINERS
116	#       Containers updated in normal mode. Aliased by detect_hosts() →
117	#       DAILY_RESTART_CONTAINERS
118	#
119	#   HOST*_WEEKLY_RESTART_CONTAINERS
120	#       Containers updated in weekly mode. Aliased by detect_hosts() →
121	#       WEEKLY_RESTART_CONTAINERS
122	#
123	# ==============================================================================================
124	# RUNTIME MODES
125	# ==============================================================================================
126	#
127	# docker_update.sh
128	#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
129	#     Called by daily_sync_maintenance.sh before docker_daily_restart.sh
130	#
131	# docker_update.sh --weekly
132	#     Weekly mode — pull latest images for WEEKLY_RESTART_CONTAINERS
133	#     Called by weekly_sync_maintenance.sh (WEEKLY_MAINTENANCE_SCRIPTS) before docker_weekly_restart.sh
134	#
135	# docker_update.sh --remainder
136	#     Remainder mode — pull all running containers not in managed lists,
137	#     restart those that received updates, prune dangling images
138	#     Called by monthly_maintenance.sh
139	#
140	# docker_update.sh --dry-run
141	#     Preview which containers would be pulled without making changes
142	#
143	# docker_update.sh --status
144	#     Show configuration and container list for current mode
145	#
146	# docker_update.sh --log
147	#     Verbose per-container pull and comparison output
148	#
149	# ==============================================================================================
150
151	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
152
153	source "$SCRIPT_DIR/../load_config.sh"
154
155	# Pre-parse mode flags before parse_args (unknown args pass through to PARSED_ARGS)
156	REMAINDER_MODE=false
157	WEEKLY_MODE=false
158	_filtered_args=()
159	for _arg in "$@"; do
160	    case "$_arg" in
161	        --remainder) REMAINDER_MODE=true ;;
162	        --weekly)    WEEKLY_MODE=true ;;
163	        *)           _filtered_args+=("$_arg") ;;
164	    esac
165	done
166	unset _arg
167
168	parse_args "${_filtered_args[@]}"
169	unset _filtered_args
170
171	# ==============================================================================================
172	# ━━━ Setup ━━━
173	# ==============================================================================================
174	if [[ "$EUID" -ne 0 ]]; then
175	    error "Must be run as root"
176	    exit 1
177	fi
178
179	acquire_lock
180
181	if ! command -v docker &>/dev/null; then
182	    error "Docker command not found"
183	    exit 1
184	fi
185
186	detect_hosts
187
188	# ==============================================================================================

... [270 lines truncated] ...


Bash

grep -rn 'log ".*✅\|log ".*done\|log ".*removed\|log ".*cleared\|log ".*restarted\|log ".*rebuilt\|log ".*reconnected\|log ".*updated' \
  /boot/config/plugins/varaverk/Docker_Essentials/

Verify Docker_Essentials is clean


/boot/config/plugins/varaverk/Docker_Essentials/docker_network_connect.sh:216:        log "$ICON_DOCKER_NET $network exists ✅ — driver: $net_driver subnet: $net_subnet"


ok, thats what we were doing last week. we were going through all scripts correcting for verbose...... lets go through and check echo vs log... goal clean informative no verbose, and all details verbo…


Run docker network script during onboarding


Docker_Essentials clean (that remaining one is a precondition check, correct as `log`). Now Media:


grep -rn 'log ".*✅\|log ".*done\|log ".*nothing to do\|log ".*clean\|log ".*removed\|log ".*cleared' \
  /boot/config/plugins/varaverk/Media/ | grep -v "^Binary"

Find log→echo candidates in Media


/boot/config/plugins/varaverk/Media/arrs_failed_stalled_recovery.sh:299:        log "$arr_name — clean ✅ no failed imports or stalled downloads"
/boot/config/plugins/varaverk/Media/media_cleaner.sh:211:        log "$FOLDER_NAME — clean ✅"
/boot/config/plugins/varaverk/Media/media_cleaner.sh:226:            log "$FOLDER_NAME — $FILE_COUNT file(s) removed"
/boot/config/plugins/varaverk/Media/radarr_tmdb_removed.sh:150:    log "No TMDb-removed movies found — nothing to do"
/boot/config/plugins/varaverk/Media/radarr_tmdb_removed.sh:201:        log "  Removed from Radarr ✅"
/boot/config/plugins/varaverk/Media/sonarr_tvdb_removed.sh:149:    log "No TVDB-removed series found — nothing to do"
/boot/config/plugins/varaverk/Media/sonarr_tvdb_removed.sh:203:        log "  Removed from Sonarr ✅"
/boot/config/plugins/varaverk/Media/arr_sync.sh:432:                log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally"
/boot/config/plugins/varaverk/Media/arr_sync.sh:442:                    not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;;
/boot/config/plugins/varaverk/Media/arr_sync.sh:742:        log "${arr_type^}: re-monitored $count items ✅"


`arr_sync.sh:432,442` are per-item in a blocklist loop (could be hundreds) — stay `log`. The rest are per-arr or per-folder outcomes:


perl -i -pe 's/\blog "(\$arr_name — clean ✅ no failed imports or stalled downloads)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/arrs_failed_stalled_recovery.sh

perl -i -pe 's/\blog "(\$FOLDER_NAME — clean ✅)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/media_cleaner.sh

perl -i -pe 's/\blog "(\$FOLDER_NAME — \$FILE_COUNT file\(s\) removed)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/media_cleaner.sh

perl -i -pe 's/\blog "(No TMDb-removed movies found — nothing to do)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/radarr_tmdb_removed.sh

perl -i -pe 's/\blog "(No TVDB-removed series found — nothing to do)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/sonarr_tvdb_removed.sh

perl -i -pe 's/\blog "  (Removed from Radarr ✅)"/echo "  $1"/' \
  /boot/config/plugins/varaverk/Media/radarr_tmdb_removed.sh

perl -i -pe 's/\blog "  (Removed from Sonarr ✅)"/echo "  $1"/' \
  /boot/config/plugins/varaverk/Media/sonarr_tvdb_removed.sh

perl -i -pe 's/\blog "(\$\{arr_type\^\}: re-monitored \$count items ✅)"/echo "$1"/' \
  /boot/config/plugins/varaverk/Media/arr_sync.sh

echo "done"

Fix Media log→echo outcome lines


done


grep -rn 'log ".*✅\|log ".*nothing to do\|log ".*clean\b\|log ".*file.s. removed\|log ".*re-monitored' \
  /boot/config/plugins/varaverk/Media/ | grep -v "^Binary\|arr_sync.*Not found\|arr_sync.*already removed"

Verify Media clean


(Bash completed with no output)


Now Tools (manual-run scripts — users need to see everything):


grep -rn 'log ".*✅\|log ".*done\|log ".*nothing\|log ".*removed\|log ".*created\|log ".*verified\|log ".*started\|log ".*updated\|log ".*flushed\|log ".*stopped\|log ".*restarted\|log ".*leaving stopped\|log ".*mounted\|log ".*exists\|log ".*applied\|log ".*placed\|log ".*fixed' \
  /boot/config/plugins/varaverk/Tools/ \
  /boot/config/plugins/varaverk/Plugin/unraid/Tools/ \
  /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/ \
  /boot/config/plugins/varaverk/System_Essentials/ \
  /boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh \
  /boot/config/plugins/varaverk/Tools/ramdisk_stop.sh | grep -v "^Binary"

Find log→echo candidates in Tools and System_Essentials


/boot/config/plugins/varaverk/Tools/container_data_export.sh:178:                log "$CONTAINER_NAME stopped ✅"
/boot/config/plugins/varaverk/Tools/container_data_export.sh:188:        log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
/boot/config/plugins/varaverk/Tools/container_data_export.sh:227:            log "Archive verified ✅"
/boot/config/plugins/varaverk/Tools/container_data_export.sh:263:                log "$CONTAINER_NAME restarted and running ✅"
/boot/config/plugins/varaverk/Tools/container_data_export.sh:280:    log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
/boot/config/plugins/varaverk/Tools/zfs_pool_scrub.sh:206:        log "$pool scrub started ✅"
/boot/config/plugins/varaverk/Tools/zfs_pool_scrub.sh:234:log "Safe to interrupt — scrubs continue in background if script is stopped"
/boot/config/plugins/varaverk/Tools/bulk_permissions_repair.sh:187:        log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:203:                log "$EMBY_CONTAINER stopped ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:266:            log "$db_name exists but is empty — no pending transactions ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:285:        log "$db_name — integrity check passed ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:314:                log "$EMBY_CONTAINER restarted and running ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:332:    log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
/boot/config/plugins/varaverk/Tools/emby_database_repair.sh:354:[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log "  $ICON_SUCCESS $db"; done
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:180:log "Ramdisk is mounted ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:202:        log "SSD fallback created ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:205:    log "SSD fallback exists: $TRANSCODE_SSD ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:256:    log "No active files on ramdisk ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:306:    log "State file updated: current_target=$TRANSCODE_SSD"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/storage_migrate.sh:169:        while IFS= read -r line; do log "$line"; done; then
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh:199:log "Config updated"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh:226:    log "Verified: pm.max_children = $APPLIED_VAL ✅"
/boot/config/plugins/varaverk/Tools/emby_to_lidarr_sync.sh:222:for a in "${MISSING[@]}"; do log "  $a"; done
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:157:log "Array is started — /mnt/user is mounted ✅"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:218:            log "$disk/$SHARE_NAME already exists — skipping"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:225:                log "Created: $DISK_PATH ✅"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:239:            log ".recovery marker already exists in $SHARE_NAME"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:245:            log "Marker placed: $MARKER_PATH ✅"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh:138:log "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:158:            log "Key pushed to $partner_host ✅"
/boot/config/plugins/varaverk/System_Essentials/server_reboot.sh:250:        log "Array stop complete ✅"
/boot/config/plugins/varaverk/System_Essentials/server_reboot.sh:312:    log "Filesystem buffers flushed ✅"
/boot/config/plugins/varaverk/System_Essentials/conf_sync.sh:75:                log "Own conf cached ✅" || warn "Failed to cache own conf"
/boot/config/plugins/varaverk/System_Essentials/conf_sync.sh:110:            log "Pulled ${partner_slot}.conf from $partner_host ✅"
/boot/config/plugins/varaverk/System_Essentials/conf_sync.sh:137:        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:180:log "Ramdisk is mounted ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:202:        log "SSD fallback created ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:205:    log "SSD fallback exists: $TRANSCODE_SSD ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:256:    log "No active files on ramdisk ✅"
/boot/config/plugins/varaverk/Tools/ramdisk_stop.sh:306:    log "State file updated: current_target=$TRANSCODE_SSD"
/boot/config/plugins/varaverk/Tools/docker_prune_images.sh:113:        log "No stopped containers"
/boot/config/plugins/varaverk/Tools/docker_prune_images.sh:119:        log "Removing $STOPPED_COUNT stopped container(s)..."
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:188:    log "Ramdisk already mounted — $USAGE used / $AVAIL available"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:225:        log "SSD fallback already exists: $TRANSCODE_SSD"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:229:            log "SSD fallback created: $TRANSCODE_SSD ✅"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:249:            log "Symlink already points to ramdisk — no change needed ✅"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:273:    [[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK → $RAMDISK_PATH ✅"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:291:        log "transcoding-temp already exists on ramdisk"
/boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh:294:            log "Created transcoding-temp on ramdisk ✅"
/boot/config/plugins/varaverk/System_Essentials/rsync_stop.sh:134:    log "$REMOTE_SERVER_NAME reachable ✅"
/boot/config/plugins/varaverk/System_Essentials/rsync_stop.sh:436:                    log "$c — running ✅"


Conversation compacted


This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.

Summary:
1. Primary Request and Intent:

   Six tasks completed this session, with one ongoing:

   a) **Remainder mode exclusion fix**: User asked "doesnt remainder catch one specifically not in daily or weekly?" — WEEKLY_RESTART_CONTAINERS were wrongly excluded from remainder; docker_weekly_restart.sh only restarts them (no image pull), so remainder was their only update path. Removed the exclusion.

   b) **Move remainder to monthly**: "daily needs to update its containers, weekly its containers and we need to move remainder to monthly" — removed `docker_update.sh --remainder` from weekly, added to `MONTHLY_MAINTENANCE_SCRIPTS`, renamed toggle `WEEKLY_REMAINING_UPDATES` → `MONTHLY_REMAINING_UPDATES`.

   c) **Add --weekly mode**: "again what does weekly call? does it get its own arg?" / "yes" — added `--weekly` mode to `docker_update.sh` targeting `WEEKLY_RESTART_CONTAINERS`, wired before `docker_weekly_restart.sh` in `WEEKLY_MAINTENANCE_SCRIPTS`, added `WEEKLY_CONTAINER_UPDATES` toggle, re-excluded WEEKLY_RESTART_CONTAINERS from remainder.

   d) **Transcode fixes**: Live TV and Direct Stream sessions showed "0 streaming, 0 active, 0.00GB" despite ramdisk having 426MB of HLS segments. Two bugs: RAM_SESSION_COUNT counted subdirs only (Live TV writes flat files); lsof misses atomically-written HLS segments. Fixed session count, added "Active" count for too-young files.

   e) **Transcode card in monitors page**: "lets add that to our transcoding card in the monitors page" — added `active_files` to PHP API, rendered "Active segments N (Live TV / Direct Stream)" in the card.

   f) **slskd connection check blank**: Connected success used `log` instead of `echo` — fixed to `echo "$ICON_DONE [OK]    slskd connected to Soulseek ✅"`.

   g) **Echo vs log audit (ONGOING)**: "lets go through and check echo vs log... goal clean informative no verbose, and all details verbose. i know we got a few folders done, cant remember which ones" — systematic audit of all scripts. Rule: `echo` for headers/outcomes/results; `log` for per-item loops, per-minute health checks, intermediate steps. Completed Orchestrators, Docker_Essentials, Media folders. Pending: Monitors (selective), Watchdogs (state transitions only), Tools, Rsync, System_Essentials, Plugin/unraid, Fallback, Partnership.

2. Key Technical Concepts:
   - Varaverk: self-healing two-server Unraid home media ecosystem; one codebase on both servers
   - Three-file config model: `master.conf` (shared), `host1.conf`/`host2.conf` (per-host)
   - `detect_hosts()` aliases `HOST*_` vars to generic names for scripts
   - `platform_rebuild_container()` recreates container from XML template (picks up new image); `docker start` reuses creation-time SHA
   - Update ownership chain: daily → `DAILY_RESTART_CONTAINERS`; weekly (inline) → emby/critical-data; weekly (--weekly) → `WEEKLY_RESTART_CONTAINERS`; monthly (--remainder) → everything else
   - HLS segments for Live TV/Direct Stream are written atomically (write then close) — lsof never sees them as open; age check is the effective guard
   - RAM_SESSION_COUNT must count both subdirectories (legacy transcode) AND unique hex session ID prefixes from flat files (Live TV HLS)
   - Echo vs log rule: `echo` = always visible (outcomes, headers, key results); `log` = verbose-only (per-item iteration, per-minute health, intermediate values)
   - Orchestrators: per-step headers echoed but per-step `done ✅` completions were silently log'd — confusing

3. Files and Code Sections:

   - **`Docker_Essentials/docker_update.sh`**
     - Three modes: normal (daily), --weekly (WEEKLY_RESTART_CONTAINERS), --remainder (monthly)
     - Pre-parse loop handles `--remainder` and `--weekly` before parse_args
     - Remainder excludes: DAILY_RESTART_CONTAINERS, WEEKLY_RESTART_CONTAINERS, emby/critical-data profiles, FALLBACK tiers
     - Weekly mode: checks WEEKLY_CONTAINER_UPDATES toggle, targets WEEKLY_RESTART_CONTAINERS
     - `log "$ICON_DONE $container rebuilt ✅"` → `echo` (echo audit fix)
     - MONTHLY_REMAINING_UPDATES toggle (renamed from WEEKLY_REMAINING_UPDATES)

   - **`Configurations/master.conf`**
     - `WEEKLY_CONTAINER_UPDATES=true` added (before docker_weekly_restart.sh)
     - `MONTHLY_REMAINING_UPDATES=true` (renamed)
     - `WEEKLY_MAINTENANCE_SCRIPTS` now: `docker_update.sh --weekly`, `docker_weekly_restart.sh`, `clear_logs.sh`, playback_aware scripts
     - `MONTHLY_MAINTENANCE_SCRIPTS`: `docker_update.sh --remainder`, `zfs_pool_scrub.sh`, `smart_long_test.sh`

   - **`Transcodes/transcode_cleanup.sh`**
     - Added `files_too_young` counter: `file_count - eligible_count` (files not yet old enough)
     - `LOCATION_TOO_YOUNG` output variable
     - Added comment: "HLS/remux segments are written atomically and immediately closed — lsof will not detect them. The age check is the effective guard"
     - Summary line: `echo "$ICON_RUNNING Active: $TOTAL_TOO_YOUNG files (< ${TRANSCODE_MAX_AGE}min old — Live TV / Direct Stream segments)"`
     - `echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open file handle — long-running transcode)"`

   - **`Transcodes/transcode_manager.sh`**
     - Fixed RAM_SESSION_COUNT and SSD_SESSION_COUNT using `_count_sessions()` helper:
       ```bash
       _count_sessions() {
           local base="$1/transcoding-temp"
           local _dirs _flat
           _dirs=$(find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
           _flat=$(find "$base" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | \
               grep -oE '^[0-9a-f]{16,}' | sort -u | wc -l)
           echo $(( _dirs + _flat ))
       }
       ```

   - **`Plugin/unraid/include/common.php`** — `vv_transcode_sessions()`
     - Fixed `$ramSessions` to add flat file prefix count:
       ```php
       $flatPrefixes = [];
       if (is_dir($ramdiskPath)) {
           foreach (new DirectoryIterator($ramdiskPath) as $f) {
               if (!$f->isFile()) continue;
               if (preg_match('/^([0-9a-f]{16,})/', $f->getFilename(), $m)) {
                   $flatPrefixes[$m[1]] = true;
               }
               if ($f->getMTime() >= $cutoff) $activeFiles++;
           }
       }
       $ramSessions += count($flatPrefixes);
       ```
     - Added `'active_files' => $activeFiles` to return array

   - **`Plugin/unraid/pages/monitor.php`**
     - Added `activeSegHtml` when `tc.active_files > 0`:
       ```javascript
       const activeFiles = tc.active_files ?? 0;
       let activeSegHtml = '';
       if (activeFiles > 0) {
         activeSegHtml = `<div style="...">
           <span style="...">Active segments</span>
           <span><span style="color:#4caf50;font-weight:600;">${activeFiles}</span>
           <span style="color:#444;font-size:9px;margin-left:4px;">Live TV / Direct Stream</span></span>
         </div>`;
       }
       ```
     - Appended `${activeSegHtml}${sessHtml}` to card output

   - **`Docker_Essentials/downloaders_reset.sh`**
     - `log "slskd connected to Soulseek ✅"` → `echo "$ICON_DONE [OK]    slskd connected to Soulseek ✅"`
     - `log "slskd reconnected after ${_ELAPSED}s ✅"` → `echo` (echo audit)

   - **Orchestrators (all)** — echo audit:
     - `run_job()` helper: `log "$script_name — done ✅"` → `echo "$script_name — done ✅"` in all orchestrators
     - `log "Running: $script_name"` kept as log (section header already announces)
     - `log "$SHARE_NAME — done ✅"` → `echo` (daily, intermediate)
     - `log "$SHARE_NAME — done in $SHARE_DUR ✅"` → `echo` (critical)
     - `log "$JOB_NAME — done in $JOB_DUR ✅"` → `echo` (weekly)
     - `log "$c — remote image updated ✅"` → `echo` (weekly)
     - `log "$_c rebuilt on new image ✅"` → `echo` (weekly)
     - `log "Partner conf cache refreshed ✅"` → `echo` (intermediate)
     - `log "$SCRIPT_NAME — completed (one-shot) ✅"` → `echo` (array_started)
     - Used perl for unicode-safe replacements (sed failed on ✅)

   - **Docker_Essentials** — echo audit:
     - `docker_daily_restart.sh:276`: container restarted → echo
     - `docker_weekly_restart.sh:241`: container restarted → echo
     - `docker_container_stop.sh:214`: container stopped → echo
     - `docker_network_connect.sh:216`: network exists — kept as log (precondition check)

   - **Media** — echo audit:
     - `arrs_failed_stalled_recovery.sh:299`: arr_name clean ✅ → echo
     - `media_cleaner.sh:211,226`: folder clean / files removed → echo
     - `radarr_tmdb_removed.sh:150,201`: nothing to do / Removed from Radarr → echo
     - `sonarr_tvdb_removed.sh:149,203`: nothing to do / Removed from Sonarr → echo
     - `arr_sync.sh:742`: re-monitored $count items ✅ → echo
     - `arr_sync.sh:432,442`: "Not found... already removed" — kept as log (per-item blocklist loop)

4. Errors and Fixes:
   - **sed failed on unicode checkmark ✅**: `sed -i` with ✅ in the pattern produced no matches. Fixed by using `perl -i -pe` which handles unicode correctly.
   - **`log "Running: $script_name"` accidentally left as log**: Correct — section headers already announce the script. The `done ✅` line is what needed to be echo.
   - **Ramdisk showing 0.00GB despite active streams**: Two issues — (1) timing: at exact cleanup moment, only a few MB of HLS data present; (2) display precision: 31MB / 1048576 = 0.03GB rounds correctly but at very low usage shows 0.00. The bigger fix was adding the "Active files" counter to explain the presence/activity.
   - **RAM_SESSION_COUNT always 0**: `glob("$ramdiskPath/*/", GLOB_ONLYDIR)` and `find -type d` both find zero because Live TV writes flat files, not per-session subdirectories. Fixed in both bash and PHP.

5. Problem Solving:
   - **Why weekly restart containers never got image updates**: daily didn't own them, weekly only restarted them, remainder was the only path — but remainder was excluded. Solution: --weekly mode pulls before restart, remainder excludes them.
   - **Why "Transcoding: 0" with active Live TV**: Emby API uses "Transcode" specifically for video re-encoding. Live TV Direct Play/Stream writes HLS segments but the API method is "DirectPlay" or "DirectStream". Not a bug — corrected the display to show "Active segments" count separately.
   - **Why lsof finds 0 streaming files**: HLS/remux segments are written atomically (ffmpeg writes complete segment then closes file handle). lsof only finds files with currently-open handles. Age check is the correct guard for HLS segments.

6. All User Messages:
   - "ok, now doesnt remainder catch one specifically not in daily or weekly?"
   - "ok, somthing is going on with thranscoding, [pasted logs showing 0 streaming/active/removed]"
   - "no, i didnt catch direct play i seldomly have all direct play. also assumed that live tv reguardless used the ramdisk"
   - "its working i forced transcode in a jf stream, really never have that many direct, i always have at least 1 transcoding"
   - "lets add that tou our transcoding card in the monitors page"
   - "━━━ slskd — Connection Check ━━━ [blank] slskd - connection check is blank"
   - "ok, thats what we were doing last week. we were going through all scripts correcting for verbose...... lets go through and check echo vs log... goal clean informative no verbose, and all details verbose. i know we got a few folders done, cant remember wich ones"

7. Pending Tasks:
   - **Echo vs log audit — ONGOING**: Continue folder by folder. Completed: Orchestrators, Docker_Essentials, Media. Still pending:
     - **Monitors**: backup_verify.sh (connectivity/parity/array mounted checks → echo; per-share match ✅ → echo); emby_session_report.sh connected → echo; smart_health.sh per-drive (stay log — many drives); system_tuning_monitor.sh (stay log); weekly_health_digest cert (stay log); zfs_memory_snapshot thresholds (stay log)
     - **Watchdogs**: network_watchdog.sh "NPM strikes cleared" → echo; resource_watchdog.sh "All pressure cleared" → echo; system_watchdog.sh "done ✅" → echo; stability_watchdog.sh per-minute metric checks → stay log; docker_watchdog.sh per-container "running" / CPU/MEM → stay log; resource_watchdog.sh "System normal" → stay log
     - **Tools**: all action outcome lines need echo (stopped, verified, restarted, scrub started, permissions applied, etc.)
     - **Rsync/rsync.sh**: container restarted → echo
     - **System_Essentials**: conf_sync.sh (cached/pulled/pushed), rsync_stop.sh (reachable), server_reboot.sh (array stop, buffers flushed) → echo
     - **Plugin/unraid/System_Essentials**: php_fpm_max_children.sh, unraid_api_key_renew.sh, recreate_shares.sh → echo for action outcomes
     - **Fallback/fallback.sh**: container started during fallback → echo (important events)
     - **Partnership**: complex multi-step — selective echo for key outcomes
   - Commit all echo audit changes when complete

8. Current Work:
   The echo vs log audit was actively in progress. The systematic approach:
   - Rule: `echo` = always visible (headers, final status, key outcomes, what changed); `log` = verbose-only (per-item iteration, per-minute health checks, intermediate values, "already correct" confirmations)
   - Identified candidates via: `grep -rn 'log ".*✅\|log ".*done\|log ".*removed...' /boot/config/plugins/varaverk --include="*.sh"`
   - Completed Orchestrators folder: all `run_job()` completion lines, per-share done lines, per-step outcomes → echo
   - Completed Docker_Essentials: restart/rebuild/stop outcomes → echo; precondition checks → stay log
   - Completed Media: per-arr/per-folder outcomes → echo; high-volume per-item loops → stay log
   - Last verified with: `grep -rn 'log ".*✅\|log ".*nothing to do...' /boot/config/plugins/varaverk/Media/` returning no output (clean)
   - **NOT YET COMMITTED** — changes to Orchestrators, Docker_Essentials, Media are unstaged

9. Optional Next Step:
   Continue the echo vs log audit with the next folder. Based on the grep output already obtained, the next folders to process are **Monitors** (selective — backup_verify.sh connectivity/parity/array checks and emby_session_report.sh connected line are clear echo candidates; per-drive SMART and per-metric threshold checks stay log) followed by **Watchdogs** (state transitions only: network_watchdog "NPM strikes cleared", resource_watchdog "All pressure cleared", system_watchdog "done ✅" → echo; per-minute per-container/per-metric confirmations stay log).

   The exact items remaining from the grep are:
   - `Monitors/backup_verify.sh:180,187,197` — log "Connectivity/Version parity/Remote array ✅" → echo
   - `Monitors/backup_verify.sh:294` — log "SHARE_NAME — all files match ✅" → echo
   - `Monitors/emby_session_report.sh:189` — log "Connected ✅" → echo
   - `Watchdogs/System/network_watchdog.sh:275` — log "NPM strikes cleared" → echo
   - `Watchdogs/resource_watchdog.sh:564` — log "All pressure cleared ✅" → echo
   - `Watchdogs/system_watchdog.sh:120` — log "$script_name — done ✅" → echo

If you need specific details from before compaction (like exact code snippets, error messages, or content you generated), read the full transcript at: /root/.claude/projects/-root/faa6d093-11ef-4bd4-9a9a-e0495100f066.jsonl
Continue the conversation from where it left off without asking the user any further questions. Resume directly — do not acknowledge the summary, do not recap what was happening, do not preface with "I'll continue" or similar. Pick up the last task as if the break never happened.


#!/bin/bash
# ==============================================================================================
# =========================== Intermediate Sync Maintenance ====================================
# ==============================================================================================
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync.
# Schedule: 0 */4 * * * (every 4 hours)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. conf_sync.sh --pull-only    — refresh partner conf cache in RAM (/tmp/.vv/)
#   2. arr_sync.sh                 — sync Lidarr/Sonarr/Radarr libraries across all nodes
#   3. Rsync window (optional)     — INTERMEDIATE_SYNC_SHARES, if any configured
#   4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
#
# ── WHY A SEPARATE ORCHESTRATOR ───────────────────────────────────────────────────────────────
# arr libraries need to converge more frequently than once a day. If a remote node adds
# something at 2am, the next daily window is 23 hours away — remote arrs search for content
# they don't know is already owned. Running every 4 hours closes that gap.
#
# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill.
# Pairing it here means artwork catches up within 4 hours of a new album landing.
#
# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config
# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media
# share sync stays in the daily window.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES.
# Each server can have a different set of mid-day shares — configure in host*.conf.
# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic.
#
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
# Same as daily_sync_maintenance.sh:
#   exit 1 = temp WARN — skip this share, continue to next
#   exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — scripts called here require root
#   acquire_lock         — prevents concurrent intermediate windows
#   check_connectivity   — verified before any rsync (skipped if no shares)
#   check_remote_rootfs  — aborts rsync if remote rootfs nearly full
#   Non-fatal jobs       — a failed arr_sync warns but does not block rsync or artwork fetch
#   Silent on success    — runs 4x/day, only failures warrant notification
#
# ── CONFIGURATION ─────────────────────────────────────────────────────────────────────────────
#   host*.conf:   HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
#   master.conf:         INTERMEDIATE_RSYNC_ENABLED     — enable/disable rsync section (default: true)
#   master.conf:         INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
#   master.conf:         ARR_SYNC_ENABLED               — toggle inside arr_sync.sh
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   intermediate_sync_maintenance.sh              — normal run
#   intermediate_sync_maintenance.sh --dry-run    — preview without changes
#   intermediate_sync_maintenance.sh --log        — verbose per-job output
#   intermediate_sync_maintenance.sh --status     — show configured shares/jobs and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

detect_hosts
resolve_remote_ip

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
run_job() {
    local script_entry="$1"
    local extra_dry=""
    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"

    read -r -a script_args <<< "$script_entry"
    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
    local script_name
    script_name=$(basename "${script_args[0]}")
    local extra_args=("${script_args[@]:1}")

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        JOB_FAIL+=("$script_name")
        return 1
    fi

    log "Running: $script_name ${extra_args[*]}"
    # shellcheck disable=SC2086
    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
        echo "$script_name — done ✅"
        JOB_PASS+=("$script_name ${extra_args[*]}")
    else
        warn "$script_name — failed (exit $?)"
        JOB_FAIL+=("$script_name ${extra_args[*]}")
    fi
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
    echo "$ICON_GEAR   Arr sync:        ${ARR_SYNC_ENABLED:-true}"
    echo ""
    echo "━━━ Intermediate Sync Shares ━━━"
    if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
        echo "  None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
    else
        for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
        done
    fi
    echo ""
    echo "━━━ Intermediate Maintenance Scripts ━━━"
    if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
PASS=()
FAIL=()
SHARE_TIMES=()

echo ""
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

# ==============================================================================================
# ━━━ Conf Pull ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Conf Pull ━━━"

CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
    warn "conf_sync.sh not found — skipping partner conf refresh"
else
    _conf_args=("--pull-only")
    [[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
    if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
        echo "Partner conf cache refreshed ✅"
        JOB_PASS+=("conf_sync.sh --pull-only")
    else
        warn "Partner conf pull failed — cache may be stale"
        JOB_FAIL+=("conf_sync.sh --pull-only")
    fi
    unset _conf_args
fi

# ==============================================================================================
# ━━━ Arr Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━"

ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
    echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
    JOB_FAIL+=("arr_sync.sh")
else
    _arr_sync_args=()
    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
        echo "Arr sync complete ✅"
        JOB_PASS+=("arr_sync.sh")
    else
        warn "Arr sync completed with errors — continuing"
        JOB_FAIL+=("arr_sync.sh")
    fi
    unset _arr_sync_args
fi

# ==============================================================================================
# ━━━ Rsync (optional) ━━━
# ==============================================================================================
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}

echo ""
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"

TOTAL_START=$(date +%s)
SHARE_INDEX=0
ABORT_ALL_SYNCS=false

if [[ "$SHARE_COUNT" -eq 0 ]]; then
    echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
    echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
elif ! check_rsync_enabled "INTERMEDIATE"; then
    warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
else
    check_connectivity
    check_remote_rootfs

    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
        (( SHARE_INDEX++ ))
        SHARE_NAME=$(basename "$SHARE")
        SHARE_START=$(date +%s)

        echo ""
        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"

        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
            FAIL+=("$SHARE_NAME:temp-critical")
            continue
        fi

        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
        RSYNC_EXIT=$?

        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")

        case "$RSYNC_EXIT" in
            0)
                PASS+=("$SHARE_NAME")
                echo "$SHARE_NAME — done ✅"
                ;;
            1)
                FAIL+=("$SHARE_NAME:temp-warn")
                warn "$SHARE_NAME skipped — drive temps too high"
                ;;
            2)
                FAIL+=("$SHARE_NAME:temp-critical")
                ABORT_ALL_SYNCS=true
                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
                notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
                    "Intermediate Sync" "warning"
                ;;
            *)
                FAIL+=("$SHARE_NAME")
                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
                ;;
        esac
    done
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Maintenance Jobs ━━━
# ==============================================================================================
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
    for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_job "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo ""

if [[ "$SHARE_COUNT" -gt 0 ]]; then
    echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
    for entry in "${SHARE_TIMES[@]}"; do
        sname="${entry%%:*}"
        sdur="${entry##*:}"
        if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
            echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
        else
            echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
        fi
    done
    echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
    echo ""
fi

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo "$ICON_GEAR  Jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
    echo ""
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
    echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
else
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Intermediate Sync" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0


#!/bin/bash
# ==============================================================================================
# ============================= Critical Sync Maintenance ======================================
# ==============================================================================================
# Orchestrator for time-sensitive syncs that run every 30 minutes.
# Keeps the mirror current between the less frequent daily and weekly windows.
# Schedule: */30 * * * * (every 30 minutes via User Scripts plugin)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. Critical-Data rsync    — auth stack, NPM config, certs (containers stopped both sides)
#   2. emby-fallback rsync    — dirty Emby sync (watch states, library — Emby stays running)
#   3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
#   4. partnership --check    — read both state files, detect changes, act accordingly
#
# ── WHY EVERY 30 MINUTES ──────────────────────────────────────────────────────────────────────
#   Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅
#   Emby watch states stay in sync — mirror users see correct playback position ✅
#   Partnership state changes detected and acted on quickly ✅
#   Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
#
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
#   RSYNC_ENABLED=false           → skips all syncs (global gate)
#   CRITICAL_RSYNC_ENABLED=false  → skips critical syncs only (per-orchestrator gate)
#   partnership --check always runs regardless — state check doesn't need rsync
#
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
#   acquire_lock "strict" — if previous 30min run still going, skip this cycle entirely
#   Critical-Data taking > 30min is a problem worth knowing about
#   Strict mode prevents pile-up without waiting — log and move on ✅
#
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
#   Runs 48 times per day — clean runs must produce zero output ✅
#   Only failures and notable events produce visible output
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   CRITICAL_RSYNC_ENABLED       — enable/disable rsync section
#   CRITICAL_SYNC_SHARES         — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES)
#   CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
#   PARTNERSHIP_ENABLED          — enable/disable partnership check
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   critical_sync_maintenance.sh              — normal run
#   critical_sync_maintenance.sh --dry-run    — preview syncs without transferring
#   critical_sync_maintenance.sh --log        — verbose per-share output
#   critical_sync_maintenance.sh --status     — show configuration and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock "strict"

detect_hosts
resolve_remote_ip

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
    echo "$ICON_HOST    Identity:           $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET     Remote:             $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC    Rsync enabled:      ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC    Critical enabled:   ${CRITICAL_RSYNC_ENABLED:-false}"
    echo "$ICON_SHIELD  Partnership:        ${PARTNERSHIP_ENABLED:-false}"
    echo ""
    echo "━━━ Critical Sync Shares ━━━"
    if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
        warn "  No CRITICAL_SYNC_SHARES configured"
    else
        for share in "${CRITICAL_SYNC_SHARES[@]}"; do
            [[ -z "$share" ]] && continue
            SHARE_PATH="${share%%|*}"
            SHARE_PROFILE="${share##*|}"
            SHARE_NAME=$(basename "$SHARE_PATH")
            [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
                echo "  $ICON_SYNC $SHARE_NAME — no profile" || \
                echo "  $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
        done
    fi
    echo ""
    echo "━━━ Critical Maintenance Scripts ━━━"
    if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -z "$entry" || "$entry" == \#* ]] && continue
            echo "  $ICON_GEAR $(basename "${entry%% *}")"
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
START=$(date +%s)
RSYNC_OK=false
PASS=()
FAIL=()

log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
    log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"

if ! check_rsync_enabled "CRITICAL"; then
    echo "Critical rsync disabled — skipping sync, running partnership check only"
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
    warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
    warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf"
else
    echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"

    # Build dry-run flag to pass through
    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for share in "${CRITICAL_SYNC_SHARES[@]}"; do
        [[ -z "$share" ]] && continue

        # Parse optional profile flag: "/path/to/share|profile-name"
        SHARE_PATH="${share%%|*}"
        SHARE_PROFILE="${share##*|}"
        SHARE_NAME=$(basename "$SHARE_PATH")

        SHARE_START=$(date +%s)

        if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
        else
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
                --profile="$SHARE_PROFILE" $RSYNC_DRY
        fi

        RSYNC_EXIT=$?
        SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))

        if [[ "$RSYNC_EXIT" -eq 0 ]]; then
            PASS+=("$SHARE_NAME")
            echo "$SHARE_NAME — done in $SHARE_DUR ✅"
            RSYNC_OK=true
        else
            FAIL+=("$SHARE_NAME")
            error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
        fi
    done
fi

# ==============================================================================================
# ━━━ Critical Maintenance Scripts ━━━
# ==============================================================================================
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue

        SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
        SCRIPT_ARGS="${script_entry#* }"
        [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
        [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"

        SCRIPT_NAME=$(basename "$SCRIPT_PATH")

        if [[ ! -f "$SCRIPT_PATH" ]]; then
            warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
            continue
        fi

        log "Running: $SCRIPT_NAME"
        bash "$SCRIPT_PATH" $SCRIPT_ARGS
        EXIT_CODE=$?
        [[ "$EXIT_CODE" -ne 0 ]] && \
            warn "$SCRIPT_NAME exited with code $EXIT_CODE"
    done
fi

# ==============================================================================================
# ━━━ Partnership Check ━━━
# ==============================================================================================
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
    PARTNER_DRY=""
    [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"

    if [[ "$RSYNC_OK" == true ]]; then
        bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
            --check --remote-seen $PARTNER_DRY
    else
        bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
            --check --remote-unseen $PARTNER_DRY
    fi
else
    echo "Partnership disabled — skipping check"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
END=$(date +%s)
DURATION=$(format_duration $(( END - START )))

# Silent when healthy — only show summary if there were failures or notable events
if [[ ${#FAIL[@]} -gt 0 ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
    echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_TIME   Duration: $DURATION"
    [[ ${#PASS[@]} -gt 0 ]] && echo "Synced:  ${PASS[*]}"
    echo "$ICON_ERROR  Failed:  ${FAIL[*]}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
        "Critical Sync" "warning"
    exit 1
else
    echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
fi

exit 0


claude claude-code-guide Explore general-purpose Plan statusline-setup

- claude: Catch-all for any task that doesn't fit a more specific agent. FleetView's default when no agent name is typed. (Tools: *)

- claude-code-guide: Use this agent when the user asks questions ("Can Claude...", "Does Claude...", "How do I...") about: (1) Claude Code (the CLI tool) - features, hooks, slash commands, MCP servers, settings, IDE integrations, keyboard shortcuts; (2) Claude Agent SDK - building custom agents; (3) Claude API (formerly Anthropic API) - API usage, tool use, Anthropic SDK usage. **IMPORTANT:** Before spawning a new agent, check if there is already a running or recently completed claude-code-guide agent that you can continue via SendMessage. (Tools: Bash, Read, WebFetch, WebSearch)

- Explore: Fast read-only search agent for locating code. Use it to find files by pattern (eg. "src/components/**/*.tsx"), grep for symbols or keywords (eg. "API endpoints"), or answer "where is X defined / which files reference Y." Do NOT use it for code review, design-doc auditing, cross-file consistency checks, or open-ended analysis — it reads excerpts rather than whole files and will miss content past its read window. When calling, specify search breadth: "quick" for a single targeted lookup, "medium" for moderate exploration, or "very thorough" to search across multiple locations and naming conventions. (Tools: All tools except Agent, ExitPlanMode, Edit, Write, NotebookEdit)

- general-purpose: General-purpose agent for researching complex questions, searching for code, and executing multi-step tasks. When you are searching for a keyword or file and are not confident that you will find the right match in the first few tries use this agent to perform the search for you. (Tools: *)

- Plan: Software architect agent for designing implementation plans. Use this when you need to plan the implementation strategy for a task. Returns step-by-step plans, identifies critical files, and considers architectural trade-offs. (Tools: All tools except Agent, ExitPlanMode, Edit, Write, NotebookEdit)

- statusline-setup: Use this agent to configure the user's Claude Code status line setting. (Tools: Read, Edit)


Let me read all the pending files in parallel to plan the changes.


170	    "$CONTAINER_NAME" 2>/dev/null)
171
172	case "$STATUS" in
173	    true)
174	        CONTAINER_WAS_RUNNING=true
175	        log "Stopping $CONTAINER_NAME for clean export..."
176	        if [[ "$DRY_RUN" == false ]]; then
177	            if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
178	                log "$CONTAINER_NAME stopped ✅"
179	            else
180	                error "Failed to stop $CONTAINER_NAME — aborting export"
181	                exit 1
182	            fi
183	        else
184	            warn "DRY RUN — would stop $CONTAINER_NAME"
185	        fi
186	        ;;
187	    false)
188	        log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
189	        ;;
190	    "")
191	        error "$CONTAINER_NAME not found — check container name"
192	        exit 1
193	        ;;
194	    *)
195	        warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
196	        ;;
197	esac
198
199	# ==============================================================================================
200	# ━━━ Archive ━━━
201	# ==============================================================================================
202	echo ""
203	echo "━━━ $ICON_SYNC Archive ━━━"
204
205	TIMESTAMP=$(date '+%Y-%m-%d_%H-%M')
206	ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz"
207	ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"
208
209	warn "Creating: $ARCHIVE_PATH"
210	warn "Source:   $APPDATA_PATH ($APPDATA_SIZE_H)"
211
212	START=$(date +%s)
213	ARCHIVE_VERIFIED=false
214
215	if [[ "$DRY_RUN" == false ]]; then
216	    if tar -czf "$ARCHIVE_PATH" \
217	        -C "$(dirname "$APPDATA_PATH")" \
218	        "$(basename "$APPDATA_PATH")" 2>/dev/null; then
219
220	        ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
221	        warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
222
223	        # Verify archive integrity before declaring success
224	        log "Verifying archive..."
225	        if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
226	           tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
227	            log "Archive verified ✅"
228	            ARCHIVE_VERIFIED=true
229	        else
230	            error "Archive verification FAILED — archive may be corrupt"
231	            error "Container will be restarted but DO NOT rely on this backup"
232	            notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \
233	                "Container Export" "warning"
234	        fi
235	    else
236	        error "tar failed — archive creation unsuccessful"
237	        exit 1
238	    fi
239	else
240	    warn "DRY RUN — would create: $ARCHIVE_PATH"
241	    ARCHIVE_VERIFIED=true
242	fi
243
244	END=$(date +%s)
245
246	# ==============================================================================================
247	# ━━━ Restart Container ━━━
248	# ==============================================================================================
249	echo ""
250	echo "━━━ $ICON_START Restart Container ━━━"
251
252	RESTART_OK=false
253
254	if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
255	    log "Restarting $CONTAINER_NAME..."
256	    if [[ "$DRY_RUN" == false ]]; then
257	        if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
258	            # Brief settle then verify
259	            sleep 3
260	            POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
261	                '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
262	            if [[ "$POST_STATUS" == "true" ]]; then
263	                log "$CONTAINER_NAME restarted and running ✅"
264	                RESTART_OK=true
265	            else
266	                error "$CONTAINER_NAME started but crashed immediately — check container logs"
267	                notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
268	                    "Container Export" "warning"
269	            fi
270	        else
271	            error "Failed to restart $CONTAINER_NAME — start it manually"
272	            notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
273	                "Container Export" "warning"
274	        fi
275	    else
276	        warn "DRY RUN — would restart $CONTAINER_NAME"
277	        RESTART_OK=true
278	    fi
279	else
280	    log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
281	    RESTART_OK=true
282	fi
283
284	# Clear trap — clean exit, cleanup_on_exit no longer needed
285	trap - EXIT
286
287	# ==============================================================================================
288	# ━━━ Summary ━━━
289	# ==============================================================================================

    "$CONTAINER_NAME" 2>/dev/null)

case "$STATUS" in
    true)
        CONTAINER_WAS_RUNNING=true
        log "Stopping $CONTAINER_NAME for clean export..."
        if [[ "$DRY_RUN" == false ]]; then
            if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
                log "$CONTAINER_NAME stopped ✅"
            else
                error "Failed to stop $CONTAINER_NAME — aborting export"
                exit 1
            fi
        else
            warn "DRY RUN — would stop $CONTAINER_NAME"
        fi
        ;;
    false)
        log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
        ;;
    "")
        error "$CONTAINER_NAME not found — check container name"
        exit 1
        ;;
    *)
        warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
        ;;
esac

# ==============================================================================================
# ━━━ Archive ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Archive ━━━"

TIMESTAMP=$(date '+%Y-%m-%d_%H-%M')
ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz"
ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"

warn "Creating: $ARCHIVE_PATH"
warn "Source:   $APPDATA_PATH ($APPDATA_SIZE_H)"

START=$(date +%s)
ARCHIVE_VERIFIED=false

if [[ "$DRY_RUN" == false ]]; then
    if tar -czf "$ARCHIVE_PATH" \
        -C "$(dirname "$APPDATA_PATH")" \
        "$(basename "$APPDATA_PATH")" 2>/dev/null; then

        ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
        warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"

        # Verify archive integrity before declaring success
        log "Verifying archive..."
        if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
           tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
            log "Archive verified ✅"
            ARCHIVE_VERIFIED=true
        else
            error "Archive verification FAILED — archive may be corrupt"
            error "Container will be restarted but DO NOT rely on this backup"
            notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \
                "Container Export" "warning"
        fi
    else
        error "tar failed — archive creation unsuccessful"
        exit 1
    fi
else
    warn "DRY RUN — would create: $ARCHIVE_PATH"
    ARCHIVE_VERIFIED=true
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Restart Container ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START Restart Container ━━━"

RESTART_OK=false

if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
    log "Restarting $CONTAINER_NAME..."
    if [[ "$DRY_RUN" == false ]]; then
        if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
            # Brief settle then verify
            sleep 3
            POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
            if [[ "$POST_STATUS" == "true" ]]; then
                log "$CONTAINER_NAME restarted and running ✅"
                RESTART_OK=true
            else
                error "$CONTAINER_NAME started but crashed immediately — check container logs"
                notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
                    "Container Export" "warning"
            fi
        else
            error "Failed to restart $CONTAINER_NAME — start it manually"
            notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
                "Container Export" "warning"
        fi
    else
        warn "DRY RUN — would restart $CONTAINER_NAME"
        RESTART_OK=true
    fi
else
    log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
    RESTART_OK=true
fi

# Clear trap — clean exit, cleanup_on_exit no longer needed
trap - EXIT

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================


198	    fi
199
200	    log "Starting scrub on $pool..."
201
202	    if [[ "$DRY_RUN" == true ]]; then
203	        warn "DRY RUN — would scrub: $pool"
204	        STARTED+=("$pool")
205	    elif zpool scrub "$pool" 2>/dev/null; then
206	        log "$pool scrub started ✅"
207	        STARTED+=("$pool")
208	    else
209	        error "Failed to start scrub on $pool"
210	        SKIPPED_POOLS+=("$pool")
211	    fi
212	done
213
214	if [[ "$DRY_RUN" == true ]]; then
215	    echo ""
216	    echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
217	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
218	    warn "DRY RUN — no scrubs started"
219	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
220	    exit 0
221	fi
222
223	if [[ ${#STARTED[@]} -eq 0 ]]; then
224	    error "No scrubs were started — check pool status"
225	    exit 1
226	fi
227
228	# ==============================================================================================
229	# ━━━ Poll Until Complete ━━━
230	# ==============================================================================================
231	echo ""
232	echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
233	log "Polling every 60 seconds — scrubs may take hours on large pools"
234	log "Safe to interrupt — scrubs continue in background if script is stopped"
235
236	while [[ "$SCRUB_RUNNING" == true ]]; do
237	    sleep 60
238
239	    STILL_RUNNING=false
240	    for pool in "${STARTED[@]}"; do
241	        IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
242	            grep "scan:" | grep -c "in progress" || true)

    fi

    log "Starting scrub on $pool..."

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would scrub: $pool"
        STARTED+=("$pool")
    elif zpool scrub "$pool" 2>/dev/null; then
        log "$pool scrub started ✅"
        STARTED+=("$pool")
    else
        error "Failed to start scrub on $pool"
        SKIPPED_POOLS+=("$pool")
    fi
done

if [[ "$DRY_RUN" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    warn "DRY RUN — no scrubs started"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ ${#STARTED[@]} -eq 0 ]]; then
    error "No scrubs were started — check pool status"
    exit 1
fi

# ==============================================================================================
# ━━━ Poll Until Complete ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
log "Polling every 60 seconds — scrubs may take hours on large pools"
log "Safe to interrupt — scrubs continue in background if script is stopped"

while [[ "$SCRUB_RUNNING" == true ]]; do
    sleep 60

    STILL_RUNNING=false
    for pool in "${STARTED[@]}"; do
        IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
            grep "scan:" | grep -c "in progress" || true)


173
174	# Bail if not mounted — nothing to do
175	if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
176	    warn "Ramdisk is not mounted at $RAMDISK_PATH — nothing to stop"
177	    exit 0
178	fi
179
180	log "Ramdisk is mounted ✅"
181
182	# Warn if transcode_manager is running — it may flip symlink back on next cycle
183	if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
184	    warn "transcode_manager.sh is currently RUNNING"
185	    warn "It may flip the symlink back to ramdisk on its next cycle"
186	    warn "Stop transcode_manager.sh first if you need the SSD redirect to hold"
187	    echo ""
188	fi
189
190	# Confirm SSD fallback exists
191	if [[ ! -d "$TRANSCODE_SSD" ]]; then
192	    if [[ "$DRY_RUN" == true ]]; then
193	        warn "DRY RUN — SSD fallback does not exist: $TRANSCODE_SSD"
194	        warn "DRY RUN — would create it before redirecting symlink"
195	    else
196	        warn "SSD fallback does not exist — creating: $TRANSCODE_SSD"
197	        mkdir -p "$TRANSCODE_SSD" || {
198	            error "Failed to create SSD fallback: $TRANSCODE_SSD"
199	            error "Cannot safely redirect symlink — aborting"
200	            exit 1
201	        }
202	        log "SSD fallback created ✅"
203	    fi
204	else
205	    log "SSD fallback exists: $TRANSCODE_SSD ✅"
206	fi
207
208	START=$(date +%s)
209	STOP_SUCCESS=true
210
211	# ==============================================================================================
212	# ━━━ Redirect Symlink → SSD ━━━
213	# ==============================================================================================
214	# Redirect BEFORE unmount — Emby continues writing to SSD with no broken path window.
215	echo ""
216	echo "━━━ $ICON_LINK Redirect Symlink → SSD ━━━"
217
218	if [[ -L "$TRANSCODE_LINK" ]]; then
219	    CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
220	    if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
221	        echo "Symlink already points to SSD — no change needed ✅"
222	    else
223	        if [[ "$DRY_RUN" == true ]]; then
224	            warn "DRY RUN — would redirect: $TRANSCODE_LINK → $TRANSCODE_SSD"
225	        else
226	            ln -sfn "$TRANSCODE_SSD" "$TRANSCODE_LINK" && \
227	                warn "Symlink redirected: $TRANSCODE_LINK → $TRANSCODE_SSD ✅" || {
228	                error "Failed to redirect symlink"
229	                STOP_SUCCESS=false
230	            }
231	        fi
232	    fi
233	elif [[ -e "$TRANSCODE_LINK" ]]; then
234	    warn "$TRANSCODE_LINK exists but is not a symlink — leaving as-is"
235	else
236	    warn "Symlink $TRANSCODE_LINK does not exist — nothing to redirect"
237	fi
238
239	# ==============================================================================================
240	# ━━━ Active Files Warning ━━━
241	# ==============================================================================================
242	echo ""
243	echo "━━━ $ICON_RAM Active Files Check ━━━"
244
245	FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
246	if [[ "$FILE_COUNT" -gt 0 ]]; then
247	    warn "⚠  $FILE_COUNT file(s) still on ramdisk — will be lost on unmount"
248	    warn "Active transcode sessions should be stopped before unmounting"
249	    warn "Proceeding regardless (this is expected for maintenance)"
250	    if [[ "$LOG" == true ]]; then
251	        find "$RAMDISK_PATH" -type f 2>/dev/null | while read -r f; do
252	            log "  $f"
253	        done
254	    fi
255	else
256	    log "No active files on ramdisk ✅"
257	fi
258
259	# ==============================================================================================
260	# ━━━ Unmount ━━━
261	# ==============================================================================================
262	echo ""
263	echo "━━━ $ICON_RAM Unmount Ramdisk ━━━"
264
265	if [[ "$DRY_RUN" == true ]]; then
266	    warn "DRY RUN — would unmount: $RAMDISK_PATH"
267	else
268	    if umount "$RAMDISK_PATH" 2>/dev/null; then
269	        warn "Ramdisk unmounted: $RAMDISK_PATH ✅"
270	    else
271	        # Regular unmount failed — check if only directory handles are open (no active writes)
272	        OPEN_FILES=$(lsof +D "$RAMDISK_PATH" 2>/dev/null | awk 'NR>1 && $5 != "DIR"' | wc -l)
273	        if [[ "$OPEN_FILES" -eq 0 ]]; then
274	            warn "Busy — only directory handles open, no active writes — trying lazy unmount"
275	            if umount -l "$RAMDISK_PATH"; then
276	                warn "Ramdisk lazy-unmounted: $RAMDISK_PATH ✅"
277	                warn "Handles will release when owning processes next check the directory"
278	            else
279	                error "Lazy unmount also failed — $RAMDISK_PATH"
280	                STOP_SUCCESS=false
281	            fi
282	        else
283	            error "Failed to unmount $RAMDISK_PATH — $OPEN_FILES file(s) still open for writing"
284	            error "Stop active transcode sessions and retry"
285	            STOP_SUCCESS=false
286	        fi
287	    fi
288	fi
289
290	# ==============================================================================================
291	# ━━━ Update State File ━━━
292	# ==============================================================================================
293	echo ""
294	echo "━━━ $ICON_GEAR State File ━━━"
295
296	if [[ "$DRY_RUN" == true ]]; then
297	    warn "DRY RUN — would update $STATE_FILE: current_target=$TRANSCODE_SSD"
298	elif [[ "$STOP_SUCCESS" == true ]]; then
299	    NOW=$(date +%s)
300	    cat > "$STATE_FILE" <<EOF
301	current_target=$TRANSCODE_SSD
302	last_flip_time=$NOW
303	flip_count_hour=0
304	flip_hour_start=$NOW
305	EOF
306	    log "State file updated: current_target=$TRANSCODE_SSD"
307	else
308	    warn "Skipping state file update — stop had errors"
309	fi
310
311	END=$(date +%s)
312

# Bail if not mounted — nothing to do
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
    warn "Ramdisk is not mounted at $RAMDISK_PATH — nothing to stop"
    exit 0
fi

log "Ramdisk is mounted ✅"

# Warn if transcode_manager is running — it may flip symlink back on next cycle
if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
    warn "transcode_manager.sh is currently RUNNING"
    warn "It may flip the symlink back to ramdisk on its next cycle"
    warn "Stop transcode_manager.sh first if you need the SSD redirect to hold"
    echo ""
fi

# Confirm SSD fallback exists
if [[ ! -d "$TRANSCODE_SSD" ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — SSD fallback does not exist: $TRANSCODE_SSD"
        warn "DRY RUN — would create it before redirecting symlink"
    else
        warn "SSD fallback does not exist — creating: $TRANSCODE_SSD"
        mkdir -p "$TRANSCODE_SSD" || {
            error "Failed to create SSD fallback: $TRANSCODE_SSD"
            error "Cannot safely redirect symlink — aborting"
            exit 1
        }
        log "SSD fallback created ✅"
    fi
else
    log "SSD fallback exists: $TRANSCODE_SSD ✅"
fi

START=$(date +%s)
STOP_SUCCESS=true

# ==============================================================================================
# ━━━ Redirect Symlink → SSD ━━━
# ==============================================================================================
# Redirect BEFORE unmount — Emby continues writing to SSD with no broken path window.
echo ""
echo "━━━ $ICON_LINK Redirect Symlink → SSD ━━━"

if [[ -L "$TRANSCODE_LINK" ]]; then
    CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
    if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
        echo "Symlink already points to SSD — no change needed ✅"
    else
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would redirect: $TRANSCODE_LINK → $TRANSCODE_SSD"
        else
            ln -sfn "$TRANSCODE_SSD" "$TRANSCODE_LINK" && \
                warn "Symlink redirected: $TRANSCODE_LINK → $TRANSCODE_SSD ✅" || {
                error "Failed to redirect symlink"
                STOP_SUCCESS=false
            }
        fi
    fi
elif [[ -e "$TRANSCODE_LINK" ]]; then
    warn "$TRANSCODE_LINK exists but is not a symlink — leaving as-is"
else
    warn "Symlink $TRANSCODE_LINK does not exist — nothing to redirect"
fi

# ==============================================================================================
# ━━━ Active Files Warning ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_RAM Active Files Check ━━━"

FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
if [[ "$FILE_COUNT" -gt 0 ]]; then
    warn "⚠  $FILE_COUNT file(s) still on ramdisk — will be lost on unmount"
    warn "Active transcode sessions should be stopped before unmounting"
    warn "Proceeding regardless (this is expected for maintenance)"
    if [[ "$LOG" == true ]]; then
        find "$RAMDISK_PATH" -type f 2>/dev/null | while read -r f; do
            log "  $f"
        done
    fi
else
    log "No active files on ramdisk ✅"
fi

# ==============================================================================================
# ━━━ Unmount ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_RAM Unmount Ramdisk ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would unmount: $RAMDISK_PATH"
else
    if umount "$RAMDISK_PATH" 2>/dev/null; then
        warn "Ramdisk unmounted: $RAMDISK_PATH ✅"
    else
        # Regular unmount failed — check if only directory handles are open (no active writes)
        OPEN_FILES=$(lsof +D "$RAMDISK_PATH" 2>/dev/null | awk 'NR>1 && $5 != "DIR"' | wc -l)
        if [[ "$OPEN_FILES" -eq 0 ]]; then
            warn "Busy — only directory handles open, no active writes — trying lazy unmount"
            if umount -l "$RAMDISK_PATH"; then
                warn "Ramdisk lazy-unmounted: $RAMDISK_PATH ✅"
                warn "Handles will release when owning processes next check the directory"
            else
                error "Lazy unmount also failed — $RAMDISK_PATH"
                STOP_SUCCESS=false
            fi
        else
            error "Failed to unmount $RAMDISK_PATH — $OPEN_FILES file(s) still open for writing"
            error "Stop active transcode sessions and retry"
            STOP_SUCCESS=false
        fi
    fi
fi

# ==============================================================================================
# ━━━ Update State File ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR State File ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would update $STATE_FILE: current_target=$TRANSCODE_SSD"
elif [[ "$STOP_SUCCESS" == true ]]; then
    NOW=$(date +%s)
    cat > "$STATE_FILE" <<EOF
current_target=$TRANSCODE_SSD
last_flip_time=$NOW
flip_count_hour=0
flip_hour_start=$NOW
EOF
    log "State file updated: current_target=$TRANSCODE_SSD"
else
    warn "Skipping state file update — stop had errors"
fi

END=$(date +%s)