Add verbose log() coverage across Docker_Essentials, unRAID_Essentials, Transcodes, and Orchestrators
- Config/threshold dumps at startup in every script (retry counts, timeouts, sizes, thresholds) - Per-item detail in verbose: container images, timing per container/share/job, image ID diffs - Orchestrators: watchdog cycle now logs array state, grace state, per-script timing; transcode_management shows ramdisk state before each cycle; critical_sync logs share list and maintenance scripts; coffee report logs server state at run time - Summary counts replaced with names in verbose where previously only counts were shown Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
8f696d85f7
commit
bc70ebe5ee
@@ -146,6 +146,8 @@ DOCKER_TIMEOUT=30
|
||||
DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally
|
||||
_RETRY_COUNT="${RETRY_COUNT:-3}"
|
||||
|
||||
log "$ICON_GEAR Config: retries=$_RETRY_COUNT sleep=${SLEEP:-5}s grace=${DOCKER_STOP_TIMEOUT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -175,6 +177,7 @@ if [[ ${#RUNNING[@]} -eq 0 ]]; then
|
||||
fi
|
||||
|
||||
echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running"
|
||||
log "$ICON_CONTAINERS Queue: ${RUNNING[*]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
@@ -183,7 +186,11 @@ FAILED=()
|
||||
|
||||
for container in "${RUNNING[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
local c_start
|
||||
c_start=$(date +%s)
|
||||
local c_image
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop $container"
|
||||
@@ -204,7 +211,7 @@ for container in "${RUNNING[@]}"; do
|
||||
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATE" != "true" ]]; then
|
||||
log "$ICON_DONE $container stopped ✅"
|
||||
log "$ICON_DONE $container stopped in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||
STOPPED+=("$container")
|
||||
success=true
|
||||
break
|
||||
|
||||
@@ -230,7 +230,6 @@ build_restart_order() {
|
||||
fi
|
||||
done
|
||||
|
||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||||
}
|
||||
|
||||
# Checks if a container is a dependent of the previously restarted container.
|
||||
@@ -279,6 +278,7 @@ echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} container(s)"
|
||||
log "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
log "$ICON_GEAR Config: sleep=${SLEEP}s delay=${CONTAINER_DELAY}s verify-wait=${RESTART_VERIFY_WAIT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
@@ -293,7 +293,11 @@ LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
local c_start
|
||||
c_start=$(date +%s)
|
||||
local c_image
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container does not exist — skipping"
|
||||
@@ -316,7 +320,7 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
if retry_docker docker restart "$container"; then
|
||||
# Verify container stayed running after restart
|
||||
if verify_running "$container"; then
|
||||
log "$ICON_STARTED $container restarted and running ✅"
|
||||
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||
RESTARTED+=("$container")
|
||||
LAST_RESTARTED="$container"
|
||||
else
|
||||
|
||||
@@ -209,7 +209,12 @@ for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do
|
||||
|
||||
# ── Step 1 — ensure network exists ───────────────────────────────────────────────────────
|
||||
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then
|
||||
log "$network exists ✅"
|
||||
local net_subnet net_driver
|
||||
net_subnet=$(timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
||||
--format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null || echo "unknown")
|
||||
net_driver=$(timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
||||
--format '{{.Driver}}' 2>/dev/null || echo "unknown")
|
||||
log "$ICON_DOCKER_NET $network exists ✅ — driver: $net_driver subnet: $net_subnet"
|
||||
else
|
||||
warn "$ICON_DOCKER_NET $network not found — creating (unRAID update may have wiped networks)"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -279,7 +284,7 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#NETWORKS_CREATED[@]} -gt 0 ]] && warn "$ICON_DOCKER_NET Created: ${NETWORKS_CREATED[*]} (networks were missing)"
|
||||
[[ ${#CONNECTED[@]} -gt 0 ]] && log "Connected: ${CONNECTED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${#SKIPPED[@]} skipped"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
|
||||
@@ -246,6 +246,7 @@ echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
@@ -298,10 +299,10 @@ for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
||||
log "$ICON_DONE $container — updated ✅"
|
||||
log "$ICON_DONE $container — updated ✅ (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
else
|
||||
log "$container — already up to date"
|
||||
log "$container — already up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
@@ -367,8 +368,8 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${#SKIPPED[@]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
|
||||
@@ -235,10 +235,10 @@ for container in "${REMAINING[@]}"; do
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
||||
log "$ICON_DONE $container — updated ✅"
|
||||
log "$ICON_DONE $container — updated ✅ (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
else
|
||||
log "$container — already up to date"
|
||||
log "$container — already up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
@@ -326,13 +326,13 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE New image: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}"
|
||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Restarted: ${#RESTARTED[@]}"
|
||||
log " ${RESTARTED[*]}"
|
||||
fi
|
||||
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${#SKIPPED_STOPPED[@]} (skipped restart)"
|
||||
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
|
||||
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
|
||||
@@ -246,6 +246,7 @@ echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
log "$ICON_GEAR Config: sleep=${SLEEP}s delay=${CONTAINER_DELAY}s verify-wait=${RESTART_VERIFY_WAIT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
@@ -259,7 +260,11 @@ LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
local c_start
|
||||
c_start=$(date +%s)
|
||||
local c_image
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container does not exist — skipping"
|
||||
@@ -281,7 +286,7 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
if verify_running "$container"; then
|
||||
log "$ICON_STARTED $container restarted and running ✅"
|
||||
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||
RESTARTED+=("$container")
|
||||
LAST_RESTARTED="$container"
|
||||
else
|
||||
@@ -319,9 +324,9 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}"
|
||||
log " ${RESTARTED[*]}"
|
||||
log " Names: ${RESTARTED[*]}"
|
||||
fi
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} (were stopped)"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
|
||||
@@ -148,6 +148,8 @@ CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
|
||||
log "$ICON_GEAR Config: retention=${DOWNLOADER_RETENTION_DAYS}d qbit-age=${QBIT_FAILSAFE_MIN_DAYS}d qbit-ratio=${QBIT_FAILSAFE_MIN_RATIO}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -304,6 +306,8 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" =
|
||||
if [[ -z "$USERNAMES" ]]; then
|
||||
success "No transfer records found ✅"
|
||||
else
|
||||
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $USER_COUNT user(s) with transfer records"
|
||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||
while IFS= read -r USER; do
|
||||
[[ -z "$USER" ]] && continue
|
||||
@@ -422,6 +426,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$COMPLETED_IDS" ]]; then
|
||||
success "No completed history found ✅"
|
||||
else
|
||||
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $HIST_TOTAL completed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
@@ -468,6 +474,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$FAILED_IDS" ]]; then
|
||||
success "No failed history found ✅"
|
||||
else
|
||||
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $FAILED_TOTAL failed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
@@ -516,6 +524,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$STALLED_IDS" ]]; then
|
||||
success "No stalled queue items found ✅"
|
||||
else
|
||||
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $QUEUE_TOTAL queue item(s) — checking status"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
@@ -575,6 +585,8 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
||||
|
||||
NOW=$(date +%s)
|
||||
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || echo 0)
|
||||
log "Found $TORRENT_TOTAL torrent(s) — applying age/ratio filter"
|
||||
DELETED=0; SKIPPED=0
|
||||
|
||||
while read -r TORRENT; do
|
||||
|
||||
@@ -119,6 +119,10 @@ RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
|
||||
[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
|
||||
log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
|
||||
@@ -87,11 +87,21 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Main ━━━
|
||||
# ==============================================================================================
|
||||
_unraid_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
|
||||
_uptime_s=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)
|
||||
_container_count=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||
_rootfs_pct=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "☕ Sunday Morning Coffee Report — $MY_ID"
|
||||
[[ "$DRY_RUN" == true ]] && echo " [DRY RUN]"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "$ICON_HOST Server: $MY_ID ($LOCAL_SERVER_NAME) — unRAID $_unraid_ver"
|
||||
log "$ICON_TIME Uptime: $(format_duration $_uptime_s)"
|
||||
log "$ICON_CONTAINERS Docker: $_container_count container(s) running"
|
||||
log "$ICON_HEALTH Rootfs: ${_rootfs_pct:-?}% used"
|
||||
log "$ICON_GEAR Scripts: ${#COFFEE_REPORT_SCRIPTS[@]} configured"
|
||||
|
||||
if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
|
||||
warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf"
|
||||
|
||||
@@ -137,22 +137,40 @@ if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-run State Snapshot ━━━
|
||||
# ==============================================================================================
|
||||
if mountpoint -q "${RAMDISK_PATH:-}" 2>/dev/null; then
|
||||
_rd_used=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
_rd_avail=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
_rd_used_gb=$(awk "BEGIN {printf \"%.2f\", ${_rd_used:-0}/1048576}")
|
||||
_rd_avail_gb=$(awk "BEGIN {printf \"%.2f\", ${_rd_avail:-0}/1048576}")
|
||||
_rd_target=$(readlink "${TRANSCODE_LINK:-}" 2>/dev/null || echo "unknown")
|
||||
log "$ICON_RAM Ramdisk: ${_rd_used_gb}GB used / ${_rd_avail_gb}GB avail — symlink → ${_rd_target##*/}"
|
||||
else
|
||||
log "$ICON_RAM Ramdisk: not mounted"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
DRY_FLAG=""
|
||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||
|
||||
_cleanup_start=$(date +%s)
|
||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
||||
CLEANUP_EXIT=$?
|
||||
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Manager ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
||||
_manager_start=$(date +%s)
|
||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
||||
MANAGER_EXIT=$?
|
||||
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
|
||||
@@ -68,6 +68,9 @@ acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||
@@ -122,6 +125,7 @@ if ! df --output=fstype /mnt/user 2>/dev/null | grep -q shfs; then
|
||||
echo "Array not started — skipping watchdog cycle"
|
||||
exit 0
|
||||
fi
|
||||
log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Startup Grace ━━━
|
||||
@@ -131,6 +135,7 @@ if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||||
exit 0
|
||||
fi
|
||||
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Watchdog Cycle ━━━
|
||||
@@ -154,12 +159,15 @@ run_watchdog() {
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||
|
||||
local _ws
|
||||
_ws=$(date +%s)
|
||||
log "$ICON_START $name"
|
||||
if bash "$script" "${extra_args[@]}"; then
|
||||
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
|
||||
PASS+=("$name")
|
||||
return 0
|
||||
else
|
||||
error "$name — non-zero exit"
|
||||
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
|
||||
FAIL+=("$name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -132,7 +132,8 @@ else
|
||||
fi
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Max age: ${TRANSCODE_MAX_AGE} minutes"
|
||||
log "$ICON_GEAR Config: max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
|
||||
log "$ICON_RAM Locations: ramdisk=$RAMDISK_PATH ssd=$TRANSCODE_SSD"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
|
||||
@@ -180,6 +180,9 @@ case "$TRANSCODE_MANAGER_MODE" in
|
||||
;;
|
||||
esac
|
||||
|
||||
log "$ICON_GEAR Config: ramdisk=$RAMDISK_PATH size=$RAMDISK_SIZE warn-at=${RAMDISK_WARN_GB}GB flip-back-at=${RAMDISK_LOW_GB}GB ssd-min-free=${RAMDISK_SSD_MIN_GB}GB flip-warn=${TRANSCODE_FLIP_WARN}/hr"
|
||||
log "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -165,6 +165,9 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear Logs ━━━
|
||||
# ==============================================================================================
|
||||
log "$ICON_GEAR Config: system-threshold=${LOG_MIN_SIZE_MB:-10}MB docker-threshold=${LOG_DOCKER_MAX_MB:-100}MB"
|
||||
log "$ICON_GEAR System logs: ${LOG_FILES[*]}"
|
||||
|
||||
START=$(date +%s)
|
||||
SYS_CLEARED=0
|
||||
SYS_SKIPPED=0
|
||||
|
||||
@@ -132,7 +132,10 @@ if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
fi
|
||||
|
||||
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
|
||||
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
|
||||
MOVER_ELAPSED=$(ps -o etimes= -p "$MOVER_PID" 2>/dev/null | tr -d ' ')
|
||||
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
log "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
|
||||
|
||||
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
|
||||
@@ -226,6 +226,9 @@ else
|
||||
log "Verified: pm.max_children = $APPLIED_VAL ✅"
|
||||
fi
|
||||
|
||||
FPM_WORKERS=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
|
||||
log "$ICON_PHP Workers running: $FPM_WORKERS"
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -194,6 +194,14 @@ if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||
log "$ICON_CONTAINERS Docker: ${CONTAINER_COUNT} container(s) running"
|
||||
|
||||
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
||||
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
||||
log "$ICON_GEAR VMs: ${VM_COUNT} running"
|
||||
fi
|
||||
|
||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||
log "Pre-flight clean — no active processes to warn about"
|
||||
else
|
||||
|
||||
@@ -40,6 +40,9 @@ detect_hosts
|
||||
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
|
||||
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
|
||||
|
||||
log "$ICON_GEAR Conf file: $CONF_FILE"
|
||||
log "$ICON_GEAR Key var: $VAR_NAME"
|
||||
|
||||
if [[ ! -f "$CONF_FILE" ]]; then
|
||||
error "Conf file not found: $CONF_FILE"
|
||||
exit 1
|
||||
@@ -54,15 +57,19 @@ fi
|
||||
# Check if key already exists in the unraid-api registry before creating.
|
||||
# --overwrite generates a new key value every time, invalidating the old one.
|
||||
# Only renew if the registry has lost it.
|
||||
log "Checking unraid-api registry for existing Varaverk key..."
|
||||
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "Varaverk" --json </dev/null 2>/dev/null)
|
||||
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
|
||||
|
||||
if [[ -n "$KEY" ]]; then
|
||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||||
log "Key found in registry — no renewal needed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Key not found in registry — creating new key..."
|
||||
|
||||
RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
|
||||
--name "Varaverk" --create --overwrite \
|
||||
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
|
||||
@@ -87,4 +94,5 @@ else
|
||||
fi
|
||||
|
||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||
log "Writing new key to: $CONF_FILE"
|
||||
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW (registry had lost it)"
|
||||
|
||||
Reference in New Issue
Block a user