Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e71c314d7 | ||
|
|
08fc551d9f | ||
|
|
bc70ebe5ee |
@@ -146,6 +146,8 @@ DOCKER_TIMEOUT=30
|
|||||||
DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally
|
DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally
|
||||||
_RETRY_COUNT="${RETRY_COUNT:-3}"
|
_RETRY_COUNT="${RETRY_COUNT:-3}"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: retries=$_RETRY_COUNT sleep=${SLEEP:-5}s grace=${DOCKER_STOP_TIMEOUT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -175,6 +177,7 @@ if [[ ${#RUNNING[@]} -eq 0 ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running"
|
echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running"
|
||||||
|
log "$ICON_CONTAINERS Queue: ${RUNNING[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
@@ -183,7 +186,11 @@ FAILED=()
|
|||||||
|
|
||||||
for container in "${RUNNING[@]}"; do
|
for container in "${RUNNING[@]}"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
local c_start
|
||||||
|
c_start=$(date +%s)
|
||||||
|
local c_image
|
||||||
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would stop $container"
|
warn "DRY RUN — would stop $container"
|
||||||
@@ -204,7 +211,7 @@ for container in "${RUNNING[@]}"; do
|
|||||||
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
|
||||||
if [[ "$STATE" != "true" ]]; then
|
if [[ "$STATE" != "true" ]]; then
|
||||||
log "$ICON_DONE $container stopped ✅"
|
log "$ICON_DONE $container stopped in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||||
STOPPED+=("$container")
|
STOPPED+=("$container")
|
||||||
success=true
|
success=true
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -230,7 +230,6 @@ build_restart_order() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Checks if a container is a dependent of the previously restarted container.
|
# Checks if a container is a dependent of the previously restarted container.
|
||||||
@@ -279,6 +278,7 @@ echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} container(s)"
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} container(s)"
|
||||||
log "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
log "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||||
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||||
|
log "$ICON_GEAR Config: sleep=${SLEEP}s delay=${CONTAINER_DELAY}s verify-wait=${RESTART_VERIFY_WAIT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
FAILED=()
|
FAILED=()
|
||||||
@@ -293,7 +293,11 @@ LAST_RESTARTED=""
|
|||||||
|
|
||||||
for container in "${ORDERED_RESTART[@]}"; do
|
for container in "${ORDERED_RESTART[@]}"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
local c_start
|
||||||
|
c_start=$(date +%s)
|
||||||
|
local c_image
|
||||||
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||||
|
|
||||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||||
warn "$container does not exist — skipping"
|
warn "$container does not exist — skipping"
|
||||||
@@ -316,7 +320,7 @@ for container in "${ORDERED_RESTART[@]}"; do
|
|||||||
if retry_docker docker restart "$container"; then
|
if retry_docker docker restart "$container"; then
|
||||||
# Verify container stayed running after restart
|
# Verify container stayed running after restart
|
||||||
if verify_running "$container"; then
|
if verify_running "$container"; then
|
||||||
log "$ICON_STARTED $container restarted and running ✅"
|
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||||
RESTARTED+=("$container")
|
RESTARTED+=("$container")
|
||||||
LAST_RESTARTED="$container"
|
LAST_RESTARTED="$container"
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -209,7 +209,12 @@ for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do
|
|||||||
|
|
||||||
# ── Step 1 — ensure network exists ───────────────────────────────────────────────────────
|
# ── Step 1 — ensure network exists ───────────────────────────────────────────────────────
|
||||||
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then
|
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then
|
||||||
log "$network exists ✅"
|
local net_subnet net_driver
|
||||||
|
net_subnet=$(timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
||||||
|
--format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null || echo "unknown")
|
||||||
|
net_driver=$(timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
||||||
|
--format '{{.Driver}}' 2>/dev/null || echo "unknown")
|
||||||
|
log "$ICON_DOCKER_NET $network exists ✅ — driver: $net_driver subnet: $net_subnet"
|
||||||
else
|
else
|
||||||
warn "$ICON_DOCKER_NET $network not found — creating (unRAID update may have wiped networks)"
|
warn "$ICON_DOCKER_NET $network not found — creating (unRAID update may have wiped networks)"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
@@ -279,7 +284,7 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|||||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
[[ ${#NETWORKS_CREATED[@]} -gt 0 ]] && warn "$ICON_DOCKER_NET Created: ${NETWORKS_CREATED[*]} (networks were missing)"
|
[[ ${#NETWORKS_CREATED[@]} -gt 0 ]] && warn "$ICON_DOCKER_NET Created: ${NETWORKS_CREATED[*]} (networks were missing)"
|
||||||
[[ ${#CONNECTED[@]} -gt 0 ]] && log "Connected: ${CONNECTED[*]}"
|
[[ ${#CONNECTED[@]} -gt 0 ]] && log "Connected: ${CONNECTED[*]}"
|
||||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${#SKIPPED[@]} skipped"
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${SKIPPED[*]}"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
|||||||
@@ -246,6 +246,7 @@ echo ""
|
|||||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||||
|
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||||
else
|
else
|
||||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||||
@@ -298,10 +299,10 @@ for container in "${TARGET_CONTAINERS[@]}"; do
|
|||||||
|
|
||||||
if [[ $_pull_rc -eq 0 ]]; then
|
if [[ $_pull_rc -eq 0 ]]; then
|
||||||
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
||||||
log "$ICON_DONE $container — updated ✅"
|
log "$ICON_DONE $container — updated ✅ (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||||
UPDATED+=("$container")
|
UPDATED+=("$container")
|
||||||
else
|
else
|
||||||
log "$container — already up to date"
|
log "$container — already up to date (${NEW_ID:7:12})"
|
||||||
UP_TO_DATE+=("$container")
|
UP_TO_DATE+=("$container")
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
@@ -367,8 +368,8 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
|||||||
fi
|
fi
|
||||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}"
|
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${#SKIPPED[@]}"
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||||
|
|
||||||
|
|||||||
@@ -235,10 +235,10 @@ for container in "${REMAINING[@]}"; do
|
|||||||
|
|
||||||
if [[ $_pull_rc -eq 0 ]]; then
|
if [[ $_pull_rc -eq 0 ]]; then
|
||||||
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
||||||
log "$ICON_DONE $container — updated ✅"
|
log "$ICON_DONE $container — updated ✅ (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||||
UPDATED+=("$container")
|
UPDATED+=("$container")
|
||||||
else
|
else
|
||||||
log "$container — already up to date"
|
log "$container — already up to date (${NEW_ID:7:12})"
|
||||||
UP_TO_DATE+=("$container")
|
UP_TO_DATE+=("$container")
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
@@ -326,13 +326,13 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
|||||||
echo "$ICON_DONE New image: ${#UPDATED[@]}"
|
echo "$ICON_DONE New image: ${#UPDATED[@]}"
|
||||||
log " ${UPDATED[*]}"
|
log " ${UPDATED[*]}"
|
||||||
fi
|
fi
|
||||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}"
|
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}"
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}"
|
||||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||||
echo "$ICON_DONE Restarted: ${#RESTARTED[@]}"
|
echo "$ICON_DONE Restarted: ${#RESTARTED[@]}"
|
||||||
log " ${RESTARTED[*]}"
|
log " ${RESTARTED[*]}"
|
||||||
fi
|
fi
|
||||||
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${#SKIPPED_STOPPED[@]} (skipped restart)"
|
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
|
||||||
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}"
|
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}"
|
||||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||||
|
|
||||||
|
|||||||
@@ -246,6 +246,7 @@ echo ""
|
|||||||
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
log "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
log "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||||
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||||
|
log "$ICON_GEAR Config: sleep=${SLEEP}s delay=${CONTAINER_DELAY}s verify-wait=${RESTART_VERIFY_WAIT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
FAILED=()
|
FAILED=()
|
||||||
@@ -259,7 +260,11 @@ LAST_RESTARTED=""
|
|||||||
|
|
||||||
for container in "${ORDERED_RESTART[@]}"; do
|
for container in "${ORDERED_RESTART[@]}"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
local c_start
|
||||||
|
c_start=$(date +%s)
|
||||||
|
local c_image
|
||||||
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||||
|
|
||||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||||
warn "$container does not exist — skipping"
|
warn "$container does not exist — skipping"
|
||||||
@@ -281,7 +286,7 @@ for container in "${ORDERED_RESTART[@]}"; do
|
|||||||
else
|
else
|
||||||
if retry_docker docker restart "$container"; then
|
if retry_docker docker restart "$container"; then
|
||||||
if verify_running "$container"; then
|
if verify_running "$container"; then
|
||||||
log "$ICON_STARTED $container restarted and running ✅"
|
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||||
RESTARTED+=("$container")
|
RESTARTED+=("$container")
|
||||||
LAST_RESTARTED="$container"
|
LAST_RESTARTED="$container"
|
||||||
else
|
else
|
||||||
@@ -319,9 +324,9 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|||||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||||
echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}"
|
echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}"
|
||||||
log " ${RESTARTED[*]}"
|
log " Names: ${RESTARTED[*]}"
|
||||||
fi
|
fi
|
||||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} (were stopped)"
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
|||||||
@@ -148,6 +148,8 @@ CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
|
|||||||
TOTAL_PASS=0
|
TOTAL_PASS=0
|
||||||
TOTAL_FAIL=0
|
TOTAL_FAIL=0
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: retention=${DOWNLOADER_RETENTION_DAYS}d qbit-age=${QBIT_FAILSAFE_MIN_DAYS}d qbit-ratio=${QBIT_FAILSAFE_MIN_RATIO}"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -304,6 +306,8 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" =
|
|||||||
if [[ -z "$USERNAMES" ]]; then
|
if [[ -z "$USERNAMES" ]]; then
|
||||||
success "No transfer records found ✅"
|
success "No transfer records found ✅"
|
||||||
else
|
else
|
||||||
|
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || echo 0)
|
||||||
|
log "Found $USER_COUNT user(s) with transfer records"
|
||||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||||
while IFS= read -r USER; do
|
while IFS= read -r USER; do
|
||||||
[[ -z "$USER" ]] && continue
|
[[ -z "$USER" ]] && continue
|
||||||
@@ -422,6 +426,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
|||||||
if [[ -z "$COMPLETED_IDS" ]]; then
|
if [[ -z "$COMPLETED_IDS" ]]; then
|
||||||
success "No completed history found ✅"
|
success "No completed history found ✅"
|
||||||
else
|
else
|
||||||
|
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||||
|
log "Found $HIST_TOTAL completed history entries"
|
||||||
DELETED=0; SKIPPED=0
|
DELETED=0; SKIPPED=0
|
||||||
while IFS= read -r NZO_ID; do
|
while IFS= read -r NZO_ID; do
|
||||||
[[ -z "$NZO_ID" ]] && continue
|
[[ -z "$NZO_ID" ]] && continue
|
||||||
@@ -468,6 +474,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
|||||||
if [[ -z "$FAILED_IDS" ]]; then
|
if [[ -z "$FAILED_IDS" ]]; then
|
||||||
success "No failed history found ✅"
|
success "No failed history found ✅"
|
||||||
else
|
else
|
||||||
|
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||||
|
log "Found $FAILED_TOTAL failed history entries"
|
||||||
DELETED=0; SKIPPED=0
|
DELETED=0; SKIPPED=0
|
||||||
while IFS= read -r NZO_ID; do
|
while IFS= read -r NZO_ID; do
|
||||||
[[ -z "$NZO_ID" ]] && continue
|
[[ -z "$NZO_ID" ]] && continue
|
||||||
@@ -516,6 +524,8 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
|||||||
if [[ -z "$STALLED_IDS" ]]; then
|
if [[ -z "$STALLED_IDS" ]]; then
|
||||||
success "No stalled queue items found ✅"
|
success "No stalled queue items found ✅"
|
||||||
else
|
else
|
||||||
|
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||||
|
log "Found $QUEUE_TOTAL queue item(s) — checking status"
|
||||||
DELETED=0; SKIPPED=0
|
DELETED=0; SKIPPED=0
|
||||||
while IFS= read -r NZO_ID; do
|
while IFS= read -r NZO_ID; do
|
||||||
[[ -z "$NZO_ID" ]] && continue
|
[[ -z "$NZO_ID" ]] && continue
|
||||||
@@ -575,6 +585,8 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
|||||||
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
||||||
|
|
||||||
NOW=$(date +%s)
|
NOW=$(date +%s)
|
||||||
|
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || echo 0)
|
||||||
|
log "Found $TORRENT_TOTAL torrent(s) — applying age/ratio filter"
|
||||||
DELETED=0; SKIPPED=0
|
DELETED=0; SKIPPED=0
|
||||||
|
|
||||||
while read -r TORRENT; do
|
while read -r TORRENT; do
|
||||||
|
|||||||
@@ -119,6 +119,10 @@ RSYNC_OK=false
|
|||||||
PASS=()
|
PASS=()
|
||||||
FAIL=()
|
FAIL=()
|
||||||
|
|
||||||
|
log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
|
||||||
|
[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
|
||||||
|
log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"
|
||||||
|
|
||||||
if ! check_rsync_enabled "CRITICAL"; then
|
if ! check_rsync_enabled "CRITICAL"; then
|
||||||
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
||||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||||
|
|||||||
@@ -87,11 +87,21 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Main ━━━
|
# ━━━ Main ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
_unraid_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
|
||||||
|
_uptime_s=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)
|
||||||
|
_container_count=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||||
|
_rootfs_pct=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
echo "☕ Sunday Morning Coffee Report — $MY_ID"
|
echo "☕ Sunday Morning Coffee Report — $MY_ID"
|
||||||
[[ "$DRY_RUN" == true ]] && echo " [DRY RUN]"
|
[[ "$DRY_RUN" == true ]] && echo " [DRY RUN]"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
log "$ICON_HOST Server: $MY_ID ($LOCAL_SERVER_NAME) — unRAID $_unraid_ver"
|
||||||
|
log "$ICON_TIME Uptime: $(format_duration $_uptime_s)"
|
||||||
|
log "$ICON_CONTAINERS Docker: $_container_count container(s) running"
|
||||||
|
log "$ICON_HEALTH Rootfs: ${_rootfs_pct:-?}% used"
|
||||||
|
log "$ICON_GEAR Scripts: ${#COFFEE_REPORT_SCRIPTS[@]} configured"
|
||||||
|
|
||||||
if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
|
if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
|
||||||
warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf"
|
warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf"
|
||||||
|
|||||||
@@ -137,22 +137,40 @@ if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Pre-run State Snapshot ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if mountpoint -q "${RAMDISK_PATH:-}" 2>/dev/null; then
|
||||||
|
_rd_used=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||||
|
_rd_avail=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||||
|
_rd_used_gb=$(awk "BEGIN {printf \"%.2f\", ${_rd_used:-0}/1048576}")
|
||||||
|
_rd_avail_gb=$(awk "BEGIN {printf \"%.2f\", ${_rd_avail:-0}/1048576}")
|
||||||
|
_rd_target=$(readlink "${TRANSCODE_LINK:-}" 2>/dev/null || echo "unknown")
|
||||||
|
log "$ICON_RAM Ramdisk: ${_rd_used_gb}GB used / ${_rd_avail_gb}GB avail — symlink → ${_rd_target##*/}"
|
||||||
|
else
|
||||||
|
log "$ICON_RAM Ramdisk: not mounted"
|
||||||
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Run Cleanup ━━━
|
# ━━━ Run Cleanup ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
DRY_FLAG=""
|
DRY_FLAG=""
|
||||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||||
|
|
||||||
|
_cleanup_start=$(date +%s)
|
||||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
||||||
CLEANUP_EXIT=$?
|
CLEANUP_EXIT=$?
|
||||||
|
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Run Manager ━━━
|
# ━━━ Run Manager ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
||||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
# No --no-log flag here — manager owns the log write for this cycle ✅
|
||||||
|
_manager_start=$(date +%s)
|
||||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
||||||
MANAGER_EXIT=$?
|
MANAGER_EXIT=$?
|
||||||
|
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Exit ━━━
|
# ━━━ Exit ━━━
|
||||||
|
|||||||
@@ -68,6 +68,9 @@ acquire_lock
|
|||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||||
|
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||||
|
|
||||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||||
@@ -122,6 +125,7 @@ if ! df --output=fstype /mnt/user 2>/dev/null | grep -q shfs; then
|
|||||||
echo "Array not started — skipping watchdog cycle"
|
echo "Array not started — skipping watchdog cycle"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Startup Grace ━━━
|
# ━━━ Startup Grace ━━━
|
||||||
@@ -131,6 +135,7 @@ if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
|||||||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Run Watchdog Cycle ━━━
|
# ━━━ Run Watchdog Cycle ━━━
|
||||||
@@ -154,12 +159,15 @@ run_watchdog() {
|
|||||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||||
|
|
||||||
|
local _ws
|
||||||
|
_ws=$(date +%s)
|
||||||
log "$ICON_START $name"
|
log "$ICON_START $name"
|
||||||
if bash "$script" "${extra_args[@]}"; then
|
if bash "$script" "${extra_args[@]}"; then
|
||||||
|
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
|
||||||
PASS+=("$name")
|
PASS+=("$name")
|
||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
error "$name — non-zero exit"
|
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
|
||||||
FAIL+=("$name")
|
FAIL+=("$name")
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ echo json_encode([
|
|||||||
'disk_io' => vv_disk_io_rates(),
|
'disk_io' => vv_disk_io_rates(),
|
||||||
'watchdog' => vv_watchdog_summary(),
|
'watchdog' => vv_watchdog_summary(),
|
||||||
'scripts' => vv_scripts_status(),
|
'scripts' => vv_scripts_status(),
|
||||||
|
'rsync' => vv_rsync_status(),
|
||||||
'thresholds' => vv_disk_thresholds(),
|
'thresholds' => vv_disk_thresholds(),
|
||||||
'vms' => vv_get_vms(),
|
'vms' => vv_get_vms(),
|
||||||
'docker_folders' => vv_get_docker_folders(),
|
'docker_folders' => vv_get_docker_folders(),
|
||||||
|
|||||||
@@ -319,3 +319,58 @@ function vv_scripts_status(): array {
|
|||||||
'error_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
|
'error_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function vv_rsync_status(): array {
|
||||||
|
$vars = vv_conf_vars();
|
||||||
|
|
||||||
|
$enabled = ($vars['RSYNC_ENABLED'] ?? 'true') !== 'false';
|
||||||
|
|
||||||
|
$windows = [
|
||||||
|
'critical' => ($vars['CRITICAL_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
||||||
|
'daily' => ($vars['DAILY_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
||||||
|
'intermediate' => ($vars['INTERMEDIATE_RSYNC_ENABLED'] ?? 'true') !== 'false',
|
||||||
|
'weekly' => ($vars['WEEKLY_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
||||||
|
];
|
||||||
|
|
||||||
|
// Active rsync profiles — from lock files
|
||||||
|
$lockDir = '/tmp/unraid_locks';
|
||||||
|
$active = [];
|
||||||
|
foreach (glob("$lockDir/rsync_*.lock") ?: [] as $lf) {
|
||||||
|
$content = trim(@file_get_contents($lf) ?: '');
|
||||||
|
[$pid, $locked_name] = array_pad(explode(':', $content, 2), 2, '');
|
||||||
|
if (!$pid || !file_exists("/proc/$pid")) continue;
|
||||||
|
$profile = preg_replace('/^rsync_/', '', $locked_name ?: basename($lf, '.lock'));
|
||||||
|
$active[] = [
|
||||||
|
'profile' => $profile,
|
||||||
|
'pid' => (int)$pid,
|
||||||
|
'elapsed' => time() - (int)filemtime($lf),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last completed run per window (orchestrator log files)
|
||||||
|
$scriptMap = [
|
||||||
|
'critical' => 'critical_sync_maintenance',
|
||||||
|
'daily' => 'daily_sync_maintenance',
|
||||||
|
'intermediate' => 'intermediate_sync_maintenance',
|
||||||
|
'weekly' => 'weekly_sync_maintenance',
|
||||||
|
];
|
||||||
|
$lastSync = [];
|
||||||
|
foreach ($scriptMap as $key => $scriptName) {
|
||||||
|
$logFile = LOG_DIR . "/$scriptName.json";
|
||||||
|
if (!file_exists($logFile)) continue;
|
||||||
|
$stat = json_decode(@file_get_contents($logFile) ?: '{}', true) ?: [];
|
||||||
|
$lastSync[$key] = [
|
||||||
|
'ts' => (int)($stat['end'] ?? $stat['start'] ?? 0),
|
||||||
|
'status' => $stat['status'] ?? 'unknown',
|
||||||
|
'duration' => isset($stat['start'], $stat['end'])
|
||||||
|
? (int)$stat['end'] - (int)$stat['start'] : null,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
'enabled' => $enabled,
|
||||||
|
'windows' => $windows,
|
||||||
|
'active' => $active,
|
||||||
|
'last_sync' => $lastSync,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|||||||
@@ -98,8 +98,18 @@
|
|||||||
<div id="vv-docker-folders-body">Loading...</div>
|
<div id="vv-docker-folders-body">Loading...</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Row 3: GPU | Transcode | Streams -->
|
<!-- Row 3: Rsync | GPU | Transcode | Streams -->
|
||||||
<div class="vv-card" id="vv-gpu-card" style="grid-column:span 2;">
|
<div class="vv-card" id="vv-rsync-card" style="grid-column:span 1;">
|
||||||
|
<h3>
|
||||||
|
<span style="display:flex;align-items:center;gap:5px;">
|
||||||
|
<span class="vv-ico"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" stroke="#aaa" stroke-width="1.1" stroke-linecap="round" stroke-linejoin="round"><polyline points="2,4 10,4 8,2"/><polyline points="10,8 2,8 4,10"/></svg></span>
|
||||||
|
Rsync
|
||||||
|
</span>
|
||||||
|
</h3>
|
||||||
|
<div id="vv-rsync-body">Loading...</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="vv-card" id="vv-gpu-card" style="grid-column:span 1;">
|
||||||
<h3>
|
<h3>
|
||||||
<span style="display:flex;align-items:center;gap:5px;">
|
<span style="display:flex;align-items:center;gap:5px;">
|
||||||
<span class="vv-ico"><svg width="15" height="10" viewBox="0 0 16 10" fill="none" stroke="#aaa" stroke-width="1.1" stroke-linecap="round" stroke-linejoin="round"><rect x="0.8" y="0.8" width="14.4" height="7" rx="1.2"/><rect x="2.5" y="2.5" width="3.5" height="3.5" rx="0.5"/><line x1="8" y1="3" x2="13" y2="3"/><line x1="8" y1="5" x2="11" y2="5"/><rect x="3" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/><rect x="6.5" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/><rect x="10" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/></svg></span>
|
<span class="vv-ico"><svg width="15" height="10" viewBox="0 0 16 10" fill="none" stroke="#aaa" stroke-width="1.1" stroke-linecap="round" stroke-linejoin="round"><rect x="0.8" y="0.8" width="14.4" height="7" rx="1.2"/><rect x="2.5" y="2.5" width="3.5" height="3.5" rx="0.5"/><line x1="8" y1="3" x2="13" y2="3"/><line x1="8" y1="5" x2="11" y2="5"/><rect x="3" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/><rect x="6.5" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/><rect x="10" y="7.8" width="2" height="1.8" rx="0.3" stroke="none" fill="#666"/></svg></span>
|
||||||
@@ -1306,6 +1316,79 @@ function vvPollMonitor() {
|
|||||||
else if (sys.array_state && sys.array_state !== 'UNKNOWN') c.classList.add('vv-accent-err');
|
else if (sys.array_state && sys.array_state !== 'UNKNOWN') c.classList.add('vv-accent-err');
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// ── Rsync ────────────────────────────────────────────────────────────────
|
||||||
|
(function() {
|
||||||
|
const rs = d.rsync ?? {};
|
||||||
|
const enabled = rs.enabled ?? true;
|
||||||
|
const windows = rs.windows ?? {};
|
||||||
|
const active = rs.active ?? [];
|
||||||
|
const lastSync = rs.last_sync ?? {};
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
|
||||||
|
const el = document.getElementById('vv-rsync-body');
|
||||||
|
if (!el) return;
|
||||||
|
|
||||||
|
const gColor = enabled ? '#4caf50' : '#555';
|
||||||
|
let html = `<div style="font-size:10px;color:${gColor};margin-bottom:7px;">● ${enabled ? 'enabled' : 'disabled'}</div>`;
|
||||||
|
|
||||||
|
// Window badges: C D I W
|
||||||
|
html += '<div style="display:flex;gap:4px;margin-bottom:8px;">';
|
||||||
|
[['C','critical'],['D','daily'],['I','intermediate'],['W','weekly']].forEach(([s,k]) => {
|
||||||
|
const on = windows[k] ?? false;
|
||||||
|
html += `<span title="${k}" style="font-size:9px;padding:1px 6px;border-radius:3px;
|
||||||
|
background:${on?'#1a2a1a':'#1a1a1a'};border:1px solid ${on?'#2a5a2a':'#252525'};
|
||||||
|
color:${on?'#4caf50':'#444'};">${s}</span>`;
|
||||||
|
});
|
||||||
|
html += '</div>';
|
||||||
|
|
||||||
|
// Active sessions
|
||||||
|
if (active.length) {
|
||||||
|
active.forEach(a => {
|
||||||
|
const sec = a.elapsed ?? 0;
|
||||||
|
const dur = sec < 60 ? sec+'s' : sec < 3600 ? Math.floor(sec/60)+'m'+String(sec%60).padStart(2,'0')+'s' : Math.floor(sec/3600)+'h'+Math.floor((sec%3600)/60)+'m';
|
||||||
|
html += `<div style="display:flex;justify-content:space-between;font-size:10px;margin-bottom:3px;">
|
||||||
|
<span style="color:#ff9800;font-weight:500;">⟳ ${a.profile}</span>
|
||||||
|
<span style="color:#ff9800;">${dur}</span>
|
||||||
|
</div>`;
|
||||||
|
});
|
||||||
|
html += `<div style="height:1px;background:#222;margin:5px 0;"></div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last sync per window
|
||||||
|
const syncRows = [['critical','critical'],['daily','daily'],['intermediate','interm'],['weekly','weekly']];
|
||||||
|
let hasSyncs = false;
|
||||||
|
let syncHtml = '';
|
||||||
|
syncRows.forEach(([key, label]) => {
|
||||||
|
const s = lastSync[key];
|
||||||
|
if (!s || !s.ts) return;
|
||||||
|
hasSyncs = true;
|
||||||
|
const diff = now - s.ts;
|
||||||
|
const ago = diff < 60 ? diff+'s' : diff < 3600 ? Math.floor(diff/60)+'m' : diff < 86400 ? Math.floor(diff/3600)+'h' : Math.floor(diff/86400)+'d';
|
||||||
|
const ok = s.status === 'ok' || s.status === 'success';
|
||||||
|
const warn = s.status === 'warn';
|
||||||
|
const run = s.status === 'running';
|
||||||
|
const col = ok ? '#4caf50' : warn ? '#ff9800' : run ? '#4fc3f7' : '#f44336';
|
||||||
|
const icon = ok ? '✓' : warn ? '!' : run ? '●' : '✗';
|
||||||
|
syncHtml += `<div style="display:flex;justify-content:space-between;align-items:center;font-size:10px;margin-bottom:3px;">
|
||||||
|
<span style="color:#555;">${label}</span>
|
||||||
|
<span style="color:#3a3a3a;">${ago}</span>
|
||||||
|
<span style="color:${col};">${icon}</span>
|
||||||
|
</div>`;
|
||||||
|
});
|
||||||
|
if (hasSyncs) {
|
||||||
|
html += `<div style="font-size:9px;color:#333;text-transform:uppercase;letter-spacing:.05em;margin-bottom:4px;">Last sync</div>`;
|
||||||
|
html += syncHtml;
|
||||||
|
}
|
||||||
|
|
||||||
|
el.innerHTML = html;
|
||||||
|
|
||||||
|
const card = document.getElementById('vv-rsync-card');
|
||||||
|
if (card) {
|
||||||
|
card.classList.remove('vv-accent-ok','vv-accent-warn','vv-accent-err');
|
||||||
|
if (active.length) card.classList.add('vv-accent-ok');
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
// ── GPU ─────────────────────────────────────────────────────────────────
|
// ── GPU ─────────────────────────────────────────────────────────────────
|
||||||
const gpu = d.gpu ?? {};
|
const gpu = d.gpu ?? {};
|
||||||
const gpuProcs = d.gpu_procs ?? [];
|
const gpuProcs = d.gpu_procs ?? [];
|
||||||
|
|||||||
@@ -132,7 +132,8 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
log "Max age: ${TRANSCODE_MAX_AGE} minutes"
|
log "$ICON_GEAR Config: max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
|
||||||
|
log "$ICON_RAM Locations: ramdisk=$RAMDISK_PATH ssd=$TRANSCODE_SSD"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
|
|||||||
@@ -180,6 +180,9 @@ case "$TRANSCODE_MANAGER_MODE" in
|
|||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: ramdisk=$RAMDISK_PATH size=$RAMDISK_SIZE warn-at=${RAMDISK_WARN_GB}GB flip-back-at=${RAMDISK_LOW_GB}GB ssd-min-free=${RAMDISK_SSD_MIN_GB}GB flip-warn=${TRANSCODE_FLIP_WARN}/hr"
|
||||||
|
log "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
@@ -124,6 +124,9 @@ detect_hosts
|
|||||||
[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
|
[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: internet=${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1} timeout=${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}s tailscale=${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true} npm-strikes=${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||||
|
log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no container} NPM: ${NPM_URL:-not configured}"
|
||||||
|
|
||||||
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
||||||
|
|
||||||
# ━━━ Strike helpers ━━━
|
# ━━━ Strike helpers ━━━
|
||||||
|
|||||||
@@ -157,6 +157,10 @@ detect_hosts
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: growth-thresh=${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle log-max=${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB truncate=${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false} strikes=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
|
||||||
|
log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
|
||||||
|
[[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]] && log "$ICON_GEAR Suppress ceilings: $(for k in "${!WATCHDOG_APPDATA_SIZES[@]}"; do printf '%s=%sMB ' "$k" "${WATCHDOG_APPDATA_SIZES[$k]}"; done)"
|
||||||
|
|
||||||
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
||||||
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
||||||
|
|
||||||
|
|||||||
@@ -163,6 +163,9 @@ log "WebGUI check — $WEBGUI_URL"
|
|||||||
|
|
||||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||||
if check_webgui; then
|
if check_webgui; then
|
||||||
|
_nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
|
||||||
|
_fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
|
||||||
|
log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
|
||||||
echo "WebGUI responding — healthy ✅"
|
echo "WebGUI responding — healthy ✅"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -246,6 +246,9 @@ fi
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s mem-soft=${SOFT_MEM_THRESHOLD}% cpu-soft=${SOFT_CPU_THRESHOLD}% cpu-hard=${HARD_CPU_THRESHOLD}% cpu-limit=${CPU_FAIL_LIMIT} http-limit=${RESP_FAIL_LIMIT} daemon-timeout=${DOCKER_TIMEOUT}s"
|
||||||
|
log "$ICON_CONTAINERS Tier1: watched=${#WATCHDOG_CONTAINERS[@]} required=${#WATCHDOG_REQUIRED_CONTAINERS[@]} urls=${#WATCHDOG_CONTAINER_URLS[@]} restart-limit=${WATCHDOG_CONTAINER_RESTART_LIMIT}/${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||||
|
|
||||||
# Validate unRAID-specific commands used by this script
|
# Validate unRAID-specific commands used by this script
|
||||||
# If rc.docker is missing or changed, daemon restart will fail — better to know now
|
# If rc.docker is missing or changed, daemon restart will fail — better to know now
|
||||||
validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed"
|
validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed"
|
||||||
@@ -623,6 +626,11 @@ CYCLE_START=$(date +%s)
|
|||||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# ── Skip list visibility ─────────────────────────────────────────────────────────────────
|
||||||
|
local _skip_contents
|
||||||
|
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||||
|
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
|
||||||
|
|
||||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||||
if ! check_docker_daemon; then
|
if ! check_docker_daemon; then
|
||||||
@@ -681,6 +689,7 @@ CYCLE_START=$(date +%s)
|
|||||||
if [[ "$STATUS" == "true" ]]; then
|
if [[ "$STATUS" == "true" ]]; then
|
||||||
# Running — clear any strikes
|
# Running — clear any strikes
|
||||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||||
|
log "$ICON_RUNNING $container — running ✅"
|
||||||
else
|
else
|
||||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||||
STRIKES=$(( STRIKES + 1 ))
|
STRIKES=$(( STRIKES + 1 ))
|
||||||
@@ -765,6 +774,7 @@ CYCLE_START=$(date +%s)
|
|||||||
else
|
else
|
||||||
# Normal — clear CPU strikes
|
# Normal — clear CPU strikes
|
||||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||||
|
log "$container — CPU ${CPU_NORM}% | MEM ${MEM_MB}MB / ${MEM_LIMIT_MB}MB ✅"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -133,6 +133,9 @@ detect_hosts
|
|||||||
|
|
||||||
DOCKER_TIMEOUT=15
|
DOCKER_TIMEOUT=15
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}"
|
||||||
|
log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}"
|
||||||
|
|
||||||
touch "$RW_STATE_FILE" 2>/dev/null || {
|
touch "$RW_STATE_FILE" 2>/dev/null || {
|
||||||
error "Cannot create state file: $RW_STATE_FILE"
|
error "Cannot create state file: $RW_STATE_FILE"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -570,7 +573,7 @@ else
|
|||||||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||||
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||||
else
|
else
|
||||||
echo "System at normal pressure ✅"
|
log "System normal — RAM ${MEM_GB}GB free | load ${LOAD} | ${TOTAL_CORES} cores | SABnzbd=${RW_SABNZBD_ENABLED:-true} qBit=${RW_QBIT_ENABLED:-true} ✅"
|
||||||
fi
|
fi
|
||||||
rm_state_set "rm_recover_cycles" 0
|
rm_state_set "rm_recover_cycles" 0
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -142,6 +142,9 @@ done
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Config: strikes=${SYS_WATCHDOG_STRIKE_LIMIT} reboot-limit=${SYS_WATCHDOG_REBOOT_LIMIT}/${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr oom-limit=${SYS_WATCHDOG_OOM_LIMIT}"
|
||||||
|
log "$ICON_GEAR Tiers: rootfs-crit=${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}% rootfs-warn=${SYS_WATCHDOG_ROOTFS_PCT}% ram-reboot=${SYS_WATCHDOG_MEM_GB}GB load=${SYS_WATCHDOG_LOAD_MULTIPLIER}x(${TOTAL_CORES}cores) cpu-temp=${SYS_WATCHDOG_CPU_TEMP_MAX}°C zombies=${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -570,6 +573,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
||||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
||||||
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||||
|
[[ "$TRIGGERED" == false ]] && log "rootfs ${ROOTFS_USED}% ✅ (warn at ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── /var/log ─────────────────────────────────────────────────────────────────────────────
|
# ── /var/log ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -579,6 +583,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
[[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
[[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
||||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
||||||
TRIGGERS+=("log=${LOG_USED}%")
|
TRIGGERS+=("log=${LOG_USED}%")
|
||||||
|
[[ "$TRIGGERED" == false ]] && log "/var/log ${LOG_USED:-?}% ✅ (warn at ${SYS_WATCHDOG_LOG_PCT}%)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── /tmp ─────────────────────────────────────────────────────────────────────────────────
|
# ── /tmp ─────────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -672,6 +677,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
||||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
||||||
TRIGGERS+=("load=${LOAD}")
|
TRIGGERS+=("load=${LOAD}")
|
||||||
|
[[ "$TRIGGERED" == false ]] && log "load avg ${LOAD} ✅ (warn at ${LOAD_THRESHOLD} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}×${TOTAL_CORES} cores)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
||||||
@@ -682,6 +688,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
||||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
||||||
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||||
|
[[ "$TRIGGERED" == false ]] && log "zombies ${ZOMBIE_COUNT} ✅ (warn at ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Array disk errors — accumulating mdstat errors ────────────────────────────────────────
|
# ── Array disk errors — accumulating mdstat errors ────────────────────────────────────────
|
||||||
@@ -709,10 +716,12 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then
|
if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then
|
||||||
NIC="${SYS_WATCHDOG_NIC:-eth0}"
|
NIC="${SYS_WATCHDOG_NIC:-eth0}"
|
||||||
NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown")
|
NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown")
|
||||||
|
NIC_SPEED=$(cat "/sys/class/net/${NIC}/speed" 2>/dev/null || echo "?")
|
||||||
TRIGGERED=false
|
TRIGGERED=false
|
||||||
[[ "$NIC_STATE" != "up" ]] && TRIGGERED=true
|
[[ "$NIC_STATE" != "up" ]] && TRIGGERED=true
|
||||||
run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \
|
run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \
|
||||||
TRIGGERS+=("nic_down=${NIC}")
|
TRIGGERS+=("nic_down=${NIC}")
|
||||||
|
[[ "$TRIGGERED" == false ]] && log "$NIC ${NIC_STATE} @ ${NIC_SPEED}Mbps ✅"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── sshd — try restart before escalating ─────────────────────────────────────────────────
|
# ── sshd — try restart before escalating ─────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -119,11 +119,12 @@ for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
_ss=$(date +%s)
|
||||||
if bash "$script_path"; then
|
if bash "$script_path"; then
|
||||||
log "$script_name — done ✅"
|
log "$script_name — done in $(format_duration $(( $(date +%s) - _ss ))) ✅"
|
||||||
PASSED+=("$script_name")
|
PASSED+=("$script_name")
|
||||||
else
|
else
|
||||||
warn "$script_name — exit non-zero (issues found or fixed) — continuing"
|
warn "$script_name — exit non-zero in $(format_duration $(( $(date +%s) - _ss ))) (issues found or fixed) — continuing"
|
||||||
FAILED+=("$script_name")
|
FAILED+=("$script_name")
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -165,6 +165,9 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Clear Logs ━━━
|
# ━━━ Clear Logs ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
log "$ICON_GEAR Config: system-threshold=${LOG_MIN_SIZE_MB:-10}MB docker-threshold=${LOG_DOCKER_MAX_MB:-100}MB"
|
||||||
|
log "$ICON_GEAR System logs: ${LOG_FILES[*]}"
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
SYS_CLEARED=0
|
SYS_CLEARED=0
|
||||||
SYS_SKIPPED=0
|
SYS_SKIPPED=0
|
||||||
|
|||||||
@@ -132,7 +132,10 @@ if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
|
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
|
||||||
|
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
|
||||||
|
MOVER_ELAPSED=$(ps -o etimes= -p "$MOVER_PID" 2>/dev/null | tr -d ' ')
|
||||||
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||||
|
log "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
|
||||||
|
|
||||||
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
|||||||
@@ -226,6 +226,9 @@ else
|
|||||||
log "Verified: pm.max_children = $APPLIED_VAL ✅"
|
log "Verified: pm.max_children = $APPLIED_VAL ✅"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
FPM_WORKERS=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
|
||||||
|
log "$ICON_PHP Workers running: $FPM_WORKERS"
|
||||||
|
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
@@ -194,6 +194,14 @@ if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||||
|
log "$ICON_CONTAINERS Docker: ${CONTAINER_COUNT} container(s) running"
|
||||||
|
|
||||||
|
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
||||||
|
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
||||||
|
log "$ICON_GEAR VMs: ${VM_COUNT} running"
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||||
log "Pre-flight clean — no active processes to warn about"
|
log "Pre-flight clean — no active processes to warn about"
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ detect_hosts
|
|||||||
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
|
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
|
||||||
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
|
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
|
||||||
|
|
||||||
|
log "$ICON_GEAR Conf file: $CONF_FILE"
|
||||||
|
log "$ICON_GEAR Key var: $VAR_NAME"
|
||||||
|
|
||||||
if [[ ! -f "$CONF_FILE" ]]; then
|
if [[ ! -f "$CONF_FILE" ]]; then
|
||||||
error "Conf file not found: $CONF_FILE"
|
error "Conf file not found: $CONF_FILE"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -54,15 +57,19 @@ fi
|
|||||||
# Check if key already exists in the unraid-api registry before creating.
|
# Check if key already exists in the unraid-api registry before creating.
|
||||||
# --overwrite generates a new key value every time, invalidating the old one.
|
# --overwrite generates a new key value every time, invalidating the old one.
|
||||||
# Only renew if the registry has lost it.
|
# Only renew if the registry has lost it.
|
||||||
|
log "Checking unraid-api registry for existing Varaverk key..."
|
||||||
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "Varaverk" --json </dev/null 2>/dev/null)
|
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "Varaverk" --json </dev/null 2>/dev/null)
|
||||||
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
|
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
|
||||||
|
|
||||||
if [[ -n "$KEY" ]]; then
|
if [[ -n "$KEY" ]]; then
|
||||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||||
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||||||
|
log "Key found in registry — no renewal needed"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
log "Key not found in registry — creating new key..."
|
||||||
|
|
||||||
RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
|
RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
|
||||||
--name "Varaverk" --create --overwrite \
|
--name "Varaverk" --create --overwrite \
|
||||||
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
|
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
|
||||||
@@ -87,4 +94,5 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||||
|
log "Writing new key to: $CONF_FILE"
|
||||||
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW (registry had lost it)"
|
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW (registry had lost it)"
|
||||||
|
|||||||
Reference in New Issue
Block a user