massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+766
-194
File diff suppressed because it is too large
Load Diff
@@ -1,64 +1,93 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Bulk Permissions Repair ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Applies correct permissions to a single share or specific path.
|
||||
# Faster than running media_shares_permissions.sh which processes all shares.
|
||||
# Use when a specific share has wrong ownership or permissions after:
|
||||
# - A failed transfer that left files owned by wrong user
|
||||
# - A container writing files as root instead of nobody:users
|
||||
# - Manual file operations that bypassed normal permission handling
|
||||
# ==============================================================================================
|
||||
# ============================= Bulk Permissions Repair ========================================
|
||||
# ==============================================================================================
|
||||
# Applies correct ownership and permissions to one or more specific paths.
|
||||
# Faster than running media_shares_permissions.sh which processes all configured shares.
|
||||
#
|
||||
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
|
||||
# Use for targeted repair after:
|
||||
# - A failed transfer that left files owned by wrong user (root:root from rsync)
|
||||
# - A container writing as root instead of nobody:users — before PUID/PGID was fixed
|
||||
# - Manual file copies that bypassed normal permission handling
|
||||
# - A new share that needs permissions applied before the next nightly run
|
||||
# - A large rsync that imported thousands of files before media_shares_permissions.sh ran
|
||||
#
|
||||
# Usage:
|
||||
# ── PERMISSIONS MODEL ─────────────────────────────────────────────────────────────────────────
|
||||
# Directories: PERMISSIONS_DIR_MODE (default 755)
|
||||
# Owner (nobody) — rwx enter, list, create files
|
||||
# Group (users) — r-x enter and list
|
||||
# Others — r-x Samba guests can browse
|
||||
#
|
||||
# Files: PERMISSIONS_FILE_MODE (default 664)
|
||||
# Owner (nobody) — rw read + write
|
||||
# Group (users) — rw arrs can import and rename
|
||||
# Others — r Samba guests can read
|
||||
# No execute bit — media files are never executable
|
||||
#
|
||||
# ── DIAGNOSTIC — HIGH WRONG OWNER COUNT ───────────────────────────────────────────────────────
|
||||
# This script counts files with wrong ownership before applying the fix.
|
||||
# A high count on a share that was recently written → a container has wrong PUID/PGID.
|
||||
# Fix: add PUID=99 PGID=100 to the container's Docker template.
|
||||
# Common culprits: SABnzbd, qBittorrent, slskd.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — required for chown
|
||||
# Path existence check — skips missing paths with error
|
||||
# Separate passes — directories and files chmod'd separately for correctness
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent on success — only failures produce visible output
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# bulk_permissions_repair.sh /mnt/user/Movies
|
||||
# bulk_permissions_repair.sh /mnt/user/Movies --dry-run
|
||||
# bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows
|
||||
#
|
||||
# Uses PERMISSIONS_MODE and PERMISSIONS_OWNER from Master.conf.
|
||||
# Supports --dry-run to show what would be changed without applying.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# bulk_permissions_repair.sh /mnt/user/Movies --dry-run
|
||||
# bulk_permissions_repair.sh /mnt/user/Movies --log
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — chown requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
if [[ ${#PARSED_ARGS[@]} -eq 0 ]]; then
|
||||
error "No paths specified"
|
||||
error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path]"
|
||||
error " bulk_permissions_repair.sh /mnt/user/Movies --dry-run"
|
||||
error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path] [--dry-run]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Mode: $PERMISSIONS_MODE"
|
||||
info "Owner: $PERMISSIONS_OWNER"
|
||||
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
log "Owner: $PERMISSIONS_OWNER"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permissions will be changed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_PERMS Apply Permissions ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Permissions ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Permissions Repair ━━━"
|
||||
echo "━━━ $ICON_PERMS Permissions Repair — $MY_ID ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
TOTAL_WRONG_OWNER=0
|
||||
|
||||
for share_path in "${PARSED_ARGS[@]}"; do
|
||||
[[ -z "$share_path" ]] && continue
|
||||
@@ -72,62 +101,94 @@ for share_path in "${PARSED_ARGS[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Count files for progress context
|
||||
# Count files for context — warn level so user knows what they're in for on large shares
|
||||
FILE_COUNT=$(find "$share_path" -type f 2>/dev/null | wc -l)
|
||||
DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l)
|
||||
DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l)
|
||||
SIZE=$(du -sh "$share_path" 2>/dev/null | cut -f1)
|
||||
warn "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)"
|
||||
|
||||
info "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)"
|
||||
# Count files with wrong ownership before fixing — diagnostic
|
||||
WRONG_OWNER=$(find "$share_path" \( ! -user nobody -o ! -group users \) \
|
||||
2>/dev/null | wc -l)
|
||||
if [[ "$WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "$WRONG_OWNER file(s) with wrong ownership — fixing..."
|
||||
if [[ "$WRONG_OWNER" -gt 500 ]]; then
|
||||
warn "High wrong-owner count — check container PUID/PGID settings (should be PUID=99 PGID=100)"
|
||||
warn "Common culprits: SABnzbd, qBittorrent, slskd"
|
||||
fi
|
||||
TOTAL_WRONG_OWNER=$(( TOTAL_WRONG_OWNER + WRONG_OWNER ))
|
||||
else
|
||||
log "Ownership already correct — applying mode only"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would apply: chmod -R $PERMISSIONS_MODE $share_path"
|
||||
warn "DRY RUN — would apply: chown -R $PERMISSIONS_OWNER $share_path"
|
||||
warn "DRY RUN — would apply:"
|
||||
warn " chown -R $PERMISSIONS_OWNER $share_path"
|
||||
warn " find -type d → chmod ${PERMISSIONS_DIR_MODE:-755}"
|
||||
warn " find -type f → chmod ${PERMISSIONS_FILE_MODE:-664}"
|
||||
PASS+=("$(basename "$share_path")")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Apply ownership first — chmod after so files are owned correctly before mode change
|
||||
info "Applying ownership: $PERMISSIONS_OWNER..."
|
||||
chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null
|
||||
CHOWN_EXIT=$?
|
||||
CHOWN_OK=true
|
||||
CHMOD_DIR_OK=true
|
||||
CHMOD_FILE_OK=true
|
||||
|
||||
info "Applying permissions: $PERMISSIONS_MODE..."
|
||||
chmod -R "$PERMISSIONS_MODE" "$share_path" 2>/dev/null
|
||||
CHMOD_EXIT=$?
|
||||
# Apply ownership first
|
||||
log "Applying ownership: $PERMISSIONS_OWNER..."
|
||||
chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
if [[ "$CHOWN_EXIT" -eq 0 && "$CHMOD_EXIT" -eq 0 ]]; then
|
||||
success "$ICON_UNLOCKED $(basename "$share_path") — permissions applied"
|
||||
# Apply directory permissions — separate pass (dirs need execute bit)
|
||||
log "Applying directory permissions: ${PERMISSIONS_DIR_MODE:-755}..."
|
||||
find "$share_path" -type d \
|
||||
-exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + 2>/dev/null || CHMOD_DIR_OK=false
|
||||
|
||||
# Apply file permissions — no execute bit on media files
|
||||
log "Applying file permissions: ${PERMISSIONS_FILE_MODE:-664}..."
|
||||
find "$share_path" -type f \
|
||||
-exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>/dev/null || CHMOD_FILE_OK=false
|
||||
|
||||
if [[ "$CHOWN_OK" == true && "$CHMOD_DIR_OK" == true && "$CHMOD_FILE_OK" == true ]]; then
|
||||
log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
|
||||
PASS+=("$(basename "$share_path")")
|
||||
else
|
||||
error "$(basename "$share_path") — permission repair failed"
|
||||
error "$(basename "$share_path") — repair failed"
|
||||
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
|
||||
FAIL+=("$(basename "$share_path")")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PERMISSIONS REPAIR SUMMARY ━━━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Pass: ${#PASS[@]} $ICON_ERROR Fail: ${#FAIL[@]}"
|
||||
echo ""
|
||||
[[ ${#PASS[@]} -gt 0 ]] && for p in "${PASS[@]}"; do echo " $ICON_UNLOCKED $p"; done
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && for f in "${FAIL[@]}"; do echo " $ICON_ERROR $f"; done
|
||||
[[ ${#PASS[@]} -gt 0 ]] && log "Pass: ${PASS[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Fail: ${FAIL[*]}"
|
||||
|
||||
if [[ "$TOTAL_WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "Total wrong-owner files fixed: $TOTAL_WRONG_OWNER"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME REPAIRS FAILED"
|
||||
notify "Permissions repair failed on $(hostname) — failed shares: ${FAIL[*]}" "Permissions Repair" "warning"
|
||||
notify "Permissions repair failed on $(hostname) — ${FAIL[*]}" \
|
||||
"Permissions Repair" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Permissions repair complete on $(hostname) — ${#PASS[@]} share(s) repaired" "Permissions Repair" "normal"
|
||||
log "$ICON_DONE Status: done — ${#PASS[@]} path(s) repaired"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+199
-84
@@ -1,50 +1,79 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Container Data Export --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Container Data Export ==========================================
|
||||
# ==============================================================================================
|
||||
# Exports a container's appdata directory to a compressed tar archive.
|
||||
# Stops the container before archiving and restarts it after — ensures clean consistent backup.
|
||||
# Verifies the archive after creation — confirms backup is valid before restarting container.
|
||||
#
|
||||
# Usage:
|
||||
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
|
||||
# - Before major container updates (roll back if update goes wrong)
|
||||
# - Before pool migrations or disk replacements
|
||||
# - When archiving a container being removed from the stack
|
||||
# - Before destructive operations on appdata (database migrations etc.)
|
||||
# - One-off backup of a specific container without running full backup
|
||||
#
|
||||
# ── OUTPUT FILE NAMING ────────────────────────────────────────────────────────────────────────
|
||||
# ContainerName_YYYY-MM-DD_HH-MM.tar.gz
|
||||
# Timestamp in filename — run multiple times safely, no overwrite ✅
|
||||
#
|
||||
# ── SPACE CHECK ───────────────────────────────────────────────────────────────────────────────
|
||||
# Estimates required space as appdata size × 1.1 (10% buffer).
|
||||
# Compressed archive will typically be much smaller — this is a conservative floor.
|
||||
# gzip compression ratio depends heavily on content — database files compress well,
|
||||
# media files do not. If output is on a media share estimate may be pessimistic.
|
||||
#
|
||||
# ── ARCHIVE VERIFICATION ──────────────────────────────────────────────────────────────────────
|
||||
# After creation the archive is tested with tar --test-file before restarting the container.
|
||||
# If verification fails the container is still restarted (data unchanged) and an error logged.
|
||||
# A corrupt archive is not a usable backup — do not assume the archive is good without this.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER_TIMEOUT — docker calls protected against hung daemon
|
||||
# Container restart rule — was running → restart | was stopped → leave stopped ✅
|
||||
# Archive cleanup — partial archive removed on tar failure
|
||||
# Archive verification — tar --test-file after creation
|
||||
# Container restart on — any failure path still restarts container if it was running
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent on success — only problems produce visible output
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir
|
||||
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/
|
||||
#
|
||||
# Output file naming:
|
||||
# ContainerName_YYYY-MM-DD_HH-MM.tar.gz
|
||||
#
|
||||
# Use before major container updates, pool migrations, or when archiving
|
||||
# a container you are removing from the stack.
|
||||
#
|
||||
# Supports --dry-run to show what would be archived without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run
|
||||
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --log
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Args
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
DOCKER_TIMEOUT=30 # longer timeout — stop can take time on large containers
|
||||
|
||||
# ── Positional args ───────────────────────────────────────────────────────────────────────────
|
||||
CONTAINER_NAME="${PARSED_ARGS[0]:-}"
|
||||
APPDATA_PATH="${PARSED_ARGS[1]:-}"
|
||||
OUTPUT_DIR="${PARSED_ARGS[2]:-}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
# Arg validation
|
||||
if [[ -z "$CONTAINER_NAME" || -z "$APPDATA_PATH" || -z "$OUTPUT_DIR" ]]; then
|
||||
error "Usage: container_data_export.sh <ContainerName> <appdata_path> <output_dir>"
|
||||
error "Example: container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/"
|
||||
@@ -58,55 +87,94 @@ fi
|
||||
|
||||
if [[ ! -d "$OUTPUT_DIR" ]]; then
|
||||
error "Output directory not found: $OUTPUT_DIR"
|
||||
error "Create it first: mkdir -p \"$OUTPUT_DIR\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check free space — rough estimate: appdata size × 1.1
|
||||
# Space check — conservative: appdata × 1.1
|
||||
APPDATA_SIZE_KB=$(du -sk "$APPDATA_PATH" 2>/dev/null | cut -f1)
|
||||
OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ')
|
||||
OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
REQUIRED_KB=$(( APPDATA_SIZE_KB * 11 / 10 ))
|
||||
|
||||
APPDATA_SIZE_H=$(du -sh "$APPDATA_PATH" 2>/dev/null | cut -f1)
|
||||
OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ')
|
||||
|
||||
info "Container: $CONTAINER_NAME"
|
||||
info "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
info "Output dir: $OUTPUT_DIR ($OUTPUT_FREE_H free)"
|
||||
OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
|
||||
if [[ "$OUTPUT_FREE_KB" -lt "$REQUIRED_KB" ]]; then
|
||||
error "Insufficient space in $OUTPUT_DIR — need ~${APPDATA_SIZE_H}, have ${OUTPUT_FREE_H}"
|
||||
error "Insufficient space in $OUTPUT_DIR"
|
||||
error "Estimated need: ~${APPDATA_SIZE_H} (×1.1 conservative) — available: ${OUTPUT_FREE_H}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Space check passed"
|
||||
log "Container: $CONTAINER_NAME"
|
||||
log "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
log "Output: $OUTPUT_DIR ($OUTPUT_FREE_H free)"
|
||||
log "Space check passed"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP Stop Container ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Ensure container is restarted on any exit if it was running ────────────────────────────────
|
||||
CONTAINER_WAS_RUNNING=false
|
||||
ARCHIVE_PATH=""
|
||||
|
||||
cleanup_on_exit() {
|
||||
local exit_code=$?
|
||||
# Remove partial archive on failure
|
||||
if [[ "$exit_code" -ne 0 && -n "$ARCHIVE_PATH" && -f "$ARCHIVE_PATH" ]]; then
|
||||
warn "Removing partial archive: $ARCHIVE_PATH"
|
||||
rm -f "$ARCHIVE_PATH" 2>/dev/null
|
||||
fi
|
||||
# Always restart container if it was running
|
||||
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$CONTAINER_NAME" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "Restarting $CONTAINER_NAME (cleanup)..."
|
||||
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
|
||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap cleanup_on_exit EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Container ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Container ━━━"
|
||||
|
||||
CONTAINER_WAS_RUNNING=false
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$CONTAINER_NAME" 2>/dev/null)
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
CONTAINER_WAS_RUNNING=true
|
||||
info "$ICON_STOP Stopping $CONTAINER_NAME for clean export..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
docker stop "$CONTAINER_NAME" >/dev/null 2>&1 && \
|
||||
success "$ICON_STOPPED $CONTAINER_NAME stopped" || \
|
||||
{ error "Failed to stop $CONTAINER_NAME"; exit 1; }
|
||||
else
|
||||
warn "DRY RUN — would stop $CONTAINER_NAME"
|
||||
fi
|
||||
else
|
||||
info "$CONTAINER_NAME is not running — archiving as-is"
|
||||
fi
|
||||
case "$STATUS" in
|
||||
true)
|
||||
CONTAINER_WAS_RUNNING=true
|
||||
log "Stopping $CONTAINER_NAME for clean export..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
log "$CONTAINER_NAME stopped ✅"
|
||||
else
|
||||
error "Failed to stop $CONTAINER_NAME — aborting export"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop $CONTAINER_NAME"
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
|
||||
;;
|
||||
"")
|
||||
error "$CONTAINER_NAME not found — check container name"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
|
||||
;;
|
||||
esac
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Archive ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Archive ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Archive ━━━"
|
||||
|
||||
@@ -114,60 +182,107 @@ TIMESTAMP=$(date '+%Y-%m-%d_%H-%M')
|
||||
ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz"
|
||||
ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"
|
||||
|
||||
info "Creating: $ARCHIVE_PATH"
|
||||
warn "Creating: $ARCHIVE_PATH"
|
||||
warn "Source: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
|
||||
START=$(date +%s)
|
||||
ARCHIVE_VERIFIED=false
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
tar -czf "$ARCHIVE_PATH" -C "$(dirname "$APPDATA_PATH")" "$(basename "$APPDATA_PATH")" 2>/dev/null
|
||||
TAR_EXIT=$?
|
||||
if tar -czf "$ARCHIVE_PATH" \
|
||||
-C "$(dirname "$APPDATA_PATH")" \
|
||||
"$(basename "$APPDATA_PATH")" 2>/dev/null; then
|
||||
|
||||
if [[ "$TAR_EXIT" -ne 0 ]]; then
|
||||
error "Archive failed (exit code $TAR_EXIT)"
|
||||
# Restart container before exiting
|
||||
[[ "$CONTAINER_WAS_RUNNING" == true ]] && docker start "$CONTAINER_NAME" >/dev/null 2>&1
|
||||
ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
|
||||
warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
|
||||
|
||||
# Verify archive integrity before declaring success
|
||||
log "Verifying archive..."
|
||||
if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
|
||||
tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
|
||||
log "Archive verified ✅"
|
||||
ARCHIVE_VERIFIED=true
|
||||
else
|
||||
error "Archive verification FAILED — archive may be corrupt"
|
||||
error "Container will be restarted but DO NOT rely on this backup"
|
||||
notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
error "tar failed — archive creation unsuccessful"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
|
||||
success "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
|
||||
else
|
||||
warn "DRY RUN — would create: $ARCHIVE_PATH"
|
||||
ARCHIVE_VERIFIED=true
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_START Restart Container ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Restart Container ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Restart Container ━━━"
|
||||
|
||||
RESTART_OK=false
|
||||
|
||||
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
|
||||
info "$ICON_START Restarting $CONTAINER_NAME..."
|
||||
log "Restarting $CONTAINER_NAME..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
docker start "$CONTAINER_NAME" >/dev/null 2>&1 && \
|
||||
success "$ICON_STARTED $CONTAINER_NAME restarted" || \
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
# Brief settle then verify
|
||||
sleep 3
|
||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
|
||||
if [[ "$POST_STATUS" == "true" ]]; then
|
||||
log "$CONTAINER_NAME restarted and running ✅"
|
||||
RESTART_OK=true
|
||||
else
|
||||
error "$CONTAINER_NAME started but crashed immediately — check container logs"
|
||||
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
||||
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would restart $CONTAINER_NAME"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
else
|
||||
info "$CONTAINER_NAME was not running — not restarting"
|
||||
log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Clear trap — clean exit, cleanup_on_exit no longer needed
|
||||
trap - EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CONTAINER EXPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_CONTAINERS Container: $CONTAINER_NAME"
|
||||
echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Container: $CONTAINER_NAME"
|
||||
echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}"
|
||||
echo "$ICON_SHIELD Verified: $([[ "$ARCHIVE_VERIFIED" == true ]] && echo "✅" || echo "❌ FAILED")"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$ARCHIVE_VERIFIED" == true && "$RESTART_OK" == true ]]; then
|
||||
log "$ICON_DONE Status: done — $ARCHIVE_NAME"
|
||||
elif [[ "$ARCHIVE_VERIFIED" == false ]]; then
|
||||
echo "$ICON_ERROR Status: archive verification FAILED — check backup before relying on it"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Container export complete — $CONTAINER_NAME archived to $ARCHIVE_NAME" "Container Export" "normal"
|
||||
warn "Status: complete with warnings — check restart status above"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$ARCHIVE_VERIFIED" == false ]] && exit 1
|
||||
exit 0
|
||||
+234
-117
@@ -1,122 +1,199 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Emby Database Repair ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Emby Database Repair ===========================================
|
||||
# ==============================================================================================
|
||||
# Stops Emby, runs SQLite integrity checks on all Emby databases, and restarts.
|
||||
# Use when Emby reports database corruption, unexpected crashes, or playback state issues.
|
||||
#
|
||||
# Checks performed:
|
||||
# integrity_check — full SQLite integrity verification per database file
|
||||
# quick_check — faster check for common corruption patterns
|
||||
# ── CHECKS PERFORMED ──────────────────────────────────────────────────────────────────────────
|
||||
# PRAGMA integrity_check — full SQLite integrity verification per database
|
||||
# Skips missing databases gracefully — not all files exist on all setups
|
||||
#
|
||||
# If corruption is found:
|
||||
# Reports which database files are corrupted
|
||||
# Does NOT automatically repair — corruption repair requires manual steps
|
||||
# Provides guidance on next steps per database type
|
||||
# ── DATABASES CHECKED ─────────────────────────────────────────────────────────────────────────
|
||||
# library.db — media library metadata (largest, most critical)
|
||||
# library.db-wal — write-ahead log (if exists — uncommitted transactions)
|
||||
# librarydb.db — legacy library database
|
||||
# users.db — user accounts and settings
|
||||
# authentication.db — API keys and sessions
|
||||
# activity.db — activity log (least critical, safe to delete)
|
||||
#
|
||||
# Emby database files checked:
|
||||
# library.db — media library metadata
|
||||
# library.db-wal — write-ahead log (if exists)
|
||||
# librarydb.db — legacy library database
|
||||
# users.db — user accounts and settings
|
||||
# authentication.db — API keys and sessions
|
||||
# activity.db — activity log
|
||||
# ── IF CORRUPTION FOUND ───────────────────────────────────────────────────────────────────────
|
||||
# Reports which databases are corrupted. Does NOT automatically repair.
|
||||
# Corruption repair requires manual steps — see guidance in summary output.
|
||||
# Always take a backup before deleting any database file.
|
||||
#
|
||||
# HOST1 repairs its own Emby (HOST1_EMBY_CONTAINER).
|
||||
# HOST2 repairs its own Emby (HOST2_EMBY_CONTAINER).
|
||||
# detect_hosts() selects the correct container at runtime.
|
||||
# Container name defined in Host Configuration in Master.conf.
|
||||
# Supports --dry-run to show what would be checked without stopping Emby.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER.
|
||||
# Each server checks its own Emby instance automatically.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# EXIT trap — Emby always restarted even if script crashes mid-check
|
||||
# DOCKER_TIMEOUT — all docker calls protected against hung daemon
|
||||
# jq validation — verifies jq available before config path detection
|
||||
# validate_unraid_cmd — sqlite3 and notify validated before use
|
||||
# Container verify — checks Emby stayed running after restart
|
||||
# Silent healthy — only corruption produces visible output
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# emby_database_repair.sh — stop Emby, check all databases, restart
|
||||
# emby_database_repair.sh --dry-run — show what would be checked, no Emby stop
|
||||
# emby_database_repair.sh --log — verbose output per database
|
||||
# emby_database_repair.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
DOCKER_TIMEOUT=30 # Emby can take time to stop cleanly
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
# Validate required tools
|
||||
validate_unraid_cmd \
|
||||
"$(command -v sqlite3 2>/dev/null || echo /usr/bin/sqlite3)" \
|
||||
"--version" "." \
|
||||
"sqlite3" || { error "sqlite3 not found — install sqlite package"; exit 1; }
|
||||
|
||||
acquire_lock
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# Select correct Emby container based on which server is running this script
|
||||
detect_hosts
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
EMBY_CONTAINER="$HOST1_EMBY_CONTAINER"
|
||||
else
|
||||
EMBY_CONTAINER="$HOST2_EMBY_CONTAINER"
|
||||
fi
|
||||
|
||||
info "Emby container: $LOCAL_SERVER_NAME → $EMBY_CONTAINER"
|
||||
|
||||
if ! command -v sqlite3 >/dev/null 2>&1; then
|
||||
error "sqlite3 not found — install sqlite package"
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required to detect Emby config path from Docker mounts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "sqlite3 available"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped"
|
||||
acquire_lock
|
||||
|
||||
# Detect Emby config path from Docker mount
|
||||
EMBY_CONFIG_HOST=$(docker inspect "$EMBY_CONTAINER" 2>/dev/null | \
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "${EMBY_CONTAINER:-}" ]]; then
|
||||
error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in master_host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Emby container: $EMBY_CONTAINER"
|
||||
|
||||
# Detect Emby config path from Docker container mounts
|
||||
EMBY_CONFIG_HOST=$(timeout "$DOCKER_TIMEOUT" docker inspect "$EMBY_CONTAINER" 2>/dev/null | \
|
||||
jq -r '.[] | .Mounts[] | select(.Destination == "/config") | .Source' 2>/dev/null)
|
||||
|
||||
if [[ -z "$EMBY_CONFIG_HOST" ]]; then
|
||||
error "Could not detect Emby config path from Docker mounts"
|
||||
error "Make sure $EMBY_CONTAINER is the correct container name in Master.conf"
|
||||
error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in master_host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Emby config path: $EMBY_CONFIG_HOST"
|
||||
log "Emby config: $EMBY_CONFIG_HOST"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP Stop Emby ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped, no checks run"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
|
||||
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
echo "━━━ Database Files ━━━"
|
||||
for db_rel in "data/library.db" "data/library.db-wal" "data/librarydb.db" \
|
||||
"data/users.db" "data/authentication.db" "data/activity.db"; do
|
||||
db_path="${EMBY_CONFIG_HOST}/${db_rel}"
|
||||
db_name=$(basename "$db_rel")
|
||||
if [[ -f "$db_path" ]]; then
|
||||
db_size=$(du -sh "$db_path" 2>/dev/null | cut -f1)
|
||||
echo " $ICON_SUCCESS $db_name ($db_size)"
|
||||
else
|
||||
echo " $ICON_SKIP $db_name — not found"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── EXIT trap — Emby always restarted if it was running ───────────────────────────────────────
|
||||
EMBY_WAS_RUNNING=false
|
||||
|
||||
cleanup_on_exit() {
|
||||
local exit_code=$?
|
||||
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$EMBY_CONTAINER" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "Restarting $EMBY_CONTAINER (cleanup)..."
|
||||
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
|
||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup_on_exit EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Emby ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Emby ━━━"
|
||||
|
||||
EMBY_WAS_RUNNING=false
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$EMBY_CONTAINER" 2>/dev/null)
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
EMBY_WAS_RUNNING=true
|
||||
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
docker stop "$EMBY_CONTAINER" >/dev/null 2>&1 && \
|
||||
success "$ICON_STOPPED $EMBY_CONTAINER stopped" || \
|
||||
{ error "Failed to stop $EMBY_CONTAINER"; exit 1; }
|
||||
sleep 3 # brief wait for file handles to release
|
||||
else
|
||||
warn "DRY RUN — would stop $EMBY_CONTAINER"
|
||||
fi
|
||||
else
|
||||
info "$EMBY_CONTAINER is not running — proceeding with checks"
|
||||
fi
|
||||
case "$STATUS" in
|
||||
true)
|
||||
EMBY_WAS_RUNNING=true
|
||||
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
||||
log "$EMBY_CONTAINER stopped ✅"
|
||||
sleep 3 # let file handles release
|
||||
else
|
||||
error "Failed to stop $EMBY_CONTAINER — aborting"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop $EMBY_CONTAINER"
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
log "$EMBY_CONTAINER is not running — proceeding with checks"
|
||||
;;
|
||||
"")
|
||||
error "$EMBY_CONTAINER not found — check container name"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
|
||||
;;
|
||||
esac
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_HEALTH Database Integrity Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Database Integrity Check ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Database Integrity Check ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Emby database files to check
|
||||
DB_FILES=(
|
||||
"data/library.db"
|
||||
"data/library.db-wal"
|
||||
"data/librarydb.db"
|
||||
"data/users.db"
|
||||
"data/authentication.db"
|
||||
@@ -138,25 +215,38 @@ for db_rel in "${DB_FILES[@]}"; do
|
||||
fi
|
||||
|
||||
DB_SIZE=$(du -sh "$db_path" 2>/dev/null | cut -f1)
|
||||
info "$ICON_HEALTH Checking $db_name ($DB_SIZE)..."
|
||||
log "Checking $db_name ($DB_SIZE)..."
|
||||
|
||||
# WAL file — different check (not a full SQLite database)
|
||||
if [[ "$db_name" == "*.wal" || "$db_name" == "library.db-wal" ]]; then
|
||||
if [[ -s "$db_path" ]]; then
|
||||
warn "$db_name exists and is non-empty (${DB_SIZE})"
|
||||
warn "Uncommitted WAL data — will be merged when Emby next starts cleanly"
|
||||
PASS_DBS+=("$db_name (WAL — see warning)")
|
||||
else
|
||||
log "$db_name exists but is empty — no pending transactions ✅"
|
||||
PASS_DBS+=("$db_name")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would check: $db_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Run integrity check
|
||||
# Full integrity check
|
||||
RESULT=$(sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null)
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
error "$db_name — sqlite3 could not open database (may be locked or corrupt)"
|
||||
error "$db_name — sqlite3 could not open database (locked or corrupt)"
|
||||
FAIL_DBS+=("$db_name")
|
||||
elif [[ "$RESULT" == "ok" ]]; then
|
||||
success "$db_name — integrity check passed"
|
||||
log "$db_name — integrity check passed ✅"
|
||||
PASS_DBS+=("$db_name")
|
||||
else
|
||||
error "$db_name — integrity check FAILED"
|
||||
error "$db_name — CORRUPTION DETECTED"
|
||||
echo "$RESULT" | head -10 | while IFS= read -r line; do
|
||||
error " $line"
|
||||
done
|
||||
@@ -166,61 +256,88 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_START Restart Emby ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Restart Emby ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Restart Emby ━━━"
|
||||
|
||||
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
||||
docker start "$EMBY_CONTAINER" >/dev/null 2>&1 && \
|
||||
success "$ICON_STARTED $EMBY_CONTAINER restarted" || \
|
||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
||||
elif [[ "$DRY_RUN" == true && "$EMBY_WAS_RUNNING" == true ]]; then
|
||||
warn "DRY RUN — would restart $EMBY_CONTAINER"
|
||||
RESTART_OK=false
|
||||
|
||||
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
log "Restarting $EMBY_CONTAINER..."
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
||||
sleep 5 # Emby takes longer to initialise than most containers
|
||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
|
||||
if [[ "$POST_STATUS" == "true" ]]; then
|
||||
log "$EMBY_CONTAINER restarted and running ✅"
|
||||
RESTART_OK=true
|
||||
else
|
||||
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
|
||||
error "Check Docker logs: docker logs $EMBY_CONTAINER"
|
||||
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
|
||||
"Emby DB Repair" "warning"
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
||||
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
|
||||
"Emby DB Repair" "warning"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would restart $EMBY_CONTAINER"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
else
|
||||
info "$EMBY_CONTAINER was not running — not restarting"
|
||||
log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Clear EXIT trap — clean exit
|
||||
trap - EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY EMBY DATABASE REPAIR SUMMARY ━━━━━"
|
||||
echo "$ICON_HEALTH Container: $EMBY_CONTAINER"
|
||||
echo "$ICON_HEALTH Config path: $EMBY_CONFIG_HOST"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
|
||||
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]} $ICON_ERROR Failed: ${#FAIL_DBS[@]} $ICON_INFO Missing: ${#MISSING_DBS[@]}"
|
||||
echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]}"
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAIL_DBS[@]}"
|
||||
[[ ${#MISSING_DBS[@]} -gt 0 ]] && log "Skipped: ${#MISSING_DBS[@]} (not found)"
|
||||
echo ""
|
||||
|
||||
if [[ ${#PASS_DBS[@]} -gt 0 ]]; then
|
||||
for db in "${PASS_DBS[@]}"; do echo " $ICON_SUCCESS $db"; done
|
||||
fi
|
||||
if [[ ${#FAIL_DBS[@]} -gt 0 ]]; then
|
||||
for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
|
||||
fi
|
||||
[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log " $ICON_SUCCESS $db"; done
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
|
||||
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no checks performed"
|
||||
warn "DRY RUN — no checks performed"
|
||||
elif [[ ${#FAIL_DBS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: CORRUPTION FOUND"
|
||||
echo "$ICON_ERROR Status: CORRUPTION FOUND — manual intervention needed"
|
||||
echo ""
|
||||
echo "$ICON_INFO Next steps for corrupted databases:"
|
||||
echo " library.db — Stop Emby, delete library.db, restart"
|
||||
echo " Emby will rebuild from media files (slow first start)"
|
||||
echo " users.db — Stop Emby, restore from backup or delete"
|
||||
echo " Deleting resets all user accounts"
|
||||
echo " authentication.db — Stop Emby, delete, restart"
|
||||
echo " API keys and sessions will be regenerated"
|
||||
echo " activity.db — Stop Emby, delete, restart — activity log only"
|
||||
echo "$ICON_INFO Next steps per corrupted database:"
|
||||
echo " library.db — Delete file, restart Emby — rebuilds from media (slow first start)"
|
||||
echo " library.db-wal — Delete WAL file, restart Emby — safe, no permanent data loss"
|
||||
echo " librarydb.db — Delete file, restart Emby — legacy, Emby recreates"
|
||||
echo " users.db — Restore from backup or delete — deleting resets all user accounts"
|
||||
echo " authentication.db — Delete file, restart Emby — API keys regenerated automatically"
|
||||
echo " activity.db — Delete file, restart Emby — activity log only, no media data"
|
||||
echo ""
|
||||
echo "$ICON_WARN Always take a backup before deleting any database file"
|
||||
notify "Emby database corruption found on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" "Emby DB Repair" "warning"
|
||||
warn "⚠️ Always take a backup before deleting any database file"
|
||||
warn " Run: container_data_export.sh $EMBY_CONTAINER <config_path> <backup_dir>"
|
||||
notify "Emby database CORRUPTION on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" \
|
||||
"Emby DB Repair" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DATABASES HEALTHY"
|
||||
notify "Emby database integrity check passed on $(hostname) — ${#PASS_DBS[@]} databases healthy" "Emby DB Repair" "normal"
|
||||
log "$ICON_DONE Status: all ${#PASS_DBS[@]} databases healthy ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+168
-66
@@ -1,127 +1,229 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Failover State Reset ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Failover State Reset ===========================================
|
||||
# ==============================================================================================
|
||||
# Resets the failover state file to NORMAL and clears all tier flags.
|
||||
# Use when the failover state file is stuck in a non-NORMAL state after testing,
|
||||
# a failed handback, or manual intervention that left state inconsistent.
|
||||
# Use when the failover state file is stuck in a non-NORMAL state after:
|
||||
# - Failover testing that left state as FAILOVER
|
||||
# - A failed handback that did not complete cleanly
|
||||
# - Manual intervention that left state inconsistent
|
||||
# - failover.sh was killed mid-cycle and state is unknown
|
||||
#
|
||||
# ── WHAT THIS DOES ────────────────────────────────────────────────────────────────────────────
|
||||
# Writes a fresh state file with:
|
||||
# state=NORMAL
|
||||
# failover_start=0
|
||||
# handback_strikes=0
|
||||
# tier2_started=false / tier3_started=false / tier4_started=false
|
||||
#
|
||||
# Does NOT start or stop any containers — state file only.
|
||||
# After reset, failover.sh will resume from NORMAL on its next cycle.
|
||||
#
|
||||
# ⚠️ Only run this when you have manually verified both servers are in their
|
||||
# correct states — right containers running on the right server, DDNS correct.
|
||||
# Resetting state without verifying the actual state can cause failover.sh
|
||||
# to make incorrect decisions on its next cycle.
|
||||
# ── ⚠️ ONLY RUN WHEN SAFE ────────────────────────────────────────────────────────────────────
|
||||
# Verify BEFORE resetting:
|
||||
# ✓ Right containers running on the right server
|
||||
# ✓ DDNS pointing at the correct server
|
||||
# ✓ No active failover actually in progress
|
||||
# ✓ Both servers can see each other
|
||||
#
|
||||
# Supports --dry-run to show what would be reset without changing anything.
|
||||
# Supports --status to show the current state file contents.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Resetting state while a real failover is happening causes failover.sh to stop
|
||||
# covering the remote server — services go offline until next detection cycle.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# failover.sh running check — warns if failover.sh is active when reset is attempted
|
||||
# acquire_lock — prevents concurrent resets
|
||||
# flock on state write — prevents race with failover.sh mid-cycle read
|
||||
# Confirmation required — interactive: type YES | non-interactive: --force flag
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# failover_state_reset.sh — interactive reset (prompts for YES)
|
||||
# failover_state_reset.sh --dry-run — show current state, show what would be written
|
||||
# failover_state_reset.sh --status — show current state file contents and exit
|
||||
# failover_state_reset.sh --force — non-interactive reset (no prompt, use in scripts)
|
||||
# failover_state_reset.sh --force --dry-run — dry run without prompt
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
# ── Handle --force flag before parse_args ─────────────────────────────────────────────────────
|
||||
FORCE=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--force) FORCE=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Current State ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary and notification
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Current State ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Current Failover State ━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SUMMARY Current State ━━━"
|
||||
|
||||
if [[ ! -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
warn "State file not found: $FAILOVER_STATE_FILE"
|
||||
warn "Will be created fresh on reset"
|
||||
CURRENT_STATE="NOT FOUND"
|
||||
else
|
||||
info "State file: $FAILOVER_STATE_FILE"
|
||||
log "State file: $FAILOVER_STATE_FILE"
|
||||
echo ""
|
||||
while IFS='=' read -r key value; do
|
||||
[[ -z "$key" ]] && continue
|
||||
echo " $ICON_INFO $key = $value"
|
||||
done < "$FAILOVER_STATE_FILE"
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
fi
|
||||
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
# Check if failover.sh is running — informational in status mode
|
||||
if pgrep -f "failover.sh" >/dev/null 2>&1; then
|
||||
warn "failover.sh is currently RUNNING — any reset would race with active cycle"
|
||||
else
|
||||
log "failover.sh is not running"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Confirmation ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
warn "$ICON_WARN This will reset the failover state to NORMAL"
|
||||
warn "Only proceed if you have verified both servers are in their correct states"
|
||||
warn " — Right containers running on the right server"
|
||||
warn " — DDNS pointing at the correct server"
|
||||
warn " — No active failover in progress"
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
# Check if failover.sh is actively running
|
||||
FAILOVER_RUNNING=false
|
||||
if pgrep -f "failover.sh" >/dev/null 2>&1; then
|
||||
FAILOVER_RUNNING=true
|
||||
warn "⚠️ failover.sh is currently RUNNING"
|
||||
warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration"
|
||||
warn "Consider stopping failover.sh first (click Abort in User Scripts)"
|
||||
warn "Then reset state, then restart failover.sh"
|
||||
echo ""
|
||||
warn "If you are sure you want to proceed anyway, confirm below"
|
||||
else
|
||||
log "failover.sh is not running — safe to reset ✅"
|
||||
fi
|
||||
|
||||
# Check current state — if already NORMAL warn user
|
||||
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
||||
warn "State is already NORMAL — reset may not be necessary"
|
||||
warn "Proceeding anyway (will refresh the state file)"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Confirmation ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
warn "This will reset failover state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
warn "Verify before proceeding:"
|
||||
warn " ✓ Right containers running on the right server"
|
||||
warn " ✓ DDNS pointing at correct server"
|
||||
warn " ✓ No real failover actually in progress"
|
||||
warn " ✓ Both servers can reach each other"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
read -r -p "Type YES to confirm reset: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
info "Reset cancelled"
|
||||
exit 0
|
||||
if [[ "$FORCE" == true ]]; then
|
||||
log "FORCE flag set — skipping confirmation prompt"
|
||||
elif [[ -t 0 ]]; then
|
||||
# Interactive terminal — prompt for confirmation
|
||||
read -r -p "Type YES to confirm reset: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
warn "Reset cancelled"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
# Non-interactive — no terminal, cannot prompt
|
||||
error "Non-interactive mode — use --force flag to skip confirmation"
|
||||
error "Usage: failover_state_reset.sh --force"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_FAILOVER Reset State File ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Reset State File ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Resetting State File ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would write:"
|
||||
echo " state=NORMAL"
|
||||
echo " failover_start=0"
|
||||
echo " handback_strikes=0"
|
||||
echo " tier2_started=false"
|
||||
echo " tier3_started=false"
|
||||
echo " tier4_started=false"
|
||||
echo " last_reset=$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
else
|
||||
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
|
||||
cat > "$FAILOVER_STATE_FILE" << EOF
|
||||
state=NORMAL
|
||||
NEW_STATE_CONTENT="state=NORMAL
|
||||
failover_start=0
|
||||
handback_strikes=0
|
||||
tier2_started=false
|
||||
tier3_started=false
|
||||
tier4_started=false
|
||||
last_reset=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
EOF
|
||||
success "State file reset to NORMAL"
|
||||
reset_by=$MY_ID"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would write to $FAILOVER_STATE_FILE:"
|
||||
echo ""
|
||||
echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
else
|
||||
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
|
||||
|
||||
# flock prevents race with failover.sh mid-cycle read/write
|
||||
(
|
||||
flock -x 200
|
||||
echo "$NEW_STATE_CONTENT" > "$FAILOVER_STATE_FILE"
|
||||
) 200>"${FAILOVER_STATE_FILE}.lock"
|
||||
|
||||
warn "State file reset to NORMAL ✅"
|
||||
log "Written to: $FAILOVER_STATE_FILE"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FAILOVER STATE RESET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_FAILOVER File: $FAILOVER_STATE_FILE"
|
||||
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS State reset to NORMAL"
|
||||
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
warn "$ICON_DONE State reset to NORMAL"
|
||||
log "failover.sh will resume from NORMAL on next cycle"
|
||||
log "No containers were started or stopped"
|
||||
echo ""
|
||||
echo "$ICON_INFO failover.sh will resume from NORMAL on next cycle"
|
||||
echo "$ICON_INFO No containers were started or stopped"
|
||||
notify "Failover state manually reset to NORMAL on $(hostname)" "Failover State Reset" "normal"
|
||||
[[ "$FAILOVER_RUNNING" == true ]] && \
|
||||
warn "⚠️ failover.sh was running during reset — monitor next cycle carefully"
|
||||
notify "Failover state manually reset to NORMAL on $(hostname) ($MY_ID)" \
|
||||
"Failover State Reset" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+232
-93
@@ -1,134 +1,273 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Recreate Shares Script -------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Reads all .cfg files from /boot/config/shares/ and creates the corresponding share
|
||||
# directories on the correct disks based on shareInclude settings.
|
||||
# Also drops a .recovery marker file in each share via /mnt/user/ so that an initial
|
||||
# rsync push can run without --delete and self-clean on the second nightly run.
|
||||
# ==============================================================================================
|
||||
# ============================= Recreate Shares ================================================
|
||||
# ==============================================================================================
|
||||
# Creates share directories on the correct disks after a fresh unRAID install or disk rebuild.
|
||||
# Reads all .cfg files from /boot/config/shares/ and creates the corresponding directories
|
||||
# on each disk listed in the shareInclude setting.
|
||||
#
|
||||
# Run this script directly on the secondary server after array is started.
|
||||
# Usage: bash recreate_shares.sh
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
|
||||
# Run directly on HOST2 after array is started following:
|
||||
# - A full disk replacement or rebuild where share folders were lost
|
||||
# - A fresh unRAID install where /boot/config/shares/*.cfg files were restored
|
||||
# - Any situation where the share folder structure exists in config but not on disk
|
||||
#
|
||||
# The array must be started before running this script — /mnt/user must be mounted.
|
||||
#
|
||||
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
|
||||
# For each share .cfg file:
|
||||
# 1. Reads shareInclude= to determine which disks own this share
|
||||
# 2. Creates /mnt/diskN/ShareName/ on each listed disk if it doesn't exist
|
||||
# 3. Places a .recovery marker file in /mnt/user/ShareName/ via the union filesystem
|
||||
#
|
||||
# ── .RECOVERY MARKER FILE ─────────────────────────────────────────────────────────────────────
|
||||
# The .recovery marker signals to rsync.sh that this is a fresh share with no existing data.
|
||||
# rsync.sh checks for .recovery before running with --delete:
|
||||
# .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed)
|
||||
# .recovery absent → rsync WITH --delete (normal — mirror mode)
|
||||
#
|
||||
# The marker self-cleans: after the first successful rsync the source side has no .recovery
|
||||
# file so the second nightly run will delete it from the mirror, restoring normal --delete
|
||||
# behaviour automatically. No manual cleanup needed. ✅
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# This script runs on the server that needs shares recreated — typically HOST2 during rebuild.
|
||||
# detect_hosts() sets MY_ID for output clarity.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents duplicate runs placing duplicate markers
|
||||
# Root check — mkdir on /mnt/diskN requires root
|
||||
# Array mount check — exits cleanly if array not started
|
||||
# Empty cfg guard — warns if no share cfg files found
|
||||
# Per-disk guards — skips missing disks with warning, continues others
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent on success — only failures produce visible output
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# recreate_shares.sh — create all shares from .cfg files
|
||||
# recreate_shares.sh --dry-run — preview what would be created, no changes
|
||||
# recreate_shares.sh --log — verbose output per disk
|
||||
# recreate_shares.sh --status — show current share state and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
SHARE_CFG_DIR="/boot/config/shares"
|
||||
MARKER_FILE=".recovery"
|
||||
|
||||
|
||||
CREATED=()
|
||||
SKIPPED=()
|
||||
FAILED=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if ! mountpoint -q /mnt/user; then
|
||||
error "Array is not started — /mnt/user is not mounted"
|
||||
info "Start the array in the unRAID UI before running this script"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — mkdir on /mnt/diskN requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Array is started"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_DISK Recreating Shares ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no directories or markers will be created"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DISK Cfg dir: $SHARE_CFG_DIR"
|
||||
echo "$ICON_DISK Marker: $MARKER_FILE"
|
||||
echo ""
|
||||
|
||||
if ! mountpoint -q /mnt/user; then
|
||||
warn "Array: NOT STARTED — /mnt/user not mounted"
|
||||
else
|
||||
echo " Array: started ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Share Config Files ━━━"
|
||||
CFG_COUNT=0
|
||||
for cfg in "$SHARE_CFG_DIR"/*.cfg; do
|
||||
[[ ! -f "$cfg" ]] && continue
|
||||
(( CFG_COUNT++ ))
|
||||
SHARE_NAME=$(basename "$cfg" .cfg)
|
||||
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
|
||||
MARKER_EXISTS="no"
|
||||
[[ -f "/mnt/user/${SHARE_NAME}/${MARKER_FILE}" ]] && MARKER_EXISTS="yes"
|
||||
echo " $ICON_DISK $SHARE_NAME — disks: ${INCLUDE:-none} — recovery marker: $MARKER_EXISTS"
|
||||
done
|
||||
[[ "$CFG_COUNT" -eq 0 ]] && warn "No .cfg files found in $SHARE_CFG_DIR"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
# Array must be started — /mnt/user must be mounted
|
||||
if ! mountpoint -q /mnt/user; then
|
||||
error "Array is not started — /mnt/user is not mounted"
|
||||
warn "Start the array in the unRAID UI before running this script"
|
||||
notify "Recreate shares failed on $(hostname) — array is not started" \
|
||||
"Recreate Shares" "warning"
|
||||
exit 1
|
||||
fi
|
||||
log "Array is started — /mnt/user is mounted ✅"
|
||||
|
||||
# Check share cfg directory exists and has files
|
||||
if [[ ! -d "$SHARE_CFG_DIR" ]]; then
|
||||
error "Share config directory not found: $SHARE_CFG_DIR"
|
||||
error "Is /boot mounted? Is this the correct server?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CFG_FILES=("$SHARE_CFG_DIR"/*.cfg)
|
||||
if [[ ! -f "${CFG_FILES[0]}" ]]; then
|
||||
warn "No share .cfg files found in $SHARE_CFG_DIR"
|
||||
warn "Nothing to recreate — are share configs present on /boot?"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Found ${#CFG_FILES[@]} share .cfg file(s) in $SHARE_CFG_DIR"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Recreate Shares ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Recreating Shares ━━━"
|
||||
echo "━━━ $ICON_DISK Recreate Shares — $MY_ID ━━━"
|
||||
echo ""
|
||||
|
||||
for cfg in "$SHARE_CFG_DIR"/*.cfg; do
|
||||
|
||||
for cfg in "${CFG_FILES[@]}"; do
|
||||
[[ ! -f "$cfg" ]] && continue
|
||||
SHARE_NAME=$(basename "$cfg" .cfg)
|
||||
INCLUDE=$(grep '^shareInclude=' "$cfg" | cut -d'"' -f2)
|
||||
|
||||
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
|
||||
|
||||
echo "━━━ $ICON_DISK $SHARE_NAME ━━━"
|
||||
|
||||
if [[ -z "$INCLUDE" ]]; then
|
||||
warn "$SHARE_NAME — no shareInclude defined, skipping"
|
||||
warn "$SHARE_NAME — no shareInclude in .cfg — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_DISK Processing $SHARE_NAME (disks: $INCLUDE)..."
|
||||
|
||||
|
||||
log "$SHARE_NAME — disks: $INCLUDE"
|
||||
|
||||
SHARE_OK=true
|
||||
|
||||
DIRS_CREATED=0
|
||||
DIRS_EXISTED=0
|
||||
|
||||
# Create directory on each listed disk
|
||||
IFS=',' read -ra DISKS <<< "$INCLUDE"
|
||||
for disk in "${DISKS[@]}"; do
|
||||
DISK_PATH="/mnt/${disk}/${SHARE_NAME}"
|
||||
|
||||
disk="${disk// /}" # trim whitespace
|
||||
[[ -z "$disk" ]] && continue
|
||||
|
||||
DISK_MOUNT="/mnt/${disk}"
|
||||
DISK_PATH="${DISK_MOUNT}/${SHARE_NAME}"
|
||||
|
||||
# Verify disk is mounted
|
||||
if ! mountpoint -q "$DISK_MOUNT" 2>/dev/null; then
|
||||
warn "$disk not mounted — skipping $DISK_PATH"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -d "$DISK_PATH" ]]; then
|
||||
echo "$ICON_RUNNING $disk/$SHARE_NAME already exists, skipping"
|
||||
log "$disk/$SHARE_NAME already exists — skipping"
|
||||
(( DIRS_EXISTED++ ))
|
||||
else
|
||||
if mkdir -p "$DISK_PATH"; then
|
||||
echo "$ICON_STARTED Created $DISK_PATH"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create: $DISK_PATH"
|
||||
(( DIRS_CREATED++ ))
|
||||
elif mkdir -p "$DISK_PATH"; then
|
||||
log "Created: $DISK_PATH ✅"
|
||||
(( DIRS_CREATED++ ))
|
||||
else
|
||||
error "Failed to create $DISK_PATH"
|
||||
error "Failed to create: $DISK_PATH"
|
||||
SHARE_OK=false
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Place .recovery marker via /mnt/user (union filesystem)
|
||||
MARKER_PATH="/mnt/user/${SHARE_NAME}/${MARKER_FILE}"
|
||||
|
||||
if [[ "$SHARE_OK" == true ]]; then
|
||||
if touch "$MARKER_PATH" 2>/dev/null; then
|
||||
echo "$ICON_DONE Marker placed: $MARKER_PATH"
|
||||
if [[ -f "$MARKER_PATH" ]]; then
|
||||
log ".recovery marker already exists in $SHARE_NAME"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would place marker: $MARKER_PATH"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
elif touch "$MARKER_PATH" 2>/dev/null; then
|
||||
log "Marker placed: $MARKER_PATH ✅"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
else
|
||||
warn "Could not place marker in $SHARE_NAME — share may not be visible yet"
|
||||
warn "$SHARE_NAME — could not place .recovery marker"
|
||||
warn "Share directory may not be visible via /mnt/user yet"
|
||||
warn "Try: touch /mnt/user/${SHARE_NAME}/.recovery manually after verifying share"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
fi
|
||||
else
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
|
||||
|
||||
[[ "$DIRS_CREATED" -gt 0 ]] && warn "$SHARE_NAME — created $DIRS_CREATED dir(s) on disk"
|
||||
[[ "$DIRS_EXISTED" -gt 0 ]] && log "$SHARE_NAME — $DIRS_EXISTED dir(s) already existed"
|
||||
echo ""
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY RECREATE SHARES SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
if [[ ${#CREATED[@]} -gt 0 ]]; then
|
||||
echo " $ICON_DONE Created & marked:"
|
||||
for s in "${CREATED[@]}"; do
|
||||
echo " $ICON_STARTED $s"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ ${#SKIPPED[@]} -gt 0 ]]; then
|
||||
echo " $ICON_WARN Skipped:"
|
||||
for s in "${SKIPPED[@]}"; do
|
||||
echo " $ICON_NOT_RUNNING $s"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo " $ICON_ERROR Failed:"
|
||||
for s in "${FAILED[@]}"; do
|
||||
echo " $ICON_ERROR $s"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo " $ICON_SUCCESS Created: ${#CREATED[@]}"
|
||||
echo " $ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]}"
|
||||
echo " $ICON_ERROR Failed: ${#FAILED[@]}"
|
||||
|
||||
[[ ${#CREATED[@]} -gt 0 ]] && warn "Created + marked: ${CREATED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI"
|
||||
echo " 2. $ICON_GEAR Remove --delete from Master.conf rsync opts"
|
||||
echo " 3. $ICON_RUN Run initial push — marker files self-clean on second nightly run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
echo " Created: ${#CREATED[@]}"
|
||||
echo " Skipped: ${#SKIPPED[@]}"
|
||||
echo " Failed: ${#FAILED[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" == false && ${#CREATED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ Next Steps ━━━"
|
||||
echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI"
|
||||
echo " 2. $ICON_SYNC Run initial rsync push from HOST1 → HOST2"
|
||||
echo " rsync.sh will detect .recovery markers and skip --delete"
|
||||
echo " Normal --delete mode restores automatically on second nightly run"
|
||||
echo " 3. $ICON_GEAR No manual config changes needed — markers self-clean ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: completed with failures"
|
||||
notify "Recreate shares failed on $(hostname) ($MY_ID) — failed: ${FAILED[*]}" \
|
||||
"Recreate Shares" "warning"
|
||||
exit 1
|
||||
else
|
||||
log "$ICON_DONE Status: done — ${#CREATED[@]} created, ${#SKIPPED[@]} skipped"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -1,175 +1,259 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Watchdog Skip List Manager ---------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# =========================== Watchdog Skip List Manager =======================================
|
||||
# ==============================================================================================
|
||||
# View and manage the persistent container skip list used by docker_watchdog.sh.
|
||||
# Containers are added to the skip list when they exceed the restart loop limit.
|
||||
# They stay there until manually cleared or until found running again automatically.
|
||||
#
|
||||
# Usage:
|
||||
# watchdog_skip_list_manager.sh --status — show current skip list and restart history
|
||||
# watchdog_skip_list_manager.sh --clear-all — clear all skip lists and restart history
|
||||
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
|
||||
# ── WHAT THE SKIP LIST IS ─────────────────────────────────────────────────────────────────────
|
||||
# docker_watchdog.sh adds a container to the skip list when it exceeds the restart loop
|
||||
# limit (WATCHDOG_CONTAINER_RESTART_LIMIT in WATCHDOG_CONTAINER_RESTART_WINDOW hours).
|
||||
# Once on the skip list the watchdog stops restarting it — prevents infinite restart loops.
|
||||
#
|
||||
# After clearing a container from the skip list:
|
||||
# Skip list persists on /boot/config — survives reboots.
|
||||
# Auto-clears when docker_watchdog.sh sees the container running on a cycle.
|
||||
# This script clears it manually when you have fixed the underlying problem.
|
||||
#
|
||||
# ── ACTIONS ───────────────────────────────────────────────────────────────────────────────────
|
||||
# --status — show skip list, container states, restart history
|
||||
# --clear ContainerName — clear a specific container from skip list + history
|
||||
# --clear-all — clear all skip lists and restart history
|
||||
#
|
||||
# ── AFTER CLEARING ────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Fix whatever was causing the container to fail
|
||||
# 2. Start the container manually: docker start ContainerName
|
||||
# 3. The watchdog will monitor it normally on the next cycle
|
||||
# 2. Start it manually: docker start ContainerName
|
||||
# 3. docker_watchdog.sh monitors it normally on the next cycle
|
||||
# 4. If it crashes again → watchdog adds it back and notifies
|
||||
#
|
||||
# Files managed:
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SKIP LIST AUTO-CLEAR ──────────────────────────────────────────────────────────────────────
|
||||
# docker_watchdog.sh auto-clears a container from the skip list when it sees it running.
|
||||
# So if a container recovers on its own (Docker restart policy eventually works),
|
||||
# the watchdog will see it running, remove it from the skip list, and resume monitoring.
|
||||
# Manual clear only needed when container is stuck stopped and needs intervention.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent access with docker_watchdog.sh writing files
|
||||
# docker_watchdog check — warns if watchdog is running during clear (could re-add instantly)
|
||||
# DOCKER_TIMEOUT — docker inspect calls protected against daemon hangs
|
||||
# Confirmation required — interactive: YES | non-interactive: --force flag
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── FILES MANAGED ─────────────────────────────────────────────────────────────────────────────
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_skip_list_manager.sh — show status
|
||||
# watchdog_skip_list_manager.sh --status — show status explicitly
|
||||
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
|
||||
# watchdog_skip_list_manager.sh --clear-all — clear everything
|
||||
# Any action supports --dry-run and --force
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# Parse action from args
|
||||
ACTION=""
|
||||
# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
|
||||
ACTION="status"
|
||||
TARGET_CONTAINER=""
|
||||
FORCE=false
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "${PARSED_ARGS[@]}"; do
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--clear-all) ACTION="clear-all" ;;
|
||||
--clear) ACTION="clear" ;;
|
||||
--status) ACTION="status" ;;
|
||||
--force) FORCE=true ;;
|
||||
*)
|
||||
[[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]] && TARGET_CONTAINER="$arg"
|
||||
if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
|
||||
TARGET_CONTAINER="$arg"
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -z "$ACTION" ]] && ACTION="status"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in output
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ STATUS ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status — always shown regardless of action ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
|
||||
|
||||
SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
|
||||
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
|
||||
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
|
||||
|
||||
# docker_watchdog.sh running check
|
||||
WATCHDOG_RUNNING=false
|
||||
if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
|
||||
WATCHDOG_RUNNING=true
|
||||
warn "docker_watchdog.sh is currently RUNNING"
|
||||
[[ "$ACTION" != "status" ]] && \
|
||||
warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$SKIP_COUNT" -eq 0 ]]; then
|
||||
success "Skip list is empty — all containers healthy"
|
||||
log "Skip list: empty — all containers monitored normally ✅"
|
||||
else
|
||||
warn "$SKIP_COUNT container(s) on skip list:"
|
||||
warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
|
||||
echo ""
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
# Check if container is currently running
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " $ICON_RUNNING $container — currently RUNNING (will auto-clear on next watchdog cycle)"
|
||||
elif [[ "$STATUS" == "false" ]]; then
|
||||
echo " $ICON_STOPPED $container — currently STOPPED — fix and start manually"
|
||||
else
|
||||
echo " $ICON_INFO $container — container not found"
|
||||
fi
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo " $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
|
||||
;;
|
||||
false)
|
||||
echo " $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
|
||||
;;
|
||||
*)
|
||||
echo " $ICON_WARN $container — not found on this server"
|
||||
;;
|
||||
esac
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Restart History ━━━"
|
||||
if [[ "$RESTART_COUNT" -eq 0 ]]; then
|
||||
success "No restart history"
|
||||
log "No restart history"
|
||||
else
|
||||
info "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
|
||||
log "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
|
||||
echo ""
|
||||
# Show per-container restart counts
|
||||
awk -F'|' '{counts[$1]++} END {for (c in counts) printf " %-30s %d restart(s)\n", c, counts[c]}' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
|
||||
awk -F'|' '{counts[$1]++} END {
|
||||
for (c in counts)
|
||||
printf " %-30s %d restart(s)\n", c, counts[c]
|
||||
}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
|
||||
fi
|
||||
|
||||
[[ "$ACTION" == "status" ]] && exit 0
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CLEAR ALL ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear All ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear-all" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
|
||||
warn "This will clear the skip list and restart history for ALL containers"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
read -r -p "Type YES to confirm: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
info "Cancelled"
|
||||
exit 0
|
||||
if [[ "$FORCE" == true ]]; then
|
||||
log "FORCE flag set — skipping confirmation"
|
||||
elif [[ -t 0 ]]; then
|
||||
read -r -p "Type YES to confirm: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
warn "Cancelled"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
error "Non-interactive mode — use --force flag to skip confirmation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
success "Skip list cleared"
|
||||
success "Restart history cleared"
|
||||
notify "Watchdog skip list manually cleared on $(hostname) — all containers will be monitored normally" "Watchdog Manager" "normal"
|
||||
warn "Skip list cleared ✅"
|
||||
warn "Restart history cleared ✅"
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — containers will be monitored on next cycle"
|
||||
notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
|
||||
"Watchdog Manager" "warning"
|
||||
else
|
||||
warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE"
|
||||
warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CLEAR SPECIFIC CONTAINER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear Specific Container ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"
|
||||
|
||||
if [[ -z "$TARGET_CONTAINER" ]]; then
|
||||
error "No container specified. Usage: --clear ContainerName"
|
||||
error "No container specified"
|
||||
error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove from skip list
|
||||
if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then
|
||||
warn "$TARGET_CONTAINER is not on the skip list"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE"
|
||||
success "$TARGET_CONTAINER removed from skip list"
|
||||
warn "$TARGET_CONTAINER removed from skip list ✅"
|
||||
else
|
||||
warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clear restart history for this container
|
||||
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"
|
||||
|
||||
if [[ "$HIST_COUNT" -gt 0 ]]; then
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
success "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER"
|
||||
warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
|
||||
else
|
||||
warn "DRY RUN — would clear $HIST_COUNT restart history entries"
|
||||
fi
|
||||
else
|
||||
info "No restart history for $TARGET_CONTAINER"
|
||||
log "No restart history for $TARGET_CONTAINER"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "$ICON_INFO Next steps:"
|
||||
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
|
||||
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
|
||||
echo " 3. Watchdog will monitor it normally on the next cycle"
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_INFO Next Steps ━━━"
|
||||
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
|
||||
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
|
||||
echo " 3. docker_watchdog.sh monitors it on the next cycle"
|
||||
echo " 4. If it crashes again → watchdog adds it back and notifies"
|
||||
notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
|
||||
"Watchdog Manager" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DONE ━━━━━"
|
||||
echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"
|
||||
+175
-91
@@ -1,84 +1,128 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- ZFS Pool Scrub ---------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= ZFS Pool Scrub ============================================
|
||||
# ==============================================================================================
|
||||
# Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion.
|
||||
# Sends a notification when scrub completes with a summary of any errors found.
|
||||
#
|
||||
# ZFS scrub reads every block on every pool and verifies checksums — it catches
|
||||
# silent data corruption that would otherwise only surface when you try to read
|
||||
# the corrupted data. Running monthly is recommended for all ZFS pools.
|
||||
# ── WHAT ZFS SCRUB DOES ───────────────────────────────────────────────────────────────────────
|
||||
# Reads every block on every pool and verifies checksums against the stored hash.
|
||||
# Catches silent data corruption that would otherwise only surface when you read the
|
||||
# corrupted data — by then it may be too late for redundancy to help.
|
||||
#
|
||||
# Usage:
|
||||
# zfs_pool_scrub.sh — scrub all pools
|
||||
# zfs_pool_scrub.sh poolname — scrub specific pool only
|
||||
# zfs_pool_scrub.sh --status — show scrub status for all pools
|
||||
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
|
||||
# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O.
|
||||
# It does consume I/O bandwidth — run during off-peak hours or maintenance windows.
|
||||
# Monthly is recommended for all pools. Quarterly minimum for large pools.
|
||||
#
|
||||
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped unless specified explicitly.
|
||||
# Scrub runs in background — script polls until complete then reports.
|
||||
# Safe to run while the pool is in use — scrub does not interrupt normal I/O.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Starts scrub on each pool then polls every 60 seconds until all complete.
|
||||
# Progress shown via warn() every poll (visible) when scrub is running.
|
||||
# Safe to leave running or interrupt — scrub continues even if script is stopped.
|
||||
# On completion reports errors per pool and notifies if any found.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
||||
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.)
|
||||
# unless specified explicitly as a positional argument.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent scrub starts on same server
|
||||
# detect_hosts() — correct pool ignore list per host
|
||||
# validate_unraid_cmd — zpool and notify validated before use
|
||||
# Scrub-in-progress check — skips pools already scrubbing rather than erroring
|
||||
# SIGTERM trap — poll loop exits cleanly on signal
|
||||
# Silent when clean — only errors produce visible output and notification
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub
|
||||
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# zfs_pool_scrub.sh — scrub all non-ignored pools
|
||||
# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list)
|
||||
# zfs_pool_scrub.sh --status — show scrub status for all pools
|
||||
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
|
||||
# zfs_pool_scrub.sh --log — verbose progress output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
TARGET_POOL="${PARSED_ARGS[0]:-}"
|
||||
SCRUB_RUNNING=true
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
|
||||
"--version" "" \
|
||||
"zpool" || {
|
||||
error "ZFS not available on this system — zpool not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v zpool >/dev/null 2>&1; then
|
||||
error "ZFS not available on this system"
|
||||
exit 1
|
||||
fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
detect_hosts
|
||||
|
||||
success "ZFS available"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
||||
|
||||
# Build ignore map
|
||||
# Build ignore pool map
|
||||
declare -A IGNORE_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
||||
|
||||
# SIGTERM trap — exit poll loop cleanly
|
||||
trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SCRUB STATUS ━━━━━"
|
||||
zpool list -H -o name 2>/dev/null | while read -r pool; do
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
echo " $ICON_ZFS $pool — $SCAN"
|
||||
done
|
||||
IGNORED=""
|
||||
[[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
|
||||
echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
|
||||
done < <(zpool list -H -o name 2>/dev/null)
|
||||
echo ""
|
||||
echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build pool list to scrub
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── Build pool list ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
POOLS_TO_SCRUB=()
|
||||
|
||||
if [[ -n "$TARGET_POOL" ]]; then
|
||||
# Specific pool requested — validate it exists
|
||||
# Specific pool — bypass ignore list, validate exists
|
||||
if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
|
||||
error "Pool not found: $TARGET_POOL"
|
||||
exit 1
|
||||
@@ -89,7 +133,7 @@ else
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
|
||||
info "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
||||
log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
||||
continue
|
||||
fi
|
||||
POOLS_TO_SCRUB+=("$pool")
|
||||
@@ -97,99 +141,139 @@ else
|
||||
fi
|
||||
|
||||
if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
|
||||
warn "No pools to scrub"
|
||||
warn "No pools to scrub — all pools may be on the ignore list"
|
||||
warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
info "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
||||
log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS Start Scrubs ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Scrubs ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Starting ZFS Scrubs ━━━"
|
||||
echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
|
||||
START=$(date +%s)
|
||||
|
||||
STARTED=()
|
||||
SKIPPED_POOLS=()
|
||||
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
info "$ICON_ZFS Starting scrub on $pool..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
zpool scrub "$pool" 2>/dev/null && \
|
||||
success "$pool scrub started" || \
|
||||
error "Failed to start scrub on $pool"
|
||||
else
|
||||
|
||||
# Check if scrub already in progress
|
||||
ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$ALREADY" -gt 0 ]]; then
|
||||
warn "$pool — scrub already in progress — joining existing scrub"
|
||||
STARTED+=("$pool")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Starting scrub on $pool..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would scrub: $pool"
|
||||
STARTED+=("$pool")
|
||||
elif zpool scrub "$pool" 2>/dev/null; then
|
||||
log "$pool scrub started ✅"
|
||||
STARTED+=("$pool")
|
||||
else
|
||||
error "Failed to start scrub on $pool"
|
||||
SKIPPED_POOLS+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_WARN Status: DRY RUN — no scrubs started"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
warn "DRY RUN — no scrubs started"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Poll until complete ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#STARTED[@]} -eq 0 ]]; then
|
||||
error "No scrubs were started — check pool status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Poll Until Complete ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_TIME Waiting for scrubs to complete ━━━"
|
||||
info "Polling every 60 seconds — this may take a while on large pools"
|
||||
info "Safe to leave running — scrub continues even if this script is stopped"
|
||||
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
|
||||
log "Polling every 60 seconds — scrubs may take hours on large pools"
|
||||
log "Safe to interrupt — scrubs continue in background if script is stopped"
|
||||
|
||||
STILL_RUNNING=true
|
||||
while [[ "$STILL_RUNNING" == true ]]; do
|
||||
while [[ "$SCRUB_RUNNING" == true ]]; do
|
||||
sleep 60
|
||||
|
||||
STILL_RUNNING=false
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
STATUS=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$STATUS" -gt 0 ]]; then
|
||||
for pool in "${STARTED[@]}"; do
|
||||
IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$IN_PROGRESS" -gt 0 ]]; then
|
||||
STILL_RUNNING=true
|
||||
REPAIRED=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -oE "[0-9]+ repaired")
|
||||
log "$pool — scrub in progress ${REPAIRED:+($REPAIRED)}"
|
||||
# Show progress — always visible so user knows it's running
|
||||
PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
|
||||
REPAIRED=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+ repaired")
|
||||
warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
success "All scrubs complete"
|
||||
warn "All scrubs complete — $(format_duration $(( END - START )))"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Results ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Results ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Scrub Results ━━━"
|
||||
|
||||
POOLS_OK=()
|
||||
POOLS_ERRORS=()
|
||||
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
for pool in "${STARTED[@]}"; do
|
||||
SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
ERRORS=$(zpool status "$pool" 2>/dev/null | grep "errors:" | grep -v "No known data errors")
|
||||
ERRORS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "errors:" | grep -v "No known data errors")
|
||||
|
||||
if [[ -n "$ERRORS" ]]; then
|
||||
error "$pool — $SCAN_LINE"
|
||||
error "$pool — $ERRORS"
|
||||
error "$pool — ERRORS FOUND"
|
||||
error " $SCAN_LINE"
|
||||
error " $ERRORS"
|
||||
POOLS_ERRORS+=("$pool")
|
||||
else
|
||||
success "$pool — $SCAN_LINE"
|
||||
log "$pool — $SCAN_LINE"
|
||||
POOLS_OK+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_ZFS Pools scrubbed: ${#POOLS_TO_SCRUB[@]}"
|
||||
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
||||
echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub"
|
||||
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
||||
[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
|
||||
notify "ZFS scrub complete on $(hostname) — ERRORS found in pools: ${POOLS_ERRORS[*]}" "ZFS Scrub" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL POOLS CLEAN"
|
||||
notify "ZFS scrub complete on $(hostname) — ${#POOLS_OK[@]} pools clean in $(format_duration $((END - START)))" "ZFS Scrub" "normal"
|
||||
notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
|
||||
"ZFS Scrub" "warning"
|
||||
elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
|
||||
log "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user