Gmer4Lfe
95151c2278
Move all watchdog scripts to a dedicated Watchdogs/ folder: Docker_Essentials/docker_watchdog.sh → Watchdogs/ unRAID_Essentials/system_watchdog.sh → Watchdogs/ unRAID_Essentials/resource_watchdog.sh → Watchdogs/ Orchestrators/watchdog_orchestrator.sh → Watchdogs/ Tools/watchdog_skip_list_manager.sh → Watchdogs/ Rename host config files: master_host1.conf → host1.conf master_host2.conf → host2.conf Update all references across the ecosystem: master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS paths → Watchdogs/ load_config.sh: host*.conf glob + all comments git_pull_execute.sh: sparse checkout glob + all comments Partnership/ssh_setup.sh: HOST_CONF path construction user_script_plug-in.sh: all script paths + per-host conf path common.sh, README.md, README-User_Script_Plug-in.md: comment refs All Partnership, Fallback, Monitors, Transcodes, Tools scripts: comment refs
298 lines
13 KiB
Bash
Executable File
298 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Docker Network Connect =====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Ensures custom Docker networks exist and connects configured containers to
|
|
# them at every array start.
|
|
#
|
|
# Called via ARRAY_START_SCRIPTS — runs early in the array start sequence,
|
|
# before watchdogs begin their first cycle. Idempotent: safe to re-run at any
|
|
# time. Silent when everything is already correct. Notifies when a network had
|
|
# to be created — that only happens after a unRAID update wipes custom networks,
|
|
# and it is worth knowing when it does.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# For each configured network:
|
|
#
|
|
# 1. Does the network exist?
|
|
# NO → create it (bridge driver, Docker assigns subnet automatically)
|
|
# → send notification — creation is unexpected outside post-update recovery
|
|
# YES → skip silently
|
|
#
|
|
# 2. For each configured container:
|
|
# Already connected → skip silently
|
|
# Not connected → connect it
|
|
# Container missing → warn and skip — may not be running yet, not fatal
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Idempotent by Design
|
|
# Running this script multiple times produces the same result as running it
|
|
# once. Skips anything already in the correct state without error or noise.
|
|
#
|
|
# Silent When Correct
|
|
# Produces no output on a clean run. The absence of output is confirmation
|
|
# that everything is already correct.
|
|
#
|
|
# Notify on Creation
|
|
# Network creation is always notified because it should only happen after a
|
|
# unRAID update. If it happens regularly something is misconfigured and the
|
|
# operator needs to know.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Lock Acquisition
|
|
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
|
# array start hooks or manually without risk of overlap.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() identifies which server is running the script and aliases
|
|
# HOST*_NETWORK_CONNECT_* arrays to the correct host's values.
|
|
#
|
|
# Docker Daemon Check
|
|
# Verifies daemon is responsive before any network operations. Network
|
|
# commands against a hung daemon hang indefinitely.
|
|
#
|
|
# Timeout Protection
|
|
# All docker commands wrapped in timeout. Daemon hangs cannot stall the
|
|
# array start sequence.
|
|
#
|
|
# Empty Array Guards
|
|
# Warns and exits cleanly if NETWORK_CONNECT_NETWORKS or
|
|
# NETWORK_CONNECT_CONTAINERS are unconfigured.
|
|
#
|
|
# Command Validation
|
|
# Validates unRAID notify script before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_NETWORK_CONNECT_NETWORKS
|
|
# Networks to ensure exist at array start. Aliased by detect_hosts() →
|
|
# NETWORK_CONNECT_NETWORKS
|
|
#
|
|
# HOST*_NETWORK_CONNECT_CONTAINERS
|
|
# Containers to connect to every configured network. Aliased by
|
|
# detect_hosts() → NETWORK_CONNECT_CONTAINERS
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# docker_network_connect.sh
|
|
# Ensure all configured networks exist and containers are connected
|
|
#
|
|
# docker_network_connect.sh --dry-run
|
|
# Preview what would be created or connected without making changes
|
|
#
|
|
# docker_network_connect.sh --status
|
|
# Show current network state and container connection status
|
|
#
|
|
# docker_network_connect.sh --log
|
|
# Verbose per-network per-container output
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker not found — cannot manage networks"
|
|
exit 1
|
|
fi
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_NETWORK_CONNECT_* arrays
|
|
detect_hosts
|
|
|
|
# Validate unRAID notify script — used for network creation alerts
|
|
validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
# Docker daemon check — network operations are useless if daemon is hung
|
|
DOCKER_TIMEOUT=15
|
|
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
|
error "Docker daemon not responding — cannot manage networks"
|
|
notify "docker_network_connect failed on $(hostname) — Docker daemon not responding" "Network Connect" "warning"
|
|
exit 1
|
|
fi
|
|
|
|
# Empty array guards
|
|
if [[ ${#NETWORK_CONNECT_NETWORKS[@]} -eq 0 ]]; then
|
|
warn "NETWORK_CONNECT_NETWORKS is empty for $MY_ID — nothing to do"
|
|
warn "Check HOST*_NETWORK_CONNECT_NETWORKS in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#NETWORK_CONNECT_CONTAINERS[@]} -eq 0 ]]; then
|
|
warn "NETWORK_CONNECT_CONTAINERS is empty for $MY_ID — no containers to connect"
|
|
warn "Check HOST*_NETWORK_CONNECT_CONTAINERS in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}"
|
|
echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}"
|
|
echo ""
|
|
echo "━━━ Network State ━━━"
|
|
for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do
|
|
[[ -z "$network" ]] && continue
|
|
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then
|
|
echo " $ICON_SUCCESS $network — exists"
|
|
timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
|
--format ' Subnet: {{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null
|
|
else
|
|
echo " $ICON_ERROR $network — missing (will be created on next run)"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo "━━━ Container Connections ━━━"
|
|
for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
echo "$ICON_CONTAINERS $container:"
|
|
timeout "$DOCKER_TIMEOUT" docker inspect "$container" \
|
|
--format '{{range $k, $v := .NetworkSettings.Networks}} {{$k}}{{"\\n"}}{{end}}' \
|
|
2>/dev/null || echo " not found"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Network Ensure + Connect ━━━
|
|
# ==============================================================================================
|
|
log "Networks: ${NETWORK_CONNECT_NETWORKS[*]}"
|
|
log "Containers: ${NETWORK_CONNECT_CONTAINERS[*]}"
|
|
|
|
START=$(date +%s)
|
|
NETWORKS_CREATED=()
|
|
CONNECTED=()
|
|
SKIPPED=()
|
|
FAILED=()
|
|
|
|
for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do
|
|
[[ -z "$network" ]] && continue
|
|
|
|
log "Processing network: $network"
|
|
|
|
# ── Step 1 — ensure network exists ───────────────────────────────────────────────────────
|
|
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then
|
|
log "$network exists ✅"
|
|
else
|
|
warn "$ICON_DOCKER_NET $network not found — creating (unRAID update may have wiped networks)"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would create: $network (bridge, auto subnet)"
|
|
else
|
|
if timeout "$DOCKER_TIMEOUT" docker network create \
|
|
--driver bridge "$network" >/dev/null 2>&1; then
|
|
warn "$ICON_DOCKER_NET $network created"
|
|
NETWORKS_CREATED+=("$network")
|
|
# Network creation is unexpected — notify so user is aware
|
|
notify "Docker network created on $(hostname) — $network (unRAID update likely wiped it)" \
|
|
"Network Connect" "normal"
|
|
else
|
|
error "$network — failed to create"
|
|
FAILED+=("$network:create")
|
|
continue
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ── Step 2 — connect containers to this network ──────────────────────────────────────────
|
|
for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
|
|
# Container not found — warn and skip, not an error
|
|
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
|
warn "$container not found — skipping (may not be running yet)"
|
|
continue
|
|
fi
|
|
|
|
# Already connected — skip cleanly and silently
|
|
if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \
|
|
--format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null \
|
|
| grep -qw "$container"; then
|
|
log "$container already on $network — skipping"
|
|
SKIPPED+=("$container→$network")
|
|
continue
|
|
fi
|
|
|
|
# Connect
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would connect $container to $network"
|
|
continue
|
|
fi
|
|
|
|
log "Connecting $container to $network..."
|
|
if timeout "$DOCKER_TIMEOUT" docker network connect "$network" "$container" 2>/dev/null; then
|
|
log "$container connected to $network"
|
|
CONNECTED+=("$container→$network")
|
|
else
|
|
error "Failed to connect $container to $network"
|
|
FAILED+=("$container:$network")
|
|
fi
|
|
done
|
|
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
# Always show summary — this runs at array start and output is useful for diagnostics
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY NETWORK CONNECT SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
[[ ${#NETWORKS_CREATED[@]} -gt 0 ]] && warn "$ICON_DOCKER_NET Created: ${NETWORKS_CREATED[*]} (networks were missing)"
|
|
[[ ${#CONNECTED[@]} -gt 0 ]] && log "Connected: ${CONNECTED[*]}"
|
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${#SKIPPED[@]} skipped"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: SOME OPERATIONS FAILED"
|
|
notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning"
|
|
elif [[ ${#NETWORKS_CREATED[@]} -gt 0 ]]; then
|
|
warn "Networks recreated — ${NETWORKS_CREATED[*]} — unRAID update likely wiped them"
|
|
else
|
|
echo "All networks healthy — no action needed"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |