194 lines
9.2 KiB
Bash
Executable File
194 lines
9.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Job Runner =====================================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Wraps every scheduled script execution with JSON status tracking and log
|
|
# management. Called by /etc/cron.d/varaverk for every scheduled job. The PHP
|
|
# dashboard polls the JSON files to show live job status without running scripts.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Invocation: bash run_job.sh <job_id> <script_path> [flags...]
|
|
#
|
|
# Writes three files per job to /var/log/varaverk/:
|
|
# <id>.json — status, timestamps, exit code, pid (polled by WebGUI)
|
|
# <id>.log — appended per run, trimmed to LOG_MAX_LINES lines
|
|
# <id>.manual_ts — sentinel: epoch of last manual completion (interval suppression)
|
|
#
|
|
# Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+)
|
|
#
|
|
# MANUAL FLAG
|
|
# --manual marks a UI-triggered run. On completion, writes a manual_ts sentinel.
|
|
# The next cron fire reads the sentinel and suppresses itself if the elapsed time
|
|
# is within the job's own cron interval — prevents double-firing after a manual run.
|
|
# Static schedules (e.g. "30 2 * * 0") are never suppressed — only */N intervals.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Suppress Double-Fire
|
|
# When a user triggers a job from the UI, the next cron fire within the job's
|
|
# own interval is skipped. A 30-minute cron job triggered at HH:14 won't fire
|
|
# again at HH:30 — it waits for HH:44. Static schedules are never suppressed.
|
|
#
|
|
# Status as Ground Truth
|
|
# The JSON file is overwritten atomically on every state change (start → end).
|
|
# The WebGUI polls it directly — no additional IPC or database needed.
|
|
#
|
|
# Log Trim on Every Write
|
|
# The log file is trimmed to LOG_MAX_LINES after every run. Never grows
|
|
# unbounded regardless of how long the server runs or how often the job fires.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# --manual stripped — flag consumed here, never passed to the wrapped script
|
|
# mkdir -p — log dir created if missing before any write
|
|
# Log trim — tail -n LOG_MAX_LINES via tmp file + mv (atomic)
|
|
# Sentinel cleanup — manual_ts removed after it's used or expired
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# LOG_DIR /var/log/varaverk (hardcoded — tmpfs on Unraid, cleared on reboot)
|
|
# LOG_MAX_LINES 1000 (hardcoded — trim threshold per job log)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# bash run_job.sh <job_id> <script_path> [script_flags...]
|
|
# Normal cron-triggered run.
|
|
#
|
|
# bash run_job.sh <job_id> <script_path> --manual [script_flags...]
|
|
# UI-triggered run. Suppresses next cron fire within the job's interval.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
JOB_ID="$1"
|
|
SCRIPT="$2"
|
|
shift 2
|
|
|
|
LOG_DIR="/var/log/varaverk"
|
|
BASE="${JOB_ID%.sh}"
|
|
LOG_FILE="$LOG_DIR/$BASE.log"
|
|
STAT_FILE="$LOG_DIR/$BASE.json"
|
|
MANUAL_TS_FILE="$LOG_DIR/$BASE.manual_ts"
|
|
LOG_MAX_LINES=1000
|
|
|
|
# Strip --manual from script args — it's for run_job.sh only
|
|
MANUAL=false
|
|
SCRIPT_ARGS=()
|
|
for _arg in "$@"; do
|
|
[[ "$_arg" == "--manual" ]] && MANUAL=true || SCRIPT_ARGS+=("$_arg")
|
|
done
|
|
unset _arg
|
|
|
|
mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$STAT_FILE")"
|
|
|
|
# Cron invocation: skip if a manual run completed recently within this job's own interval.
|
|
if [[ "$MANUAL" == false && -f "$MANUAL_TS_FILE" ]]; then
|
|
LAST_MANUAL=$(cat "$MANUAL_TS_FILE" 2>/dev/null || echo 0)
|
|
ELAPSED=$(( $(date +%s) - LAST_MANUAL ))
|
|
|
|
# Only suppress for interval-based crons (*/N * * * *).
|
|
# Static schedules like "30 2 * * 0" run at their appointed time and are never suppressed.
|
|
INTERVAL=0
|
|
_VV_CFG="/boot/config/plugins/varaverk/varaverk.cfg"
|
|
# Sourced in a subshell rather than grep|cut|tr. varaverk.cfg is shell syntax, so this is
|
|
# how bash itself reads it, and it agrees with the PHP side — include/config.php uses
|
|
# parse_ini_file(), which already honours quoting and comments.
|
|
#
|
|
# The pattern this replaces was `cut -d= -f2 | tr -d '[:space:]'`, which keeps everything
|
|
# after the first `=` and then squeezes all whitespace out of it. One trailing comment on
|
|
# that line — SCRIPTS_DIR="/mnt/user/appdata/Varaverk" # appdata mode — and the value
|
|
# becomes a path with the comment welded onto it, schedule.json is never found, INTERVAL
|
|
# stays 0, and every manual run silently stops suppressing the next cron fire. It also
|
|
# matched a bare `^SCRIPTS_DIR` prefix, so a future SCRIPTS_DIRECTORY= would win the -m1.
|
|
#
|
|
# Subshell so nothing else the cfg defines leaks into this runner or the job it wraps.
|
|
_SCRIPTS_DIR=$( . "$_VV_CFG" >/dev/null 2>&1 || true; printf '%s' "${SCRIPTS_DIR:-}" )
|
|
SCHEDULE_FILE="${_SCRIPTS_DIR:-/boot/config/plugins/varaverk}/schedule.json"
|
|
if [[ -f "$SCHEDULE_FILE" ]]; then
|
|
CRON_EXPR=$(php -r "
|
|
\$s = json_decode(file_get_contents('$SCHEDULE_FILE'), true) ?: [];
|
|
echo \$s['$JOB_ID']['cron'] ?? '';
|
|
" 2>/dev/null)
|
|
if [[ "$CRON_EXPR" =~ ^\*/([0-9]+) ]]; then
|
|
INTERVAL=$(( ${BASH_REMATCH[1]} * 60 ))
|
|
fi
|
|
fi
|
|
|
|
if (( INTERVAL > 0 && ELAPSED < INTERVAL )); then
|
|
printf '\n── %s [SKIPPED — ran manually %ds ago, interval %ds] ────────\n' \
|
|
"$(date '+%Y-%m-%d %H:%M:%S')" "$ELAPSED" "$INTERVAL" >> "$LOG_FILE"
|
|
printf '{"id":"%s","status":"skipped","manual_elapsed":%s,"interval":%s}\n' \
|
|
"$JOB_ID" "$ELAPSED" "$INTERVAL" > "$STAT_FILE"
|
|
rm -f "$MANUAL_TS_FILE"
|
|
exit 0
|
|
fi
|
|
rm -f "$MANUAL_TS_FILE"
|
|
fi
|
|
|
|
# Refuse to start when this job is already running, and leave its record untouched.
|
|
#
|
|
# The wrapped scripts take their own locks, so a second invocation was already refused — but it
|
|
# was refused *after* run_job.sh had overwritten the stat file with its own pid, and it then wrote
|
|
# its instant exit-1 over the live run's record. The job kept working while every status reader
|
|
# showed it failed. Observed for real: an onboard mid-way through deploying containers reported
|
|
# {"status":"warn","exit":1} because the operator, seeing no progress, had clicked twice.
|
|
#
|
|
# api/run.php has always had this guard; run_job.sh did not, and cron and the remote phase-2
|
|
# trigger both reach run_job.sh directly without passing through it.
|
|
if [[ -f "$STAT_FILE" ]]; then
|
|
_prev_status=$(sed -n 's/.*"status":"\([^"]*\)".*/\1/p' "$STAT_FILE" 2>/dev/null)
|
|
_prev_pid=$(sed -n 's/.*"pid":\([0-9]*\).*/\1/p' "$STAT_FILE" 2>/dev/null)
|
|
if [[ "$_prev_status" == "running" && -n "$_prev_pid" && -d "/proc/$_prev_pid" ]]; then
|
|
printf '\n── %s [REFUSED — already running as PID %s] ────────\n' \
|
|
"$(date '+%Y-%m-%d %H:%M:%S')" "$_prev_pid" >> "$LOG_FILE"
|
|
echo "$JOB_ID already running (PID $_prev_pid) — not starting a second run" >&2
|
|
exit 0
|
|
fi
|
|
unset _prev_status _prev_pid
|
|
fi
|
|
|
|
START=$(date +%s)
|
|
printf '{"id":"%s","status":"running","start":%s,"pid":%s}\n' \
|
|
"$JOB_ID" "$START" "$$" > "$STAT_FILE"
|
|
|
|
printf '\n── %s ────────────────────────────────────────────────\n' \
|
|
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE"
|
|
|
|
bash "$SCRIPT" "${SCRIPT_ARGS[@]}" >> "$LOG_FILE" 2>&1
|
|
EC=$?
|
|
|
|
END=$(date +%s)
|
|
if [ "$EC" -eq 0 ]; then STATUS="ok"
|
|
elif [ "$EC" -eq 1 ]; then STATUS="warn"
|
|
else STATUS="error"
|
|
fi
|
|
|
|
printf '{"id":"%s","status":"%s","start":%s,"end":%s,"exit":%s}\n' \
|
|
"$JOB_ID" "$STATUS" "$START" "$END" "$EC" > "$STAT_FILE"
|
|
|
|
# Write manual sentinel so cron can skip the next fire within the interval
|
|
if [[ "$MANUAL" == true ]]; then
|
|
echo "$END" > "$MANUAL_TS_FILE"
|
|
fi
|
|
|
|
# Trim to last LOG_MAX_LINES to prevent unbounded growth
|
|
line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
|
if [ "$line_count" -gt "$LOG_MAX_LINES" ]; then
|
|
tail -n "$LOG_MAX_LINES" "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
|
|
fi
|
|
|
|
exit $EC
|