Sweep finished runs for misconfigurations instead of waiting to be told
Nothing in Varaverk fires when a job ends, so this picks up run records that completed since the last pass rather than adding a hook to forty scripts. Ahead of stability in the cycle on purpose: a wrong port is not fixed by rebooting the machine.
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// One pass of the repair sweep. Reads the logs of jobs that finished since the last pass,
|
||||
// turns known error shapes into findings, probes for a correction, and either writes a proven
|
||||
// value or leaves the finding for the operator.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Called from ai_repair_sweep.sh, which the watchdog orchestrator runs. Everything it does
|
||||
// lives in include/ai_repair.php so the same code path is what the tests exercise; this file
|
||||
// owns only the lock, the log line, and the exit code.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Non-fatal, always.
|
||||
// Exits 0 on a disabled feature, a held lock, or a sweep that found nothing. Repair is an
|
||||
// enhancement — the orchestrator that runs it has real work to do either side, and this
|
||||
// must never be the reason a watchdog cycle reports failure.
|
||||
//
|
||||
// The summary is logged, not the reasoning.
|
||||
// What was scanned, what was found, what was written. The findings themselves are the
|
||||
// record; duplicating their contents into a log would be two copies to keep in step.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// One sweep at a time.
|
||||
// flock, non-blocking. A pass that overruns its fifteen-minute slot must not have a second
|
||||
// copy start probing and writing conf underneath it.
|
||||
//
|
||||
// Nothing is written unless two switches say so.
|
||||
// AI_REPAIR_ENABLED gates the sweep; AI_REPAIR_AUTOFIX_ENABLED gates writing. With only
|
||||
// the first, this reads and files and changes no configuration at all.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// ai_repair_sweep.php one pass
|
||||
// ai_repair_sweep.php --dry-run probe and report, write nothing, leave the marker alone
|
||||
// ai_repair_sweep.php --status what the last pass did, and what is open
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
$args = array_slice($argv ?? [], 1);
|
||||
$dryRun = in_array('--dry-run', $args, true);
|
||||
$status = in_array('--status', $args, true);
|
||||
|
||||
function rlog(string $msg): void {
|
||||
if (!is_dir(LOG_DIR)) return;
|
||||
@file_put_contents(LOG_DIR . '/ai_repair.log',
|
||||
date('Y-m-d H:i:s') . ' ' . $msg . "\n", FILE_APPEND | LOCK_EX);
|
||||
}
|
||||
|
||||
if ($status) {
|
||||
$last = vv_ai_sweep_last();
|
||||
$open = vv_ai_findings_list(['open', 'needs_operator']);
|
||||
printf("repair: %s\n", vv_ai_repair_enabled() ? 'enabled' : 'disabled');
|
||||
printf("autofix: %s\n", vv_ai_repair_autofix_enabled() ? 'enabled' : 'disabled (detect only)');
|
||||
printf("last pass: %s\n", $last ? date('Y-m-d H:i:s', $last) : 'never');
|
||||
printf("open findings: %d\n", count($open));
|
||||
foreach ($open as $f) {
|
||||
printf(" [%s] %-22s %-28s seen %d — %s\n",
|
||||
$f['severity'] ?? '?', $f['subject'] ?? '?', $f['conf_key'] ?? '?',
|
||||
(int)($f['seen'] ?? 0), $f['state'] ?? '?');
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (!vv_ai_repair_enabled()) exit(0); // silent: a disabled feature is not an event
|
||||
|
||||
$lockPath = VV_CACHE_ROOT . '/ai_repair.lock';
|
||||
if (!is_dir(dirname($lockPath))) @mkdir(dirname($lockPath), 0755, true);
|
||||
$lock = @fopen($lockPath, 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
|
||||
if ($lock) fclose($lock);
|
||||
rlog('skipped — a sweep is already running');
|
||||
exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
$t0 = microtime(true);
|
||||
$sum = vv_ai_repair_sweep($dryRun);
|
||||
$ms = (int)round((microtime(true) - $t0) * 1000);
|
||||
|
||||
if (!($sum['ok'] ?? false)) {
|
||||
rlog('sweep refused — ' . ($sum['error'] ?? 'unknown'));
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Nothing found and nothing to say. A line every fifteen minutes reporting no news is how a
|
||||
// log stops being read.
|
||||
if ($sum['findings'] === 0) {
|
||||
if ($sum['runs'] > 0 && $dryRun) rlog(sprintf('dry-run: %d run(s), nothing found (%dms)', $sum['runs'], $ms));
|
||||
exit(0);
|
||||
}
|
||||
|
||||
rlog(sprintf('%s%d run(s): %d finding(s), %d fixed, %d for the operator, %d resolved, %d quiet (%dms)',
|
||||
$dryRun ? 'dry-run: ' : '', $sum['runs'], $sum['findings'], $sum['fixed'],
|
||||
$sum['needs_operator'], $sum['resolved'], $sum['quiet'], $ms));
|
||||
|
||||
foreach ($sum['details'] as $d) rlog(' ' . $d);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
|
||||
exit(0);
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================== AI Repair Sweep ===========================================
|
||||
# ==============================================================================================
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Reads the logs of jobs that finished since the last pass, turns known error shapes into
|
||||
# findings, probes for a correction, and either writes a proven value or leaves the finding
|
||||
# for the operator to answer.
|
||||
# Runs from the watchdog orchestrator. Off unless AI_REPAIR_ENABLED is true.
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
# A one-line shim: exec php on ai_repair_sweep.php in the same directory.
|
||||
# The logic is PHP because everything it needs already is — the conf writer with its backups
|
||||
# and read-back verification, the findings store, and the probe layer are all functions the
|
||||
# WebGUI shares. A bash reimplementation would be a second conf writer, which is precisely the
|
||||
# drift the guarded write path exists to prevent.
|
||||
#
|
||||
# There is no post-run hook in Varaverk; nothing fires when a job finishes. The sweep picks up
|
||||
# completed run records instead, so this is one entry in an orchestrator list rather than a
|
||||
# call added to forty scripts.
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
# A Shim, Not a Program
|
||||
# This file exists only because the scheduler runs shell scripts and the work is PHP.
|
||||
# Anything added here would be logic the WebGUI cannot reach, and the operator answering a
|
||||
# finding in the browser must take exactly the same path as the sweep that filed it.
|
||||
#
|
||||
# Detecting And Repairing Are Separate Trusts
|
||||
# AI_REPAIR_ENABLED alone reads logs, files findings and proposes fixes, writing nothing.
|
||||
# AI_REPAIR_AUTOFIX_ENABLED is what allows a value to be written, and only ever one a probe
|
||||
# has answered on. Both live in master.conf; neither is set by this script.
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
# Never Fatal
|
||||
# Always exits 0 — on a disabled feature, a held lock, or a failed pass. The watchdog
|
||||
# orchestrator runs real work either side of this, and a repair sweep must never be the
|
||||
# reason a cycle reports failure.
|
||||
#
|
||||
# One Sweep At A Time
|
||||
# The PHP takes a non-blocking flock. A pass that overruns its slot cannot have a second
|
||||
# copy start probing and writing conf underneath it.
|
||||
#
|
||||
# Nothing Is Written That Has Not Answered
|
||||
# A value reaches conf only after a probe got a response from it. Toggles are never written
|
||||
# unattended at all — whether something should be switched on is a decision about intent,
|
||||
# and a probe cannot prove intent.
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# AI_ENABLED master switch; nothing here runs without it
|
||||
# AI_REPAIR_ENABLED read logs and file findings
|
||||
# AI_REPAIR_AUTOFIX_ENABLED allow a proven value to be written unattended
|
||||
# AI_PROBE_TIMEOUT seconds a single probe may take
|
||||
# AI_FINDING_RETAIN_DAYS how long closed findings are kept
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_repair_sweep.sh
|
||||
# One pass. Files findings, applies proven fixes if autofix is on.
|
||||
#
|
||||
# ai_repair_sweep.sh --dry-run
|
||||
# Probes and reports what it would do. Writes no conf and does not move the marker, so the
|
||||
# same runs are examined again next pass.
|
||||
#
|
||||
# ai_repair_sweep.sh --status
|
||||
# Both switches, when the last pass ran, and every open finding.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/ai_repair_sweep.php" "$@"
|
||||
Reference in New Issue
Block a user