Probe partner liveness separately so a blink cannot delete a cached ledger
This commit is contained in:
+25
-5
@@ -144,7 +144,21 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$partner_ip" ]]; then
|
||||
log "$partner_host — offline or unresolvable, leaving its ledger absent"
|
||||
log "$partner_host — unresolvable, leaving any cached ledger as-is"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Liveness and ledger presence are separate questions, probed separately on purpose.
|
||||
# Tailscale hands back an IP for a peer that is registered but powered off, so resolving
|
||||
# proves nothing. And a single `ssh test -f` answers both questions at once: it fails
|
||||
# identically whether the host is down or the file is simply absent. Treating that one
|
||||
# failure as "no ledger" would delete a perfectly good cached copy every time the partner
|
||||
# blinked — turning "synced 3h ago" into "not collected here" on a transient.
|
||||
if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" true 2>/dev/null; then
|
||||
log "$partner_host — not answering, keeping any cached ledger as-is"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
fi
|
||||
@@ -157,12 +171,13 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
continue
|
||||
fi
|
||||
|
||||
# A partner with AI off has no ledger at all. That is not an error — it is the same
|
||||
# "nothing collected" the tab already knows how to render.
|
||||
# Reachable, but nothing recorded there — a partner with AI off, or one that has simply
|
||||
# never been asked anything. Now that liveness is established this is a real answer, so the
|
||||
# stale copy goes: the tab should say "not collected here", not quote a number from before.
|
||||
if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" "[[ -f '$remote_db' ]]" 2>/dev/null; then
|
||||
log "$partner_host — no ledger on that host yet"
|
||||
log "$partner_host — reachable, but no ledger there yet"
|
||||
rm -f "$CACHE_DIR/${partner_slot}.tokens.db"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
@@ -183,7 +198,12 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
fi
|
||||
done
|
||||
|
||||
info "AI token sync complete — pulled $PULLED${OFFLINE:+, $OFFLINE unavailable}${FAILED:+, $FAILED failed}"
|
||||
# Zero counts are omitted rather than printed. ${VAR:+...} keeps "0" because it is a non-empty
|
||||
# string, and a summary that always ends "0 failed" is what teaches you to stop reading it.
|
||||
_summary="AI token sync complete — pulled $PULLED"
|
||||
(( OFFLINE > 0 )) && _summary+=", $OFFLINE unavailable"
|
||||
(( FAILED > 0 )) && _summary+=", $FAILED failed"
|
||||
info "$_summary"
|
||||
|
||||
# Only a partner that answered and then failed the transfer is worth an exit code. An absent
|
||||
# partner is the normal state while HOST2 is being rebuilt.
|
||||
|
||||
Reference in New Issue
Block a user