diff --git a/AI/ai_token_sync.sh b/AI/ai_token_sync.sh index 236c956..ada7c0e 100755 --- a/AI/ai_token_sync.sh +++ b/AI/ai_token_sync.sh @@ -144,7 +144,21 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true) if [[ -z "$partner_ip" ]]; then - log "$partner_host — offline or unresolvable, leaving its ledger absent" + log "$partner_host — unresolvable, leaving any cached ledger as-is" + (( OFFLINE++ )) + continue + fi + + # Liveness and ledger presence are separate questions, probed separately on purpose. + # Tailscale hands back an IP for a peer that is registered but powered off, so resolving + # proves nothing. And a single `ssh test -f` answers both questions at once: it fails + # identically whether the host is down or the file is simply absent. Treating that one + # failure as "no ledger" would delete a perfectly good cached copy every time the partner + # blinked — turning "synced 3h ago" into "not collected here" on a transient. + if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \ + "root@${partner_ip}" true 2>/dev/null; then + log "$partner_host — not answering, keeping any cached ledger as-is" (( OFFLINE++ )) continue fi @@ -157,12 +171,13 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do continue fi - # A partner with AI off has no ledger at all. That is not an error — it is the same - # "nothing collected" the tab already knows how to render. + # Reachable, but nothing recorded there — a partner with AI off, or one that has simply + # never been asked anything. Now that liveness is established this is a real answer, so the + # stale copy goes: the tab should say "not collected here", not quote a number from before. if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \ "root@${partner_ip}" "[[ -f '$remote_db' ]]" 2>/dev/null; then - log "$partner_host — no ledger on that host yet" + log "$partner_host — reachable, but no ledger there yet" rm -f "$CACHE_DIR/${partner_slot}.tokens.db" (( OFFLINE++ )) continue @@ -183,7 +198,12 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do fi done -info "AI token sync complete — pulled $PULLED${OFFLINE:+, $OFFLINE unavailable}${FAILED:+, $FAILED failed}" +# Zero counts are omitted rather than printed. ${VAR:+...} keeps "0" because it is a non-empty +# string, and a summary that always ends "0 failed" is what teaches you to stop reading it. +_summary="AI token sync complete — pulled $PULLED" +(( OFFLINE > 0 )) && _summary+=", $OFFLINE unavailable" +(( FAILED > 0 )) && _summary+=", $FAILED failed" +info "$_summary" # Only a partner that answered and then failed the transfer is worth an exit code. An absent # partner is the normal state while HOST2 is being rebuilt.