audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops

This commit is contained in:
Gmer4Lfe
2026-06-14 12:40:15 -04:00
parent 4c37ab16fd
commit 3964f6fb46
1010 changed files with 377767 additions and 132 deletions
+2 -2
View File
@@ -205,7 +205,7 @@ HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
"$GITEA_API/user" 2>/dev/null)
if [[ "$HTTP_CODE" == "200" ]]; then
log "API token valid ✅"
echo "API token valid ✅"
elif [[ "$HTTP_CODE" == "401" ]]; then
error "API token rejected (HTTP 401) — token may be expired or have wrong scope"
error "Regenerate the token in Gitea: Settings → Applications → Generate Token → scope: write:user"
@@ -436,7 +436,7 @@ PYEOF
echo " IdentityFile $GITEA_SSH_KEY"
} >> "$SSH_CONFIG"
chmod 600 "$SSH_CONFIG"
log "SSH config entry added: Host gitea-${MY_ID,,}"
echo "SSH config entry added: Host gitea-${MY_ID,,}"
fi
END=$(date +%s)
+4 -4
View File
@@ -90,7 +90,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
"sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
echo ok" 2>/dev/null | grep -q ok && {
log "HOST1 key removed from $MIRROR authorized_keys ✅"
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
H1_DONE=true
} || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
fi
@@ -102,7 +102,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would delete: $SSH_KEY and ${SSH_KEY}.pub"
else
rm -f "$SSH_KEY" "$SSH_KEY_PUB" && log "Local key pair deleted ✅" || \
rm -f "$SSH_KEY" "$SSH_KEY_PUB" && echo "Local key pair deleted ✅" || \
warn "Failed to delete local key — check permissions"
fi
@@ -110,7 +110,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
if [[ -f "$STATE_FILE" ]]; then
if [[ "$DRY_RUN" == false ]]; then
sed -i "/^${MIRROR_ID}_PHASE/d; /^${MIRROR_ID}_KEY_READY/d" "$STATE_FILE"
log "Phase flags cleared from local setup.db ✅"
echo "Phase flags cleared from local setup.db ✅"
else
warn "DRY RUN — would clear ${MIRROR_ID}_PHASE* from setup.db"
fi
@@ -133,7 +133,7 @@ if [[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]]; then
H2_DONE=true
else
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && {
log "$MIRROR key removed from HOST1 authorized_keys ✅"
echo "$MIRROR key removed from HOST1 authorized_keys ✅"
H2_DONE=true
} || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
fi
+17 -17
View File
@@ -357,7 +357,7 @@ push_state_to_remote() {
fi
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
log "State file pushed to remote ✅" || \
echo "State file pushed to remote ✅" || \
warn "Could not push state file to remote — will propagate on next sync"
}
@@ -466,7 +466,7 @@ do_ssh_key_revocation() {
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
&& echo removed" 2>/dev/null | grep -q removed; then
log "Our pubkey revoked from $REMOTE_SERVER_NAME"
echo "Our pubkey revoked from $REMOTE_SERVER_NAME"
SSH_REVOKE_REMOTE_OK=true
else
warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:"
@@ -491,7 +491,7 @@ do_ssh_key_revocation() {
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
log "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
SSH_REVOKE_LOCAL_OK=true
else
warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually"
@@ -568,7 +568,7 @@ start_own_stack() {
continue
fi
if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then
log "$container started ✅"
echo "$container started ✅"
else
warn "$container failed to start — check manually"
fi
@@ -610,7 +610,7 @@ cleanup_partner_containers() {
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
_PM_TRAP_STOPPED+=("$container")
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
log "$container removed ✅" || warn "$container rm failed"
echo "$container removed ✅" || warn "$container rm failed"
else
log "$container not found — skipping"
fi
@@ -623,7 +623,7 @@ cleanup_partner_containers() {
warn " DRY RUN — would rm -rf $path"
continue
fi
rm -rf "$path" && log " Appdata removed: $path" || warn " Failed to remove: $path"
rm -rf "$path" && echo " Appdata removed: $path" || warn " Failed to remove: $path"
done <<< "$all_appdata_paths"
}
@@ -667,7 +667,7 @@ cleanup_owner_containers_on_mirror() {
"docker stop '$container' >/dev/null 2>&1
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
log "$container removed from $MIRROR" || \
echo "$container removed from $MIRROR" || \
warn "Failed to remove $container from $MIRROR"
# Delete appdata on remote after container removal
@@ -676,7 +676,7 @@ cleanup_owner_containers_on_mirror() {
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
log " Appdata removed on $MIRROR: $path" || \
echo " Appdata removed on $MIRROR: $path" || \
warn " Failed to remove appdata on $MIRROR: $path"
done <<< "$appdata_paths"
done <<< "$container_list"
@@ -710,7 +710,7 @@ start_mirror_own_stack() {
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
grep -q started && \
log "$container started on $MIRROR" || \
echo "$container started on $MIRROR" || \
warn "$container failed to start on $MIRROR — check manually"
done
}
@@ -801,7 +801,7 @@ provision_emby_admin() {
"${emby_url}/Users/${user_id}/Password" 2>/dev/null)
if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then
log "Emby admin '$username' created (id: $user_id) ✅"
echo "Emby admin '$username' created (id: $user_id) ✅"
else
warn "User created but password set failed (HTTP $pw_code) — set password manually"
fi
@@ -812,7 +812,7 @@ provision_emby_admin() {
-H "Content-Type: application/json" \
-d '{"IsAdministrator": true, "IsDisabled": false}' \
"${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \
log "$username granted admin policy ✅" || \
echo "$username granted admin policy ✅" || \
warn "Could not set admin policy — grant manually in Emby dashboard"
}
@@ -867,7 +867,7 @@ revoke_emby_admin() {
"${emby_url}/Users/${user_id}" 2>/dev/null)
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
log "Emby admin '$username' removed ✅"
echo "Emby admin '$username' removed ✅"
else
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
fi
@@ -922,7 +922,7 @@ update_master_conf() {
return 1
fi
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
log "master.conf updated: ${key}=${value}"
echo "master.conf updated: ${key}=${value}"
return 0
else
error "Failed to update master.conf: ${key}=${value}"
@@ -1168,7 +1168,7 @@ if [[ "$MODE" == "onboard" ]]; then
else
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
fi
log "PARTNERSHIP_ENABLED=true in master.conf ✅"
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
platform_push_conf | while IFS= read -r line; do log "$line"; done
else
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
@@ -1184,7 +1184,7 @@ if [[ "$MODE" == "onboard" ]]; then
echo "${flag_key}=true" >> "$local_state_file"
fi
platform_push_setup_state
log "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
echo "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
else
warn "DRY RUN — would write ${MY_ID}_LOCAL_DONE=true"
fi
@@ -1259,7 +1259,7 @@ if [[ "$MODE" == "onboard" ]]; then
container="${entry%%|*}"
port="${entry##*|}"
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
log "$container reachable at http://${OWNER_IP}:${port}/ ✅"
echo "$container reachable at http://${OWNER_IP}:${port}/ ✅"
else
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
fi
@@ -1438,7 +1438,7 @@ if false; then
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
log "Remote master.conf updated ✅" || \
echo "Remote master.conf updated ✅" || \
error "Failed to update remote master.conf — update manually"
else
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
+4 -4
View File
@@ -205,7 +205,7 @@ revoke_local_emby_admin() {
"${emby_url}/Users/${user_id}" 2>/dev/null)
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
log "Emby admin '$username' removed ✅"
echo "Emby admin '$username' removed ✅"
else
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
fi
@@ -243,7 +243,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ "$DRY_RUN" == false ]]; then
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
log "Rsync stopped ✅"
echo "Rsync stopped ✅"
else
warn "DRY RUN — would stop rsync"
fi
@@ -379,7 +379,7 @@ echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
if [[ "$DRY_RUN" == false ]]; then
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
log "Rsync stopped ✅"
echo "Rsync stopped ✅"
else
warn "DRY RUN — would stop rsync"
fi
@@ -474,7 +474,7 @@ NOW=$(date '+%Y-%m-%d %H:%M:%S')
if [[ "$DRY_RUN" == false ]]; then
write_state_file "$LOCAL_STATE_FILE" \
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
log "Local state: INACTIVE ✅"
echo "Local state: INACTIVE ✅"
add_to_blocklist "$MIRROR" "$REASON"
[[ "$MIRROR_REACHABLE" == true ]] && \
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
+8 -8
View File
@@ -278,7 +278,7 @@ stop_mirror_stack() {
"docker stop '$container' 2>/dev/null
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
grep -q removed && \
log " $container removed ✅" || \
echo " $container removed ✅" || \
log " $container not found on $MIRROR — skipping"
done
}
@@ -295,7 +295,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ "$SKIP_SSH" == true ]]; then
warn "Skipping SSH setup (--skip-ssh)"
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH key ready ✅"
echo "SSH key ready ✅"
else
error "SSH key setup failed"
exit 1
@@ -323,7 +323,7 @@ if [[ "$AM_MIRROR" == true ]]; then
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
2>/dev/null | grep -q triggered; then
log "Phase 2 triggered on $OWNER"
echo "Phase 2 triggered on $OWNER"
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
PHASE2_TRIGGERED=true
else
@@ -387,12 +387,12 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
# hanging for a password prompt with no TTY.
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
log "SSH to $MIRROR already works ✅ — skipping key install"
echo "SSH to $MIRROR already works ✅ — skipping key install"
STEP_SSH_OK=true
else
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH keys ready ✅"
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
# Soft-fail: generate key locally if not present, then tell user to install manually
@@ -423,7 +423,7 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
fi
fi
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH keys ready ✅"
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
error "SSH key setup failed — aborting"
@@ -472,7 +472,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then
[[ -n "$push_output" ]] && echo "$push_output"
platform_push_setup_state
if [[ $push_rc -eq 0 ]]; then
log "Conf push complete ✅"
echo "Conf push complete ✅"
CONF_PUSH_OK=true
else
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
@@ -514,7 +514,7 @@ if [[ "$DRY_RUN" == true ]]; then
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"bash '$_net_script'" 2>/dev/null; then
log "Docker network ready on $MIRROR"
echo "Docker network ready on $MIRROR"
STEP_NETWORK_OK=true
else
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
+1 -1
View File
@@ -296,7 +296,7 @@ if [[ "$DRY_RUN" == false ]]; then
-o StrictHostKeyChecking=no \
"$SCRIPTS_ROOT/Configurations/master.conf" \
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
log "master.conf pushed to $NEW_OWNER" || \
echo "master.conf pushed to $NEW_OWNER" || \
error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
else
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
+4 -4
View File
@@ -141,7 +141,7 @@ update_conf_key_path() {
if grep -q "^[[:space:]]*${KEY_CONF_VAR}=" "$HOST_CONF" 2>/dev/null; then
sed -i "s|^[[:space:]]*${KEY_CONF_VAR}=.*| ${KEY_CONF_VAR}=\"${SSH_KEY_PATH}\"|" "$HOST_CONF" && \
log "${KEY_CONF_VAR} updated in $(basename "$HOST_CONF")" || \
echo "${KEY_CONF_VAR} updated in $(basename "$HOST_CONF")" || \
warn "Failed to update ${KEY_CONF_VAR} in $(basename "$HOST_CONF") — update manually"
else
warn "${KEY_CONF_VAR} not found in $(basename "$HOST_CONF") — add manually:"
@@ -237,9 +237,9 @@ if [[ "$MODE" == "validate" ]]; then
STRIKES=$(read_strikes)
if [[ "$STRIKES" -gt 0 ]]; then
write_strike_file 0 "" "$NOW"
log "SSH validate — auth restored to $REMOTE_SERVER_NAME ✅ (strikes reset)"
echo "SSH validate — auth restored to $REMOTE_SERVER_NAME ✅ (strikes reset)"
else
log "SSH validate — $REMOTE_SERVER_NAME SSH auth OK ✅"
echo "SSH validate — $REMOTE_SERVER_NAME SSH auth OK ✅"
fi
exit 0
fi
@@ -301,7 +301,7 @@ else
warn "Generating ed25519 keypair: $SSH_KEY_PATH"
if [[ "$DRY_RUN" == false ]]; then
ssh-keygen -t ed25519 -N "" -f "$SSH_KEY_PATH" -C "${SSH_KEY_NAME}@${LOCAL_SERVER_NAME}" && \
log "Keypair generated ✅" || {
echo "Keypair generated ✅" || {
error "Failed to generate keypair"
exit 1
}