#!/usr/bin/env bash
# scripts/cluster-ping — is every cluster host up? One line per host.
#
# Hosts come from the gitignored overlay deploy/inventory/hosts.yml (same
# lookup as scripts/lib/pgb-host.sh; no host lives in this public repo). Each
# host gets one ICMP ping to its ansible_host and `ssh <name> uptime`, all
# hosts in parallel. The ssh options bound a wedged host (TCP accepts, sshd
# never answers) to about 15 s: ConnectTimeout=8 for the connect, then
# ServerAliveInterval=5 x ServerAliveCountMax=2 once connected. macOS has no
# `timeout`, so ssh options are the only bound. BatchMode never prompts.
# A connect timeout (exit 255, "timed out") is retried once, since a tailnet
# path can stall on first contact: a dead host takes about 30 s, and a
# second-try success prints "ssh ok (retry)" so a flaky link stays visible.
#
# Usage:
#   scripts/cluster-ping              # every inventory host
#   scripts/cluster-ping NODE...      # only these inventory names
#   scripts/cluster-ping --addr       # ssh the ansible_host, not the name
#
# Output:  <name>  ping ok|fail  ssh ok|ok (retry)|fail  <uptime, or ssh's last error>
# Exit:    0 all hosts answered ssh; 1 any host failed ssh (down); 2 no
#          inventory. Ping alone does not count: some networks drop ICMP.
#
# Env: CLUSTER_INVENTORY=<hosts.yml> overrides the overlay path;
#      CLUSTER_PING_SSH_OPTS adds ssh options (word-split).
set -euo pipefail

. "$(dirname "$0")/lib/pgb-host.sh"

USE_ADDR=0
WANT=()
for arg in "$@"; do
    case "$arg" in
    --addr) USE_ADDR=1 ;;
    -h | --help)
        sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'
        exit 0
        ;;
    -*)
        echo "cluster-ping: unknown option $arg" >&2
        exit 2
        ;;
    *) WANT+=("$arg") ;;
    esac
done

HOSTS="$(list_cluster_hosts)" || exit 2
if [[ -z "$HOSTS" ]]; then
    echo "cluster-ping: no hosts with ansible_host in the inventory" >&2
    exit 2
fi

if [[ "$(uname -s)" == Darwin ]]; then
    PING=(ping -c 1 -t 3) # -t: whole-run timeout in seconds
else
    PING=(ping -c 1 -w 3) # -w: deadline in seconds
fi
# shellcheck disable=SC2206 # deliberate word-split of extra options
SSH=(ssh -n -o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=5
    -o ServerAliveCountMax=2 -o IdentityAgent=none ${CLUSTER_PING_SSH_OPTS:-})

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

probe() {
    local name="$1" addr="$2" out="$3" ping_s ssh_s detail target rc
    if "${PING[@]}" "$addr" >/dev/null 2>&1; then ping_s=ok; else ping_s=fail; fi
    target="$name"
    [[ $USE_ADDR -eq 1 ]] && target="$addr"
    rc=0
    detail="$("${SSH[@]}" "$target" uptime 2>&1)" || rc=$?
    # A tailnet path can stall past ConnectTimeout on first contact; retry
    # once on a connect timeout only (auth and other failures fail at once).
    retried=""
    if [[ $rc -eq 255 && "$detail" == *"timed out"* ]]; then
        retried=" (retry)"
        rc=0
        detail="$("${SSH[@]}" "$target" uptime 2>&1)" || rc=$?
    fi
    detail="$(printf '%s' "$detail" | tr -s '\n' ' ' | sed 's/^ *//; s/ *$//')"
    if [[ $rc -eq 0 ]]; then
        ssh_s="ok$retried"
    else
        ssh_s=fail
        detail="exit $rc: ${detail:-no output}"
    fi
    printf '%s\tping %s\tssh %s\t%s\n' "$name" "$ping_s" "$ssh_s" "$detail" >"$out"
}

NAMES=()
n=0
while read -r name addr; do
    [[ -z "$name" ]] && continue
    if [[ ${#WANT[@]} -gt 0 ]]; then
        hit=0
        for w in "${WANT[@]}"; do [[ "$w" == "$name" ]] && hit=1; done
        [[ $hit -eq 1 ]] || continue
    fi
    NAMES+=("$name")
    probe "$name" "$addr" "$WORK/$n" &
    n=$((n + 1))
done <<<"$HOSTS"

if [[ $n -eq 0 ]]; then
    echo "cluster-ping: none of ${WANT[*]} is in the inventory" >&2
    exit 2
fi
wait

down=0
for ((i = 0; i < n; i++)); do
    line="$(cat "$WORK/$i")"
    printf '%s\n' "$line" | awk -F'\t' '{ printf "%-14s %-10s %-9s %s\n", $1, $2, $3, $4 }'
    [[ "$line" == *$'\tssh ok'* ]] || down=1
done
exit "$down"
