#!/usr/bin/env bash
# scripts/deploy — non-interactive prod deploy: reinstall precis-mcp@<ref>
# into every cluster venv and bounce every daemon.
#
# The deterministic deploy backbone (the twin of scripts/ship). No LLM in the
# loop: it wraps `ansible-playbook redeploy-precis.yml` from the in-repo
# `deploy/` tree (default since the slice-12a cutover). The private overlay
# (real inventory + `.vault-pass`) is gitignored and LOCAL-only; it lives in the
# main checkout's `deploy/inventory` and is resolved from there even when a
# deploy runs from a worktree (e.g. /go), so there are no symlinks and no
# per-worktree secret copies. Chained after a green ship by the /go command.
#
# Steps:
#   1. reachability — `ansible all -m ping`; abort (non-zero) if any host is
#      down (a partial redeploy leaves nodes on mixed code — never do that).
#   2. deploy       — run the redeploy playbook for the ref.
#   3. report       — the play's final debug task prints installed sha +
#                     bounced daemons; a `failed=` on any host exits non-zero.
#
# Usage:  scripts/deploy            # deploy main (default)
#         scripts/deploy some-branch
#         scripts/deploy <old-sha> --force-rollback   # deliberate rollback
#
# gr338201 (2026-09-13 incident): a stale worktree pinned to an ancestor
# commit of what was already live ran scripts/deploy and rolled the whole
# cluster BACKWARD — the freshness guard below only catches drift on the
# *local checkout* (`git rev-parse "$REF"` vs origin), and no-ops the moment
# REF is a literal sha (there is no `origin/<sha>` ref to compare against),
# which is exactly the shape of that incident. A second, independent guard
# (below the freshness guard) resolves the target sha and refuses when it is
# a strict ancestor of either (a) the sha recorded as currently deployed
# (the shared deploy-state marker — see scripts/lib/deploy-state.sh) or (b)
# origin/main freshly fetched. Equal-sha (no-op redeploy) is allowed; a
# missing marker (first-ever deploy) is not a refusal. Deliberate rollback:
# pass --force-rollback (anywhere in argv).
#
# --pinned (anywhere in argv): the target is deliberately behind origin/main.
# /go pins the deploy to the sha its gate actually validated (scripts/ship
# writes it to .ship-sha), because `main` is a BRANCH NAME resolved at deploy
# time — a sibling --quick ship landing in between substitutes an ungated tree
# for the gated one, and the deploy reports success over code nobody ran the
# suite against. Being behind origin/main is therefore the expected steady
# state, not a rollback, so --pinned drops that leg of the guard only; the
# deployed-sha leg stays (it is the one that catches the real gr338201 shape)
# and an absent marker refuses outright. Expect a nonzero deploy-lag footer
# from the next scripts/ship: that count IS the ungated backlog on main.
#
# --ignore-backup-window (anywhere in argv): the prod DB node reboots at 03:00
# UTC and the nightly pg_dump runs 03:30 → ~04:18 UTC holding ACCESS SHARE on
# every table, so a migration's ALTER TABLE waits behind the whole dump
# (2026-09-08: a /go at 03:22 UTC hung 17+ min with a silent log). Inside that
# window a deploy that carries a pending migration — one ADDED under
# src/precis/migrations/ between origin/prod and the target, computed from
# git, never by asking prod — is refused before anything is touched; a deploy
# with no pending migration warns and proceeds. The bounds live next to the
# backup cron in deploy/roles/backups/defaults/main.yml and the guard in
# scripts/lib/backup-window.sh. --ignore-backup-window is for an operator who
# has checked the dump is not running. DEPLOY_NOW_UTC=HH:MM fakes the clock
# (tests, rehearsing the message).
#
# PRECIS_DEPLOY_SKIP_HOSTS — comma-separated hosts left out of the ping and
# the rollout (a wedged host); they keep their old sha. Loud, deliberate only.
#
# PRECIS_DEPLOY_EXTRA_VARS — optional extra `-e key=value` pairs passed through
# to ansible-playbook, space-separated (e.g. "precis_embedder_trace=true" for
# the gr172390 pass-1 diagnosis runbook). Playbook-run only, never persisted.
#
# Every run's full output (incl. the profile_tasks timing table) is tee'd to
# the main checkout's .deploy-logs/<timestamp>-<ref>.log (gitignored, newest
# 50 kept) for later analysis. PRECIS_DEPLOY_NO_LOG=1 opts out;
# PRECIS_DEPLOY_LOG_DIR overrides the directory.
#
# PRECIS_DEPLOY_CANARY=<ansible-host-name> — opt-in canary staging (unset ⇒
# byte-identical single-pass behavior, the default). When set: resolve the
# target sha once (a literal sha this checkout contains is taken as-is;
# a ref name goes through git ls-remote, the same resolution the playbook's
# step-0 pin does), run the playbook against `--limit <canary>` ONLY first, poll
# `scripts/prod-psql` for that host's `host_heartbeat` row to freshen (proof
# the deployed worker came up on the new code), then fan out to the rest of
# the fleet with `--limit '!<canary>'` — same explicit sha pins on both
# phases, so the two-run pin race (redeploy-precis.yml §step 0) can't bite. A
# canary that never freshens aborts non-zero BEFORE the fleet phase and
# prints a rollback line; a `scripts/prod-psql` failure during verify is
# treated as red (fail closed).
#   PRECIS_DEPLOY_CANARY_TIMEOUT_S  — verify timeout, seconds (default 300).
#   PRECIS_DEPLOY_CANARY_DB_HOST    — override the host_heartbeat.host value
#     to poll, if it differs from the ansible host name (fqdn vs short, or a
#     custom `precis heartbeat --host`); default is the canary name itself.
set -euo pipefail

# --help / -h : print usage and exit BEFORE any side effect (gr344875). Scans
# the whole argv (not just $1) since --force-rollback can precede or follow
# the ref positional — must stay the first thing checked, ahead of the
# lock acquire, the freshness/rollback guards, and every ansible/ssh call.
usage() {
    cat <<'USAGE'
scripts/deploy — non-interactive prod deploy: reinstall precis-mcp@<ref>
into every cluster venv and bounce every daemon.

Usage:
  scripts/deploy                              # deploy main (default)
  scripts/deploy <ref>                        # deploy a branch/tag/sha
  scripts/deploy <old-sha> --force-rollback   # deliberate rollback
  scripts/deploy <.ship-sha sha> --pinned     # /go: deploy the GATED sha
  scripts/deploy <ref> --ignore-pin           # deliberately go off-pin
  scripts/deploy <ref> --ignore-backup-window # migration deploy inside the
                                              # 03:00–04:20 UTC backup window
  scripts/deploy --help | -h                  # this message; no reachability
                                               # check, no ansible, no ssh

<ref> is a positional git ref (branch/tag/sha); default 'main'.
--force-rollback bypasses the ancestor-sha rollback guard (gr338201) — use
only for a deliberate rollback.
--pinned says "this target is deliberately behind origin/main": it drops ONLY
the origin/main leg of the rollback guard, keeps the currently-deployed-sha
leg, and refuses outright when no deploy-state marker exists (fail closed —
with no marker that leg has nothing to compare against).

A .ship-sha in this worktree is an UNCONSUMED gated pin: any target that is
not that sha is refused until the pin is deployed (which removes it) or the
run passes --ignore-pin. --force-rollback also bypasses.

Inside the nightly backup window (bounds in deploy/roles/backups/defaults/
main.yml, 03:00–04:20 UTC) a deploy that adds a src/precis/migrations/*.sql
relative to origin/prod is refused: the pg_dump holds ACCESS SHARE on every
table and the migration's DDL would hang behind it. A deploy with no pending
migration warns and proceeds. --ignore-backup-window overrides, for an
operator who has checked the dump is not running.

Notable env vars (see the header comment in this script for full docs):
  PRECIS_CLUSTER_DIR, PRECIS_DEPLOY_EXTRA_VARS, PRECIS_DEPLOY_ALLOW_STALE,
  PRECIS_DEPLOY_NO_LOG, PRECIS_DEPLOY_LOG_DIR, PRECIS_DEPLOY_FROM_TREE,
  PRECIS_OVERLAY_DIR, PRECIS_DEPLOY_SKIP_CATPATH_WHEEL, PRECIS_CATPATH_DIR,
  PRECIS_DEPLOY_SKIP_WHEEL_SMOKE, PRECIS_DEPLOY_LOCK_WAIT, PRECIS_DEPLOY_CANARY,
  PRECIS_DEPLOY_CANARY_TIMEOUT_S, PRECIS_DEPLOY_CANARY_DB_HOST, DEPLOY_NOW_UTC.
USAGE
}
for _a in "$@"; do
    case "$_a" in
        --help|-h) usage; exit 0 ;;
    esac
done

FORCE_ROLLBACK=0
PINNED=0
IGNORE_PIN=0
IGNORE_BACKUP_WINDOW=0
# A scalar, not ${#_POSITIONAL[@]}: bash 3.2 (the macOS system bash) errors on
# the length of an empty array under `set -u`, which is why the REF line below
# uses the ${arr[0]:-default} idiom too.
_HAVE_TARGET=0
_POSITIONAL=()
for _a in "$@"; do
    case "$_a" in
        --force-rollback) FORCE_ROLLBACK=1 ;;
        --pinned) PINNED=1 ;;
        --ignore-pin) IGNORE_PIN=1 ;;
        --ignore-backup-window) IGNORE_BACKUP_WINDOW=1 ;;
        *) _POSITIONAL+=("$_a"); _HAVE_TARGET=1 ;;
    esac
done
REF="${_POSITIONAL[0]:-main}"
EXTRA_ARGS=()
for _v in ${PRECIS_DEPLOY_EXTRA_VARS:-}; do
    EXTRA_ARGS+=(-e "$_v")
done
CLUSTER_DIR="${PRECIS_CLUSTER_DIR:-${HOME}/work/cluster}"
# Repo root captured BEFORE any `cd` below (install-from-tree changes CLUSTER_DIR
# to a subdir, and legacy mode's CLUSTER_DIR can be an entirely different
# checkout) — the deploy-state marker always belongs at THIS repo's root, the
# one scripts/ship reads it from, regardless of where ansible itself runs.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# A short sha is expanded to the full 40-char commit here: the playbook's pin
# step treats anything that is not ^[0-9a-f]{40}$ as a ref NAME and resolves
# it with `git ls-remote`, which never matches a bare abbreviated sha, so
# `scripts/deploy 08f79a3a` died in preflight after the wheel build. Ref
# names (main, a tag) are left alone; the playbook resolves those itself.
# Resolved against REPO_ROOT, not the cwd, so a deploy launched from another
# directory does not silently skip the expansion.
if [[ "$REF" =~ ^[0-9a-f]{7,39}$ ]]; then
    _full="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    [[ -n "$_full" ]] && REF="$_full"
fi

say() { printf '\n\033[1m▶ %s\033[0m\n' "$*"; }
die() { printf '\n\033[31m✖ %s\033[0m\n' "$*" >&2; exit 1; }

# ── the gated pin is a ONE-SHOT TOKEN: ship mints it, deploy consumes it ───
# `--pinned` only helps if the caller actually types the pinned sha, and /go
# is executed by an agent following prose in .claude/commands/go.md. One run
# that types a bare `scripts/deploy` instead resolves `main` at deploy time
# and ships whatever ungated sibling qland landed since the gate — silently,
# reporting success. So the check does not depend on anyone remembering:
# while an UNCONSUMED .ship-sha sits in this worktree, any target that is not
# that sha is refused outright. Loud-refuse, never silent-substitute: an
# operator who asked for `main` and got the pin instead would be the same
# class of bug pointed the other way.
#
# Consumed (removed) once a deploy of that exact sha records success — both
# at step 3 and at the gr332009 convergence graft, since either can be the
# write that means "the fleet is on this sha now". Without the removal a
# spent pin would block every later ordinary `scripts/deploy` from the tree
# that shipped it.
#
# Bypasses: --ignore-pin (deliberate off-pin deploy, e.g. a cluster-admin
# redeploy from a tree with a pin still pending) and --force-rollback (an
# incident hatch must not be gated behind a second flag).
#
# `--pinned` with no positional is refused outright. The documented recovery
# command is `scripts/deploy "$(cat .ship-sha)" --pinned`, and a `cat` of a
# missing/emptied pin expands to nothing — which `REF="${_POSITIONAL[0]:-main}"`
# would read as "no target given" and silently substitute `main` for, WITH
# --pinned still set. That is the exact substitution this whole mechanism
# exists to prevent, arriving through its own retry path.
if [[ "$PINNED" == 1 && "$_HAVE_TARGET" != 1 ]]; then
    die "--pinned with no target. It means 'deploy exactly this sha'; there is no default for that. If this came from scripts/deploy \"\$(cat .ship-sha)\" --pinned, the pin file is gone — the deploy that consumed it already put that sha on the fleet, so check the deploy-state marker before re-running anything."
fi

SHIP_PIN_FILE="${REPO_ROOT}/.ship-sha"
SHIP_PIN=""
if [[ -f "$SHIP_PIN_FILE" ]]; then
    SHIP_PIN="$(tr -d '[:space:]' < "$SHIP_PIN_FILE" 2>/dev/null || true)"
fi
# An unusable pin must not become an unrecoverable refusal. If the file holds
# something this repo cannot resolve to a commit (a partial write from a
# killed ship, a disk-full truncation), then the target can never equal it and
# the generic refusal below would print a remedy that replays the same garbage
# and dies identically — an operator following the instructions loops forever.
# Still refuse (a pin we cannot read is not evidence that deploying is safe),
# but say the thing that actually works.
if [[ -n "$SHIP_PIN" && "$IGNORE_PIN" != 1 && "$FORCE_ROLLBACK" != 1 ]] \
    && ! git -C "$REPO_ROOT" rev-parse -q --verify "${SHIP_PIN}^{commit}" >/dev/null 2>&1; then
    die "this worktree's .ship-sha is unusable — it does not resolve to a commit here (contents: '${SHIP_PIN:0:40}'). It cannot be deployed and it is blocking other targets. Delete it (rm .ship-sha) and re-run the gate, or pass --ignore-pin if you know what you are deploying."
fi
if [[ -n "$SHIP_PIN" && "$IGNORE_PIN" != 1 && "$FORCE_ROLLBACK" != 1 ]]; then
    _pin_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    if [[ "$_pin_target" != "$SHIP_PIN" ]]; then
        die "this worktree holds an unconsumed gated pin (.ship-sha = ${SHIP_PIN:0:8}) but the target is '${REF}'${_pin_target:+ (${_pin_target:0:8})}. The gate validated ${SHIP_PIN:0:8}; deploying anything else ships code no gate ran against. Deploy the pin:
    scripts/deploy ${SHIP_PIN} --pinned
or, if going off-pin is deliberate, re-run with --ignore-pin."
    fi
fi

# Remove the pin once this exact sha is recorded as deployed. Called from
# both success-marker writers; a no-op when there is no pin, when the pin is
# for a different sha, or when the target could not be resolved.
_consume_ship_pin() {
    local sha="$1"
    [[ -n "$SHIP_PIN" && -n "$sha" && "$sha" == "$SHIP_PIN" ]] || return 0
    rm -f "$SHIP_PIN_FILE" 2>/dev/null || true
}

# ── gr332009 rollout-convergence graft (on top of the attempt-stamp/no-legacy
# design above) — write the SUCCESS marker (and clear the attempt stamp) as
# soon as the code ROLLOUT itself converges, even if a LATER, unrelated play
# in the same `ansible-playbook` invocation dies ───────────────────────────
# redeploy-precis.yml runs its residual, non-rollout plays (agent-sandbox
# image rebuilds etc.) in the SAME invocation as the core rollout (venv
# installs, daemon bounce, convergence assert) — so a persistent failure on a
# host those residual plays alone touch (the known balthazar sandbox
# podman-pull residual, commit 5fa7ff29) makes the whole command exit non-zero even
# though every host that matters to "what is the fleet actually running"
# landed cleanly. Without this graft, step 3 below never runs on that red
# exit, the attempt stamp never clears, and every ship footer says "deploy
# state uncertain" forever — alarm fatigue in the other direction. A genuine
# rollout-host failure must still leave the stamp in place (fail-safe).
#
# _write_deploy_state_marker duplicates step 3's success write (marker +
# clear-stamp) on purpose rather than factoring it out — step 3 itself stays
# byte-identical to the no-graft design so the ordinary all-green path is
# untouched. _rollout_converged reads the ALREADY-CAPTURED ansible-playbook
# output for the PLAY RECAP and requires every rollout host (ROLLOUT_HOSTS,
# set below from the reachability ping's own output) to show
# `failed=0 unreachable=0` — a host missing from the recap entirely, or
# showing any failure, means NOT converged. Scoped to the single-pass path
# only (PRECIS_DEPLOY_CANARY unset, the default) — the canary path's marker
# already means "the WHOLE fleet is on this sha", and writing it after only
# the canary phase converges would be actively wrong, so that path's existing
# all-or-nothing `|| die` behavior is left untouched.
_write_deploy_state_marker() {
    local sha
    sha="$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || true)"
    [[ -n "$sha" ]] || return 0
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    printf '%s %s %s%s\n' "$sha" "$(date +%s)" "success" "${OVERLAY_HASH:+ overlay=${OVERLAY_HASH}}" > "$(deploy_state_path "$REPO_ROOT")"
    rm -f "$(deploy_attempt_path "$REPO_ROOT")" 2>/dev/null
    _consume_ship_pin "$sha"
    _move_prod_pointer "$sha"
}

# Publish "what the cluster runs" as origin's `prod` ref, from both success
# writers (the graft above and step 3). The marker file says it to this
# machine only; the ref says it to every session and to GitHub. Fast-forward
# only — it goes backward solely on a deliberate --force-rollback — and
# best-effort: a refused push is a warning, never a failed deploy. See
# scripts/lib/env-pointers.sh.
_move_prod_pointer() {
    local sha="$1" mode=""
    # Both writers run on an all-green deploy; move (and announce) once.
    [[ "${_PROD_POINTER_AT:-}" != "$sha" ]] || return 0
    _PROD_POINTER_AT="$sha"
    [[ -f "${REPO_ROOT}/scripts/lib/env-pointers.sh" ]] || return 0
    . "${REPO_ROOT}/scripts/lib/env-pointers.sh"
    [[ "$FORCE_ROLLBACK" == 1 ]] && mode="force"
    if env_pointer_move "$REPO_ROOT" prod "$sha" "$mode"; then
        say "origin/prod → ${sha:0:8}"
    fi
    # The clone the shared session MCP serves, when this machine has one.
    if env_prod_checkout_sync "$REPO_ROOT" "$sha"; then
        say "prod clone → ${sha:0:8} ($(env_prod_checkout_dir "$REPO_ROOT"))"
    fi
    _confirm_prod_clone "$sha"
    return 0
}

# Runtime confirmation for the served clone (gr482226). env_prod_checkout_sync
# above already tried to move it and warned on failure; a clone that resolves
# but still does not name the deployed sha means the session MCP keeps serving
# the previous release while every deploy report reads green — that was the
# 2026-10-13 r32 incident (served 18f9c9f43 under prod ec345b895). Failing
# here fails the deploy's success path on purpose: the marker and the pin are
# already consistent, and the fix is one command, not a silent divergence.
# No clone resolved at all -> nothing to confirm (machines that deploy but
# do not host the session server); the sync's warning already said so.
_confirm_prod_clone() {
    local sha="$1" dir head
    dir="$(env_prod_checkout_dir "$REPO_ROOT" 2>/dev/null || true)"
    [[ -n "$dir" && -f "${dir}/.git/HEAD" ]] || return 0
    head="$(git -C "$dir" rev-parse HEAD 2>/dev/null || true)"
    [[ "$head" == "$sha" ]] || die "prod clone ${dir} is at ${head:-unknown} but this deploy put ${sha} on the fleet — the session MCP keeps serving the old sha. Fix it with:
    bash ~/work/infrastructure/precis-mcp/scripts/precis-mcp-http-ensure.sh
or  git -C ${dir} fetch origin && git -C ${dir} checkout --detach ${sha}"
}

_rollout_converged() {
    local play_out="$1" host line failed unreachable recap
    [[ -n "${ROLLOUT_HOSTS:-}" ]] || return 1
    recap="$(awk '/^PLAY RECAP/{f=1; next} f' "$play_out")"
    [[ -n "$recap" ]] || return 1
    for host in $ROLLOUT_HOSTS; do
        line="$(awk -v h="$host" '$1==h' <<< "$recap")"
        [[ -n "$line" ]] || return 1
        failed="$(grep -oE 'failed=[0-9]+' <<< "$line" | cut -d= -f2)"
        unreachable="$(grep -oE 'unreachable=[0-9]+' <<< "$line" | cut -d= -f2)"
        [[ "${failed:-1}" == 0 && "${unreachable:-1}" == 0 ]] || return 1
    done
    return 0
}

# A render worktree deleted under a running ansible surfaces only as a bare
# `[Errno 2] No such file or directory` on some playbook path (2026-09-27, and
# again 2026-09-29 as `os.getcwd()`), and the ping step below reads it as "a
# host is unreachable". Name the real cause whenever ansible fails.
_say_if_render_tree_gone() {
    [[ -n "$_DEPLOY_TREE" && ! -d "$_DEPLOY_TREE" ]] || return 0
    say "the render worktree ${_DEPLOY_TREE} is GONE — ansible lost its playbooks mid-run, so the missing-file error above is that, not a host fault. Something outside this deploy deleted it (a worktree reaper, or a run that judged this one dead). Hosts may be half-converged: re-run the same pinned sha."
}

# gr256065: name what failed, and say whether it can have split the fleet.
# Reads the captured ansible-playbook output and classifies each `fatal:` by
# whether the host had already finished the precis install (an
# `Install precis-mcp[...]` task result of ok/changed):
#   prereq  — the host never reached the install: still entirely on the old sha
#   install — the install task itself failed: possibly half-updated (the real
#             mixed-versions case)
#   post    — the install had landed; a later task failed (code is deployed)
# A `...ignoring` line after a fatal (ignore_errors) drops that failure.
# FAIL_MIXED=1 only for an install failure.
FAIL_MIXED=0
FAIL_SUMMARY=""
_analyze_failures() {
    local play_out="$1" rows host cls play task
    rows="$(awk '
        /^PLAY \[/ { play = $0; sub(/^PLAY \[/, "", play); sub(/\] \**$/, "", play) }
        /^TASK \[/ { task = $0; sub(/^TASK \[/, "", task); sub(/\] \**$/, "", task); inst = (task ~ /Install precis-mcp\[/) }
        /^(ok|changed): \[/ && inst { h = $0; sub(/^[a-z]+: \[/, "", h); sub(/[] ].*$/, "", h); done[h] = 1 }
        /^fatal: \[/ {
            h = $0; sub(/^fatal: \[/, "", h); sub(/[] ].*$/, "", h)
            if (h in seen) next
            seen[h] = 1; n++; fh[n] = h; fp[n] = play; ft[n] = task; fi[n] = inst; fd[n] = (h in done)
        }
        /^\.\.\.ignoring/ { if (n > 0) { delete seen[fh[n]]; n-- } }
        END { for (i = 1; i <= n; i++) printf "%s\t%s\t%s\t%s\n", fh[i], (fi[i] ? "install" : (fd[i] ? "post" : "prereq")), fp[i], ft[i] }
    ' "$play_out" 2>/dev/null || true)"
    FAIL_MIXED=0
    FAIL_SUMMARY=""
    [[ -n "$rows" ]] || return 0
    say "failed hosts:"
    while IFS=$'\t' read -r host cls play task; do
        case "$cls" in
            install) FAIL_MIXED=1; cls="precis install failed (possibly half-updated)" ;;
            post)    cls="failed after the precis install landed (code deployed)" ;;
            *)       cls="prerequisite failed before the precis install (still entirely on the old sha)" ;;
        esac
        printf '  %s — %s\n      play: %s\n      task: %s\n' "$host" "$cls" "$play" "$task"
        FAIL_SUMMARY+="${FAIL_SUMMARY:+, }${host}"
    done <<< "$rows"
}

# The die text for a red rollout: only an install failure may claim mixed
# versions (gr256065).
_redeploy_failure_msg() {
    if [[ -z "$FAIL_SUMMARY" ]]; then
        printf '%s' "redeploy failed — see the failing task above (nodes may be on mixed versions; re-run once fixed)."
    elif [[ "$FAIL_MIXED" == 1 ]]; then
        printf '%s' "redeploy failed on ${FAIL_SUMMARY} — the precis install itself failed (nodes may be on mixed versions; re-run once fixed)."
    else
        printf '%s' "redeploy failed on ${FAIL_SUMMARY} — not a precis install failure (see the failed-host list above); this failure leaves no mixed versions. Re-run once fixed."
    fi
}

# Run one redeploy-precis.yml invocation. Its combined output is ALSO tee'd
# to $play_out (a scratch capture, separate from the whole-script tee already
# flowing to console + the deploy log) purely so _rollout_converged can parse
# the PLAY RECAP — nothing about what reaches the console/log changes. The
# `if !` form (not `|| die`) is deliberate: it lets this function inspect the
# recap and possibly write the marker BEFORE the caller's own `|| die` fires.
_run_rollout_playbook() {
    local play_out rc=0
    play_out="$(mktemp)"
    # PYTHONUNBUFFERED: ansible is python, and python block-buffers stdout
    # when it is a pipe — which it is here (and again at the whole-script
    # `exec > >(tee …)`). The effect is that during a single long task the
    # live log receives NOTHING, including the `ASYNC POLL` lines that prove
    # an async task is still being polled. That is why `tail -F` on a deploy
    # log looked dead through the 2026-09-26 agent-image builds and `ps` was
    # the only liveness check. Line-buffering costs nothing and makes the
    # live log honest.
    if ! PYTHONUNBUFFERED=1 ansible-playbook "$@" 2>&1 | tee "$play_out"; then
        rc=1
    fi
    [[ "$rc" -eq 0 ]] || _say_if_render_tree_gone
    [[ "$rc" -eq 0 ]] || _analyze_failures "$play_out"
    if [[ "$rc" -ne 0 ]] && _rollout_converged "$play_out"; then
        say "redeploy-precis.yml exited non-zero, but every rollout host (${ROLLOUT_HOSTS}) converged clean in its own PLAY RECAP line — recording the deploy-state marker before reporting the failure (gr332009)"
        _write_deploy_state_marker
    fi
    rm -f "$play_out"
    return "$rc"
}

# ── mutual exclusion: one deploy at a time, machine-wide (gr203786) ──────
# Two concurrent deploys (e.g. sibling /go sessions in different worktrees)
# interleave their installs: each run pins its own sha, so the later run's
# installs move the venvs under the earlier run's convergence assert →
# spurious "DEPLOY DID NOT CONVERGE" on a healthy cluster (observed twice
# 2026-08-11). The playbook's per-run pinning is correct; the missing piece
# is that runs must not overlap. Lock dir lives in the git COMMON dir so
# every worktree of this repo shares one lock. mkdir is the atomic acquire
# (macOS ships no flock); a dead holder's lock is stolen.
_LOCK_DIR="$(git -C "$REPO_ROOT" rev-parse --path-format=absolute --git-common-dir)/precis-deploy.lock"
# Does pid $1 exist, whoever owns it? `kill -0` alone answers "may I signal
# it": for a live pid owned by another user it fails with EPERM, which a bare
# `kill -0` check reads as dead — and then steals a live deploy's lock or
# deletes its render tree. This controller runs deploys as both `reto` and
# `deploy`. EPERM means the process exists; only ESRCH means it is gone.
_pid_alive() {
    local err
    err="$(export LC_ALL=C; kill -0 "$1" 2>&1)" && return 0
    [[ "$err" == *"not permitted"* ]]
}
_acquire_deploy_lock() {
    local wait="${PRECIS_DEPLOY_LOCK_WAIT:-1800}" waited=0 holder
    while ! mkdir "$_LOCK_DIR" 2>/dev/null; do
        holder="$(cat "${_LOCK_DIR}/pid" 2>/dev/null || true)"
        if [[ -n "$holder" ]] && ! _pid_alive "$holder"; then
            say "stealing deploy lock from dead pid ${holder}"
            rm -rf "$_LOCK_DIR"
            continue
        fi
        (( waited < wait )) || die "another deploy (pid ${holder:-unknown}) still holds the lock after ${wait}s — investigate it, or rm -rf ${_LOCK_DIR}"
        (( waited % 60 == 0 )) && say "another deploy (pid ${holder:-starting}) is in flight — waiting (${waited}s/${wait}s)"
        sleep 10; waited=$((waited + 10))
    done
    echo $$ > "${_LOCK_DIR}/pid"
    trap '_deploy_cleanup' EXIT
    trap 'exit 130' INT TERM
}
# Set once the render worktree below exists; the EXIT trap has to clean up
# BOTH it and the lock, and a second `trap ... EXIT` would silently replace
# the first rather than add to it.
_DEPLOY_TREE=""
# Every command here is no-fail, and the triggering status is captured and
# re-raised explicitly. Under `set -e` a failing command inside an EXIT trap
# aborts the REST OF THE TRAP and overwrites the shell's exit status — so a
# worktree teardown that loses a race with a leftover .ansible/tmp file would
# both leak the lock (the rm below never runs) and report a completed deploy
# as failed. Verified: `x() { echo A; false || false; echo B; }; trap x EXIT`
# under `set -euo pipefail` with `exit 0` prints A, never B, and exits 1.
_deploy_cleanup() {
    local rc=$?
    if [[ -n "$_DEPLOY_TREE" ]]; then
        # Unlock first: `worktree remove` refuses a locked tree even with --force.
        git -C "$REPO_ROOT" worktree unlock "$_DEPLOY_TREE" 2>/dev/null || true
        git -C "$REPO_ROOT" worktree remove --force "$_DEPLOY_TREE" 2>/dev/null \
            || rm -rf "$_DEPLOY_TREE" 2>/dev/null || true
    fi
    rm -rf "$_LOCK_DIR" 2>/dev/null || true
    exit "$rc"
}
_acquire_deploy_lock

# ── deploy log: persist every run's full output — including the per-task
# profile_tasks timing table (deploy/ansible.cfg) — so deploys can be compared
# later instead of dying with the terminal scrollback. One timestamped file per
# run, in the MAIN checkout's .deploy-logs/ (gitignored) so every worktree's
# deploys land in one place. Deliberately NOT under deploy/: the logs name real
# hosts and tests/test_deploy_tree_no_secrets.py walks that whole tree. Keep
# the newest 50. PRECIS_DEPLOY_NO_LOG=1 opts out (e.g. for probe runs).
if [[ -z "${PRECIS_DEPLOY_NO_LOG:-}" ]]; then
    _main_root="$(dirname "$(git -C "$REPO_ROOT" rev-parse --path-format=absolute --git-common-dir)")"
    DEPLOY_LOG_DIR="${PRECIS_DEPLOY_LOG_DIR:-${_main_root}/.deploy-logs}"
    mkdir -p "$DEPLOY_LOG_DIR"
    DEPLOY_LOG="${DEPLOY_LOG_DIR}/$(date -u +%Y%m%d-%H%M%S)-${REF//\//_}.log"
    exec > >(tee -a "$DEPLOY_LOG") 2>&1
    say "logging this deploy to ${DEPLOY_LOG}"
    ls -1t "$DEPLOY_LOG_DIR"/*.log 2>/dev/null | tail -n +51 | xargs rm -f 2>/dev/null || true
fi

# ── render worktree: templates come from the TARGET SHA, not this checkout ──
# ansible renders deploy/ roles+templates from a filesystem tree while the
# venvs install precis-mcp@<ref> from git. Those were two different sources of
# truth: the invoking checkout's HEAD, and the pinned sha. Every consequence
# of that split was a rule someone had to remember — sync before deploying,
# never edit the tree mid-deploy, and (the reason this exists) a sibling
# landing on main could invalidate a deploy that had already pinned its sha.
#
# For a literal-sha target — the /go path, since ship pins the gated sha — we
# check that sha out into a throwaway detached worktree and render from there.
# HEAD == target by construction, so the skew class is gone rather than
# guarded, and main is free to move under a running deploy.
#
# A ref-NAME target (main, a tag) still renders from this checkout: the
# playbook resolves the name itself via ls-remote at pin time, so resolving it
# a second time here could disagree. The freshness guard below still covers
# that path.
if [[ -n "${PRECIS_DEPLOY_FROM_TREE-1}" && "$REF" =~ ^[0-9a-f]{40}$ ]]; then
    _wt_common="$(git -C "$REPO_ROOT" rev-parse --path-format=absolute --git-common-dir)"
    # PID-suffixed, not a fixed path (2026-09-29). The old fixed
    # `precis-deploy-tree` was shared by every run, and the deploy lock does NOT
    # serialise access to it: the window between one deploy's process dying and
    # its EXIT trap finishing is enough for the next deploy to acquire the freed
    # lock, create its tree, and then have the previous run's trailing trap
    # delete that tree out from under it. Ansible, whose cwd IS that directory,
    # then dies on `os.getcwd()` with a bare `[Errno 2]`, no PLAY RECAP, and —
    # worst of all — `scripts/deploy` reports it as "a host is unreachable",
    # sending the reader off to check the network. Cost two failed deploys in
    # one session before the traceback named `getcwd`.
    _wt_root="${_wt_common}/precis-deploy-tree-$$"
    # Reap dead runs' trees. With a unique path per run, a crashed deploy would
    # otherwise leak its tree forever; with the pid in the name we can tell a
    # leftover from a live sibling's. `worktree remove` first so git forgets the
    # registration — rm -rf alone leaves a stale entry that blocks the re-add.
    for _stale in "${_wt_common}"/precis-deploy-tree-*; do
        [[ -d "$_stale" ]] || continue
        _stale_pid="${_stale##*-}"
        # Still running (and not us): another deploy legitimately owns it.
        if [[ "$_stale_pid" != "$$" ]] && _pid_alive "$_stale_pid"; then
            continue
        fi
        git -C "$REPO_ROOT" worktree unlock "$_stale" 2>/dev/null || true
        git -C "$REPO_ROOT" worktree remove --force "$_stale" 2>/dev/null || true
        rm -rf "$_stale"
    done
    git -C "$REPO_ROOT" worktree add --detach --quiet "$_wt_root" "$REF" 2>/dev/null \
        || die "could not check out ${REF:0:12} into a render worktree at ${_wt_root} — is the commit present in this repo? (fetch it, or set PRECIS_DEPLOY_FROM_TREE= to render from this checkout)"
    _DEPLOY_TREE="$_wt_root"
    # Lock it with a live-pid reason (2026-09-29). scripts/inflight buckets a
    # clean detached tree at a sha that is in main as `safe_remove`, and every
    # sibling session's SessionStart runs scripts/reap-worktrees over that
    # bucket — which deleted a render tree mid-deploy; ansible then died with a
    # bare `[Errno 2] No such file or directory` on its next template task. A
    # `locked … pid N` reason with N alive is exactly what inflight reads as a
    # live session, so the tree is never a reap candidate while this runs.
    git -C "$REPO_ROOT" worktree lock --reason "pid $$ scripts/deploy render tree" "$_wt_root" 2>/dev/null || true
    say "rendering deploy/ templates from a detached worktree at ${REF:0:12} (this checkout's HEAD is not used)"
fi

# ── freshness guard: ansible renders roles/templates from THIS checkout, while
# the venvs install precis-mcp@<ref> from git — a local tree behind origin/<ref>
# deploys NEW code with STALE units (bit us 2026-08-03: workers restarted
# without the §F-b env exports). Refuse the skew unless explicitly overridden.
# Inert when the render worktree above took over: there is no skew to find.
if [[ -z "${PRECIS_DEPLOY_ALLOW_STALE:-}" && -z "$_DEPLOY_TREE" ]]; then
    git -C "$REPO_ROOT" fetch -q origin "$REF" 2>/dev/null || true
    _remote="$(git -C "$REPO_ROOT" rev-parse -q --verify "origin/${REF}" 2>/dev/null || true)"
    if [[ -n "$_remote" ]] && ! git -C "$REPO_ROOT" merge-base --is-ancestor "$_remote" HEAD; then
        die "local tree does not contain origin/${REF} ($(git -C "$REPO_ROOT" rev-parse --short HEAD) vs ${_remote:0:8}) — templates would render stale. Sync this checkout (git merge origin/${REF}) or set PRECIS_DEPLOY_ALLOW_STALE=1."
    fi
    # REF given as a literal sha (or a tag): `origin/<sha>` does not resolve, so
    # the branch comparison above no-ops ENTIRELY and a sha deploy gets no skew
    # check at all — the hole the gr338201 header calls out, and now the normal
    # path since /go pins the gated sha. Compare the target commit against this
    # checkout directly instead: ansible renders deploy/ templates from HEAD
    # while the venvs install the target, so HEAD must at minimum CONTAIN the
    # target. Any divergence beyond that is said out loud rather than refused —
    # deploying an older-than-HEAD sha is exactly what --pinned is for.
    if [[ -z "$_remote" ]]; then
        _fr_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
        _fr_head="$(git -C "$REPO_ROOT" rev-parse -q --verify HEAD 2>/dev/null || true)"
        if [[ -n "$_fr_target" ]]; then
            git -C "$REPO_ROOT" merge-base --is-ancestor "$_fr_target" HEAD 2>/dev/null \
                || die "local tree does not contain ${REF} (${_fr_target:0:8}; HEAD is ${_fr_head:0:8}) — templates would render from a tree that never saw this commit. Sync this checkout, or set PRECIS_DEPLOY_ALLOW_STALE=1."
            [[ "$_fr_target" == "$_fr_head" ]] \
                || echo "NOTE: installing ${_fr_target:0:8} but rendering deploy/ templates from HEAD ${_fr_head:0:8} — fine when the delta touches no template, otherwise check out the target sha first."
        fi
    fi
    _dirty="$(git -C "$REPO_ROOT" status --porcelain -- deploy/ 2>/dev/null || true)"
    [[ -z "$_dirty" ]] || die "deploy/ tree has uncommitted changes — templates would render unshipped state. Commit/ship first, or set PRECIS_DEPLOY_ALLOW_STALE=1."
fi

# ── rollback guard (gr338201): refuse an ancestor-sha deploy ─────────────
# Independent of the freshness guard above (which only compares the LOCAL
# checkout to origin/$REF and no-ops for a literal-sha REF). This compares
# the resolved TARGET sha itself against the sha already recorded as
# deployed (the shared deploy-state marker) and against a freshly-fetched
# origin/main — the two things that jointly define "already ahead of this".
# Not gated by PRECIS_DEPLOY_ALLOW_STALE (a different problem: template
# skew, not direction). Only --force-rollback bypasses it.
if [[ "$FORCE_ROLLBACK" != 1 ]]; then
    _rb_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    if [[ -n "$_rb_target" ]]; then
        . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
        _rb_marker="$(deploy_state_read_path "$REPO_ROOT")"
        _rb_deployed=""
        if [[ -n "$_rb_marker" && -f "$_rb_marker" ]]; then
            _rb_deployed="$(awk '{print $1}' "$_rb_marker" 2>/dev/null || true)"
            git -C "$REPO_ROOT" cat-file -e "${_rb_deployed}^{commit}" 2>/dev/null || _rb_deployed=""
        fi
        git -C "$REPO_ROOT" fetch -q origin main 2>/dev/null || true
        _rb_origin_main="$(git -C "$REPO_ROOT" rev-parse -q --verify origin/main 2>/dev/null || true)"

        # --pinned: the target is deliberately behind origin/main (it is the sha
        # a full gate validated; siblings have qlanded ungated commits on top
        # since). Drop ONLY the origin/main leg — the deployed-sha leg below
        # still catches the actual gr338201 incident shape, which was a stale
        # worktree deploying an ancestor of WHAT WAS ALREADY LIVE. With no
        # marker that leg has nothing to compare against, so fail closed rather
        # than let --pinned degrade into a guard-free deploy.
        if [[ "$PINNED" == 1 && -z "$_rb_deployed" ]]; then
            die "--pinned needs the deploy-state marker to check direction, and there is none (no successful deploy on record). Deploy the branch normally first, or pass --force-rollback if you know this target is correct."
        fi

        _rb_against=""
        if [[ -n "$_rb_deployed" && "$_rb_target" != "$_rb_deployed" ]] \
            && git -C "$REPO_ROOT" merge-base --is-ancestor "$_rb_target" "$_rb_deployed" 2>/dev/null; then
            _rb_against="the currently-deployed sha ${_rb_deployed:0:8} (deploy-state marker)"
        elif [[ "$PINNED" != 1 ]] && [[ -n "$_rb_origin_main" && "$_rb_target" != "$_rb_origin_main" ]] \
            && git -C "$REPO_ROOT" merge-base --is-ancestor "$_rb_target" "$_rb_origin_main" 2>/dev/null; then
            _rb_against="origin/main (${_rb_origin_main:0:8})"
        fi

        # Say how much ungated trunk this pin is leaving behind — the whole
        # point of pinning is that main moved on, and the operator should see
        # by how much rather than infer it from a silent success.
        if [[ "$PINNED" == 1 && -z "$_rb_against" && -n "$_rb_origin_main" && "$_rb_target" != "$_rb_origin_main" ]]; then
            _rb_behind="$(git -C "$REPO_ROOT" rev-list --count "${_rb_target}..${_rb_origin_main}" 2>/dev/null || true)"
            # An `if`, not a trailing `[[ … ]] && echo`: nothing behind main is
            # an ordinary pin (a /go that won the CAS race cleanly, or a target
            # ahead of origin/main), and as the last command of this block the
            # AND-list form returns non-zero on that path. Harmless where this
            # sits today — `set -e` exempts it at top level — but it exits the
            # moment anyone wraps this code in a function.
            if [[ -n "$_rb_behind" && "$_rb_behind" != 0 ]]; then
                echo "NOTE: --pinned — deploying the gated ${_rb_target:0:8}, which is ${_rb_behind} commit(s) behind origin/main (${_rb_origin_main:0:8}). Those commits are NOT going out; the next full gate certifies them."
            fi
        fi

        if [[ -n "$_rb_against" ]]; then
            {
                . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
                printf '%s %s %s\n' "$_rb_target" "$(date +%s)" "refused" \
                    > "$(deploy_attempt_path "$REPO_ROOT")"
            } 2>/dev/null || true
            die "REFUSING TO DEPLOY: target '${REF}' (${_rb_target:0:8}) is an ANCESTOR of ${_rb_against} — this would roll the cluster BACKWARD (gr338201: this is exactly the 2026-09-13 stale-worktree incident shape). deployed=${_rb_deployed:-unknown} target=${_rb_target} origin/main=${_rb_origin_main:-unknown}. If this rollback is deliberate, re-run with --force-rollback."
        fi
    fi
fi

# ── backup-window guard: no migration deploy while the nightly pg_dump may
# be running ──────────────────────────────────────────────────────────────
# Decision and bounds live in scripts/lib/backup-window.sh (one place, next
# to the cron it mirrors — see the header). Here: keep origin/prod fresh (the
# "what runs" end of the pending-migration diff), run the guard, and stop on
# a refusal. Sits after the rollback guard so a backward deploy is named as
# that first, and before the overlay/`cd`, so nothing has been rendered or
# pinged yet. The real clock applies to every deploy — the guard says nothing
# at all outside the window. Deliberately NOT written to the attempt stamp as
# "refused": scripts/ship reads that outcome as the rollback guard.
# Tolerant of a tree without the lib (the throwaway-repo fixtures in tests/
# copy only scripts/deploy + deploy-state.sh), same as env-pointers.sh.
if [[ -f "${REPO_ROOT}/scripts/lib/backup-window.sh" ]]; then
    . "${REPO_ROOT}/scripts/lib/backup-window.sh"
    git -C "$REPO_ROOT" fetch -q origin prod 2>/dev/null || true
    _bw_rc=0
    backup_window_guard "$REPO_ROOT" "$REF" "$IGNORE_BACKUP_WINDOW" || _bw_rc=$?
    case "$_bw_rc" in
        0) ;;
        1) die "backup-window guard refused this deploy (see above) — no host was touched." ;;
        *) die "backup-window guard could not read the window bounds (see above) — fix deploy/roles/backups/defaults/main.yml; --ignore-backup-window does not bypass a broken bounds file." ;;
    esac
fi

# ── slice 12a: install-from-tree is the DEFAULT (cutover complete 2026-07-19).
# Deploy runs ansible from the portable in-repo `deploy/` tree. Overlay (private
# inventory + vault) resolution, symlink-free and checkout-independent:
#   • if THIS checkout has deploy/inventory, use it (ansible.cfg's relative paths);
#   • else fall back to the CANONICAL overlay in the main checkout (found via
#     git --git-common-dir), or $PRECIS_OVERLAY_DIR — so a deploy works from any
#     worktree (/go ships+deploys from one) with the overlay stored in ONE place.
# Legacy escape (pre-cutover private checkout): `PRECIS_DEPLOY_FROM_TREE=`.
INV_ARGS=()
OVERLAY_DIR=""
PRECIS_DEPLOY_FROM_TREE="${PRECIS_DEPLOY_FROM_TREE-1}"
if [[ -n "${PRECIS_DEPLOY_FROM_TREE}" ]]; then
    if [[ -n "$_DEPLOY_TREE" ]]; then
        CLUSTER_DIR="${_DEPLOY_TREE}/deploy"
    else
        _script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
        CLUSTER_DIR="$(cd "${_script_dir}/.." && pwd)/deploy"
    fi
    if [[ -d "${CLUSTER_DIR}/inventory" ]]; then
        OVERLAY_DIR="${CLUSTER_DIR}/inventory"
        say "install-from-tree: deploying from ${CLUSTER_DIR} (local overlay)"
    else
        _main="$(dirname "$(git -C "${CLUSTER_DIR}" rev-parse --path-format=absolute --git-common-dir 2>/dev/null)")"
        _overlay="${PRECIS_OVERLAY_DIR:-${_main}/deploy/inventory}"
        _vpass="$(dirname "${_overlay}")/.vault-pass"
        [[ -d "${_overlay}" && -f "${_vpass}" ]] || die "install-from-tree: no overlay — this checkout's ${CLUSTER_DIR}/inventory is empty and the canonical overlay (${_overlay}) is missing. Put your private inventory in the main checkout's deploy/inventory, or set PRECIS_OVERLAY_DIR (see deploy/README.md)."
        INV_ARGS=(-i "${_overlay}/hosts.yml" --vault-password-file "${_vpass}")
        OVERLAY_DIR="${_overlay}"
        say "install-from-tree: deploying from ${CLUSTER_DIR} (canonical overlay: ${_overlay})"
    fi
fi

# ── overlay change detection (gr239588) ─────────────────────────────────
# deploy/inventory is the gitignored private overlay: an edit there moves no
# sha and touches no tracked path, so the playbook's sha-move gate and its
# git-diff bounce classifier both read "nothing changed" and render nothing.
# Hash the tree here on the controller, compare with the hash recorded in the
# deploy-state marker (4th field, `overlay=<hash>`), and pass
# precis_overlay_changed=true when they differ: redeploy-precis.yml then
# counts the deploy as changed with bounce scope `full`. Only the hash is
# ever printed or stored — never file contents or a vault value. A marker with
# no recorded hash (older than this) is not a change: the first deploy after
# this lands records one without forcing a bounce.
_sha256_stdin() {
    if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}'
    else shasum -a 256 | awk '{print $1}'; fi
}
_overlay_hash() {
    local dir="$1" f
    [[ -d "$dir" ]] || return 0
    (
        cd "$dir" || exit 0
        LC_ALL=C find . \( -type f -o -type l \) ! -name .DS_Store ! -name '*.retry' ! -name '*.swp' ! -name '*~' ! -name '#*#' -print | LC_ALL=C sort \
            | while IFS= read -r f; do
                printf '%s ' "$f"
                _sha256_stdin < "$f"
            done | _sha256_stdin
    ) 2>/dev/null || true
}
OVERLAY_HASH=""
if [[ -n "$OVERLAY_DIR" ]]; then
    OVERLAY_HASH="$(_overlay_hash "$OVERLAY_DIR")"
fi
if [[ -n "$OVERLAY_HASH" ]]; then
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    _ov_marker="$(deploy_state_read_path "$REPO_ROOT")"
    _ov_prev=""
    if [[ -n "$_ov_marker" && -f "$_ov_marker" ]]; then
        _ov_prev="$(awk '{for (i = 4; i <= NF; i++) if ($i ~ /^overlay=/) { sub(/^overlay=/, "", $i); print $i; exit }}' "$_ov_marker" 2>/dev/null || true)"
    fi
    if [[ -n "$_ov_prev" && "$_ov_prev" != "$OVERLAY_HASH" ]]; then
        EXTRA_ARGS+=(-e precis_overlay_changed=true)
        say "overlay changed since the last recorded deploy (hash ${_ov_prev:0:12} → ${OVERLAY_HASH:0:12}) — counting this deploy as changed, bounce scope full."
        say "Roles outside redeploy-precis.yml (backups, monitoring, pgbouncer, ...) also read the overlay and are NOT re-rendered by this deploy: re-run their numbered playbooks in deploy/playbooks/ if the edit touches them."
    fi
fi

[[ -d "$CLUSTER_DIR" ]] || die "cluster dir not found at ${CLUSTER_DIR} (set PRECIS_CLUSTER_DIR)."
[[ -f "${CLUSTER_DIR}/redeploy-precis.yml" ]] || die "redeploy-precis.yml not in ${CLUSTER_DIR}."

cd "$CLUSTER_DIR"

# PRECIS_DEPLOY_SKIP_HOSTS=h1,h2 — deliberately leave named hosts out of the
# ping and the rollout (Reto 2026-10-09: castor wedged out of memory, the rest
# of the cluster still gets the fix). The skipped hosts stay on their old sha
# — mixed versions until the next full deploy — so it is loud, never a default.
if [[ -n "${PRECIS_DEPLOY_SKIP_HOSTS:-}" ]]; then
    [[ -z "${PRECIS_DEPLOY_CANARY:-}" ]] || die "PRECIS_DEPLOY_SKIP_HOSTS and PRECIS_DEPLOY_CANARY both set — the canary path owns --limit; use one."
    _skip_limit="all"
    IFS=',' read -r -a _skip_hosts <<< "${PRECIS_DEPLOY_SKIP_HOSTS}"
    for _h in "${_skip_hosts[@]}"; do
        if [[ -n "$_h" ]]; then _skip_limit+=":!${_h}"; fi
    done
    INV_ARGS+=(--limit "$_skip_limit")
    say "WARNING: skipping ${PRECIS_DEPLOY_SKIP_HOSTS} (PRECIS_DEPLOY_SKIP_HOSTS) — those hosts keep their old sha until the next full deploy"
fi

# ── 1. reachability ─────────────────────────────────────────────────────
# Ping only the groups redeploy-precis.yml actually targets, NOT `all` — the
# inventory can carry bootstrap-only hosts with no `deploy` SSH user that would
# abort a deploy they play no part in (gripe 196641). `serving` (castor/pollux)
# IS a redeploy target now — 20c provisions the heartbeat-only unit there — so
# it must be pinged too: a twin down mid-deploy should abort, not half-
# provision. Keep this in sync with the plays' `hosts:` in redeploy-precis.yml.
PING_TARGETS="gateway:scheduler:data:inference:serving"
say "checking cluster reachability (ansible ping — ${PING_TARGETS})"
_PING_OUT="/tmp/precis-deploy-ping.$$"
ansible "$PING_TARGETS" -m ping ${INV_ARGS[@]+"${INV_ARGS[@]}"} >"$_PING_OUT" 2>&1 || {
    cat "$_PING_OUT" >&2
    rm -f "$_PING_OUT"
    _say_if_render_tree_gone
    die "a host is unreachable — aborting (a partial redeploy mixes code versions)."
}
# gr332009: the rollout-host set _run_rollout_playbook checks convergence
# against below — every host that just answered the ping, read straight back
# out of ansible's own SUCCESS lines rather than re-querying the inventory.
ROLLOUT_HOSTS="$(grep -oE '^[^[:space:]]+ \| SUCCESS' "$_PING_OUT" | awk '{print $1}')"
rm -f "$_PING_OUT"

# ── 1a. autocatpath wheel (gr263082) ────────────────────────────────────
# Same reason the ping above aborts: a partial redeploy mixes code versions.
# autocatpath is release-gated off PyPI past 0.13.0, so the only channel to a
# host is `--find-links /opt/precis/wheels`, seeded from a controller-side
# wheel passed as `-e autocatpath_wheel=<path>`. Nothing ever set that, so a
# floor bump in THIS repo's pyproject (the >=0.18.0 move) silently outran the
# newest built wheel (0.17.0) and the `precis-mcp[paper,catalyst]` install
# died on the autocatpath_plugin hosts alone, mid-run, after four other hosts
# had already moved. Resolve the wheel here, before touching anything.
#
# Building is in scope on purpose. The floor lives in this repo and the
# artifact lives in the catpath checkout, so they drift by construction; a fix
# that still needs a hand-run `uv build` after every bump just relocates the
# same failure to the next one. Version equality with the floor is what keeps
# this honest — a checkout too old to satisfy the floor is reported, never
# built and hoped over.
#
#   PRECIS_CATPATH_DIR            — catpath checkout. Unset, the sibling
#                                   layout is probed: <repo>/../catpath,
#                                   ~/work/projects/code/catpath, ~/catpath
#   PRECIS_DEPLOY_SKIP_CATPATH_WHEEL=1 — bypass entirely (hosts already seeded
#                                   by hand, or a fleet with no plugin hosts)
if [[ -z "${PRECIS_DEPLOY_SKIP_CATPATH_WHEEL:-}" \
    && "${PRECIS_DEPLOY_EXTRA_VARS:-}" != *autocatpath_wheel* ]]; then
    source "${REPO_ROOT}/scripts/lib/autocatpath-wheel.sh"
    # Default search, first hit wins. `~/catpath` was the only candidate and it
    # does not exist on this controller — the real checkout lives beside the
    # other repos — so every deploy died in preflight with "no wheel that new
    # exists ... and there is no catpath checkout there", naming a path the
    # operator had never used. Probing the conventional sibling locations makes
    # the common layout work with no env var; PRECIS_CATPATH_DIR still wins.
    if [[ -n "${PRECIS_CATPATH_DIR:-}" ]]; then
        _CATPATH_DIR="$PRECIS_CATPATH_DIR"
    else
        _CATPATH_DIR="${HOME}/catpath"
        for _cand in \
            "${REPO_ROOT}/../catpath" \
            "${HOME}/work/projects/code/catpath" \
            "${HOME}/catpath"; do
            if [[ -f "${_cand}/pyproject.toml" ]]; then
                _CATPATH_DIR="$(cd "$_cand" && pwd)"
                break
            fi
        done
    fi
    _FLOOR="$(autocatpath_floor "${REPO_ROOT}/pyproject.toml" || true)"

    if [[ -z "$_FLOOR" ]]; then
        : # no autocatpath floor declared — nothing to seed
    else
        _WHEEL="$(newest_autocatpath_wheel "${_CATPATH_DIR}/dist" || true)"
        if [[ -z "$_WHEEL" ]] || ! version_ge "$(autocatpath_wheel_version "$_WHEEL")" "$_FLOOR"; then
            # No local wheel clears the floor — try to build one, but only
            # from a checkout that actually declares a satisfying version.
            _PROJ_V="$(autocatpath_project_version "${_CATPATH_DIR}/pyproject.toml" || true)"
            [[ -n "$_PROJ_V" ]] || die "autocatpath floor is >=${_FLOOR} but no wheel that new exists in ${_CATPATH_DIR}/dist and there is no catpath checkout there to build one from. Set PRECIS_CATPATH_DIR, or PRECIS_DEPLOY_SKIP_CATPATH_WHEEL=1 if the hosts are already seeded. (gr263082)"
            # The checkout's own version is NOT the gate: uv.lock may pin a
            # newer catpath commit than the checkout has checked out (the
            # throwaway-worktree path below builds the pin, not HEAD), so the
            # floor is checked once, after the build source is resolved.

            # Only build from a checkout that matches what precis actually
            # depends on. catpath reuses one version number across many
            # commits — 0.18.0 already spans nine, including a minimum-image
            # convention fix — so the wheel FILENAME cannot tell a stale build
            # from a current one. Two wheels named autocatpath-0.18.0 can carry
            # different code, and the wheelhouse keeps whichever landed last.
            # A deploy is the wrong place to be relaxed about that: refuse
            # rather than ship a plausible-looking wheel nobody can identify.
            git -C "$_CATPATH_DIR" fetch -q origin 2>/dev/null || true
            [[ -z "$(git -C "$_CATPATH_DIR" status --porcelain 2>/dev/null)" ]] \
                || die "catpath at ${_CATPATH_DIR} has uncommitted changes — a wheel built from it would be named autocatpath-${_PROJ_V} but contain code no one else has. Commit/stash there, or build the wheel on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"

            # uv.lock already names the exact commit precis resolved against,
            # so ask the precise question — "would this tree build the code we
            # locked?" — instead of the proxy "is this tree level with its
            # upstream?". The proxy is not the same question and gets it wrong
            # in the normal case: catpath moves ahead between `uv lock -P`
            # runs, so a checkout at the pinned commit is CORRECT while being
            # behind upstream, and the proxy would refuse it.
            #
            # Compare only what ships. Commits touching docs/ or tests/ cannot
            # change the wheel, and refusing them would be the same false
            # alarm in a new place; src/ plus pyproject.toml (metadata, deps,
            # packaging config) is what a build actually reads.
            _BUILD_DIR="$_CATPATH_DIR"
            _PIN_WT=""
            _LOCKED="$(autocatpath_locked_sha "${REPO_ROOT}/uv.lock" || true)"
            if [[ -n "$_LOCKED" ]]; then
                git -C "$_CATPATH_DIR" cat-file -e "${_LOCKED}^{commit}" 2>/dev/null \
                    || die "uv.lock pins autocatpath at ${_LOCKED}, but that commit is not in ${_CATPATH_DIR} even after a fetch — the checkout is on a different remote, or the pin references an unpushed commit. Fix the checkout, or build on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"
                if ! git -C "$_CATPATH_DIR" diff --quiet "$_LOCKED" HEAD -- src pyproject.toml 2>/dev/null; then
                    # The checkout is clean but its packaged code is not the
                    # pinned commit's: catpath moved ahead of the pin, or the
                    # operator is mid-work on another branch. Refusing here
                    # was the first answer, and it turned every deploy after
                    # a catpath release into a "check the pin out, deploy,
                    # check main back out" dance done by hand. The pin IS
                    # what precis depends on, so build exactly that from a
                    # throwaway detached worktree and remove it afterwards.
                    # The dirty-tree check above still applies — a worktree
                    # cannot rescue code nobody has committed.
                    _PIN_WT="$(mktemp -d "${TMPDIR:-/tmp}/autocatpath-pin.XXXXXX")"
                    git -C "$_CATPATH_DIR" worktree add --detach -q "$_PIN_WT" "$_LOCKED" 2>/dev/null \
                        || die "could not check the pinned autocatpath commit ${_LOCKED} out into a throwaway worktree at ${_PIN_WT}. (gr263082)"
                    _BUILD_DIR="$_PIN_WT"
                    _PIN_V="$(autocatpath_project_version "${_PIN_WT}/pyproject.toml" || true)"
                    if [[ -z "$_PIN_V" ]] || ! version_ge "$_PIN_V" "$_FLOOR"; then
                        git -C "$_CATPATH_DIR" worktree remove --force "$_PIN_WT" 2>/dev/null || rm -rf "$_PIN_WT"
                        die "uv.lock pins autocatpath at ${_LOCKED}, which declares version ${_PIN_V:-?} — below the floor >=${_FLOOR} this repo's pyproject demands. Run 'uv lock -P autocatpath' here so the lock and the floor agree. (gr263082)"
                    fi
                    _PROJ_V="$_PIN_V"
                    say "catpath at ${_CATPATH_DIR} is not at the pinned commit — building ${_PROJ_V} from a throwaway worktree at ${_LOCKED:0:12}"
                fi
            else
                # No git pin to check against (autocatpath resolved from an
                # index, or a stale lock). Fall back to the weaker signal
                # rather than dropping provenance checking entirely.
                _BEHIND="$(git -C "$_CATPATH_DIR" rev-list --count 'HEAD..@{upstream}' 2>/dev/null || echo 0)"
                [[ "$_BEHIND" == "0" ]] \
                    || die "catpath at ${_CATPATH_DIR} is ${_BEHIND} commit(s) behind its upstream, all still labelled ${_PROJ_V} — building here would ship a wheel that silently differs from the released one. Pull catpath first, or build on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"
            fi

            version_ge "$_PROJ_V" "$_FLOOR" || die "autocatpath floor is >=${_FLOOR} but the catpath source to build from (${_BUILD_DIR}) declares only ${_PROJ_V} — pull/bump catpath (or 'uv lock -P autocatpath' so the pin clears the floor) and release the wheel first. (gr263082)"

            say "building autocatpath ${_PROJ_V} wheel (floor >=${_FLOOR}, nothing that new in ${_CATPATH_DIR}/dist)"
            # The wheel always lands in the checkout's own dist/ — a worktree
            # build's dist/ would vanish with the worktree.
            if (cd "$_BUILD_DIR" && uv build --wheel --out-dir "${_CATPATH_DIR}/dist"); then
                _BUILD_RC=0
            else
                _BUILD_RC=$?
            fi
            if [[ -n "$_PIN_WT" ]]; then
                git -C "$_CATPATH_DIR" worktree remove --force "$_PIN_WT" 2>/dev/null || rm -rf "$_PIN_WT"
            fi
            [[ "$_BUILD_RC" == "0" ]] \
                || die "autocatpath wheel build failed in ${_BUILD_DIR} — the cluster cannot resolve autocatpath>=${_FLOOR} without it. (gr263082)"
            _WHEEL="$(newest_autocatpath_wheel "${_CATPATH_DIR}/dist" || true)"
            [[ -n "$_WHEEL" ]] && version_ge "$(autocatpath_wheel_version "$_WHEEL")" "$_FLOOR" \
                || die "built autocatpath in ${_CATPATH_DIR} but still no wheel >=${_FLOOR} in dist/. (gr263082)"
        fi
        say "autocatpath wheel → $(basename "$_WHEEL") (floor >=${_FLOOR})"
        EXTRA_ARGS+=(-e autocatpath_wheel="$_WHEEL")
    fi
fi

# ── 1b. wheel smoke (gr451360) ──────────────────────────────────────────
# scripts/wheel-smoke builds the wheel this deploy is about to ship,
# installs it into a scratch venv, and imports precis_web.app from a cwd
# with no repo src/ on sys.path — the check that would have caught the
# 2026-09-26 outage (a package missing from
# [tool.hatch.build.targets.wheel] packages) before it went out, instead of
# after nginx started 502ing. Same reason the ping and the autocatpath wheel
# above abort here: better to fail before touching a host.
# PRECIS_DEPLOY_SKIP_WHEEL_SMOKE=1 bypasses (e.g. a hand-verified emergency
# redeploy where the extra ~30s matters more than the check).
# The wheel is built from the render worktree at the TARGET sha when there
# is one (709faf58: a literal-sha deploy renders from a detached tree, not
# from this checkout), so the smoke tests the artifact that ships; a
# ref-name target has no render tree and builds this checkout, which is
# also what it deploys. Absolute paths only — cwd is CLUSTER_DIR here.
if [[ -z "${PRECIS_DEPLOY_SKIP_WHEEL_SMOKE:-}" ]]; then
    say "wheel smoke: build + scratch-venv install + import (gr451360)"
    "${REPO_ROOT}/scripts/wheel-smoke" "${_DEPLOY_TREE:-$REPO_ROOT}" \
        || die "scripts/wheel-smoke failed — the wheel this deploy would ship does not import cleanly outside the source tree. Fix it, or PRECIS_DEPLOY_SKIP_WHEEL_SMOKE=1 to bypass (not recommended). (gr451360)"
fi

# ── 2. deploy ───────────────────────────────────────────────────────────
# Attempt stamp (gr332009): past this point hosts get touched. Record the
# attempt now; the success marker at step 3 replaces it and removes this. If
# any play dies red, the stamp survives with no newer marker and scripts/ship
# reports "deploy state uncertain" instead of counting lag from a stale sha.
# Best-effort — never blocks a deploy.
{
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    printf '%s %s %s\n' "$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || printf '%s' "$REF")" "$(date +%s)" "attempt" \
        > "$(deploy_attempt_path "$REPO_ROOT")"
} 2>/dev/null || true

CANARY="${PRECIS_DEPLOY_CANARY:-}"
if [[ -z "$CANARY" ]]; then
    # unchanged single-pass path (PRECIS_DEPLOY_CANARY unset). Routed through
    # _run_rollout_playbook (gr332009 graft) so a rollout that genuinely
    # converged still gets its success marker written (and attempt stamp
    # cleared) even if a later, unrelated play in this same invocation dies —
    # see that function's header comment.
    if [[ "$REF" == "main" ]]; then
        say "deploying main to the cluster"
        _run_rollout_playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
            ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
            || die "$(_redeploy_failure_msg)"
    else
        say "deploying ref '${REF}' to the cluster"
        _run_rollout_playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
            -e precis_worker_git_ref="$REF" -e precis_web_git_ref="$REF" \
            ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
            || die "$(_redeploy_failure_msg)"
    fi

    say "deployed — cluster is running '${REF}'."
else
    # ── canary path (PRECIS_DEPLOY_CANARY set): one host first, verified,
    # then the fleet. Resolve the sha ONCE (same ls-remote the playbook's
    # step-0 pin does) and pass it as explicit `-e` pins to BOTH phases — an
    # explicit `-e` wins over the playbook's own set_fact pin, so both
    # ansible-playbook invocations below install the identical sha even if
    # main advances between phase 1 and phase 2.
    #
    # `git ls-remote` matches ref NAMES, so a literal commit sha — the COMMON
    # target since /go deploys `$(cat .ship-sha) --pinned` — never resolved
    # and the canary path died before touching a host (gr346747). A REF this
    # checkout already contains as a commit is taken from the local object
    # store instead; it must also be reachable from a remote-tracking branch,
    # since the hosts install precis-mcp@<sha> from GitHub.
    _target_sha=""
    if [[ "$REF" =~ ^[0-9a-f]{7,40}$ ]]; then
        _target_sha="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    fi
    if [[ ${#_target_sha} -eq 40 ]]; then
        [[ -n "$(git -C "$REPO_ROOT" branch -r --contains "$_target_sha" 2>/dev/null)" ]] \
            || die "'${REF}' (${_target_sha:0:8}) is not on any remote-tracking branch — the hosts install from GitHub, so it must be pushed first."
        say "canary deploy: '${REF}' is a commit this checkout contains — pinned ${_target_sha:0:8}"
    else
        say "canary deploy: resolving '${REF}' to a single commit (git ls-remote)"
        _target_sha="$(git ls-remote https://github.com/retospect/precis-mcp "$REF" | awk 'NR==1{print $1}')"
        [[ ${#_target_sha} -eq 40 ]] || die "could not resolve '${REF}' to a commit via git ls-remote (got '${_target_sha:-empty}') — refusing to deploy an unpinned/unresolvable ref."
    fi
    SHA_ARGS=(-e precis_worker_git_ref="$_target_sha" -e precis_web_git_ref="$_target_sha" -e precis_embedder_git_ref="$_target_sha")
    _rollback_hint="rollback: scripts/deploy <previous-sha> with PRECIS_DEPLOY_CANARY=${CANARY} (targets just the canary)."

    say "canary deploy: phase 1 — ${CANARY} only (pinned ${_target_sha:0:8})"
    _phase1_start="$(date +%s)"
    PYTHONUNBUFFERED=1 ansible-playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
        --limit "$CANARY" \
        "${SHA_ARGS[@]}" \
        ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
        || die "canary phase 1 (${CANARY}) failed — see the failing task above; the rest of the fleet was never touched."

    CANARY_TIMEOUT_S="${PRECIS_DEPLOY_CANARY_TIMEOUT_S:-300}"
    DB_HOST="${PRECIS_DEPLOY_CANARY_DB_HOST:-$CANARY}"
    [[ "$DB_HOST" =~ ^[A-Za-z0-9_.-]+$ ]] || die "PRECIS_DEPLOY_CANARY_DB_HOST/'${DB_HOST}': not a plausible hostname"
    say "canary deploy: verifying ${CANARY}'s heartbeat (host_heartbeat.host='${DB_HOST}') freshens (timeout ${CANARY_TIMEOUT_S}s)"
    _elapsed=0
    _green=""
    [[ -x "${REPO_ROOT}/scripts/prod-psql" ]] \
        || die "canary verify: could not run ${REPO_ROOT}/scripts/prod-psql (missing or not executable) — treated as red (fail closed).
mixed state: ${CANARY} may be on new code (${_target_sha}), the rest of the fleet is untouched (still on old code).
${_rollback_hint}"
    while (( _elapsed < CANARY_TIMEOUT_S )); do
        _ts_epoch="$(PRECIS_PROD_PSQL_OPTS="-At" "${REPO_ROOT}/scripts/prod-psql" \
            "SELECT extract(epoch from ts)::bigint FROM host_heartbeat WHERE host = '${DB_HOST}';")" \
            || die "canary verify: scripts/prod-psql query failed reaching prod — treated as red (fail closed).
mixed state: ${CANARY} may be on new code (${_target_sha}), the rest of the fleet is untouched (still on old code).
${_rollback_hint}"
        _now="$(date +%s)"
        if [[ -n "$_ts_epoch" ]] && (( _ts_epoch > _phase1_start )) && (( _now - _ts_epoch < 120 )); then
            _green=1
            break
        fi
        sleep 15
        _elapsed=$((_elapsed + 15))
    done

    [[ -n "$_green" ]] || die "canary verify: ${CANARY}'s host_heartbeat (host='${DB_HOST}') did not freshen within ${CANARY_TIMEOUT_S}s.
mixed state: ${CANARY} may be on new code (${_target_sha}), the rest of the fleet is untouched (still on old code).
${_rollback_hint}"

    say "canary deploy: ${CANARY} verified fresh on new code — phase 2 (rest of the fleet)"
    PYTHONUNBUFFERED=1 ansible-playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
        --limit "all:!${CANARY}" \
        "${SHA_ARGS[@]}" \
        ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
        || die "canary phase 2 (fleet) failed — see the failing task above. ${CANARY} is on new code (${_target_sha}); the rest of the fleet may be mixed — re-run once fixed."

    REF="$_target_sha"
    say "deployed — cluster is running '${REF}' (canary ${CANARY} verified first, then the fleet)."
fi

# ── 3. record the deploy-state marker (for scripts/ship's lag report and
# scripts/deploy's own rollback guard) ────────────────────────────────────
# Gitignored, local-only: `<sha> <epoch> <outcome>` of what's now actually
# running on the cluster, so a later `scripts/ship` can tell how many commits
# + how much time have accumulated since, and a later `scripts/deploy` can
# refuse to roll it backward (gr338201). Best-effort — never fails a
# successful deploy.
{
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    DEPLOYED_SHA="$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || true)"
    if [[ -n "$DEPLOYED_SHA" ]]; then
        # Shared across worktrees (git common dir): the fleet's sha is a global
        # fact, and a per-worktree marker made every OTHER tree's ship
        # over-report the lag. See scripts/lib/deploy-state.sh.
        . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
        printf '%s %s %s%s\n' "$DEPLOYED_SHA" "$(date +%s)" "success" "${OVERLAY_HASH:+ overlay=${OVERLAY_HASH}}" \
            > "$(deploy_state_path "$REPO_ROOT")"
    fi
    # Success: clear the attempt stamp so ship stops reporting "uncertain"
    # (gr332009). Unconditional — a deploy that got here ran green.
    rm -f "$(deploy_attempt_path "$REPO_ROOT")" 2>/dev/null
    # The pin has done its job: this sha is what the fleet runs now.
    _consume_ship_pin "$DEPLOYED_SHA"
    _move_prod_pointer "$DEPLOYED_SHA"
} || true
