#!/usr/bin/env bash
#MISE description="Deploy to bigblack (system-systemd + odb service account) with safety gates (v2.1)"
#
# Implements: odb-ship Phases 3-4 + sidecar-restart-zero-overlap-zero-gap-recovery-playbook Pattern 1
# Sequence: git sync → staging venv → monit unmonitor → stop → atomic swap → systemd sync → start sidecar → verify → start kintsugi → monit monitor
#
# MIGRATION (v2.0): production runs as the dedicated `odb` system service account under
# system-level systemd, from /opt/opendeviationbar-py. Any operator in group `odb`
# (tca, nasimubd) can run this deploy. Repo/venv mutations execute as `sudo -u odb`
# (odb owns the checkout); services are controlled via `sudo systemctl` (system units);
# unit files are installed by the fixed root helper `/usr/local/sbin/odb-install-units`.

set -euo pipefail

REMOTE="bigblack"
DEPLOY_DIR="/opt/opendeviationbar-py"           # fixed neutral path (was /home/tca/eon/opendeviationbar-py)
ODB_UV="/var/lib/opendeviationbar/.local/bin/uv" # uv installed in the odb service account's home
VERSION=$(grep -A5 '\[workspace.package\]' Cargo.toml | grep '^version' | head -1 | sed 's/.*= "\(.*\)"/\1/')

echo "=== Deploy v${VERSION} to $REMOTE (odb system-systemd) ==="
echo "✓ Deploy directory: $DEPLOY_DIR"

# ─────────────────────────────────────────────────────────────
# Phase 3.1: Git sync on bigblack (as the odb service account, which owns the checkout)
# ─────────────────────────────────────────────────────────────
echo "→ Git sync on $REMOTE (as odb)..."
ssh "$REMOTE" "sudo -u odb git -C $DEPLOY_DIR fetch origin main --tags && sudo -u odb git -C $DEPLOY_DIR reset --hard origin/main && sudo -u odb git -C $DEPLOY_DIR clean -fd -e .venv -e .venv-staging -e .venv-old" 2>&1 | tail -3

# ─────────────────────────────────────────────────────────────
# Phase 3.3-3.5: Build cp314 wheel LOCALLY on bigblack + verify staging venv
# (BEFORE stopping services; a build/verify failure aborts before the swap).
#
# Production runs Python 3.14 (policy: 3.14 ONLY). The cp314 Rust extension is built
# on-host via maturin (toolchain lives in the OPERATOR's ~/.cargo — tca/nasimubd), then
# installed into a 3.14 venv owned by odb. This replaces the old Mac-zig cp313 path so a
# deploy never downgrades the runtime. `--compatibility linux` skips the manylinux audit.
# ─────────────────────────────────────────────────────────────
MATURIN="$(ssh "$REMOTE" 'command -v maturin || echo \$HOME/.cargo/bin/maturin')"
echo "→ Creating 3.14 staging venv (as odb)..."
ssh "$REMOTE" "sudo -u odb rm -rf $DEPLOY_DIR/.venv-staging && cd $DEPLOY_DIR && sudo -u odb env HOME=/var/lib/opendeviationbar $ODB_UV venv --python 3.14 $DEPLOY_DIR/.venv-staging" 2>&1 | tail -1
echo "→ Building cp314 wheel on-host via maturin (operator toolchain)..."
ssh "$REMOTE" "cd $DEPLOY_DIR && rm -rf /tmp/odb-wheels && CARGO_TARGET_DIR=/tmp/odb-build-target $MATURIN build --release -i $DEPLOY_DIR/.venv-staging/bin/python3 --compatibility linux -o /tmp/odb-wheels" 2>&1 | tail -3
WHEEL=$(ssh "$REMOTE" "ls -t /tmp/odb-wheels/opendeviationbar-*cp314*.whl 2>/dev/null | head -1")
if [ -z "$WHEEL" ]; then echo "❌ ABORT: cp314 wheel not produced by maturin"; exit 1; fi
echo "✓ Built $(basename "$WHEEL")"
ssh "$REMOTE" "chmod 0644 '$WHEEL'"
echo "→ Installing cp314 wheel + dependencies into fresh staging venv (as odb)..."
ssh "$REMOTE" "cd $DEPLOY_DIR && sudo -u odb env HOME=/var/lib/opendeviationbar $ODB_UV pip install --python $DEPLOY_DIR/.venv-staging/bin/python3 '$WHEEL'" 2>&1 | tail -3

# Verify staging before proceeding
echo "→ Verifying staging venv..."
ssh "$REMOTE" "sudo -u odb $DEPLOY_DIR/.venv-staging/bin/python3 -c '
import opendeviationbar; v = opendeviationbar.__version__
from opendeviationbar import OpenDeviationBarProcessor
from pathlib import Path
import opendeviationbar.clickhouse.cache as ch
assert (Path(ch.__file__).parent / \"schema.sql\").exists()
import opendeviationbar.symbol_registry as sr
assert (Path(sr.__file__).parent / \"data\" / \"symbols.toml\").exists()
print(f\"✓ Staging verified: v{v}\")
'"

# DEPL-04: Fail-fast .pth detection — editable install means wheel was installed from inside project dir
ssh "$REMOTE" "
SITE_PKG=\$(sudo -u odb $DEPLOY_DIR/.venv-staging/bin/python3 -c 'import site; print(site.getsitepackages()[0])')
PTH=\"\$SITE_PKG/opendeviationbar.pth\"
if [ -f \"\$PTH\" ]; then
  echo '❌ FATAL: .pth shadowing detected — wheel installed as editable!'
  echo '   Content:' && cat \"\$PTH\"
  echo '   Fix: install wheel from /tmp/, not from inside project dir'
  exit 1
fi
echo '✓ No .pth shadowing'
rm -f /tmp/opendeviationbar-*.whl
"

# ─────────────────────────────────────────────────────────────
# Phase 4.1: Monit unmonitor THEN stop (CRITICAL: playbook Pattern 1)
# Without unmonitor, monit auto-restarts services within 120s
# ─────────────────────────────────────────────────────────────
echo "→ Unmonitoring from monit..."
ssh "$REMOTE" 'sudo monit unmonitor sidecar 2>/dev/null; sudo monit unmonitor kintsugi 2>/dev/null; true'

echo "→ Stopping services..."
ssh "$REMOTE" 'sudo systemctl stop opendeviationbar-sidecar opendeviationbar-kintsugi opendeviationbar-kintsugi-catchup 2>/dev/null; true'

# ─────────────────────────────────────────────────────────────
# Phase 4.2: Atomic venv swap (same-filesystem mv is atomic; as odb)
# ─────────────────────────────────────────────────────────────
echo "→ Atomic venv swap..."
ssh "$REMOTE" "sudo -u odb bash -c 'cd $DEPLOY_DIR && rm -rf .venv-old && mv .venv .venv-old 2>/dev/null; mv .venv-staging .venv && rm -rf .venv-old'"
echo "✓ Venv swapped"

# ─────────────────────────────────────────────────────────────
# Phase 4.3: Sync system-level systemd units + reload (via fixed root helper)
# ─────────────────────────────────────────────────────────────
echo "→ Syncing system systemd units..."
ssh "$REMOTE" 'sudo /usr/local/sbin/odb-install-units'

# ─────────────────────────────────────────────────────────────
# Phase 4.5: Start sidecar FIRST (provides health endpoint + real-time data)
# ─────────────────────────────────────────────────────────────
echo "→ Starting sidecar..."
ssh "$REMOTE" 'sudo systemctl start opendeviationbar-sidecar'

# DEPL-03: Post-start telemetry gate — wait for sidecar startup confirmation
echo "→ Waiting for sidecar startup marker..."
for i in $(seq 1 12); do
    if ssh "$REMOTE" "journalctl -u opendeviationbar-sidecar --since '60s ago' --no-pager 2>/dev/null | grep -q 'sidecar_startup\|StreamManager started\|engine started'" 2>/dev/null; then
        echo "✓ Sidecar confirmed running (attempt $i)"
        break
    fi
    if [ "$i" -eq 12 ]; then
        echo "❌ ABORT: Sidecar not started after 60s. Check logs:"
        echo "   ssh $REMOTE 'journalctl -u opendeviationbar-sidecar -n 50 --no-pager'"
        echo ""
        echo "Rollback: ssh $REMOTE 'sudo -u odb bash -c \"cd $DEPLOY_DIR && mv .venv .venv-bad && mv .venv-old .venv 2>/dev/null\"' && ssh $REMOTE 'sudo systemctl restart opendeviationbar-sidecar'"
        exit 1
    fi
    sleep 5
done

# Verify sidecar health endpoint
ssh "$REMOTE" 'curl -sf http://localhost:8081/health | python3 -c "
import sys,json; d=json.load(sys.stdin)
print(f\"✓ Sidecar health: {d.get(chr(115)+chr(116)+chr(97)+chr(116)+chr(117)+chr(115),chr(63))}\")" 2>/dev/null' || echo "⚠ Health endpoint not ready (may need a few more seconds)"

# ─────────────────────────────────────────────────────────────
# Phase 4.5: Start kintsugi AFTER sidecar is confirmed running
# ─────────────────────────────────────────────────────────────
echo "→ Starting kintsugi..."
ssh "$REMOTE" 'sudo systemctl start opendeviationbar-kintsugi'
sleep 3

# ─────────────────────────────────────────────────────────────
# Phase 4.6: Re-monitor in monit
# ─────────────────────────────────────────────────────────────
echo "→ Re-monitoring in monit..."
ssh "$REMOTE" 'sudo monit monitor sidecar 2>/dev/null; sudo monit monitor kintsugi 2>/dev/null; true'

# ─────────────────────────────────────────────────────────────
# Phase 4.7: Final service status (system units; is-active is readable without sudo)
# ─────────────────────────────────────────────────────────────
echo ""
ssh "$REMOTE" '
for svc in opendeviationbar-sidecar opendeviationbar-kintsugi opendeviationbar-heartbeat.timer opendeviationbar-seeder.timer; do
  svc_state=$(systemctl is-active $svc 2>/dev/null || echo "inactive")
  echo "  $svc: $svc_state"
done
'

echo ""
echo "==========================================="
echo " Deploy v${VERSION} complete!"
echo "==========================================="
echo ""
echo "Monitor:"
echo "  ssh $REMOTE 'journalctl -u opendeviationbar-sidecar -f --no-pager'"
echo "  ssh $REMOTE 'journalctl -u opendeviationbar-kintsugi -f --no-pager'"
echo ""
echo "Verify overlap self-heal (wait ~5 min):"
echo "  ssh $REMOTE 'journalctl -u opendeviationbar-sidecar --since \"10 min ago\" --no-pager | grep overlap_heal'"
