# 3.22 is what the pinned Kokkos (cmake/kokkos_dependency.cmake) itself
# requires; this project needed 3.20 before the fetch existed.
cmake_minimum_required(VERSION 3.22)
project(nrhjkokkos LANGUAGES CXX)

# Build with the Kokkos-provided nvcc_wrapper and the CUDA toolkit Kokkos
# was built against (nvcc must be found in PATH):
#   export PATH=$HOME/soft/cuda/12.8/bin:$PATH
#   export NVCC_WRAPPER_DEFAULT_COMPILER=$HOME/soft/gcc/14.2.0/bin/g++
#   cmake -S cpp -B cpp/build -DCMAKE_BUILD_TYPE=Release \
#     -DCMAKE_CXX_COMPILER=$HOME/Installations/kokkos/4.6.0/bin/nvcc_wrapper \
#     -DCMAKE_CXX_FLAGS=-arch=sm_75 \
#     -DKokkos_ROOT=$HOME/Installations/kokkos/4.6.0 \
#     -Dpybind11_DIR=$(python -m pybind11 --cmakedir) \
#     -DPython_EXECUTABLE=$(which python)
#   cmake --build cpp/build -j
# (-arch is required: CUDA 13 nvcc rejects nvcc_wrapper's sm_70 default.)

# Kokkos: an installed copy (Kokkos_ROOT / Kokkos_DIR / CMAKE_PREFIX_PATH)
# when there is one, else the pinned release is fetched and built in.  The
# whole policy, the pin and its reason live in that one file.
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkos_dependency.cmake)
find_package(pybind11 CONFIG REQUIRED)
# Compiler-specific spellings of -march=native and -ffp-contract=off (MSVC and
# Apple Silicon do not take the GCC forms).  Sets NRHJ_NATIVE_ISA_FLAG and
# NRHJ_FP_CONTRACT_OFF_FLAG, each empty when the compiler has no safe form.
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/compiler_flags.cmake)

# pack.cpp includes the AUTOGENERATED 4PN series headers under generated/
# (emitted by scripts/gen_pn_*_partials.py --cpp-only; committed).
# glue.cpp is the compiled aa_batch.py waveform-path glue (fmin solve,
# span packing, scatter, grid bounds), templated on ExecSpace and
# registered for every available space -- see its header comment.
# mr.cpp is the fused merger-ringdown segment blend (aa/mr-kokkos).
# residency.cpp is the fence/copy ledger probe (aa/workspace-views).
# ---------------------------------------------------------------------------
# BUILD PROVENANCE STAMP.  Why this exists: on 2026-08-26 a speed campaign ran
# against a module built 2026-08-11, eight days before the shared
# phase-interval hoist landed, and reported its pre-optimisation timings as the
# model's cost.  NOTHING caught it.  The existing stale-.so guard
# (assert_normaliser_matches) only catches a CORRECTNESS-class mismatch -- that
# module was numerically correct, merely slow -- and the campaign rows recorded
# the repo HEAD at RUN time plus the .so PATH, neither of which ties the binary
# to a source commit.
#
# So the module now carries the cpp/ commit it was compiled from, and whether
# cpp/ was dirty at build time.  A harness can then assert the binary matches
# the source it claims to measure.  Empty strings when git is unavailable, so a
# tarball build still compiles.
# The last commit that TOUCHED cpp/, not the repo HEAD.  `rev-parse HEAD` in a
# subdirectory returns the whole-repo HEAD, so with that every unrelated commit
# -- a findings log, a script edit -- invalidated a perfectly good module and
# the guard aborted a benchmark that had nothing wrong with it.  Measured the
# hard way: it fired on this branch after four commits that never went near
# cpp/.  What actually matters is whether the KERNEL SOURCE moved.
execute_process(
  COMMAND git -C "${CMAKE_CURRENT_SOURCE_DIR}" log -1 --format=%H -- .
  OUTPUT_VARIABLE NRHJ_BUILD_SOURCE_COMMIT
  OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
execute_process(
  COMMAND git -C "${CMAKE_CURRENT_SOURCE_DIR}" status --porcelain -- .
  OUTPUT_VARIABLE NRHJ_BUILD_CPP_STATUS
  OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
if(NRHJ_BUILD_CPP_STATUS STREQUAL "")
  set(NRHJ_BUILD_CPP_DIRTY "0")
else()
  set(NRHJ_BUILD_CPP_DIRTY "1")
endif()
message(STATUS "NRHJ build provenance: cpp commit "
               "${NRHJ_BUILD_SOURCE_COMMIT} dirty=${NRHJ_BUILD_CPP_DIRTY}")

# swsh.cpp is the spin-weight -2 spherical harmonic kernel (aa/kokkos-swsh),
# ExecSpace-templated and registered for every available space like glue.cpp.
pybind11_add_module(nrhjkokkos evaluator.cpp pack.cpp gpr.cpp anchor.cpp
                    glue.cpp
                    residency.cpp
                    mr.cpp
                    swsh.cpp)

# UNCONDITIONAL: this must not sit inside any option() branch.
target_compile_definitions(nrhjkokkos PRIVATE
  NRHJ_BUILD_SOURCE_COMMIT="${NRHJ_BUILD_SOURCE_COMMIT}"
  NRHJ_BUILD_CPP_DIRTY="${NRHJ_BUILD_CPP_DIRTY}"
  NRHJ_KOKKOS_PROVENANCE="${NRHJ_KOKKOS_PROVENANCE}")
# Where the finished module goes.  Unset (the developer route) leaves it in
# the build directory, where the Python loader searches cpp/build*.  setup.py
# sets it to the wheel staging directory so the module lands at the
# site-packages root as a top-level `nrhjkokkos`, which the loader tries
# FIRST.  Both the generic and the per-configuration property are set because
# Visual Studio and Xcode generators use the latter.
set(NRHJ_MODULE_OUTPUT_DIRECTORY "" CACHE PATH
    "Directory to place the built nrhjkokkos module in (empty: the build directory)")
if(NRHJ_MODULE_OUTPUT_DIRECTORY)
  set_target_properties(nrhjkokkos PROPERTIES
    LIBRARY_OUTPUT_DIRECTORY "${NRHJ_MODULE_OUTPUT_DIRECTORY}"
    LIBRARY_OUTPUT_DIRECTORY_RELEASE "${NRHJ_MODULE_OUTPUT_DIRECTORY}"
    LIBRARY_OUTPUT_DIRECTORY_DEBUG "${NRHJ_MODULE_OUTPUT_DIRECTORY}"
    LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO "${NRHJ_MODULE_OUTPUT_DIRECTORY}")
endif()
# anchor.cpp is the canonical reproducible anchor contraction: FP
# contraction must be COMPILER-OFF there (the source also pins the STDC
# pragma; explicit fma is the only contracted op).  nvcc_wrapper does
# not forward the flag portably -- the pragma + the explicit-op
# discipline cover the CUDA path, enforced by the FMA/AVX canary in
# tests/test_anchor_canonical.py.
# The flag's spelling per compiler comes from cmake/compiler_flags.cmake;
# empty on nvcc, exactly as the old "NOT nvcc" guard behaved.
if(NRHJ_FP_CONTRACT_OFF_FLAG)
  set_source_files_properties(anchor.cpp PROPERTIES
                              COMPILE_OPTIONS "${NRHJ_FP_CONTRACT_OFF_FLAG}")
endif()
# ==================================================================== #
# tau-inversion Newton SEED.  RESOLUTION of the codex/aa-n2-eval-fastpath
# vs psc/isa-b1a-step1 conflict on this file (INTEGRATION_PROTOCOL step 2).
#
# aa/tau-sweep shipped this as a default-OFF *build option*, because the
# seed is the one part of that branch that is NOT bitwise against trunk
# and the branch's whole asset was the bitwise gate.
# codex/aa-n2-eval-fastpath DELETED the option and made the seed a
# *runtime argument*, default ON.
#
# Neither extreme is right, and the two are not actually in conflict:
#   * runtime argument  -> KEPT (codex's good idea).  One build carries
#     BOTH arms, which is what an in-process paired A/B needs, and it lets
#     a bitwise-sensitive caller opt out per call without a rebuild.
#   * the build option  -> KEPT TOO, but repurposed: it now selects the
#     DEFAULT of that runtime argument rather than compiling one arm away.
#     OFF gives every caller the historical midpoint seed, which is what a
#     NRHJ_MARCH_NATIVE=OFF / NRHJ_PACK_NO_CONTRACT=ON bitwise-reference
#     build wants -- WITHOUT making the fast arm unreachable for tests.
# So the conflict resolves by UNION, not by choosing a side, and the ISA
# option block below (step1's) is untouched.
#
# NOT BITWISE when ON.  Characterised in TAU_SWEEP_STATUS.md section 6.1
# (cost on the OLD generic-ISA default), WARMSTART_VERDICT_STATUS.md, and
# WARMSTART_MERGE_FINDINGS.md on this branch (re-derived on trunk 4d55f10
# under the mrsafe default, itself non-bitwise).
#
# DEFAULT ON -- THE OWNER'S DECISION, 2026-08-07 (A3).  This was an open
# conflict until then, and the resolution is recorded rather than assumed.
#
# The owner's standing steer is: lean to speed; override the conservative
# call whenever the cost is bits or negligible accuracy.  A3's cost is bits.
# The gain is 1.058-1.142x on the evaluate_time kernel and ~1-3% end-to-end
# where it resolves at all (kernel share measured by stage-stub null test at
# 21.3% / 9.2% / 21.7% / unresolved / 8.8% across the five rungs).
#
# WHAT THE USER PAID, AND NO LONGER PAYS.  This block used to read: "the
# seed's history restarts at every tau chunk boundary (`TauCursor cu;` is
# declared INSIDE the work item) and `tau_chunks(B, N, concurrency)` moves
# those boundaries with the THREAD COUNT.  So with the default ON, *A USER WHO
# CHANGES OMP_NUM_THREADS CHANGES THEIR WAVEFORM'S LAST BITS.*"  That was
# true and measured -- trunk 4d55f10, node r403, a 15-point OMP sweep, bitwise
# at OMP=1 and 3, worst 168.5 ULP of array peak at OMP=32.
#
# IT IS NO LONGER TRUE.  Leaving it standing would justify the escape hatch
# below on a premise that has since been fixed.  `TauCursor` is now declared
# inside the SEED-BLOCK loop, not the work item, so restarts land on absolute
# multiples of kTauSeedBlockSamples and `tau_chunks` only chooses how those
# blocks are GROUPED into work items.  Measured after the change (job
# 43171138):
#
#     batch invariance   the previously-red row-vs-batch gate PASSES
#     thread invariance  OMP=1 vs OMP=32 BITWISE IDENTICAL
#     suite              back to the pre-A3 known-red set
#
# Dependence on N survives -- a different requested grid length still moves
# the block boundaries -- but dependence on B, concurrency and OMP_NUM_THREADS
# does not.
#
# The off switch below therefore now buys something NARROWER than it did:
# -DNRHJ_TAU_WARM_START=OFF gives the historical midpoint seed, for byte
# identity with a pre-A3 build.  It is no longer protection against
# thread-count drift, because that drift no longer exists.  The runtime
# `warm_start` argument, both arms, and every gate work either way, and BOTH
# defaults are measured -- so this is a default, not a lock-in.
option(NRHJ_TAU_WARM_START
       "Default value of evaluate_time's warm_start argument (NOT bitwise)" ON)
if(NOT NRHJ_TAU_WARM_START)
  target_compile_definitions(nrhjkokkos PRIVATE NRHJ_TAU_WARM_START_DEFAULT_OFF)
endif()
# ==================================================================== #
# aa/alg-speedup item B0 -- ISA.  The deployed CPU build is BASELINE
# SCALAR SSE2: objdump of the default .so shows 0 %ymm, 0 vfmadd, and
# ~93% of FP multiplies scalar, on an EPYC 7742 that has AVX2 and two
# FMA units.  aa/march-native confirms this is still true "after ten
# merges".
#
# THIS IS NOT NEW GROUND -- aa/march-native measured it end to end and
# its numbers govern.  See the march-native status document in the
# development tree (MARCH_NATIVE_STATUS.md),
# whose RECOMMENDATION (2026-08-03) OVERTURNED an earlier do-not-adopt:
# -march=native is worth 8-17% end to end at EVERY configuration
# measured, including the B=1 PE path the original verdict existed to
# protect.  packsafe at OMP=32: 1.101x / 1.103x / 1.063x at B = 64 /
# 256 / 1024, all 18 cells resolved (non-overlapping p16-p84).
#
# THE ONLY SAFE FORM IS "packsafe" -- native PLUS BOTH per-source
# contraction opt-outs:
#   plain -march=native ....... 21 gates FAILED
#   + MR_NO_CONTRACT .......... 1 gate FAILED   ("mrsafe", 1.17x)
#   + PACK_NO_CONTRACT too .... 179/179 GREEN   ("packsafe", 1.12x)
# so NRHJ_MARCH_NATIVE below turns BOTH opt-outs on with it.  packsafe
# is ~5 points slower than mrsafe and buys a green gate for that; the
# two produce BITWISE-IDENTICAL waveforms, so packsafe's cost is free in
# accuracy terms.
#
# DEFAULT ON, on the owner's explicit instruction: this module is
# compiled FROM SOURCE PER SITE, so the shipped-binary portability
# objection that made aa/march-native recommend opt-in does not apply,
# and `native` picks up AVX-512 on hardware that has it where a fixed
# -march=x86-64-v3 would leave it on the table.  Turn it OFF for a
# portable or cross-machine-reproducible build.
#
# *** WAVEFORMS ARE NO LONGER BIT-IDENTICAL ACROSS MACHINES. ***
# FMA contraction rounds once instead of twice -- MORE accurate, not
# bitwise; aa/march-native measured the packsafe deviation at 3.6e-11 of
# peak.  BUT anchor.cpp is EXCLUDED from all of this (its
# -ffp-contract=off property below is applied per-source and survives
# -march=native and LTO, verified bitwise by the FMA/AVX canary in
# tests/test_anchor_canonical.py), so THE CANONICAL ANCHOR REMAINS
# BITWISE-REPRODUCIBLE ACROSS MACHINES even though the rest of the path
# now differs site to site.  That is a deliberate property, not an
# accident -- do not "simplify" it away.
# *** DEFAULT CHANGED 2026-08-05, on the owner's explicit instruction: ***
# "keep the bitwise path for tests later, but default should be avx and fma
# and speed."  So NRHJ_MARCH_NATIVE no longer forces BOTH contraction
# opt-outs.  The default is now the "mrsafe" arm:
#
#   arm        FMA    ymm    gates      speed   selected by
#   base         0      0    179/179    1.00x   NRHJ_MARCH_NATIVE=OFF
#   packsafe   543   2311    179/179    1.12x   + NRHJ_PACK_NO_CONTRACT=ON
#   mrsafe    1622   2308    178/179    1.17x   *** THE DEFAULT ***
#   native    1892   2276    158/179      --    + NRHJ_MR_NO_CONTRACT=OFF
#
# WHY mrsafe AND NOT PLAIN native: mr.cpp's contraction breaks 20 host-side
# np.array_equal gates and buys only 270 of 1892 FMAs (14%).  Turning it off
# costs almost nothing and recovers 20 of the 21 red tests.  Plain native is
# available but is not recommended.
#
# WHAT THE DEFAULT NOW COSTS -- exactly one gate, and it is a CONFORMITY
# gate, not a safety one:
#   tests/test_kokkos_hybrid.py::test_reach_planner_is_the_builders_quantity
# It asserts the compiled reach-edge frequency om0 = 1/dtau(Phi0 + eps), built
# in pack.cpp, is BITWISE equal to a NumPy reference, "no tolerance granted
# anywhere".  The obvious worry is that a 1-ulp shift in the reach edge could
# FLIP a routing decision (native vs PN-extended) rather than merely perturb
# an output.  IT CANNOT, and the margins say so: _reach_classify compares
# f22[:,0] * (1.0 + margin) > f_min_geo * (1.0 + 1e-12), and the tri-state arm
# passes REACH_CANDIDATE_MARGIN = 2.0e-2.  A 2% candidate margin and a 1e-12
# comparison guard sit 14 and 4 orders of magnitude above an FMA-scale
# perturbation (~2.2e-16).  So the red gate is bit-conformity only.
#
# THE BITWISE PATH IS RETAINED, TWO WAYS, both for tests:
#   -DNRHJ_MARCH_NATIVE=OFF     true bitwise reference, 179/179, no AVX
#   -DNRHJ_PACK_NO_CONTRACT=ON  packsafe: 179/179 AND keeps AVX2 (highest ymm
#                               of any arm) -- the right arm for a CI job that
#                               must stay byte-reproducible without giving up
#                               vectorisation
# anchor.cpp keeps -ffp-contract=off unconditionally either way, so the
# CANONICAL ANCHOR STAYS BITWISE-REPRODUCIBLE ACROSS MACHINES under every arm
# above.  Do not "simplify" that away.
option(NRHJ_MARCH_NATIVE
       "Compile for the build host's ISA (AVX2+FMA where available)"
       ON)
option(NRHJ_MR_NO_CONTRACT
       "-ffp-contract=off on mr.cpp (recovers 20 host bitwise gates, ~14% of FMA)"
       ON)
option(NRHJ_PACK_NO_CONTRACT
       "-ffp-contract=off on pack.cpp (recovers the 179th gate; costs ~5 points)"
       OFF)
# swsh.cpp: the SWSH term sum is compared term for term against numpy, whose
# C is compiled without contraction; keeping FMA out of this one file costs
# nothing measurable (B*K scalars per call) and keeps the comparison honest.
option(NRHJ_SWSH_NO_CONTRACT
       "-ffp-contract=off on swsh.cpp (term-for-term comparability with numpy)"
       ON)
# PORTABILITY (2026-09-02, checklist B1).  The option's MEANING is unchanged:
# compile for the build host's own CPU.  Its SPELLING is no longer the bare
# `-march=native`, which is GCC/Clang syntax that MSVC cannot compile and that
# Apple Silicon rejects in favour of `-mcpu=native`.  cmake/compiler_flags.cmake
# picks the form for the compiler and target architecture, confirms the
# compiler accepts it, and leaves NRHJ_NATIVE_ISA_FLAG empty (with a STATUS
# line saying why) where there is no safe form: MSVC, nvcc, a universal
# macOS build, an unknown architecture.  On this cluster (x86-64, GCC) the
# result is `-march=native` exactly as before, so every measured arm above
# is unchanged; the gate-count table is about THAT flag on THAT compiler and
# says nothing about MSVC or AArch64, where no arm has been measured.
if(NRHJ_MARCH_NATIVE AND NRHJ_NATIVE_ISA_FLAG)
  target_compile_options(nrhjkokkos PRIVATE ${NRHJ_NATIVE_ISA_FLAG})
  # THE CLUSTER TRAP.  This flag compiles for the ISA of the machine running
  # cmake, which on a cluster is the LOGIN node.  If the compute nodes are an
  # older or simply different microarchitecture, the module builds cleanly,
  # passes its smoke test here, and then dies with SIGILL on the first job --
  # with nothing in the error connecting it to a build-time choice.  Warn at
  # the one moment the user is present and the decision is still free.
  # STATUS, not WARNING: pip discards build-backend output on a SUCCESSFUL
  # install (measured -- zero cmake lines reach the pip log), so a warning
  # here is invisible to exactly the user who needs it, the one whose login
  # node build worked.  The warning that reaches them is raised at IMPORT, by
  # _warn_once_about_native_isa in nrhjsurrogate/__init__.py, and only when
  # the running CPU differs from the one recorded here.
  message(STATUS
      "nrhjsurrogate: compiling for this machine's ISA "
      "(${NRHJ_NATIVE_ISA_FLAG}); see the cluster note in INSTALL.md")
elseif(NOT NRHJ_MARCH_NATIVE)
  # THE RECORD MUST DESCRIBE THE BINARY, NOT THE INTENTION.
  # compiler_flags.cmake resolves the ISA flag and decides host-specificity
  # BEFORE this file declares NRHJ_MARCH_NATIVE, so it cannot know the flag
  # will not be applied. Left alone, a -DNRHJ_MARCH_NATIVE=OFF build records
  # native_isa_flag='-mcpu=native' and host_specific=ON for a binary that
  # carries neither.
  #
  # MEASURED 2026-09-06 on macos-14: the wheel job passed
  # -DNRHJ_MARCH_NATIVE=OFF, the module was built with no ISA flag, and the
  # record still claimed a host-specific build -- which
  # scripts/release_wheel_gate.py then refused, exactly as it should have
  # given what the record said. The build was right and the record was wrong.
  set(NRHJ_NATIVE_ISA_FLAG "" CACHE INTERNAL
      "The ISA flag this build applied, or empty" FORCE)
  set(NRHJ_ISA_HOST_SPECIFIC OFF CACHE INTERNAL
      "Whether that flag means this build host's own CPU" FORCE)
  set(NRHJ_NATIVE_ISA_WHY "NRHJ_MARCH_NATIVE=OFF; no ISA flag applied"
      CACHE INTERNAL "Why that ISA flag, or why none" FORCE)
  message(STATUS "nrhjsurrogate: NRHJ_MARCH_NATIVE=OFF, portable baseline")
endif()
if(NRHJ_MR_NO_CONTRACT AND NRHJ_FP_CONTRACT_OFF_FLAG)
  set_source_files_properties(mr.cpp PROPERTIES
                              COMPILE_OPTIONS "${NRHJ_FP_CONTRACT_OFF_FLAG}")
endif()
if(NRHJ_PACK_NO_CONTRACT AND NRHJ_FP_CONTRACT_OFF_FLAG)
  set_source_files_properties(pack.cpp PROPERTIES
                              COMPILE_OPTIONS "${NRHJ_FP_CONTRACT_OFF_FLAG}")
endif()
# swsh.cpp: MEASURED 2026-09-03 (audit_artifacts/kokkos_swsh/FINDINGS.log
# [KS-004], [KS-005]): GCC combined the kernel's cos(x) + sin(x) pair into
# glibc's sincos(), which is NOT bitwise with separate cos()/sin() on about
# 1 input in 1000.  -fno-builtin-sin/-cos here did NOT survive -flto=auto
# (the link stage re-merged the declarations with the builtins and the two
# sincos@plt call sites stayed), so the fix lives in the source
# (swsh.cpp keep_separate), not in a flag.  Only the contraction setting is
# applied from here.
if(NRHJ_SWSH_NO_CONTRACT AND NRHJ_FP_CONTRACT_OFF_FLAG)
  set_source_files_properties(swsh.cpp PROPERTIES
                              COMPILE_OPTIONS "${NRHJ_FP_CONTRACT_OFF_FLAG}")
endif()
# THE THIRD OPT-OUT, added 2026-08-05.  MEASURED REASON, from a 2x2 on trunk
# b1a2226 (gate set = the same 179 tests aa/march-native used):
#
#   ISA       B1a   gates
#   generic   off   179 passed        <- baseline, clean
#   generic   ON    179 passed        <- B1a alone is gate-clean
#   native    off   179 passed        <- the ISA alone is gate-clean
#   native    ON    6 FAILED          <- ONLY THE INTERACTION BREAKS
#
# So B1a and -march=native are each safe alone and NOT safe together, and
# gpr.cpp was in neither opt-out list.  The RB=4 tile gives the compiler four
# independent accumulators it can contract/vectorise differently from trunk's
# single-dot-product loop, so the two arms' dot products round differently by
# ~1 ulp (confirmed directly: B1a is bitwise 6/6 at generic ISA and 0/6 under
# native, max|dh|/peak 1.3e-16..3.4e-16, white mismatch exactly 0.0).  That
# 1 ulp then propagates into the pack bitwise references, which is why the
# failures are test_native_gpr_batch_bitwise + the test_pack_batch_bitwise
# family rather than anything in gpr's own tests.
#
# Turn this ON to use B1a together with -march=native and keep the gates green.
option(NRHJ_GPR_NO_CONTRACT
       "-ffp-contract=off on gpr.cpp (required to combine B1a with native)" OFF)
if(NRHJ_GPR_NO_CONTRACT AND NRHJ_FP_CONTRACT_OFF_FLAG)
  set_source_files_properties(gpr.cpp PROPERTIES
                              COMPILE_OPTIONS "${NRHJ_FP_CONTRACT_OFF_FLAG}")
endif()
# ==================================================================== #

# aa/alg-speedup items B1a / B1c on gpr.cpp's `gpr_batch_contract`, both
# default OFF so the DEFAULT build stays bitwise-identical to trunk.
#   NRHJ_GPR_REGBLOCK -- register-block the batched mean contraction over
#     the batch axis (a tile of RB=4 rows per task).  BITWISE-INERT: the
#     accumulators are independent and each keeps its own ascending j
#     order, so no sum is reassociated.  Gate it with np.array_equal on
#     raw hp/hc bits, NOT with a tolerance.
#   NRHJ_GPR_SIMD -- implies REGBLOCK, and additionally vectorises the
#     length-n_train kernel reduction with `omp simd` over an SoA mirror
#     of alpha.  NOT bitwise (a vector reduction splits the accumulator
#     per lane and sums the partials), and it doubles the alpha memory
#     footprint.  Determinism WITHIN a run is preserved.
# NEW-1, ENABLED 2026-08-08 on the owner's decision. Register blocking ON,
# and the GUARD below ON with it -- the two are not independent, see there.
# Worth 1.088x [1.084, 1.106] at B=256; the B=1 floor is protected by the
# guard, which dispatches to the unblocked path below MIN_B.
option(NRHJ_GPR_REGBLOCK
       "Register-block gpr_batch_contract over the batch axis (BITWISE)" ON)
option(NRHJ_GPR_SIMD
       "Vectorise gpr_batch_contract's kernel reduction (NOT bitwise)" OFF)
#   NRHJ_GPR_REGBLOCK_GUARD -- with REGBLOCK, fall back to the UNBLOCKED
#     contraction below NRHJ_GPR_REGBLOCK_MIN_B rows.  The tile is a
#     batch-axis optimisation and at B < RB every lane clamps onto row B-1,
#     so it runs RB x the arithmetic and stores one result in RB.  MEASURED
#     on r403: speedup 0.95 at B=1, 1.09 at B=256.
#     *** THIS MAKES THE KERNEL CALL-SHAPE DEPENDENT: the same theta can
#     return different BITS at B=1 than at B>=MIN_B on ONE machine.  That is
#     the same class of cost as the warm-start secant seed.  Any gate
#     asserting row-vs-batch invariance must be re-run against this arm. ***
# ON, AND NOT OPTIONAL WHILE REGBLOCK IS ON. Measured on two machines:
# unguarded register blocking breaks row-vs-batch invariance at EVERY batch
# size, not just across MIN_B, because the lane-clamp pattern differs between
# B=1 and B=2 and contraction follows it. So REGBLOCK=ON with GUARD=OFF costs
# BOTH the 0.9546 at the B=1 floor AND invariance everywhere -- strictly worse
# than either shipping decision. CMake already refuses GUARD without REGBLOCK;
# this comment is the other half, which it cannot enforce.
option(NRHJ_GPR_REGBLOCK_GUARD
       "With REGBLOCK: use the unblocked path below NRHJ_GPR_REGBLOCK_MIN_B rows (CALL-SHAPE DEPENDENT)" ON)
set(NRHJ_GPR_REGBLOCK_MIN_B "3" CACHE STRING
    "Batch size at/above which the register-blocked tile is used. MEASURED crossover on Zen 2 / EPYC 7742 (job 43152313): B=2 speedup 0.941 [0.914,0.965], B=3 speedup 1.048 [1.024,1.062]. MACHINE-SPECIFIC -- a tuned default, NOT a portable constant; retune with scripts/k1_crossover.slurm on other hardware.")
if(NRHJ_GPR_REGBLOCK)
  target_compile_definitions(nrhjkokkos PRIVATE NRHJ_GPR_REGBLOCK)
endif()
if(NRHJ_GPR_REGBLOCK_GUARD)
  if(NOT NRHJ_GPR_REGBLOCK)
    message(FATAL_ERROR
      "NRHJ_GPR_REGBLOCK_GUARD=ON requires NRHJ_GPR_REGBLOCK=ON; without it "
      "the unblocked path is already the only path and the guard is a no-op.")
  endif()
  target_compile_definitions(nrhjkokkos PRIVATE
      NRHJ_GPR_REGBLOCK_GUARD
      NRHJ_GPR_REGBLOCK_MIN_B=${NRHJ_GPR_REGBLOCK_MIN_B})
endif()
if(NRHJ_GPR_SIMD)
  target_compile_definitions(nrhjkokkos PRIVATE NRHJ_GPR_SIMD)
endif()
target_include_directories(nrhjkokkos PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(nrhjkokkos PRIVATE Kokkos::kokkos)
target_compile_features(nrhjkokkos PRIVATE cxx_std_17)
# Works around the glibc 2.42+ vs CUDA <= 12.x noexcept clash on
# sinpi/cospi/rsqrt; see glibc_cuda_shim/math.h.
#
# NOT ON MSVC.  The shim is a glibc header interposer and works by
# `#include_next`, which is a GCC/Clang extension that cl.exe rejects
# outright:
#   glibc_cuda_shim/math.h(29,1): error C1021: invalid preprocessor command
#   'include_next'
# MEASURED 2026-09-06 on windows-latest, where it failed every translation
# unit. There is no glibc on Windows and no CUDA-vs-glibc clash to work
# around, so the guard costs nothing anywhere it used to apply -- the shim
# stays exactly as it was on the platform it was written for.
if(NOT MSVC)
  target_include_directories(nrhjkokkos BEFORE PRIVATE
                             ${CMAKE_CURRENT_SOURCE_DIR}/glibc_cuda_shim)
endif()
