cmake_minimum_required(VERSION 3.20)
project(opencuda_lab VERSION 0.1.0 LANGUAGES C)

set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED ON)
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
endif()

option(OCL_ENABLE_CUDA   "Build the CUDA runtime and VMM backends, tools and benchmarks" OFF)
option(OCL_ENABLE_TESTS  "Build the test suite" ON)
option(OCL_BUILD_APPS    "Build replay drivers, quickstart and CUDA tools" ON)
option(OCL_ENABLE_PYTORCH_INTEROP "Build ocl_pytorch_alloc / ocl_pytorch_free (CUDAPluggableAllocator)" ON)
option(OCL_ENABLE_ASAN   "Build with address and undefined-behaviour sanitizers" OFF)
option(OCL_ENABLE_TSAN   "Build with the thread sanitizer" OFF)
option(OCL_WERROR        "Treat warnings as errors" ON)

if(DEFINED SKBUILD)
  set(OCL_ENABLE_TESTS OFF CACHE BOOL "" FORCE)
  set(OCL_BUILD_APPS OFF CACHE BOOL "" FORCE)
  set(OCL_ENABLE_PYTORCH_INTEROP ON CACHE BOOL "" FORCE)
  set(OCL_WERROR OFF CACHE BOOL "" FORCE)
  # pip install: use CUDA backends when nvcc can actually compile. On Windows
  # nvcc requires MSVC cl.exe; MinGW gcc cannot be the CUDA host compiler.
  # OCL_FORCE_MOCK=1 keeps the host mock (also set by the build backend when
  # only MinGW is available).
  set(_ocl_try_cuda ON)
  if(DEFINED ENV{OCL_FORCE_MOCK})
    set(_ocl_try_cuda OFF)
  endif()
  if(_ocl_try_cuda AND WIN32 AND CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
    find_program(_OCL_CL NAMES cl cl.exe)
    if(NOT _OCL_CL)
      message(WARNING
        "nvcc on Windows needs cl.exe (Visual Studio C++). "
        "C compiler is ${CMAKE_C_COMPILER}. Building the host-mock library. "
        "Install VS 2022 Build Tools and reinstall for GPU backends.")
      set(_ocl_try_cuda OFF)
    endif()
  endif()
  if(_ocl_try_cuda)
    include(CheckLanguage)
    check_language(CUDA)
    if(CMAKE_CUDA_COMPILER)
      set(OCL_ENABLE_CUDA ON)
      message(STATUS "CUDA compiler found; pip package will include GPU backends")
    else()
      message(WARNING "nvcc not usable; pip package will be host-mock only. "
                      "On Windows install VS C++ Build Tools so nvcc can find cl.exe.")
    endif()
  endif()
endif()

# CUDA is optional so the allocator core, planner, estimator, calibrator and the
# whole test suite can be developed and validated on machines without a GPU.
if(OCL_ENABLE_CUDA)
  include(CheckLanguage)
  check_language(CUDA)
  if(CMAKE_CUDA_COMPILER)
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 14)
    if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
      set(CMAKE_CUDA_ARCHITECTURES 70 75 80 86 89 90)
    endif()
    find_package(CUDAToolkit REQUIRED)
    set(OCL_HAVE_CUDA TRUE)
  else()
    message(WARNING "OCL_ENABLE_CUDA requested but no CUDA compiler was found; "
                    "building the host-only configuration")
    set(OCL_HAVE_CUDA FALSE)
  endif()
else()
  set(OCL_HAVE_CUDA FALSE)
endif()

set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)

add_library(ocl_flags INTERFACE)
if(MSVC)
  target_compile_options(ocl_flags INTERFACE
    $<$<COMPILE_LANGUAGE:C>:/W3 /wd4996>)
  if(OCL_WERROR)
    target_compile_options(ocl_flags INTERFACE $<$<COMPILE_LANGUAGE:C>:/WX>)
  endif()
else()
  target_compile_options(ocl_flags INTERFACE
    $<$<COMPILE_LANGUAGE:C>:-Wall -Wextra -Wshadow -Wpointer-arith -Wcast-qual>)
  if(OCL_WERROR)
    target_compile_options(ocl_flags INTERFACE $<$<COMPILE_LANGUAGE:C>:-Werror>)
  endif()
endif()
if(OCL_ENABLE_ASAN AND NOT MSVC)
  target_compile_options(ocl_flags INTERFACE -fsanitize=address,undefined -fno-omit-frame-pointer -g)
  target_link_options(ocl_flags INTERFACE -fsanitize=address,undefined)
endif()
if(OCL_ENABLE_TSAN AND NOT MSVC)
  target_compile_options(ocl_flags INTERFACE -fsanitize=thread -g)
  target_link_options(ocl_flags INTERFACE -fsanitize=thread)
endif()

set(OCL_SOURCES
  src/core/runtime.c
  src/core/allocator.c
  src/core/planner.c
  src/core/util.c
  src/core/ocl_vm.c
  src/estimator/estimator.c
  src/estimator/conformal.c
  src/telemetry/telemetry.c
  src/backend/mock/provider_mock.c
  src/interop/pytorch_alloc.c)

if(OCL_HAVE_CUDA)
  list(APPEND OCL_SOURCES
    src/backend/runtime/provider_runtime.c
    src/backend/vmm/provider_vmm.c)
endif()

add_library(opencuda SHARED ${OCL_SOURCES})
add_library(opencuda_static STATIC ${OCL_SOURCES})
if(WIN32)
  # Keep CUDAPluggableAllocator symbols even if the linker GC's unused objects.
  target_sources(opencuda PRIVATE src/interop/opencuda.def)
  if(MSVC)
    target_link_options(opencuda PRIVATE
      "/EXPORT:ocl_pytorch_alloc"
      "/EXPORT:ocl_pytorch_free"
      "/EXPORT:ocl_pytorch_built_with_cuda")
  endif()
endif()

foreach(tgt opencuda opencuda_static)
  target_include_directories(${tgt} PUBLIC
    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
    $<INSTALL_INTERFACE:include>)
  target_include_directories(${tgt} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
  target_link_libraries(${tgt} PUBLIC ocl_flags)
  find_library(OCL_LIBM m)
  if(OCL_LIBM)
    target_link_libraries(${tgt} PUBLIC ${OCL_LIBM})
  endif()
  find_package(Threads REQUIRED)
  target_link_libraries(${tgt} PUBLIC Threads::Threads)
  if(OCL_HAVE_CUDA)
    target_compile_definitions(${tgt} PUBLIC OCL_WITH_CUDA=1)
    target_link_libraries(${tgt} PUBLIC CUDA::cudart CUDA::cuda_driver)
  endif()
endforeach()

set_target_properties(opencuda PROPERTIES
  OUTPUT_NAME opencuda VERSION ${PROJECT_VERSION} SOVERSION 0)
set_target_properties(opencuda_static PROPERTIES OUTPUT_NAME opencuda_static)

# pip install puts the DLL/.so in site-packages/opencudalab.
# A source checkout copies it next to the package so `import opencudalab` works
# without a second CMAKE install step.
if(NOT DEFINED SKBUILD)
  add_custom_command(TARGET opencuda POST_BUILD
    COMMAND ${CMAKE_COMMAND} -E copy_if_different
      "$<TARGET_FILE:opencuda>"
      "${CMAKE_CURRENT_SOURCE_DIR}/python/opencudalab/$<TARGET_FILE_NAME:opencuda>"
    COMMENT "Copy opencuda shared library into python/opencudalab")
endif()

# ------------------------------------------------------- benchmark support --
if(OCL_BUILD_APPS)
# The workload generators are shared by the host-only replay driver and the CUDA
# drivers, so that a result produced in CI and one produced on the GPU box refer
# to the same allocation sequence.
add_library(ocl_bench STATIC
  bench/workloads/workloads.c
  bench/baselines/baselines.c)
target_include_directories(ocl_bench PUBLIC
  ${CMAKE_CURRENT_SOURCE_DIR}/bench/workloads
  ${CMAKE_CURRENT_SOURCE_DIR}/bench/baselines)
target_link_libraries(ocl_bench PUBLIC opencuda_static)

add_executable(ocl_replay bench/replay/replay.c)
target_link_libraries(ocl_replay PRIVATE ocl_bench)
target_include_directories(ocl_replay PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

add_executable(ocl_calibrate experiments/drivers/calibrate.c)
target_link_libraries(ocl_calibrate PRIVATE ocl_bench)
target_include_directories(ocl_calibrate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

add_executable(ocl_admission experiments/drivers/admission.c)
target_link_libraries(ocl_admission PRIVATE ocl_bench)
target_include_directories(ocl_admission PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

add_executable(ocl_capacity experiments/drivers/capacity.c)
target_link_libraries(ocl_capacity PRIVATE ocl_bench)
target_include_directories(ocl_capacity PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

add_executable(ocl_baselines experiments/drivers/baselines.c)
target_link_libraries(ocl_baselines PRIVATE ocl_bench)
target_include_directories(ocl_baselines PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

add_executable(ocl_multitenant experiments/drivers/multitenant.c)
target_link_libraries(ocl_multitenant PRIVATE ocl_bench)
target_include_directories(ocl_multitenant PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

# The README's usage example, built and run so that a rename in the public
# header cannot leave documentation that no longer compiles.
add_executable(ocl_quickstart examples/quickstart.c)
target_link_libraries(ocl_quickstart PRIVATE opencuda_static)

if(OCL_HAVE_CUDA)
  add_executable(ocl_probe tools/ocl_probe.cu)
  target_link_libraries(ocl_probe PRIVATE CUDA::cudart CUDA::cuda_driver)

  add_executable(ocl_micro tools/ocl_micro.cu)
  target_link_libraries(ocl_micro PRIVATE CUDA::cudart CUDA::cuda_driver)

  add_executable(ocl_gap tools/ocl_gap.cu)
  target_link_libraries(ocl_gap PRIVATE ocl_bench CUDA::cudart CUDA::cuda_driver)
  target_include_directories(ocl_gap PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)

  add_executable(ocl_kernels bench/kernels/kernels.cu)
  target_link_libraries(ocl_kernels PRIVATE ocl_bench CUDA::cudart CUDA::cuda_driver)
  target_include_directories(ocl_kernels PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
endif()
endif() # OCL_BUILD_APPS

# ------------------------------------------------------------------- tests --
if(OCL_ENABLE_TESTS)
  enable_testing()
  set(OCL_TESTS
    test_allocator
    test_realloc
    test_planner
    test_estimator
    test_conformal
    test_admission
    test_telemetry
    test_stress
    test_threads)
  list(APPEND OCL_TESTS test_pytorch_alloc)
  if(OCL_BUILD_APPS)
    list(APPEND OCL_TESTS test_baselines)
  endif()
  foreach(t ${OCL_TESTS})
    add_executable(${t} tests/${t}.c)
    target_link_libraries(${t} PRIVATE opencuda_static)
    target_include_directories(${t} PRIVATE
      ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tests)
    add_test(NAME ${t} COMMAND ${t})
  endforeach()
  if(OCL_BUILD_APPS)
    add_test(NAME quickstart COMMAND ocl_quickstart)
    target_link_libraries(test_baselines PRIVATE ocl_bench)
  endif()
endif()

if(DEFINED SKBUILD)
  install(TARGETS opencuda
    RUNTIME DESTINATION opencudalab
    LIBRARY DESTINATION opencudalab
    ARCHIVE DESTINATION opencudalab)
  install(FILES examples/python/resnet_cifar10.py DESTINATION opencudalab)
else()
  install(TARGETS opencuda opencuda_static
          RUNTIME DESTINATION bin
          LIBRARY DESTINATION lib
          ARCHIVE DESTINATION lib)
  install(DIRECTORY include/ DESTINATION include)

  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/opencuda.pc.in
                 ${CMAKE_CURRENT_BINARY_DIR}/opencuda.pc @ONLY)
  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/opencuda.pc
          DESTINATION lib/pkgconfig)
endif()

message(STATUS "OpenCUDA-Lab ${PROJECT_VERSION}")
message(STATUS "  CUDA backends : ${OCL_HAVE_CUDA}")
message(STATUS "  tests         : ${OCL_ENABLE_TESTS}")
message(STATUS "  apps          : ${OCL_BUILD_APPS}")
message(STATUS "  pytorch hook  : ${OCL_ENABLE_PYTORCH_INTEROP}")
message(STATUS "  asan/ubsan    : ${OCL_ENABLE_ASAN}")
