fix(dht): pass communicator to DHT_create

ext/DHT

@@ -1 +1 @@
-Subproject commit 19d4510fdcb9cecf3d8df427eea4966053835a82
+Subproject commit 67e43586a8b746c8795646e974db07cc2a840932

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_library(poetlib
+set(POET_SOURCES 
   Base/Grid.cpp
   Base/SimParams.cpp
   Chemistry/ChemistryModule.cpp
@@ -6,58 +6,80 @@ add_library(poetlib
   Chemistry/WorkerFunctions.cpp 
   Chemistry/SurrogateModels/DHT_Wrapper.cpp 
   Chemistry/SurrogateModels/HashFunctions.cpp 
-#  Chemistry/SurrogateModels/InterpolationModule.cpp 
-#  Chemistry/SurrogateModels/ProximityHashTable.cpp
   DataStructures/Field.cpp
   Transport/DiffusionModule.cpp
   Transport/AdvectionModule.cpp
 )
 
-target_link_libraries(poetlib PUBLIC
+# option(POET_USE_DHT_MPI "Use MPI for DHT" OFF)
+#
+# if (NOT POET_USE_DHT_MPI)
+#   target_compile_definitions(poetlib PUBLIC POET_DHT_UCX)
+#   target_link_libraries(poetlib PUBLIC DHT_UCX)
+# else()
+#   target_link_libraries(poetlib PUBLIC DHT_MPI)
+# endif()
+#
+# target_compile_definitions(poetlib PUBLIC STRICT_R_HEADERS OMPI_SKIP_MPICXX)
+#
+# mark_as_advanced(PHREEQCRM_BUILD_MPI PHREEQCRM_DISABLE_OPENMP)
+# set(PHREEQCRM_DISABLE_OPENMP ON CACHE BOOL "" FORCE)
+#
+# option(POET_DHT_DEBUG "Build with DHT debug info" OFF)
+#
+# if(POET_DHT_DEBUG)
+#   target_compile_definitions(poetlib PRIVATE DHT_STATISTICS)
+# endif()
+#
+# option(POET_PHT_ADDITIONAL_INFO "Enables additional information in the PHT" OFF)
+#
+# if (POET_PHT_ADDITIONAL_INFO)
+#   target_compile_definitions(poetlib PRIVATE POET_PHT_ADD)
+# endif()
+#
+# if (OpenMP_FOUND)
+#   target_link_libraries(poetlib PUBLIC OpenMP::OpenMP_CXX)
+# endif()
+
+file(READ "${PROJECT_SOURCE_DIR}/R_lib/kin_r_library.R" R_KIN_LIB )
+
+configure_file(poet.hpp.in poet.hpp @ONLY)
+
+add_executable(poet_coarse poet.cpp ${POET_SOURCES})
+target_link_libraries(poet_coarse PRIVATE 
   MPI::MPI_C 
   ${MATH_LIBRARY} 
   RRuntime 
   PhreeqcRM 
   tug
   OpenSSL::Crypto
+  LMPI_DHT_OLD
 )
+target_include_directories(poet_coarse PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+install(TARGETS poet_coarse DESTINATION bin)
 
-option(POET_USE_DHT_MPI "Use MPI for DHT" OFF)
+add_executable(poet_fine poet.cpp ${POET_SOURCES})
+target_link_libraries(poet_fine PRIVATE 
+  MPI::MPI_C 
+  ${MATH_LIBRARY} 
+  RRuntime 
+  PhreeqcRM 
+  tug
+  OpenSSL::Crypto
+  LFINE_LOCK
+)
+target_include_directories(poet_fine PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+install(TARGETS poet_fine DESTINATION bin)
 
-if (NOT POET_USE_DHT_MPI)
-  target_compile_definitions(poetlib PUBLIC POET_DHT_UCX)
-  target_link_libraries(poetlib PUBLIC DHT_UCX)
-else()
-  target_link_libraries(poetlib PUBLIC DHT_MPI)
-endif()
-
-target_compile_definitions(poetlib PUBLIC STRICT_R_HEADERS OMPI_SKIP_MPICXX)
-
-mark_as_advanced(PHREEQCRM_BUILD_MPI PHREEQCRM_DISABLE_OPENMP)
-set(PHREEQCRM_DISABLE_OPENMP ON CACHE BOOL "" FORCE)
-
-option(POET_DHT_DEBUG "Build with DHT debug info" OFF)
-
-if(POET_DHT_DEBUG)
-  target_compile_definitions(poetlib PRIVATE DHT_STATISTICS)
-endif()
-
-option(POET_PHT_ADDITIONAL_INFO "Enables additional information in the PHT" OFF)
-
-if (POET_PHT_ADDITIONAL_INFO)
-  target_compile_definitions(poetlib PRIVATE POET_PHT_ADD)
-endif()
-
-if (OpenMP_FOUND)
-  target_link_libraries(poetlib PUBLIC OpenMP::OpenMP_CXX)
-endif()
-
-file(READ "${PROJECT_SOURCE_DIR}/R_lib/kin_r_library.R" R_KIN_LIB )
-
-configure_file(poet.hpp.in poet.hpp @ONLY)
-
-add_executable(poet poet.cpp)
-target_link_libraries(poet PRIVATE poetlib MPI::MPI_C RRuntime)
-target_include_directories(poet PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
-
-install(TARGETS poet DESTINATION bin)
+add_executable(poet_no poet.cpp ${POET_SOURCES})
+target_link_libraries(poet_no PRIVATE 
+  MPI::MPI_C 
+  ${MATH_LIBRARY} 
+  RRuntime 
+  PhreeqcRM 
+  tug
+  OpenSSL::Crypto
+  LMPI_DHT
+)
+target_include_directories(poet_no PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+install(TARGETS poet_no DESTINATION bin)

src/Chemistry/SurrogateModels/DHT_Wrapper.cpp

@@ -21,8 +21,8 @@
 */
 
 #include "DHT_Wrapper.hpp"
-#include "DHT_ucx/DHT.h"
 #include "HashFunctions.hpp"
+#include "LUCX/DHT.h"
 
 #include <algorithm>
 #include <cassert>
@@ -69,8 +69,13 @@ DHT_Wrapper::DHT_Wrapper(MPI_Comm dht_comm, std::uint64_t dht_size,
       .bcast_func_args = &ucx_bcast_mpi_args};
   dht_object = DHT_create(&dht_init);
 #else
-  dht_object = DHT_create(dht_comm, buckets_per_process, data_size, key_size,
-                          poet::md5_sum);
+  const DHT_init_t dht_init = {
+      .key_size = static_cast<int>(key_size),
+      .data_size = static_cast<int>(data_size),
+      .bucket_count = static_cast<unsigned int>(buckets_per_process),
+      .hash_func = &poet::md5_sum,
+      .comm = dht_comm};
+  dht_object = DHT_create(&dht_init);
 #endif
 
   if (dht_object == nullptr) {
@@ -104,7 +109,7 @@ DHT_Wrapper::~DHT_Wrapper() {
 #ifdef POET_DHT_UCX
   DHT_free(dht_object, NULL, NULL, NULL);
 #else
-  DHT_free(dht_object, NULL, NULL);
+  DHT_free(dht_object, NULL);
 #endif
 }
 auto DHT_Wrapper::checkDHT(WorkPackage &work_package)

src/Chemistry/SurrogateModels/DHT_Wrapper.hpp

@@ -39,12 +39,7 @@
 #include <utility>
 #include <vector>
 
-#ifdef POET_DHT_UCX
-#include <DHT_ucx/DHT.h>
-#include <DHT_ucx/UCX_bcast_functions.h>
-#else
-#include <DHT_mpi/DHT.h>
-#endif
+#include <LUCX/DHT.h>
 
 #include <mpi.h>