Daos

2025-12-15 20:38:23 +01:00 · 2023-09-09 01:07:52 +02:00
29 changed files with 11295 additions and 0 deletions
--- a/bench/CMakeLists.txt
+++ b/bench/CMakeLists.txt
@ -1,3 +1,4 @@
 add_subdirectory(dolo)
 add_subdirectory(surfex)
 add_subdirectory(barite)
+add_subdirectory(DAOSBenchmark)
--- a/bench/DAOSBenchmark/CMakeLists.txt
+++ b/bench/DAOSBenchmark/CMakeLists.txt
@ -0,0 +1,7 @@
+install(FILES
+    dolo_diffu_edge.R
+    dolo_inner.pqi
+    phreeqc_kin.dat
+DESTINATION
+    share/poet/bench/daos
+)
--- a/bench/DAOSBenchmark/dolo_diffu_edge.R
+++ b/bench/DAOSBenchmark/dolo_diffu_edge.R
@ -0,0 +1,205 @@
+## Time-stamp: "Last modified 2023-09-05 14:42:20 mluebke"
+
+database <- normalizePath("../share/poet/bench/dolo/phreeqc_kin.dat")
+input_script <- normalizePath("../share/poet/bench/dolo/dolo_inner.pqi")
+
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 400
+m <- 200
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0,
+  "O2g" = 0.499957,
+  "Calcite" = 2.07e-4,
+  "Dolomite" = 0
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(5, 2.5),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+init_diffu <- list(
+  "H" = 1.110124E+02,
+  "O" = 5.550833E+01,
+  "Charge" = -1.216307659761E-09,
+  "C(4)" = 1.230067028174E-04,
+  "Ca" = 1.230067028174E-04,
+  "Cl" = 0,
+  "Mg" = 0
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-6,
+  "O" = 1E-6,
+  "Charge" = 1E-6,
+  "C(4)" = 1E-6,
+  "Ca" = 1E-6,
+  "Cl" = 1E-6,
+  "Mg" = 1E-6
+)
+
+## list of boundary conditions/inner nodes
+vecinj_diffu <- list(
+  list(
+    "H" = 1.110124E+02,
+    "O" = 5.550796E+01,
+    "Charge" = -3.230390327801E-08,
+    "C(4)" = 0,
+    "Ca" = 0,
+    "Cl" = 0.002,
+    "Mg" = 0.001
+  ),
+  list(
+    "H" = 110.683,
+    "O" = 55.3413,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0.0,
+    "Cl" = 0.004,
+    "Mg" = 0.002
+  ),
+  init_diffu
+)
+
+vecinj_inner <- list(
+  l1 = c(1, 1, 1)
+  # l2 = c(2,1400,800),
+  # l3 = c(2,1600,800)
+)
+
+boundary <- list(
+  #  "N" = c(1, rep(0, n-1)),
+  "N" = rep(0, n),
+  "E" = rep(0, m),
+  "S" = rep(0, n),
+  "W" = rep(0, m)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+## # optional when using DHT
+dht_species <- c(
+  "H" = 3,
+  "O" = 3,
+  "Charge" = 3,
+  "C(4)" = 6,
+  "Ca" = 6,
+  "Cl" = 3,
+  "Mg" = 5,
+  "Calcite" = 4,
+  "Dolomite" = 4
+)
+
+## # Optional when using Interpolation (example with less key species and custom
+## # significant digits)
+
+# pht_species <- c(
+#  "C(4)" = 3,
+#  "Ca" = 3,
+#  "Mg" = 2,
+#  "Calcite" = 2,
+#  "Dolomite" = 2
+# )
+
+check_sign_cal_dol_dht <- function(old, new) {
+  if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    return(TRUE)
+  }
+  if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    return(TRUE)
+  }
+  return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+  return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+  data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+  names(data_set) <- names(dht_species)
+  cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+  dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+  cal_dol_same_sig <- cal == dol
+  return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+  neg_sign <- (result["Calcite"] <- 0) || (result["Dolomite"] < 0)
+  return(any(neg_sign))
+}
+
+hooks <- list(
+  dht_fill = check_sign_cal_dol_dht,
+  dht_fuzz = fuzz_input_dht_keys,
+  interp_pre_func = check_sign_cal_dol_interp,
+  interp_post_func = check_neg_cal_dol
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species,
+  hooks = hooks
+  #  pht_species = pht_species
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 1500
+dt <- 500
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = c(1, seq(50, iterations, by = 50))
+)
--- a/bench/DAOSBenchmark/dolo_inner.pqi
+++ b/bench/DAOSBenchmark/dolo_inner.pqi
@ -0,0 +1,28 @@
+SELECTED_OUTPUT
+         -high_precision true
+         -reset false
+         -kinetic_reactants Calcite Dolomite
+         -equilibrium O2g
+
+SOLUTION 1
+         units mol/kgw
+         temp 25.0
+         water 1
+         pH 9.91 charge
+         pe 4.0
+         C   1.2279E-04
+         Ca  1.2279E-04
+         Cl 1E-12
+         Mg 1E-12
+PURE 1
+         O2g -0.1675 10
+KINETICS 1
+         Calcite
+            -m      0.00020
+            -parms  0.05
+            -tol    1e-10
+         Dolomite
+            -m      0.0
+            -parms  0.005
+            -tol    1e-10
+END
--- a/bench/DAOSBenchmark/phreeqc_kin.dat
+++ b/bench/DAOSBenchmark/phreeqc_kin.dat
--- a/bin/RunDaosBenchmarks.sh
+++ b/bin/RunDaosBenchmarks.sh
@ -0,0 +1,8 @@
+
+
+
+
+
+taskset --cpu-list 0-23:2 mpirun --allow-run-as-root -x DAOS_POOL=test_pool -n 12 ./poet ../share/poet/bench/daos/dolo_diffu_edge.R ../../POETR >> ../../POET.out
+#    taskset --cpu-list 0-23:2 mpirun --allow-run-as-root -x DAOS_POOL=test_pool -n $i ./build/src/kivibench-DAOSKV -x 10000 -y 3 -m 1000 -k 10 -v 12000 --csv >> benchmarks/clientbig$i.csv
+
--- a/bin/poet
+++ b/bin/poet
--- a/include/poet/DaosKeyValue.h
+++ b/include/poet/DaosKeyValue.h
@ -0,0 +1,105 @@
+
+/**
+ * @file DaosKeyValue.h
+ * @author Nico Sauerbrei (nico.sauerbrei@uni-potsdam.de)
+ * @brief API to interact with DAOS
+ * @version 0.1
+ * @date 01 Jun 2023
+ *
+ * This file implements the communication between POET and the DAOS
+ * Key-Value Store
+ */
+
+#ifndef DAOS_KEY_VALUE_H
+#define DAOS_KEY_VALUE_H
+
+#include <mpi.h>
+#include <stdint.h>
+
+#include <daos.h>
+
+#define DAOS_SUCCESS 0
+#define DAOS_ERROR -1
+#define DAOS_MPI_ERROR -2
+#define DAOS_READ_MISS -3
+#define DHT_STATISTICS 1
+
+/**
+ * Internal struct to store statistics about read and write accesses and also
+ * read misses and evictions.
+ * <b>All values will be resetted to zero after a call of
+ * DHT_print_statistics().</b>
+ * Internal use only!
+ *
+ */
+typedef struct
+{
+  /** Count of writes to specific process this process did. */
+  int *writes_local;
+  /** Writes after last call of DHT_print_statistics. */
+  int old_writes;
+  /** How many read misses occur? */
+  int read_misses;
+  /** How many read hits occur? */
+  int read_hits;
+  /** How many buckets where evicted? */
+  int evictions;
+  /** How many calls of DHT_write() did this process? */
+  int w_access;
+  /** How many calls of DHT_read() did this process? */
+  int r_access;
+} DAOSKV_stats;
+
+/**
+ * Struct which serves as a handler or so called \a DHT-object. Will
+ * be created by DHT_create and must be passed as a parameter to every following
+ * function. Stores all relevant data.
+ * Do not touch outside DHT functions!
+ */
+typedef struct
+{
+
+  /** MPI communicator of all participating processes. */
+  MPI_Comm communicator;
+  /** Size of the MPI communicator respectively all participating processes. */
+  int comm_size;
+  /** Rank of the process in the MPI communicator. */
+  int rank;
+  /** Count of read misses over all time. */
+  int read_misses;
+  /** Count of evictions over all time. */
+  int evictions;
+
+  /**  Label of the DAOS container.*/
+  char *cont_label;
+  /**  DAOS pool handle.*/
+  daos_handle_t poh;
+  /**  DAOS container handle.*/
+  daos_handle_t coh;
+  /**  DAOS object handle.*/
+  daos_handle_t oh;
+
+#ifdef DHT_STATISTICS
+  /** Detailed statistics of the usage of the DHT. */
+  DAOSKV_stats *stats;
+#endif
+} DAOSKV;
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+  extern DAOSKV *DAOSKV_create(MPI_Comm comm);
+  extern int DAOSKV_free(DAOSKV *object);
+  extern int DAOSKV_write(DAOSKV *object, void *key, int key_size, void *send_data, int send_size);
+  extern int DAOSKV_read(DAOSKV *object, void *key, int key_size, void *recv_data, int recv_size);
+  extern int DAOSKV_remove(DAOSKV *object, void *key, int key_size);
+  extern int DAOSKV_print_statistics(DAOSKV *object);
+  extern int enumerate_key(DAOSKV *object, int *total_nr, int key_size);
+  extern struct daos_space get_pool_size(DAOSKV *object);
+  extern int trim_Space(DAOSKV *object, float deletePercentage, int dataSize, int keySize);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* DAOS_KEY_VALUE_H */
--- a/share/poet/bench/barite/barite.R
+++ b/share/poet/bench/barite/barite.R
@ -0,0 +1,147 @@
+## Time-stamp: "Last modified 2023-08-02 13:59:22 mluebke"
+
+database <- normalizePath("../share/poet/bench/barite/db_barite.dat")
+input_script <- normalizePath("../share/poet/bench/barite/barite.pqi")
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 20
+m <- 20
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.0124,
+  "O" = 55.5087,
+  "Charge" = -1.217E-09,
+  "Ba" = 1.E-10,
+  "Cl" = 2.E-10,
+  "S" = 6.205E-4,
+  "Sr" = 6.205E-4,
+  "Barite" = 0.001,
+  "Celestite" = 1
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(1, 1),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+
+init_diffu <- list(
+  #"H" = 110.0124,
+  "H" = 0.00000028904,
+  #"O" = 55.5087,
+  "O" = 0.000000165205,
+  #"Charge" = -1.217E-09,
+  "Charge" = -3.337E-08,
+  "Ba" = 1.E-10,
+  "Cl" = 1.E-10,
+  "S(6)" = 6.205E-4,
+  "Sr" = 6.205E-4
+)
+
+injection_diff <- list(
+    list(
+        #"H" = 111.0124,
+        "H" = 0.0000002890408,
+        #"O" = 55.50622,
+        "O" = 0.00002014464,
+        #"Charge" = -3.337E-08,
+        "Charge" = -3.337000004885E-08,
+        "Ba" = 0.1,
+        "Cl" = 0.2,
+        "S(6)"  = 0,
+        "Sr" = 0)
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H"  =  1E-06,
+  "O"  =  1E-06,
+  "Charge" = 1E-06,
+  "Ba" = 1E-06,
+  "Cl" = 1E-06,
+  "S(6)"  = 1E-06,
+  "Sr" = 1E-06
+)
+
+## vecinj_inner <- list(
+##   l1 = c(1,20,20),
+##   l2 = c(2,80,80),
+##   l3 = c(2,60,80)
+## )
+
+boundary <- list(
+  "N" = rep(1, n),
+##  "N" = rep(0, n),
+  "E" = rep(0, n),
+  "S" = rep(0, n),
+  "W" = rep(0, n)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, injection_diff)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+#  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+## # Needed when using DHT
+dht_species <- c(
+  "H" = 10,
+  "O" = 10,
+  "Charge" = 3,
+  "Ba" = 5,
+  "Cl" = 5,
+  "S(6)" = 5,
+  "Sr" = 5
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 4
+dt <- 100
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = seq(1, iterations)
+)
--- a/share/poet/bench/barite/barite.pqi
+++ b/share/poet/bench/barite/barite.pqi
@ -0,0 +1,25 @@
+SELECTED_OUTPUT
+  -high_precision true
+  -reset false
+  -kinetic_reactants Barite Celestite
+  -saturation_indices Barite Celestite
+SOLUTION 1
+units	mol/kgw
+water	1
+temperature	25
+pH	7
+pe	10.799
+Ba	0.1
+Cl	0.2
+S	1e-9
+Sr	1e-9
+KINETICS 1
+Barite
+-m 0.001
+-parms 50.  # reactive surface area
+-tol 1e-9
+Celestite
+-m 1
+-parms 10.0  # reactive surface area
+-tol 1e-9
+END
--- a/share/poet/bench/barite/barite_interp_eval.R
+++ b/share/poet/bench/barite/barite_interp_eval.R
@ -0,0 +1,151 @@
+## Time-stamp: "Last modified 2023-07-21 15:04:49 mluebke"
+
+database <- normalizePath("../share/poet/bench/barite/db_barite.dat")
+input_script <- normalizePath("../share/poet/bench/barite/barite.pqi")
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 400
+m <- 200
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.0124,
+  "O" = 55.5087,
+  "Charge" = -1.217E-09,
+  "Ba" = 1.E-10,
+  "Cl" = 2.E-10,
+  "S" = 6.205E-4,
+  "Sr" = 6.205E-4,
+  "Barite" = 0.001,
+  "Celestite" = 1
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(n / 10, m / 10),
+  type = types[1],
+  init_cell = as.data.frame(init_cell, check.names = FALSE),
+  props = names(init_cell),
+  database = database,
+  input_script = input_script
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+
+init_diffu <- list(
+  # "H" = 110.0124,
+  "H" = 0.00000028904,
+  # "O" = 55.5087,
+  "O" = 0.000000165205,
+  # "Charge" = -1.217E-09,
+  "Charge" = -3.337E-08,
+  "Ba" = 1.E-10,
+  "Cl" = 1.E-10,
+  "S(6)" = 6.205E-4,
+  "Sr" = 6.205E-4
+)
+
+injection_diff <- list(
+  list(
+    # "H" = 111.0124,
+    "H" = 0.0000002890408,
+    # "O" = 55.50622,
+    "O" = 0.00002014464,
+    # "Charge" = -3.337E-08,
+    "Charge" = -3.337000004885E-08,
+    "Ba" = 0.1,
+    "Cl" = 0.2,
+    "S(6)" = 0,
+    "Sr" = 0
+  )
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-06,
+  "O" = 1E-06,
+  "Charge" = 1E-06,
+  "Ba" = 1E-06,
+  "Cl" = 1E-06,
+  "S(6)" = 1E-06,
+  "Sr" = 1E-06
+)
+
+vecinj_inner <- list(
+  l1 = c(1, floor(n / 2), floor(m / 2))
+  ##   l2 = c(2,80,80),
+  ##   l3 = c(2,60,80)
+)
+
+boundary <- list(
+  #  "N" = rep(1, n),
+  "N" = rep(0, n),
+  "E" = rep(0, n),
+  "S" = rep(0, n),
+  "W" = rep(0, n)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, injection_diff)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+## # Needed when using DHT
+dht_species <- c(
+  "H" = 10,
+  "O" = 10,
+  "Charge" = 3,
+  "Ba" = 5,
+  "Cl" = 5,
+  "S(6)" = 5,
+  "Sr" = 5
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 200
+dt <- 250
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = seq(1, iterations)
+)
--- a/share/poet/bench/barite/db_barite.dat
+++ b/share/poet/bench/barite/db_barite.dat
@ -0,0 +1,195 @@
+DATABASE
+	###########################
+SOLUTION_MASTER_SPECIES
+	H      H+     -1            H         1.008  # phreeqc/
+	H(0)   H2      0            H                # phreeqc/
+	H(1)   H+     -1     0.0                     # phreeqc/
+	E      e-      0     0.0                0.0  # phreeqc/
+	O      H2O     0            O          16.0  # phreeqc/
+	O(0)   O2      0            O                # phreeqc/
+	O(-2)  H2O     0     0.0                     # phreeqc/
+	Na     Na+     0            Na      22.9898  # phreeqc/
+	Ba     Ba+2    0            Ba       137.34  # phreeqc/
+	Sr     Sr+2    0            Sr        87.62  # phreeqc/
+	Cl     Cl-     0            Cl       35.453  # phreeqc/
+	S      SO4-2   0            SO4      32.064  # phreeqc/
+	S(6)   SO4-2   0            SO4              # phreeqc/
+	S(-2)  HS-     1            S                # phreeqc/
+SOLUTION_SPECIES
+	H+ = H+
+		-gamma 9 0
+		-dw 9.31e-09
+		# source: phreeqc
+	e- = e-
+		# source: phreeqc
+	H2O = H2O
+		# source: phreeqc
+	Na+ = Na+
+		-gamma 4.08 0.082
+		-dw 1.33e-09
+		-Vm 2.28 -4.38 -4.1 -0.586 0.09 4 0.3 52 -0.00333 0.566
+		# source: phreeqc
+	Ba+2 = Ba+2
+		-gamma 4 0.153
+		-dw 8.48e-10
+		-Vm 2.063 -10.06 1.9534 -2.36 0.4218 5 1.58 -12.03 -0.00835 1
+		# source: phreeqc
+	Sr+2 = Sr+2
+		-gamma 5.26 0.121
+		-dw 7.94e-10
+		-Vm -0.0157 -10.15 10.18 -2.36 0.86 5.26 0.859 -27 -0.0041 1.97
+		# source: phreeqc
+	Cl- = Cl-
+		-gamma 3.63 0.017
+		-dw 2.03e-09
+		-Vm 4.465 4.801 4.325 -2.847 1.748 0 -0.331 20.16 0 1
+		# source: phreeqc
+	SO4-2 = SO4-2
+		-gamma 5 -0.04
+		-dw 1.07e-09
+		-Vm 8 2.3 -46.04 6.245 3.82 0 0 0 0 1
+		# source: phreeqc
+	H2O = OH- + H+
+		-analytical_expression 293.29227 0.1360833 -10576.913 -123.73158 0 -6.996455e-05
+		-gamma 3.5 0
+		-dw 5.27e-09
+		-Vm -9.66 28.5 80 -22.9 1.89 0 1.09 0 0 1
+		# source: phreeqc
+	2 H2O = O2 + 4 H+ + 4 e-
+		-log_k -86.08
+		-delta_h 134.79 kcal
+		-dw 2.35e-09
+		-Vm 5.7889 6.3536 3.2528 -3.0417 -0.3943     
+		# source: phreeqc
+	2 H+ + 2 e- = H2
+		-log_k -3.15
+		-delta_h -1.759 kcal
+		-dw 5.13e-09
+		-Vm 6.52 0.78 0.12       
+		# source: phreeqc
+	SO4-2 + H+ = HSO4-
+		-log_k 1.988
+		-delta_h 3.85 kcal
+		-analytical_expression -56.889 0.006473 2307.9 19.8858  
+		-dw 1.33e-09
+		-Vm 8.2 9.259 2.1108 -3.1618 1.1748 0 -0.3 15 0 1
+		# source: phreeqc
+	HS- = S-2 + H+
+		-log_k -12.918
+		-delta_h 12.1 kcal
+		-gamma 5 0
+		-dw 7.31e-10
+		# source: phreeqc
+	SO4-2 + 9 H+ + 8 e- = HS- + 4 H2O
+		-log_k 33.65
+		-delta_h -60.140 kcal
+		-gamma 3.5 0
+		-dw 1.73e-09
+		-Vm 5.0119 4.9799 3.4765 -2.9849 1.441     
+		# source: phreeqc
+	HS- + H+ = H2S
+		-log_k 6.994
+		-delta_h -5.30 kcal
+		-analytical_expression -11.17 0.02386 3279   
+		-dw 2.1e-09
+		-Vm 7.81 2.96 -0.46       
+		# source: phreeqc
+	Na+ + OH- = NaOH
+		-log_k -10
+		# source: phreeqc
+	Na+ + SO4-2 = NaSO4-
+		-log_k 0.7
+		-delta_h 1.120 kcal
+		-gamma 5.4 0
+		-dw 1.33e-09
+		-Vm 1e-05 16.4 -0.0678 -1.05 4.14 0 6.86 0 0.0242 0.53
+		# source: phreeqc
+	Ba+2 + H2O = BaOH+ + H+
+		-log_k -13.47
+		-gamma 5 0
+		# source: phreeqc
+	Ba+2 + SO4-2 = BaSO4
+		-log_k 2.7
+		# source: phreeqc
+	Sr+2 + H2O = SrOH+ + H+
+		-log_k -13.29
+		-gamma 5 0
+		# source: phreeqc
+	Sr+2 + SO4-2 = SrSO4
+		-log_k 2.29
+		-delta_h 2.08 kcal
+		-Vm 6.791 -0.9666 6.13 -2.739 -0.001     
+		# source: phreeqc
+PHASES
+	Barite
+		BaSO4 = Ba+2 + SO4-2
+		-log_k -9.97
+		-delta_h 6.35 kcal
+		-analytical_expression -282.43 -0.08972 5822 113.08  
+		-Vm 52.9
+		# source: phreeqc
+		# comment: 
+	Celestite
+		SrSO4 = Sr+2 + SO4-2
+		-log_k -6.63
+		-delta_h -4.037 kcal
+		-analytical_expression -7.14 0.00611 75 0 0 -1.79e-05
+		-Vm 46.4
+		# source: phreeqc
+		# comment: 
+RATES
+	Celestite  # Palandri & Kharaka 2004<--------------------------------change me
+# PARM(1): reactive surface area
+# am: acid mechanism, nm: neutral mechanism, bm: base mechanism
+    -start
+        10 sr_i = SR("Celestite")  # saturation ratio, (-)<----------change me
+        20 moles = 0  # init target variable, (mol)
+        30 IF ((M <= 0) AND (sr_i < 1)) OR (sr_i = 1.0) THEN GOTO 310
+        40 sa = PARM(1)  # reactive surface area, (m2)
+
+       100 r = 8.314462  # gas constant, (J K-1 mol-1)
+       110 dTi = (1 / TK) - (1 / 298.15)  # (K-1)
+       120 ea_am = 23800  # activation energy am, (J mol-1)<-----------change me
+       130 ea_nm = 0  # activation energy nm, (J mol-1)<-----------change me
+       140 ea_bm = 0  # activation energy bm, (J mol-1)<-----------change me
+       150 log_k_am = -5.66  # reaction constant am<-------------------change me
+       rem log_k_nm = -99  # reaction constant nm<-------------------change me
+       rem log_k_bm = -99  # reaction constant bm<-------------------change me
+       180 n_am = 0.109  # H+ reaction order am<-----------------------change me
+       rem n_bm = 0  # H+ reaction order bm<-----------------------change me
+       200 am = (10 ^ log_k_am) * EXP(-ea_am * dTi / r) * ACT("H+") ^ n_am
+       rem nm = (10 ^ log_k_nm) * EXP(-ea_nm * dTi / r)
+       rem bm = (10 ^ log_k_bm) * EXP(-ea_bm * dTi / r) * ACT("H+") ^ n_bm
+
+       300 moles = sa * (am) * (1 - sr_i)
+       310 save moles * time
+    -end
+
+	Barite  # Palandri & Kharaka 2004<-----------------------------------change me
+# PARM(1): reactive surface area
+# am: acid mechanism, nm: neutral mechanism, bm: base mechanism
+    -start
+        10 sr_i = SR("Barite")  # saturation ratio, (-)<----------change me
+        20 moles = 0  # init target variable, (mol)
+        30 IF ((M <= 0) AND (sr_i < 1)) OR (sr_i = 1.0) THEN GOTO 310
+        40 sa = PARM(1)  # reactive surface area, (m2)
+
+       100 r = 8.314462  # gas constant, (J K-1 mol-1)
+       110 dTi = (1 / TK) - (1 / 298.15)  # (K-1)
+       120 ea_am = 30800  # activation energy am, (J mol-1)<---------change me
+       130 ea_nm = 30800  # activation energy nm, (J mol-1)<---------change me
+       rem ea_bm = 0  # activation energy bm, (J mol-1)<---------change me
+       150 log_k_am = -6.90  # reaction constant am<-----------------change me
+       160 log_k_nm = -7.90  # reaction constant nm<-----------------change me
+       rem log_k_bm = -99  # reaction constant bm<-------------------change me
+       180 n_am = 0.22  # H+ reaction order am<----------------------change me
+       rem n_bm = 0  # H+ reaction order bm<-----------------------change me
+       200 am = (10 ^ log_k_am) * EXP(-ea_am * dTi / r) * ACT("H+") ^ n_am
+       210 nm = (10 ^ log_k_nm) * EXP(-ea_nm * dTi / r)
+       rem bm = (10 ^ log_k_bm) * EXP(-ea_bm * dTi / r) * ACT("H+") ^ n_bm
+
+       300 moles = sa * (am + nm) * (1 - sr_i)
+       310 save moles * time
+    -end
+
+END
--- a/share/poet/bench/daos/dolo_diffu_edge.R
+++ b/share/poet/bench/daos/dolo_diffu_edge.R
@ -0,0 +1,205 @@
+## Time-stamp: "Last modified 2023-09-05 14:42:20 mluebke"
+
+database <- normalizePath("../share/poet/bench/dolo/phreeqc_kin.dat")
+input_script <- normalizePath("../share/poet/bench/dolo/dolo_inner.pqi")
+
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 400
+m <- 200
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0,
+  "O2g" = 0.499957,
+  "Calcite" = 2.07e-4,
+  "Dolomite" = 0
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(5, 2.5),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+init_diffu <- list(
+  "H" = 1.110124E+02,
+  "O" = 5.550833E+01,
+  "Charge" = -1.216307659761E-09,
+  "C(4)" = 1.230067028174E-04,
+  "Ca" = 1.230067028174E-04,
+  "Cl" = 0,
+  "Mg" = 0
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-6,
+  "O" = 1E-6,
+  "Charge" = 1E-6,
+  "C(4)" = 1E-6,
+  "Ca" = 1E-6,
+  "Cl" = 1E-6,
+  "Mg" = 1E-6
+)
+
+## list of boundary conditions/inner nodes
+vecinj_diffu <- list(
+  list(
+    "H" = 1.110124E+02,
+    "O" = 5.550796E+01,
+    "Charge" = -3.230390327801E-08,
+    "C(4)" = 0,
+    "Ca" = 0,
+    "Cl" = 0.002,
+    "Mg" = 0.001
+  ),
+  list(
+    "H" = 110.683,
+    "O" = 55.3413,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0.0,
+    "Cl" = 0.004,
+    "Mg" = 0.002
+  ),
+  init_diffu
+)
+
+vecinj_inner <- list(
+  l1 = c(1, 1, 1)
+  # l2 = c(2,1400,800),
+  # l3 = c(2,1600,800)
+)
+
+boundary <- list(
+  #  "N" = c(1, rep(0, n-1)),
+  "N" = rep(0, n),
+  "E" = rep(0, m),
+  "S" = rep(0, n),
+  "W" = rep(0, m)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+## # optional when using DHT
+dht_species <- c(
+  "H" = 3,
+  "O" = 3,
+  "Charge" = 3,
+  "C(4)" = 6,
+  "Ca" = 6,
+  "Cl" = 3,
+  "Mg" = 5,
+  "Calcite" = 4,
+  "Dolomite" = 4
+)
+
+## # Optional when using Interpolation (example with less key species and custom
+## # significant digits)
+
+# pht_species <- c(
+#  "C(4)" = 3,
+#  "Ca" = 3,
+#  "Mg" = 2,
+#  "Calcite" = 2,
+#  "Dolomite" = 2
+# )
+
+check_sign_cal_dol_dht <- function(old, new) {
+  if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    return(TRUE)
+  }
+  if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    return(TRUE)
+  }
+  return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+  return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+  data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+  names(data_set) <- names(dht_species)
+  cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+  dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+  cal_dol_same_sig <- cal == dol
+  return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+  neg_sign <- (result["Calcite"] <- 0) || (result["Dolomite"] < 0)
+  return(any(neg_sign))
+}
+
+hooks <- list(
+  dht_fill = check_sign_cal_dol_dht,
+  dht_fuzz = fuzz_input_dht_keys,
+  interp_pre_func = check_sign_cal_dol_interp,
+  interp_post_func = check_neg_cal_dol
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species,
+  hooks = hooks
+  #  pht_species = pht_species
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 1500
+dt <- 500
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = c(1, seq(50, iterations, by = 50))
+)
--- a/share/poet/bench/daos/dolo_inner.pqi
+++ b/share/poet/bench/daos/dolo_inner.pqi
@ -0,0 +1,28 @@
+SELECTED_OUTPUT
+         -high_precision true
+         -reset false
+         -kinetic_reactants Calcite Dolomite
+         -equilibrium O2g
+
+SOLUTION 1
+         units mol/kgw
+         temp 25.0
+         water 1
+         pH 9.91 charge
+         pe 4.0
+         C   1.2279E-04
+         Ca  1.2279E-04
+         Cl 1E-12
+         Mg 1E-12
+PURE 1
+         O2g -0.1675 10
+KINETICS 1
+         Calcite
+            -m      0.00020
+            -parms  0.05
+            -tol    1e-10
+         Dolomite
+            -m      0.0
+            -parms  0.005
+            -tol    1e-10
+END
--- a/share/poet/bench/daos/phreeqc_kin.dat
+++ b/share/poet/bench/daos/phreeqc_kin.dat
--- a/share/poet/bench/dolo/dolo_diffu_inner.R
+++ b/share/poet/bench/dolo/dolo_diffu_inner.R
@ -0,0 +1,190 @@
+## Time-stamp: "Last modified 2023-08-16 17:04:42 mluebke"
+
+database <- normalizePath("../share/poet/bench/dolo/phreeqc_kin.dat")
+input_script <- normalizePath("../share/poet/bench/dolo/dolo_inner.pqi")
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 100
+m <- 100
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C(4)" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0,
+  "O2g" = 0.499957,
+  "Calcite" = 2.07e-4,
+  "Dolomite" = 0
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(1, 1),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+init_diffu <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C(4)" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-6,
+  "O" = 1E-6,
+  "Charge" = 1E-6,
+  "C(4)" = 1E-6,
+  "Ca" = 1E-6,
+  "Cl" = 1E-6,
+  "Mg" = 1E-6
+)
+
+## list of boundary conditions/inner nodes
+vecinj_diffu <- list(
+  list(
+    "H" = 110.683,
+    "O" = 55.3413,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0,
+    "Cl" = 0.002,
+    "Mg" = 0.001
+  ),
+  list(
+    "H" = 110.683,
+    "O" = 55.3413,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0.0,
+    "Cl" = 0.004,
+    "Mg" = 0.002
+  )
+)
+
+vecinj_inner <- list(
+  l1 = c(1, 20, 20),
+  l2 = c(2, 80, 80),
+  l3 = c(2, 60, 80)
+)
+
+boundary <- list(
+  #  "N" = c(1, rep(0, n-1)),
+  "N" = rep(0, n),
+  "E" = rep(0, n),
+  "S" = rep(0, n),
+  "W" = rep(0, n)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+## # Needed when using DHT
+dht_species <- c(
+  "H" = 10,
+  "O" = 10,
+  "Charge" = 3,
+  "C(4)" = 5,
+  "Ca" = 5,
+  "Cl" = 5,
+  "Mg" = 5,
+  "Calcite" = 5,
+  "Dolomite" = 5
+)
+
+check_sign_cal_dol_dht <- function(old, new) {
+  if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    return(TRUE)
+  }
+  if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    return(TRUE)
+  }
+  return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+  return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+  data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+  names(data_set) <- names(dht_species)
+  cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+  dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+  cal_dol_same_sig <- cal == dol
+  return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+  neg_sign <- (result["Calcite"] <- 0) || (result["Dolomite"] < 0)
+  return(any(neg_sign))
+}
+
+hooks <- list(
+  dht_fill = check_sign_cal_dol_dht,
+  dht_fuzz = fuzz_input_dht_keys,
+  interp_pre_func = check_sign_cal_dol_interp,
+  interp_post_func = check_neg_cal_dol
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species,
+  hooks = hooks
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 10
+dt <- 200
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE
+)
--- a/share/poet/bench/dolo/dolo_diffu_inner_large.R
+++ b/share/poet/bench/dolo/dolo_diffu_inner_large.R
@ -0,0 +1,190 @@
+## Time-stamp: "Last modified 2023-08-16 17:05:04 mluebke"
+
+database <- normalizePath("../share/poet/bench/dolo/phreeqc_kin.dat")
+input_script <- normalizePath("../share/poet/bench/dolo/dolo_inner.pqi")
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 2000
+m <- 1000
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0,
+  "O2g" = 0.499957,
+  "Calcite" = 2.07e-4,
+  "Dolomite" = 0
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(2, 1),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+init_diffu <- list(
+  "H" = 0.000211313883539788,
+  "O" = 0.00398302904424952,
+  "Charge" = -5.0822e-19,
+  "C(4)" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-6,
+  "O" = 1E-6,
+  "Charge" = 1E-6,
+  "C(4)" = 1E-6,
+  "Ca" = 1E-6,
+  "Cl" = 1E-6,
+  "Mg" = 1E-6
+)
+
+## list of boundary conditions/inner nodes
+vecinj_diffu <- list(
+  list(
+    "H" = 0.0001540445,
+    "O" = 0.002148006,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0,
+    "Cl" = 0.002,
+    "Mg" = 0.001
+  ),
+  list(
+    "H" = 0.0001610193,
+    "O" = 0.002386934,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0.0,
+    "Cl" = 0.004,
+    "Mg" = 0.002
+  )
+)
+
+vecinj_inner <- list(
+  l1 = c(1, 400, 200),
+  l2 = c(2, 1400, 800),
+  l3 = c(2, 1600, 800)
+)
+
+boundary <- list(
+  #  "N" = c(1, rep(0, n-1)),
+  "N" = rep(0, n),
+  "E" = rep(0, m),
+  "S" = rep(0, n),
+  "W" = rep(0, m)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+## # Needed when using DHT
+dht_species <- c(
+  "H" = 10,
+  "O" = 10,
+  "Charge" = 3,
+  "C(4)" = 5,
+  "Ca" = 5,
+  "Cl" = 5,
+  "Mg" = 5,
+  "Calcite" = 5,
+  "Dolomite" = 5
+)
+
+check_sign_cal_dol_dht <- function(old, new) {
+  if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    return(TRUE)
+  }
+  if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    return(TRUE)
+  }
+  return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+  return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+  data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+  names(data_set) <- names(dht_species)
+  cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+  dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+  cal_dol_same_sig <- cal == dol
+  return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+  neg_sign <- (result["Calcite"] <- 0) || (result["Dolomite"] < 0)
+  return(any(neg_sign))
+}
+
+hooks <- list(
+  dht_fill = check_sign_cal_dol_dht,
+  dht_fuzz = fuzz_input_dht_keys,
+  interp_pre_func = check_sign_cal_dol_interp,
+  interp_post_func = check_neg_cal_dol
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species,
+  hooks = hooks
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 500
+dt <- 50
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = seq(5, iterations, by = 5)
+)
--- a/share/poet/bench/dolo/dolo_inner.pqi
+++ b/share/poet/bench/dolo/dolo_inner.pqi
@ -0,0 +1,28 @@
+SELECTED_OUTPUT
+         -high_precision true
+         -reset false
+         -kinetic_reactants Calcite Dolomite
+         -equilibrium O2g
+
+SOLUTION 1
+         units mol/kgw
+         temp 25.0
+         water 1
+         pH 9.91 charge
+         pe 4.0
+         C   1.2279E-04
+         Ca  1.2279E-04
+         Cl 1E-12
+         Mg 1E-12
+PURE 1
+         O2g -0.1675 10
+KINETICS 1
+         Calcite
+            -m      0.00020
+            -parms  0.05
+            -tol    1e-10
+         Dolomite
+            -m      0.0
+            -parms  0.005
+            -tol    1e-10
+END
--- a/share/poet/bench/dolo/dolo_interp_long.R
+++ b/share/poet/bench/dolo/dolo_interp_long.R
@ -0,0 +1,204 @@
+## Time-stamp: "Last modified 2023-08-16 14:57:25 mluebke"
+
+database <- normalizePath("../share/poet/bench/dolo/phreeqc_kin.dat")
+input_script <- normalizePath("../share/poet/bench/dolo/dolo_inner.pqi")
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 400
+m <- 200
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(
+  "H" = 110.683,
+  "O" = 55.3413,
+  "Charge" = -5.0822e-19,
+  "C" = 1.2279E-4,
+  "Ca" = 1.2279E-4,
+  "Cl" = 0,
+  "Mg" = 0,
+  "O2g" = 0.499957,
+  "Calcite" = 2.07e-4,
+  "Dolomite" = 0
+)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(5, 2.5),
+  type = types[1]
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+## initial conditions
+init_diffu <- list(
+  "H" = 1.110124E+02,
+  "O" = 5.550833E+01,
+  "Charge" = -1.216307659761E-09,
+  "C(4)" = 1.230067028174E-04,
+  "Ca" = 1.230067028174E-04,
+  "Cl" = 0,
+  "Mg" = 0
+)
+
+## diffusion coefficients
+alpha_diffu <- c(
+  "H" = 1E-6,
+  "O" = 1E-6,
+  "Charge" = 1E-6,
+  "C(4)" = 1E-6,
+  "Ca" = 1E-6,
+  "Cl" = 1E-6,
+  "Mg" = 1E-6
+)
+
+## list of boundary conditions/inner nodes
+vecinj_diffu <- list(
+  list(
+    "H" = 1.110124E+02,
+    "O" = 5.550796E+01,
+    "Charge" = -3.230390327801E-08,
+    "C(4)" = 0,
+    "Ca" = 0,
+    "Cl" = 0.002,
+    "Mg" = 0.001
+  ),
+  list(
+    "H" = 110.683,
+    "O" = 55.3413,
+    "Charge" = 1.90431e-16,
+    "C(4)" = 0,
+    "Ca" = 0.0,
+    "Cl" = 0.004,
+    "Mg" = 0.002
+  ),
+  init_diffu
+)
+
+vecinj_inner <- list(
+  l1 = c(1, floor(n / 2), floor(m / 2))
+  # l2 = c(2,1400,800),
+  # l3 = c(2,1600,800)
+)
+
+boundary <- list(
+  #  "N" = c(1, rep(0, n-1)),
+  "N" = rep(3, n),
+  "E" = rep(3, m),
+  "S" = rep(3, n),
+  "W" = rep(3, m)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_diffu)
+
+diffusion <- list(
+  init = as.data.frame(init_diffu, check.names = FALSE),
+  vecinj = vecinj,
+  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+## # optional when using DHT
+dht_species <- c(
+  "H" = 3,
+  "O" = 3,
+  "Charge" = 3,
+  "C(4)" = 6,
+  "Ca" = 6,
+  "Cl" = 3,
+  "Mg" = 5,
+  "Calcite" = 4,
+  "Dolomite" = 4
+)
+
+## # Optional when using Interpolation (example with less key species and custom
+## # significant digits)
+
+# pht_species <- c(
+#  "C(4)" = 3,
+#  "Ca" = 3,
+#  "Mg" = 2,
+#  "Calcite" = 2,
+#  "Dolomite" = 2
+# )
+
+check_sign_cal_dol_dht <- function(old, new) {
+  if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    return(TRUE)
+  }
+  if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    return(TRUE)
+  }
+  return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+  return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+  data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+  names(data_set) <- names(dht_species)
+  cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+  dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+  cal_dol_same_sig <- cal == dol
+  return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+  neg_sign <- (result["Calcite"] <- 0) || (result["Dolomite"] < 0)
+  return(any(neg_sign))
+}
+
+hooks <- list(
+  dht_fill = check_sign_cal_dol_dht,
+  dht_fuzz = fuzz_input_dht_keys,
+  interp_pre_func = check_sign_cal_dol_interp,
+  interp_post_func = check_neg_cal_dol
+)
+
+chemistry <- list(
+  database = database,
+  input_script = input_script,
+  dht_species = dht_species,
+  hooks = hooks
+  #  pht_species = pht_species
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 20000
+dt <- 200
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = c(1, seq(50, iterations, by = 50))
+)
--- a/share/poet/bench/dolo/phreeqc_kin.dat
+++ b/share/poet/bench/dolo/phreeqc_kin.dat
--- a/share/poet/bench/surfex/ExBase.pqi
+++ b/share/poet/bench/surfex/ExBase.pqi
@ -0,0 +1,39 @@
+##  Time-stamp: "Last modified 2023-03-21 11:49:43 mluebke"
+KNOBS
+ -logfile            false
+ -iterations         10000
+ -convergence_tolerance  1E-12
+ -step_size          2
+ -pe_step_size       2
+SELECTED_OUTPUT
+ -reset false
+ -high_precision true
+ -solution true
+ -state true
+ -step true 
+ -pH true 
+ -pe true
+ -ionic_strength true 
+ -water true
+SOLUTION 1
+temp         13
+units        mol/kgw
+pH           7.06355
+pe          -2.626517
+C(4)         0.001990694
+Ca           0.02172649
+Cl           0.3227673 charge
+Fe           0.0001434717
+K            0.001902357
+Mg           0.01739704
+Na           0.2762882
+S(6)         0.01652701
+Sr           0.0004520361
+U(4)         8.147792e-12
+U(6)         2.237946e-09
+-water       0.00133
+EXCHANGE 1
+ -equil 1
+  Z     0.0012585
+  Y     0.0009418
+END
--- a/share/poet/bench/surfex/SMILE_2021_11_01_TH.dat
+++ b/share/poet/bench/surfex/SMILE_2021_11_01_TH.dat
--- a/share/poet/bench/surfex/SurfExBase.pqi
+++ b/share/poet/bench/surfex/SurfExBase.pqi
@ -0,0 +1,56 @@
+##  Time-stamp: "Last modified 2023-02-27 14:31:11 delucia"
+KNOBS
+ -logfile            false
+ -iterations         10000
+ -convergence_tolerance  1E-12
+ -step_size          2
+ -pe_step_size       2
+SELECTED_OUTPUT
+ -reset false
+ -high_precision true
+ -solution true
+ -state true
+ -step true 
+ -pH true 
+ -pe true
+ -ionic_strength true 
+ -water true
+USER_PUNCH
+-head total_o total_h cb  C(-4) C(4) Ca Cl Fe(2) Fe(3) H(0) K Mg Na S(-2) S(2) S(4) S(6) Sr U(4) U(5) U(6) UO2(am,hyd) KdU
+-start
+5 w=TOT("water")
+10 PUNCH TOTMOLE("O"), TOTMOLE("H"), CHARGE_BALANCE, w*TOT("C(-4)"), w*TOT("C(4)"), w*TOT("Ca"), w*TOT("Cl"), w*TOT("Fe(2)"), w*TOT("Fe(3)"), w*TOT("H(0)"), w*TOT("K"), w*TOT("Mg"), w*TOT("Na"), w*TOT("S(-2)"), w*TOT("S(2)"), w*TOT("S(4)"), w*TOT("S(6)"), w*TOT("Sr"), w*TOT("U(4)"), w*TOT("U(5)"), w*TOT("U(6)"), EQUI("UO2(am,hyd)")
+20 PUNCH ((SURF("U, Ill")+SURF("U, Mll")+SURF("U, Kln")+EDL("U, Ill")+EDL("U, Mll")+EDL("U, Kln"))/((TOT("U")*1.01583)))/(0.002251896406*1000)
+-end
+SOLUTION 1
+temp         13
+units        mol/kgw
+pH           7.06355
+pe          -2.626517
+C(4)         0.001990694
+Ca           0.02172649
+Cl           0.3227673 charge
+Fe           0.0001434717
+K            0.001902357
+Mg           0.01739704
+Na           0.2762882
+S(6)         0.01652701
+Sr           0.0004520361
+U(4)         8.147792e-12
+U(6)         2.237946e-09
+-water       0.00133
+SURFACE 1
+-equil 1
+-sites_units density
+-donnan 4.9e-10
+  Kln_aOH  1.155    11.  5.0518
+  Kln_siOH 1.155
+  Ill_sOH  0.05    100.  5.5931
+  Ill_wOH  2.26
+  Mll_sOH  0.05    100.  1.0825
+  Mll_wOH  2.26
+EXCHANGE 1
+ -equil 1
+  Z     0.0012585
+  Y     0.0009418
+END
--- a/share/poet/bench/surfex/ex.R
+++ b/share/poet/bench/surfex/ex.R
@ -0,0 +1,140 @@
+## Time-stamp: "Last modified 2023-08-02 13:59:35 mluebke"
+
+database <- normalizePath("./SMILE_2021_11_01_TH.dat")
+input_script <- normalizePath("./ExBase.pqi")
+
+cat(paste(":: R This is a test 1\n"))
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 100
+m <- 100
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(H = 1.476571028625e-01,
+                  O = 7.392297218936e-02,
+                  Charge = -1.765225732724e-18,
+                  `C(-4)` = 2.477908970828e-21,
+                  `C(4)` = 2.647623016916e-06,
+                  Ca = 2.889623169138e-05,
+                  Cl = 4.292806181039e-04,
+                  `Fe(2)` =1.908142472666e-07,
+                  `Fe(3)` =3.173306589931e-12,
+                  `H(0)` =2.675642675119e-15,
+                  K = 2.530134809667e-06,
+                  Mg =2.313806319294e-05,
+                  Na =3.674633059628e-04,
+                  `S(-2)` = 8.589766637180e-15,
+                  `S(2)` = 1.205284362720e-19,
+                  `S(4)` = 9.108958772790e-18,
+                  `S(6)` = 2.198092329098e-05,
+                  Sr = 6.012080128154e-07,
+                  `U(4)` = 1.039668623852e-14,
+                  `U(5)` = 1.208394829796e-15,
+                  `U(6)` = 2.976409147150e-12)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(1, 1),
+  type = "scratch"
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+vecinj_diffu <- list(
+    list(H = 0.147659686316291,
+         O = 0.0739242798146046,
+         Charge = 7.46361643222701e-20,
+         `C(-4)` = 2.92438561098248e-21,
+         `C(4)` = 2.65160558871092e-06,
+         Ca = 2.89001071336443e-05,
+         Cl = 0.000429291158114428,
+         `Fe(2)` = 1.90823391198114e-07,
+         `Fe(3)` = 3.10832423034763e-12,
+         `H(0)` = 2.7888235127385e-15,
+         K = 2.5301787e-06,
+         Mg = 2.31391999937907e-05,
+         Na = 0.00036746969,
+         `S(-2)` = 1.01376078438546e-14,
+         `S(2)` = 1.42247026981542e-19,
+         `S(4)` = 9.49422092568557e-18,
+         `S(6)` = 2.19812504654191e-05,
+         Sr = 6.01218519999999e-07,
+         `U(4)` = 4.82255946569383e-12,
+         `U(5)` = 5.49050615347901e-13,
+         `U(6)` = 1.32462838991902e-09)
+)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- grid$props
+
+## diffusion coefficients
+alpha_diffu <- c(H = 1E-6, O = 1E-6, Charge = 1E-6, `C(-4)` = 1E-6,
+                 `C(4)` = 1E-6, Ca = 1E-6, Cl = 1E-6, `Fe(2)` = 1E-6,
+                 `Fe(3)` = 1E-6, `H(0)` = 1E-6, K = 1E-6, Mg = 1E-6,
+                 Na = 1E-6, `S(-2)` = 1E-6, `S(2)` = 1E-6,
+                 `S(4)` = 1E-6, `S(6)` = 1E-6, Sr = 1E-6,
+                 `U(4)` = 1E-6, `U(5)` = 1E-6, `U(6)` = 1E-6)
+
+## list of boundary conditions/inner nodes
+
+## vecinj_inner <- list(
+##    list(1,1,1)
+## )
+
+boundary <- list(
+  "N" = rep(1, n),
+  "E" = rep(0, n),
+  "S" = rep(0, n),
+  "W" = rep(0, n)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_cell)
+
+diffusion <- list(
+  init = as.data.frame(init_cell, check.names = FALSE),
+  vecinj = vecinj,
+#  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+chemistry <- list(
+  database = database,
+  input_script = input_script
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 10
+dt <- 200
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE
+)
--- a/share/poet/bench/surfex/surfex.R
+++ b/share/poet/bench/surfex/surfex.R
@ -0,0 +1,141 @@
+## Time-stamp: "Last modified 2023-08-02 13:59:44 mluebke"
+
+database <- normalizePath("../share/poet/bench/surfex/SMILE_2021_11_01_TH.dat")
+input_script <- normalizePath("../share/poet/bench/surfex/SurfExBase.pqi")
+
+cat(paste(":: R This is a test 1\n"))
+
+#################################################################
+##                          Section 1                          ##
+##                     Grid initialization                     ##
+#################################################################
+
+n <- 10
+m <- 10
+
+types <- c("scratch", "phreeqc", "rds")
+
+init_cell <- list(H = 1.476571028625e-01,
+                  O = 7.392297218936e-02,
+                  Charge = -1.765225732724e-18,
+                  `C(-4)` = 2.477908970828e-21,
+                  `C(4)` = 2.647623016916e-06,
+                  Ca = 2.889623169138e-05,
+                  Cl = 4.292806181039e-04,
+                  `Fe(2)` =1.908142472666e-07,
+                  `Fe(3)` =3.173306589931e-12,
+                  `H(0)` =2.675642675119e-15,
+                  K = 2.530134809667e-06,
+                  Mg =2.313806319294e-05,
+                  Na =3.674633059628e-04,
+                  `S(-2)` = 8.589766637180e-15,
+                  `S(2)` = 1.205284362720e-19,
+                  `S(4)` = 9.108958772790e-18,
+                  `S(6)` = 2.198092329098e-05,
+                  Sr = 6.012080128154e-07,
+                  `U(4)` = 1.039668623852e-14,
+                  `U(5)` = 1.208394829796e-15,
+                  `U(6)` = 2.976409147150e-12)
+
+grid <- list(
+  n_cells = c(n, m),
+  s_cells = c(1, 1),
+  type = "scratch"
+)
+
+
+##################################################################
+##                          Section 2                           ##
+##         Diffusion parameters and boundary conditions         ##
+##################################################################
+
+vecinj_diffu <- list(
+    list(H = 0.147659686316291,
+         O = 0.0739242798146046,
+         Charge = 7.46361643222701e-20,
+         `C(-4)` = 2.92438561098248e-21,
+         `C(4)` = 2.65160558871092e-06,
+         Ca = 2.89001071336443e-05,
+         Cl = 0.000429291158114428,
+         `Fe(2)` = 1.90823391198114e-07,
+         `Fe(3)` = 3.10832423034763e-12,
+         `H(0)` = 2.7888235127385e-15,
+         K = 2.5301787e-06,
+         Mg = 2.31391999937907e-05,
+         Na = 0.00036746969,
+         `S(-2)` = 1.01376078438546e-14,
+         `S(2)` = 1.42247026981542e-19,
+         `S(4)` = 9.49422092568557e-18,
+         `S(6)` = 2.19812504654191e-05,
+         Sr = 6.01218519999999e-07,
+         `U(4)` = 4.82255946569383e-12,
+         `U(5)` = 5.49050615347901e-13,
+         `U(6)` = 1.32462838991902e-09)
+)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- grid$props
+
+## diffusion coefficients
+alpha_diffu <- c(H = 1E-6, O = 1E-6, Charge = 1E-6, `C(-4)` = 1E-6,
+                 `C(4)` = 1E-6, Ca = 1E-6, Cl = 1E-6, `Fe(2)` = 1E-6,
+                 `Fe(3)` = 1E-6, `H(0)` = 1E-6, K = 1E-6, Mg = 1E-6,
+                 Na = 1E-6, `S(-2)` = 1E-6, `S(2)` = 1E-6,
+                 `S(4)` = 1E-6, `S(6)` = 1E-6, Sr = 1E-6,
+                 `U(4)` = 1E-6, `U(5)` = 1E-6, `U(6)` = 1E-6)
+
+## list of boundary conditions/inner nodes
+
+## vecinj_inner <- list(
+##    list(1,1,1)
+## )
+
+boundary <- list(
+  "N" = rep(1, n),
+  "E" = rep(0, n),
+  "S" = rep(0, n),
+  "W" = rep(0, n)
+)
+
+diffu_list <- names(alpha_diffu)
+
+vecinj <- do.call(rbind.data.frame, vecinj_diffu)
+names(vecinj) <- names(init_cell)
+
+diffusion <- list(
+  init = as.data.frame(init_cell, check.names = FALSE),
+  vecinj = vecinj,
+#  vecinj_inner = vecinj_inner,
+  vecinj_index = boundary,
+  alpha = alpha_diffu
+)
+
+#################################################################
+##                          Section 3                          ##
+##                  Chemistry module (Phreeqc)                 ##
+#################################################################
+
+
+chemistry <- list(
+  database = database,
+  input_script = input_script
+)
+
+#################################################################
+##                          Section 4                          ##
+##              Putting all those things together              ##
+#################################################################
+
+
+iterations <- 10
+dt <- 200
+
+setup <- list(
+  grid = grid,
+  diffusion = diffusion,
+  chemistry = chemistry,
+  iterations = iterations,
+  timesteps = rep(dt, iterations),
+  store_result = TRUE,
+  out_save = c(5, iterations)
+)
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -1,12 +1,55 @@
+option(POET_DAOSKV "Build with DAOS, using the KV-API" OFF)
+option(POET_DAOSOB "Build with DAOS, using the Object-API" OFF)
+
 file(GLOB_RECURSE poet_lib_SRC
  CONFIGURE_DEPENDS
  "*.cpp" "*.c")

+if(POET_DAOSKV OR POET_DAOSOB)
+  #list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/SurrogateModels/DHT.c")
+  list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/SurrogateModels/DHT_Wrapper.cpp")
+
+  if (POET_DAOSKV)
+    message(NOTICE "Using DAOS KV")
+    list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/DAOS/OB/DaosKeyValue.c")
+  else () 
+    message(NOTICE "Using DAOS OB")
+    list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/DAOS/KV/DaosKeyValue.c")
+  endif()
+
+  
+    # ML: Fuer Nico: Includes + linking fuer Daos bereitstellen
+    find_library(DAOS_LIB libdaos.so
+      PATH_SUFFIXES lib lib64
+    )
+    find_path(DAOS_INCLUDE daos.h)
+
+    add_library(DAOS INTERFACE IMPORTED)
+    set_target_properties(DAOS PROPERTIES
+      INTERFACE_LINK_LIBRARIES "${DAOS_LIB}"
+      INTERFACE_INCLUDE_DIRECTORIES "${DAOS_INCLUDE}"
+    )
+    set(PHREEQCRM_BUILD_MPI OFF CACHE BOOL "" FORCE)
+
+else()
+  list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/DAOS/DHT_Wrapper.cpp")
+  list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/DAOS/KV/DaosKeyValue.c")
+  list(REMOVE_ITEM poet_lib_SRC "${CMAKE_CURRENT_SOURCE_DIR}/ChemistryModule/DAOS/OB/DaosKeyValue.c")
+endif()
+
+foreach(src IN LISTS poet_lib_SRC)
+  message(STATUS "  ${src}")
+endforeach()
+
 add_library(poet_lib ${poet_lib_SRC})
 target_include_directories(poet_lib PUBLIC ${PROJECT_SOURCE_DIR}/include)
 target_link_libraries(poet_lib PUBLIC
  MPI::MPI_CXX ${MATH_LIBRARY} RRuntime PhreeqcRM tug)

+  if(POET_DAOSKV OR POET_DAOSOB)
+  target_link_libraries(poet_lib PUBLIC DAOS)
+  endif()
+
 target_compile_definitions(poet_lib PUBLIC STRICT_R_HEADERS OMPI_SKIP_MPICXX)

 mark_as_advanced(PHREEQCRM_BUILD_MPI PHREEQCRM_DISABLE_OPENMP)
--- a/src/ChemistryModule/DAOS/DHT_Wrapper.cpp
+++ b/src/ChemistryModule/DAOS/DHT_Wrapper.cpp
@ -0,0 +1,351 @@
+//  Time-stamp: "Last modified 2023-09-08 22:09:03 mluebke"
+
+/*
+** Copyright (C) 2018-2021 Alexander Lindemann, Max Luebke (University of
+** Potsdam)
+**
+** Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include "poet/DHT_Wrapper.hpp"
+#include "poet/HashFunctions.hpp"
+#include "poet/Interpolation.hpp"
+#include "poet/LookupKey.hpp"
+#include "poet/Rounding.hpp"
+#include "poet/enums.hpp"
+
+#include "poet/DaosKeyValue.h"
+
+#include "poet/RInsidePOET.hpp"
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <iostream>
+#include <stdexcept>
+#include <vector>
+
+using namespace std;
+
+namespace poet {
+
+DAOSKV* daos_object;
+uint32_t key_size;
+uint32_t data_size;
+DHT_Wrapper::DHT_Wrapper(MPI_Comm dht_comm, std::uint64_t dht_size,
+                         const NamedVector<std::uint32_t> &key_species,
+                         const std::vector<std::int32_t> &key_indices,
+                         const std::vector<std::string> &_output_names,
+                         const ChemistryParams::Chem_Hook_Functions &_hooks,
+                         uint32_t data_count, bool _with_interp)
+    : key_count(key_indices.size()), data_count(data_count),
+      input_key_elements(key_indices), communicator(dht_comm),
+      key_species(key_species), output_names(_output_names), hooks(_hooks),
+      with_interp(_with_interp) {
+  // initialize DHT object
+  // key size = count of key elements + timestep
+  key_size = (key_count + 1) * sizeof(Lookup_Keyelement);
+  data_size =
+      (data_count + (with_interp ? input_key_elements.size() : 0)) *
+      sizeof(double);
+  uint32_t buckets_per_process =
+      static_cast<std::uint32_t>(dht_size / (data_size + key_size));
+  daos_object = DAOSKV_create(dht_comm);
+
+  dht_signif_vector = key_species.getValues();
+
+  // this->dht_signif_vector.resize(key_size, DHT_KEY_SIGNIF_DEFAULT);
+
+  this->dht_prop_type_vector.resize(key_count, DHT_TYPE_DEFAULT);
+
+  const auto key_names = key_species.getNames();
+
+  auto tot_h = std::find(key_names.begin(), key_names.end(), "H");
+  if (tot_h != key_names.end()) {
+    this->dht_prop_type_vector[tot_h - key_names.begin()] = DHT_TYPE_TOTAL;
+  }
+  auto tot_o = std::find(key_names.begin(), key_names.end(), "O");
+  if (tot_o != key_names.end()) {
+    this->dht_prop_type_vector[tot_o - key_names.begin()] = DHT_TYPE_TOTAL;
+  }
+  auto charge = std::find(key_names.begin(), key_names.end(), "Charge");
+  if (charge != key_names.end()) {
+    this->dht_prop_type_vector[charge - key_names.begin()] = DHT_TYPE_CHARGE;
+  }
+}
+
+DHT_Wrapper::~DHT_Wrapper() {
+  // free DHT
+  DAOSKV_free(daos_object);
+}
+auto DHT_Wrapper::checkDHT(WorkPackage &work_package)
+    -> const DHT_ResultObject & {
+
+  const auto length = work_package.size;
+
+  std::vector<double> bucket_writer(
+      this->data_count + (with_interp ? input_key_elements.size() : 0));
+
+  // loop over every grid cell contained in work package
+  for (int i = 0; i < length; i++) {
+    // point to current grid cell
+    auto &key_vector = dht_results.keys[i];
+
+    // overwrite input with data from DHT, IF value is found in DHT
+    int res = DAOSKV_read(daos_object,
+                      key_vector.data(),key_size,
+                      bucket_writer.data(), data_size);
+
+
+    switch (res) {
+    case DAOS_SUCCESS:
+      work_package.output[i] =
+          (with_interp
+               ? inputAndRatesToOutput(bucket_writer, work_package.input[i])
+               : bucket_writer);
+      work_package.mapping[i] = CHEM_DHT;
+      this->dht_hits++;
+      break;
+    case DAOS_READ_MISS:
+      break;
+    }
+  }
+
+  return dht_results;
+}
+
+void DHT_Wrapper::fillDHT(const WorkPackage &work_package) {
+
+  const auto length = work_package.size;
+
+  // loop over every grid cell contained in work package
+  dht_results.locations.resize(length);
+  dht_results.filledDHT = std::vector<bool>(length, false);
+  for (int i = 0; i < length; i++) {
+    // If true grid cell was simulated, needs to be inserted into dht
+    if (work_package.mapping[i] == CHEM_PQC) {
+
+      // check if calcite or dolomite is absent and present, resp.n and vice
+      // versa in input/output. If this is the case -> Do not write to DHT!
+      // HACK: hardcoded, should be fixed!
+      if (hooks.dht_fill.isValid()) {
+        NamedVector<double> old_values(output_names, work_package.input[i]);
+        NamedVector<double> new_values(output_names, work_package.output[i]);
+
+        if (hooks.dht_fill(old_values, new_values)) {
+          continue;
+        }
+      }
+
+      uint32_t proc, index;
+      auto &key = dht_results.keys[i];
+      const auto data =
+          (with_interp
+               ? outputToInputAndRates(work_package.input[i],
+                                       work_package.output[i])
+               : work_package.output[i]);
+      // void *data = (void *)&(work_package[i * this->data_count]);
+      // fuzz data (round, logarithm etc.)
+
+      // insert simulated data with fuzzed key into DHT
+      int res = DAOSKV_write(daos_object,
+                    (void *) key.data(), key_size,
+                    (void *) data.data(), data_size);
+
+
+      dht_results.locations[i] = {proc, index};
+
+      // if data was successfully written ...
+      if ((res != DAOS_SUCCESS) && (res == DHT_WRITE_SUCCESS_WITH_EVICTION)) {
+        dht_evictions++;
+      }
+
+      dht_results.filledDHT[i] = true;
+    }
+  }
+}
+
+inline std::vector<double>
+DHT_Wrapper::outputToInputAndRates(const std::vector<double> &old_results,
+                                   const std::vector<double> &new_results) {
+  const int prefix_size = this->input_key_elements.size();
+
+  std::vector<double> output(prefix_size + this->data_count);
+  std::copy(new_results.begin(), new_results.end(),
+            output.begin() + prefix_size);
+
+  for (int i = 0; i < prefix_size; i++) {
+    const int data_elem_i = input_key_elements[i];
+    output[i] = old_results[data_elem_i];
+    output[prefix_size + data_elem_i] -= old_results[data_elem_i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::inputAndRatesToOutput(const std::vector<double> &dht_data,
+                                   const std::vector<double> &input_values) {
+  const int prefix_size = this->input_key_elements.size();
+
+  std::vector<double> output(input_values);
+
+  for (int i = 0; i < prefix_size; i++) {
+    const int data_elem_i = input_key_elements[i];
+    output[data_elem_i] += dht_data[i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::outputToRates(const std::vector<double> &old_results,
+                           const std::vector<double> &new_results) {
+  std::vector<double> output(new_results);
+
+  for (const auto &data_elem_i : input_key_elements) {
+    output[data_elem_i] -= old_results[data_elem_i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::ratesToOutput(const std::vector<double> &dht_data,
+                           const std::vector<double> &input_values) {
+  std::vector<double> output(input_values);
+
+  for (const auto &data_elem_i : input_key_elements) {
+    output[data_elem_i] += dht_data[data_elem_i];
+  }
+
+  return output;
+}
+
+// void DHT_Wrapper::resultsToWP(std::vector<double> &work_package) {
+//   for (int i = 0; i < dht_results.length; i++) {
+//     if (!dht_results.needPhreeqc[i]) {
+//       std::copy(dht_results.results[i].begin(), dht_results.results[i].end(),
+//                 work_package.begin() + (data_count * i));
+//     }
+//   }
+// }
+
+int DHT_Wrapper::tableToFile(const char *filename) {
+  int res = 1; //DHT_to_file(dht_object, filename);
+  return res;
+}
+
+int DHT_Wrapper::fileToTable(const char *filename) {
+  int res = 1; //DHT_from_file(dht_object, filename);
+  //if (res != DHT_SUCCESS)
+  //  return res;
+
+#ifdef DHT_STATISTICS
+  DAOSKV_print_statistics(daos_object);
+#endif
+
+  return DHT_SUCCESS;
+}
+
+void DHT_Wrapper::printStatistics() {
+  int res;
+
+  res = DAOSKV_print_statistics(daos_object);
+
+  if (res != DAOS_SUCCESS) {
+    // MPI ERROR ... WHAT TO DO NOW?
+    // RUNNING CIRCLES WHILE SCREAMING
+  }
+}
+
+LookupKey DHT_Wrapper::fuzzForDHT_R(const std::vector<double> &cell,
+                                    double dt) {
+  const auto c_zero_val = std::pow(10, AQUEOUS_EXP);
+
+  NamedVector<double> input_nv(this->output_names, cell);
+
+  const std::vector<double> eval_vec = hooks.dht_fuzz(input_nv);
+  assert(eval_vec.size() == this->key_count);
+  LookupKey vecFuzz(this->key_count + 1, {.0});
+
+  DHT_Rounder rounder;
+
+  int totals_i = 0;
+  // introduce fuzzing to allow more hits in DHT
+  // loop over every variable of grid cell
+  for (std::uint32_t i = 0; i < eval_vec.size(); i++) {
+    double curr_key = eval_vec[i];
+    if (curr_key != 0) {
+      if (this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL) {
+        curr_key -= base_totals[totals_i++];
+      }
+      vecFuzz[i] =
+          rounder.round(curr_key, dht_signif_vector[i],
+                        this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL);
+    }
+  }
+  // add timestep to the end of the key as double value
+  vecFuzz[this->key_count].fp_element = dt;
+
+  return vecFuzz;
+}
+
+LookupKey DHT_Wrapper::fuzzForDHT(const std::vector<double> &cell, double dt) {
+  const auto c_zero_val = std::pow(10, AQUEOUS_EXP);
+
+  LookupKey vecFuzz(this->key_count + 1, {.0});
+  DHT_Rounder rounder;
+
+  int totals_i = 0;
+  // introduce fuzzing to allow more hits in DHT
+  // loop over every variable of grid cell
+  for (std::uint32_t i = 0; i < input_key_elements.size(); i++) {
+    if (input_key_elements[i] == DHT_KEY_INPUT_CUSTOM) {
+      continue;
+    }
+    double curr_key = cell[input_key_elements[i]];
+    if (curr_key != 0) {
+      if (curr_key < c_zero_val &&
+          this->dht_prop_type_vector[i] == DHT_TYPE_DEFAULT) {
+        continue;
+      }
+      if (this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL) {
+        curr_key -= base_totals[totals_i++];
+      }
+      vecFuzz[i] =
+          rounder.round(curr_key, dht_signif_vector[i],
+                        this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL);
+    }
+  }
+  // add timestep to the end of the key as double value
+  vecFuzz[this->key_count].fp_element = dt;
+
+  return vecFuzz;
+}
+
+void poet::DHT_Wrapper::SetSignifVector(std::vector<uint32_t> signif_vec) {
+  if (signif_vec.size() != this->key_count) {
+    throw std::runtime_error(
+        "Significant vector size mismatches count of key elements.");
+  }
+
+  this->dht_signif_vector = signif_vec;
+}
+} // namespace poet
--- a/src/ChemistryModule/DAOS/KV/DaosKeyValue.c
+++ b/src/ChemistryModule/DAOS/KV/DaosKeyValue.c
@ -0,0 +1,572 @@
+/*
+** Copyright (C) 2017-2021 Max Luebke (University of Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include "poet/DaosKeyValue.h"
+
+#include <inttypes.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <daos.h>
+
+enum handleType
+{
+  HANDLE_POOL,
+  HANDLE_CO,
+};
+
+static inline void
+handle_share(DAOSKV *object, int type)
+{
+  d_iov_t ghdl = {NULL, 0, 0};
+  int rc;
+
+  if (object->rank == 0)
+  {
+    /** fetch size of global handle */
+    if (type == HANDLE_POOL)
+      rc = daos_pool_local2global(object->poh, &ghdl);
+    else
+      rc = daos_cont_local2global(object->coh, &ghdl);
+  }
+
+  /** broadcast size of global handle to all peers */
+  MPI_Bcast(&ghdl.iov_buf_len, 1, MPI_UINT64_T, 0, object->communicator);
+
+  /** allocate buffer for global pool handle */
+  ghdl.iov_buf = malloc(ghdl.iov_buf_len);
+  ghdl.iov_len = ghdl.iov_buf_len;
+
+  if (object->rank == 0)
+  {
+    /** generate actual global handle to share with peer tasks */
+    if (type == HANDLE_POOL)
+      rc = daos_pool_local2global(object->poh, &ghdl);
+    else
+      rc = daos_cont_local2global(object->coh, &ghdl);
+  }
+
+  /** broadcast global handle to all peers */
+  MPI_Bcast(ghdl.iov_buf, ghdl.iov_len, MPI_BYTE, 0, object->communicator);
+
+  if (object->rank != 0)
+  {
+    /** unpack global handle */
+    if (type == HANDLE_POOL)
+    {
+      /* NB: Only pool_global2local are different */
+      rc = daos_pool_global2local(ghdl, &object->poh);
+    }
+    else
+    {
+      rc = daos_cont_global2local(object->poh, ghdl, &object->coh);
+    }
+  }
+
+  free(ghdl.iov_buf);
+
+  MPI_Barrier(object->communicator);
+}
+
+DAOSKV *DAOSKV_create(MPI_Comm comm)
+{
+
+  DAOSKV *object;
+  int rc;
+
+  object = (DAOSKV *)malloc(sizeof(DAOSKV));
+  if (object == NULL)
+    return NULL;
+
+  // fill daos-object
+  object->communicator = comm;
+  object->read_misses = 0;
+  object->evictions = 0;
+
+  MPI_Comm_rank(comm, &object->rank);
+  MPI_Comm_size(comm, &object->comm_size);
+
+  // if set, initialize dht_stats
+#ifdef DHT_STATISTICS
+  DAOSKV_stats *stats;
+
+  stats = (DAOSKV_stats *)malloc(sizeof(DAOSKV_stats));
+  if (stats == NULL)
+    return NULL;
+
+  object->stats = stats;
+  object->stats->writes_local = (int *)calloc(object->comm_size, sizeof(int));
+  object->stats->old_writes = 0;
+  object->stats->read_misses = 0;
+  object->stats->read_hits = 0;
+  object->stats->evictions = 0;
+  object->stats->w_access = 0;
+  object->stats->r_access = 0;
+#endif
+
+  /** initialize the local DAOS stack */
+  if (daos_init() != 0)
+    return NULL;
+
+  /** Call connect on rank 0 only and broadcast handle to others */
+  if (object->rank == 0)
+  {
+    char *pool_name = getenv("DAOS_POOL");
+    if (pool_name == NULL)
+      printf("Pool label invalid \n");
+    if (daos_pool_connect(pool_name, NULL, DAOS_PC_RW, &object->poh,
+                          NULL, NULL) != 0)
+      return NULL;
+  }
+
+  /** share pool handle with peer tasks */
+  handle_share(object, HANDLE_POOL);
+
+  /*
+   * Create and open container on rank 0 and share the handle.
+   *
+   * Alternatively, one could create the container outside of this program
+   * using the daos utility: daos cont create --pool=puuid
+   * and pass the uuid to the app.
+   */
+  if (object->rank == 0)
+  {
+    /** create container */
+    if (getenv("DAOS_CONT") != NULL)
+    {
+      object->cont_label = getenv("DAOS_CONT");
+    }
+    else
+    {
+      object->cont_label = "Poet_Container";
+    }
+
+    /** check & open container if it already exist */
+    if (0 != daos_cont_open(object->poh, object->cont_label, DAOS_COO_RW, &object->coh, NULL, NULL))
+    {
+      /** create & open container*/
+      daos_cont_create_with_label(object->poh, object->cont_label, NULL, NULL, NULL);
+
+      /** open container */
+      if (daos_cont_open(object->poh, object->cont_label, DAOS_COO_RW, &object->coh, NULL, NULL) != 0)
+        return NULL;
+    }
+  }
+  /** share container handle with peer tasks */
+  handle_share(object, HANDLE_CO);
+
+  /** open object */
+
+  daos_obj_id_t oid;
+
+  oid.hi = 0;
+  oid.lo = 4;
+
+  daos_obj_generate_oid(object->coh, &oid, DAOS_OT_KV_HASHED, OC_SX, 0, 0);
+
+  daos_kv_open(object->coh, oid, DAOS_OO_RW, &object->oh, NULL);
+
+  return object;
+}
+
+int DAOSKV_free(DAOSKV *object)
+{
+  MPI_Barrier(object->communicator);
+
+  if (daos_kv_close(object->oh, NULL) != 0)
+    return DAOS_ERROR;
+  if (daos_cont_close(object->coh, NULL) != 0)
+    return DAOS_ERROR;
+
+  if (object->rank == 0)
+  {
+    daos_cont_destroy(object->poh, object->cont_label, 0, NULL);
+  }
+  if (daos_pool_disconnect(object->poh, NULL) != 0)
+    return DAOS_ERROR;
+
+  if (daos_fini() != 0)
+    return DAOS_ERROR;
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_write(DAOSKV *object, void *key, int key_size, void *send_data, int send_size)
+{
+#ifdef DHT_STATISTICS
+  object->stats->w_access++;
+#endif
+
+  // Turn key into a string
+  char *keyString[(key_size * 2) + 1];
+
+  keyToString(keyString, key, key_size);
+
+  int rc;
+  rc =daos_kv_put(object->oh, DAOS_TX_NONE, 0, keyString, send_size, send_data, NULL);
+
+  // No space left in storage
+  if (rc == -DER_NOSPACE && object->rank == 0)
+  {
+    trim_Space(object, 10, send_size, key_size);
+  }
+  if (rc != 0)
+    return DAOS_ERROR;
+
+#ifdef DHT_STATISTICS
+  object->stats->writes_local[object->rank]++;
+#endif
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_read(DAOSKV *object, void *key, int key_size, void *recv_data, int recv_size)
+{
+
+#ifdef DHT_STATISTICS
+  object->stats->r_access++;
+#endif
+
+  // Turn key into a string
+  char *keyString[(key_size * 2) + 1];
+
+  keyToString(keyString, key, key_size);
+
+  daos_size_t size = recv_size;
+  int rc;
+
+  rc = daos_kv_get(object->oh, DAOS_TX_NONE, DAOS_COND_DKEY_FETCH, keyString, &size, recv_data, NULL);
+
+  if (rc == -DER_NONEXIST)
+  {
+#ifdef DHT_STATISTICS
+    object->stats->read_misses += 1;
+#endif
+    return DAOS_READ_MISS;
+  }
+  else if (rc != 0)
+    return DAOS_ERROR;
+#ifdef DHT_STATISTICS
+  object->stats->read_hits += 1;
+#endif
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_remove(DAOSKV *object, void *key, int key_size)
+{
+
+  // Turn key into a string
+  char *keyString[(key_size * 2) + 1];
+
+  keyToString(keyString, key, key_size);
+  int rc;
+
+  if (daos_kv_remove(object->oh, DAOS_TX_NONE, 0, keyString, NULL) != 0)
+    return DAOS_ERROR;
+
+  return DAOS_SUCCESS;
+}
+
+int keyToString(char *output, void *key, int key_size)
+{
+
+  int i;
+  int offset = 0;
+  for (i = 0; i < key_size; i++)
+  {
+    sprintf((char *)output + offset, "%02X", ((char *)key)[i]);
+    offset += 2;
+  }
+  output[offset++] = '\0';
+
+  return 0;
+}
+
+int enumerate_key(DAOSKV *object, int *total_nr, int key_size)
+{
+  int actual_key_size = (key_size * 2) + 1;
+  char *buf;
+  daos_key_desc_t kds[5];
+  daos_anchor_t anchor = {0};
+  int key_nr = 0;
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+
+  buf = malloc(actual_key_size);
+  d_iov_set(&sg_iov, buf, actual_key_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+
+  while (!daos_anchor_is_eof(&anchor))
+  {
+    uint32_t nr = 5;
+    int rc;
+
+    memset(buf, 0, actual_key_size);
+    rc = daos_kv_list(object->oh, DAOS_TX_NONE, &nr, kds, &sgl, &anchor,
+                      NULL);
+        //If there is no key, break the loop
+    if(buf[0] == '\0'){
+      break;
+    }
+    printf("Enumareted over dkey: %s\n", buf);
+
+    if (rc != 0)
+    {
+      printf("Error retrieving Key %d \n", rc);
+      return DAOS_ERROR;
+    }
+
+    if (nr == 0)
+      continue;
+    key_nr += nr;
+  }
+  *total_nr = key_nr;
+
+  return DAOS_SUCCESS;
+}
+
+int delete_n_entries(DAOSKV *object, int toDelete, int key_size)
+{
+
+  int actual_key_size = (key_size * 2) + 1;
+
+  daos_handle_t th = DAOS_TX_NONE;
+
+  char *buf;
+  daos_key_desc_t kds[5];
+  daos_anchor_t anchor = {0};
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+
+  int rc;
+
+  buf = malloc(actual_key_size);
+  d_iov_set(&sg_iov, buf, actual_key_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+  memset(buf, 0, key_size);
+
+  /* allocate transaction */
+  rc = daos_tx_open(object->coh, &th, 0, NULL);
+
+  int key_nr = 0;
+  while (!daos_anchor_is_eof(&anchor) && key_nr < toDelete)
+  {
+    uint32_t nr = 5;
+
+    rc = daos_kv_list(object->oh, DAOS_TX_NONE, &nr, kds, &sgl, &anchor,
+                      NULL);
+
+    //If there is no key, break the loop
+    if(buf[0] == '\0'){
+      break;
+    }
+
+    // Add delete of key to transaction th
+    printf("Delete dkey: %s\n", buf);
+
+    if (daos_kv_remove(object->oh, th, 0, buf, NULL) != 0)
+      printf("Delete n Key Error");
+
+    if (rc != 0)
+    {
+      printf("Error retrieving Key %d \n", rc);
+      return DAOS_ERROR;
+    }
+    if (nr == 0)
+      continue;
+    key_nr += nr;
+  }
+
+  // commit transaction, retry if failure
+  rc = daos_tx_commit(th, NULL);
+  if (rc)
+  {
+    printf("Commit error: %d\n", rc);
+    if (rc == -DER_TX_RESTART)
+    {
+      /* conflict with another transaction, try again */
+      rc = daos_tx_restart(th, NULL);
+    }
+  }
+
+  // free transaction resources
+  rc = daos_tx_close(th, NULL);
+  return DAOS_SUCCESS;
+}
+
+struct daos_space get_pool_size(DAOSKV *object)
+{
+  int rc;
+  daos_pool_info_t pinfo = {0};
+  struct daos_pool_space *ps = &pinfo.pi_space;
+
+  // query only the space, replace with DPI_ALL for all infos
+  pinfo.pi_bits = DPI_SPACE;
+  rc = daos_pool_query(object->poh, NULL, &pinfo, NULL, NULL);
+  // size of storage
+  // printf("Total Size:%d\n", ps->ps_space.s_total[DAOS_MEDIA_SCM]+ps->ps_space.s_total[DAOS_MEDIA_NVME]);
+  // printf("Free Size:%d\n", ps->ps_space.s_free[DAOS_MEDIA_SCM]+ ps->ps_space.s_free[DAOS_MEDIA_NVME]);
+
+  return ps->ps_space;
+}
+
+int trim_Space(DAOSKV *object, float deletePercentage, int dataSize, int keySize)
+{
+  // Get current usage of the storage space
+  struct daos_space space = get_pool_size(object);
+
+  long int total_size = space.s_total[DAOS_MEDIA_SCM] + space.s_total[DAOS_MEDIA_NVME];
+
+  // Estimate, total number of entries
+  int totalNumberOfEntries = total_size / (dataSize + keySize);
+  // Calculate how many keys to delete
+  int toDeleteEntries = totalNumberOfEntries * deletePercentage / 100;
+
+  delete_n_entries(object, toDeleteEntries, keySize);
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_print_statistics(DAOSKV *object)
+{
+#ifdef DHT_STATISTICS
+  int *written_buckets;
+  int *read_misses, sum_read_misses;
+  int *read_hits, sum_read_hits;
+  int *evictions, sum_evictions;
+  int sum_w_access, sum_r_access, *w_access, *r_access;
+  int rank;
+  MPI_Comm_rank(object->communicator, &rank);
+
+// disable possible warning of unitialized variable, which is not the case
+// since we're using MPI_Gather to obtain all values only on rank 0
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+
+  // obtaining all values from all processes in the communicator
+  if (rank == 0)
+    read_misses = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->read_misses, 1, MPI_INT, read_misses, 1,
+                 MPI_INT, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->read_misses, &sum_read_misses, 1, MPI_INT,
+                 MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->read_misses = 0;
+
+  if (rank == 0)
+    read_hits = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->read_hits, 1, MPI_INT, read_hits, 1,
+                 MPI_INT, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->read_hits, &sum_read_hits, 1, MPI_INT,
+                 MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->read_hits = 0;
+
+  if (rank == 0)
+    evictions = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->evictions, 1, MPI_INT, evictions, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->evictions, &sum_evictions, 1, MPI_INT, MPI_SUM,
+                 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->evictions = 0;
+
+  if (rank == 0)
+    w_access = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->w_access = 0;
+
+  if (rank == 0)
+    r_access = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->r_access = 0;
+
+  if (rank == 0)
+    written_buckets = (int *)calloc(object->comm_size, sizeof(int));
+  if (MPI_Reduce(object->stats->writes_local, written_buckets, object->comm_size,
+                 MPI_INT, MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+
+  if (rank == 0)
+  { // only process with rank 0 will print out results as a
+    // object
+    int sum_written_buckets = 0;
+
+    for (int i = 0; i < object->comm_size; i++)
+    {
+      sum_written_buckets += written_buckets[i];
+    }
+
+    int members = 7;
+    int padsize = (members * 12) + 1;
+    char pad[padsize + 1];
+
+    memset(pad, '-', padsize * sizeof(char));
+    pad[padsize] = '\0';
+    printf("\n");
+    printf("%-35s||resets with every call of this function\n", " ");
+    printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s|%-11s\n", "rank", "occupied",
+           "free", "w_access", "r_access", "read misses", "read hits", "evictions");
+    printf("%s\n", pad);
+    for (int i = 0; i < object->comm_size; i++)
+    {
+      printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d|%-11d\n", i,
+             written_buckets[i], 0,
+             w_access[i], r_access[i], read_misses[i], read_hits[i], evictions[i]);
+    }
+    printf("%s\n", pad);
+    printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d|%-11d\n", "sum",
+           sum_written_buckets,
+           0,
+           sum_w_access, sum_r_access, sum_read_misses, sum_read_hits, sum_evictions);
+
+    printf("%s\n", pad);
+    printf("%s %d\n",
+           "new entries:", sum_written_buckets - object->stats->old_writes);
+
+    printf("\n");
+    fflush(stdout);
+
+    object->stats->old_writes = sum_written_buckets;
+  }
+
+// enable warning again
+#pragma GCC diagnostic pop
+
+  MPI_Barrier(object->communicator);
+  return DAOS_SUCCESS;
+#endif
+}
--- a/src/ChemistryModule/DAOS/OB/DaosKeyValue.c
+++ b/src/ChemistryModule/DAOS/OB/DaosKeyValue.c
@ -0,0 +1,597 @@
+/*
+** Copyright (C) 2017-2021 Max Luebke (University of Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include "poet/DaosKeyValue.h"
+
+#include <inttypes.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <daos.h>
+
+enum handleType
+{
+  HANDLE_POOL,
+  HANDLE_CO,
+};
+
+static inline void
+handle_share(DAOSKV *object, int type)
+{
+  d_iov_t ghdl = {NULL, 0, 0};
+  int rc;
+
+  if (object->rank == 0)
+  {
+    /** fetch size of global handle */
+    if (type == HANDLE_POOL)
+      rc = daos_pool_local2global(object->poh, &ghdl);
+    else
+      rc = daos_cont_local2global(object->coh, &ghdl);
+  }
+
+  /** broadcast size of global handle to all peers */
+  MPI_Bcast(&ghdl.iov_buf_len, 1, MPI_UINT64_T, 0, object->communicator);
+
+  /** allocate buffer for global pool handle */
+  ghdl.iov_buf = malloc(ghdl.iov_buf_len);
+  ghdl.iov_len = ghdl.iov_buf_len;
+
+  if (object->rank == 0)
+  {
+    /** generate actual global handle to share with peer tasks */
+    if (type == HANDLE_POOL)
+      rc = daos_pool_local2global(object->poh, &ghdl);
+    else
+      rc = daos_cont_local2global(object->coh, &ghdl);
+  }
+
+  /** broadcast global handle to all peers */
+  MPI_Bcast(ghdl.iov_buf, ghdl.iov_len, MPI_BYTE, 0, object->communicator);
+
+  if (object->rank != 0)
+  {
+    /** unpack global handle */
+    if (type == HANDLE_POOL)
+    {
+      /* NB: Only pool_global2local are different */
+      rc = daos_pool_global2local(ghdl, &object->poh);
+    }
+    else
+    {
+      rc = daos_cont_global2local(object->poh, ghdl, &object->coh);
+    }
+  }
+
+  free(ghdl.iov_buf);
+
+  MPI_Barrier(object->communicator);
+}
+
+DAOSKV *DAOSKV_create(MPI_Comm comm)
+{
+
+  DAOSKV *object;
+  int rc;
+
+  object = (DAOSKV *)malloc(sizeof(DAOSKV));
+  if (object == NULL)
+    return NULL;
+
+  // fill daos-object
+  object->communicator = comm;
+  object->read_misses = 0;
+  object->evictions = 0;
+
+  MPI_Comm_rank(comm, &object->rank);
+  MPI_Comm_size(comm, &object->comm_size);
+
+  // if set, initialize dht_stats
+#ifdef DHT_STATISTICS
+  DAOSKV_stats *stats;
+
+  stats = (DAOSKV_stats *)malloc(sizeof(DAOSKV_stats));
+  if (stats == NULL)
+    return NULL;
+
+  object->stats = stats;
+  object->stats->writes_local = (int *)calloc(object->comm_size, sizeof(int));
+  object->stats->old_writes = 0;
+  object->stats->read_misses = 0;
+  object->stats->read_hits = 0;
+  object->stats->evictions = 0;
+  object->stats->w_access = 0;
+  object->stats->r_access = 0;
+#endif
+
+  /** initialize the local DAOS stack */
+  if (daos_init() != 0)
+    return NULL;
+
+  /** Call connect on rank 0 only and broadcast handle to others */
+  if (object->rank == 0)
+  {
+    char *pool_name = getenv("DAOS_POOL");
+    if (pool_name == NULL)
+      printf("Pool label invalid \n");
+    if (daos_pool_connect(pool_name, NULL, DAOS_PC_RW, &object->poh,
+                          NULL, NULL) != 0)
+      return NULL;
+  }
+
+  /** share pool handle with peer tasks */
+  handle_share(object, HANDLE_POOL);
+
+  /*
+   * Create and open container on rank 0 and share the handle.
+   *
+   * Alternatively, one could create the container outside of this program
+   * using the daos utility: daos cont create --pool=puuid
+   * and pass the uuid to the app.
+   */
+  if (object->rank == 0)
+  {
+    /** create container */
+    if (getenv("DAOS_CONT") != NULL)
+    {
+      object->cont_label = getenv("DAOS_CONT");
+    }
+    else
+    {
+      object->cont_label = "Poet_Container";
+    }
+
+    /** check & open container if it already exist */
+    if (0 != daos_cont_open(object->poh, object->cont_label, DAOS_COO_RW, &object->coh, NULL, NULL))
+    {
+      /** create & open container*/
+      daos_cont_create_with_label(object->poh, object->cont_label, NULL, NULL, NULL);
+
+      /** open container */
+      if (daos_cont_open(object->poh, object->cont_label, DAOS_COO_RW, &object->coh, NULL, NULL) != 0)
+        return NULL;
+    }
+  }
+  /** share container handle with peer tasks */
+  handle_share(object, HANDLE_CO);
+
+  /** open object */
+
+  daos_obj_id_t oid;
+
+  oid.hi = 0;
+  oid.lo = 4;
+
+  daos_obj_generate_oid(object->coh, &oid, 0, OC_SX, 0, 0);
+
+  daos_obj_open(object->coh, oid, DAOS_OO_RW, &object->oh, NULL);
+
+  return object;
+}
+
+int DAOSKV_free(DAOSKV *object)
+{
+  MPI_Barrier(object->communicator);
+
+  if (daos_obj_close(object->oh, NULL) != 0)
+    return DAOS_ERROR;
+  if (daos_cont_close(object->coh, NULL) != 0)
+    return DAOS_ERROR;
+
+  if (object->rank == 0)
+  {
+    daos_cont_destroy(object->poh, object->cont_label, 0, NULL);
+  }
+  if (daos_pool_disconnect(object->poh, NULL) != 0)
+    return DAOS_ERROR;
+
+  if (daos_fini() != 0)
+    return DAOS_ERROR;
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_write(DAOSKV *object, void *key, int key_size, void *send_data, int send_size)
+{
+  int rc;
+
+#ifdef DHT_STATISTICS
+  object->stats->w_access++;
+#endif
+
+  d_iov_t dkey;
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+  daos_iod_t iod;
+
+  // set dkey
+  d_iov_set(&dkey, key, key_size);
+
+  d_iov_set(&sg_iov, send_data, send_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+
+  // set akey
+  // Here for all the same value since dkey differ
+  // Maybe other way around is better? Or a mix?
+  int akey = 1;
+  d_iov_set(&iod.iod_name, &akey, sizeof(int));
+
+  iod.iod_nr = 1;                 /** has to be 1 for single value */
+  iod.iod_size = send_size;       /** size of the single value */
+  iod.iod_recxs = NULL;           /** recx is ignored for single value */
+  iod.iod_type = DAOS_IOD_SINGLE; /** value type of the akey */
+
+  rc = daos_obj_update(object->oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl,
+                       NULL);
+
+  // No space left in storage
+  if (rc == -DER_NOSPACE && object->rank == 0)
+  {
+    trim_Space(object, 10, send_size, key_size);
+  }
+  if (rc != 0)
+    return DAOS_ERROR;
+
+#ifdef DHT_STATISTICS
+  object->stats->writes_local[object->rank]++;
+#endif
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_read(DAOSKV *object, void *key, int key_size, void *recv_data, int recv_size)
+{
+  int rc;
+
+#ifdef DHT_STATISTICS
+  object->stats->r_access++;
+#endif
+
+  d_iov_t dkey;
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+  daos_iod_t iod;
+
+  // set dkey
+  d_iov_set(&dkey, key, key_size);
+
+  d_iov_set(&sg_iov, recv_data, recv_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+
+  // set akey
+  // Here for all the same value since dkey differ
+  // Maybe other way around is better? Or a mix?
+  int akey = 1;
+  d_iov_set(&iod.iod_name, &akey, sizeof(int));
+
+  iod.iod_nr = 1;                 /** 1 for single value */
+  iod.iod_size = DAOS_REC_ANY;    /** size of the single value, set to DAOS_REC_ANY to check if key was written or not */
+  iod.iod_recxs = NULL;           /** recx is ignored for single value */
+  iod.iod_type = DAOS_IOD_SINGLE; /** value type of the akey */
+
+  /** fetch a dkey */
+  rc = daos_obj_fetch(object->oh, DAOS_TX_NONE, 0, &dkey, 1, &iod, &sgl,
+                      NULL, NULL);
+  if (rc != 0)
+  {
+    return DAOS_ERROR;
+  }
+
+  if (iod.iod_size == 0)
+  {
+#ifdef DHT_STATISTICS
+    object->stats->read_misses += 1;
+#endif
+    return DAOS_READ_MISS;
+  }
+#ifdef DHT_STATISTICS
+  object->stats->read_hits += 1;
+#endif
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_remove(DAOSKV *object, void *key, int key_size)
+{
+  d_iov_t dkey;
+
+  // set dkey
+  d_iov_set(&dkey, key, key_size);
+  if (daos_obj_punch_dkeys(object->oh, DAOS_TX_NONE, 0, 1, &dkey, NULL) != 0)
+    return DAOS_ERROR;
+
+  return DAOS_SUCCESS;
+}
+
+int enumerate_key(DAOSKV *object, int *total_nr, int key_size)
+{
+
+  char *buf;
+  daos_key_desc_t kds[5];
+  daos_anchor_t anchor = {0};
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+  int key_nr = 0;
+  int rc;
+
+  buf = malloc(key_size);
+  d_iov_set(&sg_iov, buf, key_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+
+  while (!daos_anchor_is_eof(&anchor))
+  {
+    uint32_t nr = 5;
+
+    memset(buf, 0, key_size);
+
+    rc = daos_obj_list_dkey(object->oh, DAOS_TX_NONE, &nr, kds,
+                            &sgl, &anchor, NULL);
+
+        //If there is no key, break the loop
+    if(buf[0] == '\0'){
+      break;
+    }
+    printf("Enumareted over dkey: %s\n", buf);
+
+    if (rc != 0)
+    {
+      printf("Error retrieving Key %d \n", rc);
+      return DAOS_ERROR;
+    }
+    if (nr == 0)
+      continue;
+    key_nr += nr;
+  }
+
+  *total_nr = key_nr;
+
+  return DAOS_SUCCESS;
+}
+
+int delete_n_entries(DAOSKV *object, int toDelete, int key_size)
+{
+  daos_handle_t th = DAOS_TX_NONE;
+
+  char *buf;
+  daos_key_desc_t kds[5];
+  daos_anchor_t anchor = {0};
+  d_sg_list_t sgl;
+  d_iov_t sg_iov;
+
+  int rc;
+
+  buf = malloc(key_size);
+  d_iov_set(&sg_iov, buf, key_size);
+  sgl.sg_nr = 1;
+  sgl.sg_nr_out = 0;
+  sgl.sg_iovs = &sg_iov;
+  memset(buf, 0, key_size);
+
+  /* allocate transaction */
+  rc = daos_tx_open(object->coh, &th, 0, NULL);
+
+  int key_nr = 0;
+  while (!daos_anchor_is_eof(&anchor) && key_nr < toDelete)
+  {
+    uint32_t nr = 5;
+
+    rc = daos_obj_list_dkey(object->oh, DAOS_TX_NONE, &nr, kds,
+                            &sgl, &anchor, NULL);
+    
+        //If there is no key, break the loop
+    if(buf[0] == '\0'){
+      break;
+    }
+
+    // Add delete of key to transaction th
+    printf("Delete dkey: %s\n", buf);
+    d_iov_t dkey;
+    // set dkey
+    d_iov_set(&dkey, buf, key_size);
+    if (daos_obj_punch_dkeys(object->oh, th, 0, 1, &dkey, NULL) != 0)
+      printf("Delete n Key Error");
+
+    if (rc != 0)
+    {
+      printf("Error retrieving Key %d \n", rc);
+      return DAOS_ERROR;
+    }
+    if (nr == 0)
+      continue;
+    key_nr += nr;
+  }
+
+  // commit transaction, retry if failure
+  rc = daos_tx_commit(th, NULL);
+  if (rc)
+  {
+    printf("Commit error: %d\n", rc);
+    if (rc == -DER_TX_RESTART)
+    {
+      /* conflict with another transaction, try again */
+      rc = daos_tx_restart(th, NULL);
+    }
+  }
+
+  // free transaction resources
+  rc = daos_tx_close(th, NULL);
+  return DAOS_SUCCESS;
+}
+
+struct daos_space get_pool_size(DAOSKV *object)
+{
+  int rc;
+  daos_pool_info_t pinfo = {0};
+  struct daos_pool_space *ps = &pinfo.pi_space;
+
+  // query only the space, replace with DPI_ALL for all infos
+  pinfo.pi_bits = DPI_SPACE;
+  rc = daos_pool_query(object->poh, NULL, &pinfo, NULL, NULL);
+  // size of storage
+  // printf("Total Size:%d\n", ps->ps_space.s_total[DAOS_MEDIA_SCM]+ps->ps_space.s_total[DAOS_MEDIA_NVME]);
+  // printf("Free Size:%d\n", ps->ps_space.s_free[DAOS_MEDIA_SCM]+ ps->ps_space.s_free[DAOS_MEDIA_NVME]);
+
+  return ps->ps_space;
+}
+
+int trim_Space(DAOSKV *object, float deletePercentage, int dataSize, int keySize)
+{
+  // Get current usage of the storage space
+  struct daos_space space = get_pool_size(object);
+
+  long int total_size = space.s_total[DAOS_MEDIA_SCM] + space.s_total[DAOS_MEDIA_NVME];
+
+  // Estimate, total number of entries
+  int totalNumberOfEntries = total_size / (dataSize + keySize);
+  // Calculate how many keys to delete
+  int toDeleteEntries = totalNumberOfEntries * deletePercentage / 100;
+
+  delete_n_entries(object, toDeleteEntries, keySize);
+
+  return DAOS_SUCCESS;
+}
+
+int DAOSKV_print_statistics(DAOSKV *object)
+{
+#ifdef DHT_STATISTICS
+  int *written_buckets;
+  int *read_misses, sum_read_misses;
+  int *read_hits, sum_read_hits;
+  int *evictions, sum_evictions;
+  int sum_w_access, sum_r_access, *w_access, *r_access;
+  int rank;
+  MPI_Comm_rank(object->communicator, &rank);
+
+// disable possible warning of unitialized variable, which is not the case
+// since we're using MPI_Gather to obtain all values only on rank 0
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+
+  // obtaining all values from all processes in the communicator
+  if (rank == 0)
+    read_misses = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->read_misses, 1, MPI_INT, read_misses, 1,
+                 MPI_INT, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->read_misses, &sum_read_misses, 1, MPI_INT,
+                 MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->read_misses = 0;
+
+  if (rank == 0)
+    read_hits = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->read_hits, 1, MPI_INT, read_hits, 1,
+                 MPI_INT, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->read_hits, &sum_read_hits, 1, MPI_INT,
+                 MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->read_hits = 0;
+
+  if (rank == 0)
+    evictions = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->evictions, 1, MPI_INT, evictions, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->evictions, &sum_evictions, 1, MPI_INT, MPI_SUM,
+                 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->evictions = 0;
+
+  if (rank == 0)
+    w_access = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->w_access = 0;
+
+  if (rank == 0)
+    r_access = (int *)malloc(object->comm_size * sizeof(int));
+  if (MPI_Gather(&object->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  if (MPI_Reduce(&object->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0,
+                 object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+  object->stats->r_access = 0;
+
+  if (rank == 0)
+    written_buckets = (int *)calloc(object->comm_size, sizeof(int));
+  if (MPI_Reduce(object->stats->writes_local, written_buckets, object->comm_size,
+                 MPI_INT, MPI_SUM, 0, object->communicator) != 0)
+    return DAOS_MPI_ERROR;
+
+  if (rank == 0)
+  { // only process with rank 0 will print out results as a
+    // object
+    int sum_written_buckets = 0;
+
+    for (int i = 0; i < object->comm_size; i++)
+    {
+      sum_written_buckets += written_buckets[i];
+    }
+
+    int members = 7;
+    int padsize = (members * 12) + 1;
+    char pad[padsize + 1];
+
+    memset(pad, '-', padsize * sizeof(char));
+    pad[padsize] = '\0';
+    printf("\n");
+    printf("%-35s||resets with every call of this function\n", " ");
+    printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s|%-11s\n", "rank", "occupied",
+           "free", "w_access", "r_access", "read misses", "read hits", "evictions");
+    printf("%s\n", pad);
+    for (int i = 0; i < object->comm_size; i++)
+    {
+      printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d|%-11d\n", i,
+             written_buckets[i], 0,
+             w_access[i], r_access[i], read_misses[i], read_hits[i], evictions[i]);
+    }
+    printf("%s\n", pad);
+    printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d|%-11d\n", "sum",
+           sum_written_buckets,
+           0,
+           sum_w_access, sum_r_access, sum_read_misses, sum_read_hits, sum_evictions);
+
+    printf("%s\n", pad);
+    printf("%s %d\n",
+           "new entries:", sum_written_buckets - object->stats->old_writes);
+
+    printf("\n");
+    fflush(stdout);
+
+    object->stats->old_writes = sum_written_buckets;
+  }
+
+// enable warning again
+#pragma GCC diagnostic pop
+
+  MPI_Barrier(object->communicator);
+  return DAOS_SUCCESS;
+#endif
+}