Merge branch 'mdl/defaultstorelease' into 'main'

DEFAULT_BUILD_TYPE set to Release

See merge request naaice/poet!60

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,16 @@
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,41 @@
+style: gitlab
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://git.gfz-potsdam.de/sec34/port
+options:
+  commits:
+     filters:
+       Type:
+         - feat
+         - fix
+         - perf
+         - refactor
+         - data
+         - build
+         - util
+         - doc
+         - chore
+         - ci
+         - test
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       data: Simulation Input
+       build: Build System
+       util: Evaluation Scripts
+       doc: Documentation
+       chore: Householding and Cleanup
+       ci: CI
+       test: Software Testing
+  header:
+    pattern: "^(\\w*)\\:\\s(.*)$"
+    pattern_maps:
+      - Type
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.devcontainer/Dockerfile

@@ -0,0 +1,108 @@
+FROM gcc:11.2.0 AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get install -y \
+    sudo \
+    git \
+    ninja-build \
+    libmpfr-dev \
+    python3-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp
+
+ARG OPENMPI_VERSION=4.1.1
+ADD https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION%.*}/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi.tar.gz
+
+RUN mkdir openmpi && \ 
+    tar xf openmpi.tar.gz -C openmpi --strip-components 1 && \
+    cd openmpi && \
+    ./configure --prefix=/usr/local && \
+    make -j $(nproc) && \
+    make install && \
+    rm -rf /tmp/openmpi tmp/openmpi.tar.gz
+
+ARG CMAKE_VERSION=3.30.5
+ADD https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh /tmp/cmake.sh
+
+RUN bash ./cmake.sh --skip-license --prefix=/usr/local \
+    && rm cmake.sh
+
+ARG LAPACK_VERSION=3.12.0
+ADD https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v${LAPACK_VERSION}.tar.gz /tmp/lapack.tar.gz
+
+RUN mkdir lapack && \
+    tar xf lapack.tar.gz -C lapack --strip-components 1 && \
+    cd lapack && \
+    mkdir build && \
+    cd build && \
+    cmake .. -G Ninja -DBUILD_SHARED_LIBS=ON && \
+    ninja install && \
+    rm -rf /tmp/lapack tmp/lapack.tar.gz
+
+ARG R_VERSION=4.4.2
+ADD https://cran.r-project.org/src/base/R-${R_VERSION%%.*}/R-${R_VERSION}.tar.gz /tmp/R.tar.gz
+
+RUN mkdir R && \
+    tar xf R.tar.gz -C R --strip-components 1 && \
+    cd R && \
+    ./configure --prefix=/usr/local --enable-R-shlib --with-blas --with-lapack && \
+    make -j $(nproc) && \
+    make install && \
+    rm -rf /tmp/R tmp/R.tar.gz
+
+RUN  /usr/local/bin/R -q -e "install.packages(c('Rcpp', 'RInside', 'qs'), repos='https://cran.rstudio.com/')"
+
+ARG EIGEN_VERSION=3.4.0
+ADD https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.bz2 /tmp/eigen.tar.bz2
+
+RUN mkdir eigen && \
+    tar xf eigen.tar.bz2 -C eigen --strip-components 1 && \
+    cd eigen && \
+    mkdir build && \
+    cd build && \
+    cmake .. -G Ninja && \
+    ninja install && \
+    rm -rf /tmp/eigen tmp/eigen.tar.bz2
+
+ARG GDB_VERSION=15.2
+ADD https://ftp.gnu.org/gnu/gdb/gdb-${GDB_VERSION}.tar.xz /tmp/gdb.tar.xz
+
+RUN mkdir gdb && \
+    tar xf gdb.tar.xz -C gdb --strip-components 1 && \
+    cd gdb && \
+    ./configure --prefix=/usr/local && \
+    make -j $(nproc) && \
+    make install && \
+    rm -rf /tmp/gdb tmp/gdb.tar.xz
+
+RUN useradd -m -s /bin/bash -G sudo vscode \
+    && echo "vscode ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+USER vscode
+
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+RUN sudo apt-get update && \
+    sudo apt-get install -y zsh && \
+    sudo apt-get clean && \
+    sudo rm -rf /var/lib/apt/lists/*
+
+RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/download/v1.2.1/zsh-in-docker.sh)" -- \ 
+    -t agnoster \
+    -p zsh-syntax-highlighting
+
+RUN zsh -c "git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting"
+
+RUN zsh -c "git clone --depth 1 https://github.com/junegunn/fzf.git ~/.fzf && ~/.fzf/install"
+
+RUN mkdir -p /home/vscode/.config/gdb \
+    && echo "set auto-load safe-path /" > /home/vscode/.config/gdb/gdbinit
+
+ENV CMAKE_GENERATOR=Ninja
+ENV CMAKE_EXPORT_COMPILE_COMMANDS=ON
+
+WORKDIR /home/vscode

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"twxs.cmake",
+				"llvm-vs-code-extensions.vscode-clangd"
+			]
+		}
+	},
+	// in case you want to push/pull from remote repositories using ssh
+	"mounts": [
+		"source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached",
+		"source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached"
+	]
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.gitignore

@@ -140,3 +140,7 @@ vignettes/*.pdf
 # End of https://www.toptal.com/developers/gitignore/api/c,c++,r,cmake
 
 build/
+/.cache/
+
+.vscode
+.codechecker

.gitlab-ci.yml

@@ -0,0 +1,119 @@
+# This file is a template, and might need editing before it works on your project.
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
+#
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+
+image: git.gfz-potsdam.de:5000/naaice/poet:ci
+
+stages:          # List of stages for jobs, and their order of execution
+  - release
+  - test
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  SOURCE_ARCHIVE_NAME: 'poet_${CI_COMMIT_TAG}_sources.tar.gz'
+  CHANGELOG_FILE: 'commit_changelog.md'
+
+test:       # This job runs in the build stage, which runs first.
+  stage: test
+  script:
+    - mkdir -p build && cd build
+    - cmake -DPOET_ENABLE_TESTING=ON -DPOET_PREPROCESS_BENCHS=OFF -DCMAKE_BUILD_TYPE=Release ..
+    - make -j$(nproc) check
+
+pages:
+  stage: release
+  before_script:
+    - apt-get update && apt-get install -y doxygen graphviz
+    - mkdir {build_pages,public}
+  script:
+    - pushd build_pages
+    - cmake .. && make doxygen
+    - popd && mv build_pages/docs/html/* public/
+  artifacts:
+    paths:
+      - public
+  rules:
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
+
+push:
+  stage: release
+  variables:
+    GITHUB_REPOSITORY: 'git@github.com:POET-Simulator/POET.git'
+  before_script:
+    # I know that there is this file env variable in gitlab, but somehow it does not work for me (still complaining about white spaces ...)
+    # Therefore, the ssh key is stored as a base64 encoded string
+    - mkdir -p ~/.ssh && echo $GITHUB_SSH_PRIVATE_KEY | base64 -d > ~/.ssh/id_ed25519 && chmod 0600 ~/.ssh/id_ed25519
+    - ssh-keyscan github.com >> ~/.ssh/known_hosts
+    - echo $MIRROR_SCRIPT | base64 -d > mirror.sh && chmod +x mirror.sh
+  script:
+    - if [[-d poet.git ]]; then rm -rf poet.git; fi
+    - git clone --mirror "https://git.gfz-potsdam.de/naaice/poet.git" "poet.git" && cd poet.git
+    - git push --mirror $GITHUB_REPOSITORY
+  allow_failure: true
+
+#archive-sources:       # This job runs in the build stage, which runs first.
+#  image: python:3
+#  stage: release
+#
+#  before_script:
+#    - pip install git-archive-all
+#    - echo ARCHIVE_JOB_ID=${CI_JOB_ID} >> archives.env
+#  script:
+#    - git-archive-all ${SOURCE_ARCHIVE_NAME}
+#  artifacts:
+#    paths:
+#      - ${SOURCE_ARCHIVE_NAME}
+#    expire_in: never
+#    reports:
+#      dotenv: archives.env
+#  rules:
+#    - if: $CI_COMMIT_TAG
+
+#release-description:
+#  image: golang:bullseye
+#  stage: release
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  before_script:
+#    - go install github.com/git-chglog/git-chglog/cmd/git-chglog@v0.15.2
+#  script:
+#    - git-chglog -o ${CHANGELOG_FILE} ${CI_COMMIT_TAG}
+#  artifacts:
+#    paths:
+#      - ${CHANGELOG_FILE}
+#
+#
+#release-create:
+#  stage: release
+#  image: registry.gitlab.com/gitlab-org/release-cli:latest
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  script:
+#    - echo "Running release job"
+#  needs:
+#    - job: archive-sources
+#      artifacts: true
+#    - job: release-description
+#      artifacts: true
+#  release:
+#    tag_name: $CI_COMMIT_TAG
+#    name: 'POET $CI_COMMIT_TAG'
+#    description: ${CHANGELOG_FILE}
+#    assets:
+#      links:
+#        - name: '${SOURCE_ARCHIVE_NAME}'
+#          url: 'https://git.gfz-potsdam.de/naaice/poet/-/jobs/${ARCHIVE_JOB_ID}/artifacts/file/${SOURCE_ARCHIVE_NAME}'

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "ext/tug"]
+	path = ext/tug
+	url = ../tug.git
+
+[submodule "ext/litephreeqc"]
+	path = ext/litephreeqc
+	url = ../litephreeqc.git

CITATION.cff

@@ -0,0 +1,51 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: 'POET: POtsdam rEactive Transport'
+message: >-
+  If you use this software, please cite it using these
+  metadata.
+type: software
+authors:
+  - given-names: Max
+    family-names: Lübke
+    email: mluebke@uni-potsdam.de
+    affiliation: University of Potsdam
+    orcid: 'https://orcid.org/0009-0008-9773-3038'
+  - given-names: Marco
+    family-names: De Lucia
+    email: delucia@gfz.de
+    affiliation: GFZ Helmholtz Centre for Geosciences
+    orcid: 'https://orcid.org/0000-0002-1186-4491'
+  - given-names: Alexander
+    family-names: Lindemann
+  - given-names: Hannes
+    family-names: Signer
+    email: signer@uni-potsdam.de
+    orcid: 'https://orcid.org/0009-0000-3058-8472'
+  - given-names: Bettina
+    family-names: Schnor
+    email: schnor@cs.uni-potsdam.de
+    affiliation: University of Potsdam
+    orcid: 'https://orcid.org/0000-0001-7369-8057'
+  - given-names: Hans
+    family-names: Straile
+identifiers:
+  - type: doi
+    value: 10.5194/gmd-14-7391-2021
+    description: >-
+      POET (v0.1): speedup of many-core parallel reactive
+      transport simulations with fast DHT lookups
+repository-code: 'https://git.gfz-potsdam.de/naaice/poet'
+abstract: >-
+  Massively parallel reactive transport simulator exploring
+  acceleration strategies such as embedding of AI/ML and
+  cache of results in Distributed Hash Tables. Developed in
+  cooperation with computer scientists of University of
+  Potsdam.
+keywords:
+  - Reactive Transport
+  - Geochemistry
+  - AI/ML Surrogate Modelling
+license: GPL-2.0

CMake/FindR.cmake

@@ -1,25 +0,0 @@
-# prepare R environment (Rcpp + RInside)
-find_program(R_EXE "R")
-
-# search for R executable, R header file and library path
-if(R_EXE)
-  execute_process(COMMAND ${R_EXE} RHOME
-    OUTPUT_VARIABLE R_ROOT_DIR
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-
-  find_path(R_INCLUDE_DIR R.h
-    HINTS ${R_ROOT_DIR}
-    PATHS /usr/inlcude /usr/local/include /usr/share
-    PATH_SUFFIXES include/R R/include
-  )
-
-  find_library(R_LIBRARY R
-    HINTS ${R_ROOT_DIR}/lib
-  )
-else()
-  message(FATAL_ERROR "No R runtime found!")
-endif()
-
-set(R_LIBRARIES ${R_LIBRARY})
-set(R_INCLUDE_DIRS ${R_INCLUDE_DIR})

CMake/FindRInside.cmake

@@ -1,23 +0,0 @@
-# find RInside libraries and include path
-execute_process(COMMAND echo "cat(find.package('RInside'))"
-  COMMAND ${R_EXE} --vanilla --slave
-  RESULT_VARIABLE RINSIDE_NOT_FOUND
-  ERROR_QUIET
-  OUTPUT_VARIABLE RINSIDE_PATH
-  OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-
-if(RInside_NOT_FOUND)
-  message(FATAL_ERROR "RInside not found!")
-endif()
-
-find_library(R_RInside_LIBRARY libRInside.so
-  HINTS ${RINSIDE_PATH}/lib)
-
-list(APPEND R_LIBRARIES ${R_RInside_LIBRARY})
-
-find_path(R_RInside_INCLUDE_DIR RInside.h
-  HINTS ${RINSIDE_PATH}
-  PATH_SUFFIXES include)
-
-list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR})

CMake/FindRRuntime.cmake

@@ -0,0 +1,77 @@
+# prepare R environment (Rcpp + RInside)
+find_program(R_EXE "R"
+  REQUIRED
+)
+
+# search for R executable, R header file and library path
+execute_process(
+  COMMAND ${R_EXE} RHOME
+  OUTPUT_VARIABLE R_ROOT_DIR
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+find_path(
+  R_INCLUDE_DIR R.h
+  HINTS /usr/include /usr/local/include /usr/share ${R_ROOT_DIR}/include
+  PATH_SUFFIXES R/include R
+  REQUIRED
+)
+
+find_library(
+  R_LIBRARY libR.so
+  HINTS ${R_ROOT_DIR}/lib
+  REQUIRED
+)
+
+set(R_LIBRARIES ${R_LIBRARY})
+set(R_INCLUDE_DIRS ${R_INCLUDE_DIR})
+
+# find Rcpp include directory
+execute_process(COMMAND Rscript -e "cat(system.file(package='Rcpp'))"
+  RESULT_VARIABLE RCPP_NOT_FOUND
+  ERROR_QUIET
+  OUTPUT_VARIABLE RCPP_PATH
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+if(RCPP_NOT_FOUND)
+  message(FATAL_ERROR "Rcpp not found!")
+endif()
+
+find_path(R_Rcpp_INCLUDE_DIR Rcpp.h
+  HINTS ${RCPP_PATH}
+  PATH_SUFFIXES include)
+
+list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR})
+
+# find RInside libraries and include path
+execute_process(COMMAND Rscript -e "cat(system.file(package='RInside'))"
+  RESULT_VARIABLE RINSIDE_NOT_FOUND
+  ERROR_QUIET
+  OUTPUT_VARIABLE RINSIDE_PATH
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+if(RInside_NOT_FOUND)
+  message(FATAL_ERROR "RInside not found!")
+endif()
+
+find_library(R_RInside_LIBRARY libRInside.so
+  HINTS ${RINSIDE_PATH}/lib)
+
+
+find_path(R_RInside_INCLUDE_DIR RInside.h
+  HINTS ${RINSIDE_PATH}
+  PATH_SUFFIXES include)
+
+list(APPEND R_LIBRARIES ${R_RInside_LIBRARY})
+list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR})
+
+# putting all together into interface library
+
+add_library(RRuntime INTERFACE IMPORTED)
+target_link_libraries(RRuntime INTERFACE ${R_LIBRARIES})
+target_include_directories(RRuntime INTERFACE ${R_INCLUDE_DIRS})
+
+unset(R_LIBRARIES)
+unset(R_INCLUDE_DIRS)

CMake/FindRcpp.cmake

@@ -1,23 +0,0 @@
-# find Rcpp include directory
-execute_process(COMMAND echo "cat(find.package('Rcpp'))"
-  COMMAND ${R_EXE} --vanilla --slave
-  RESULT_VARIABLE RCPP_NOT_FOUND
-  ERROR_QUIET
-  OUTPUT_VARIABLE RCPP_PATH
-  OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-
-if(RCPP_NOT_FOUND)
-  message(FATAL_ERROR "Rcpp not found!")
-endif()
-
-# find_library(R_Rcpp_LIBRARY Rcpp.so
-#   HINTS ${RCPP_PATH}/libs)
-
-# list(APPEND R_LIBRARIES ${R_Rcpp_LIBRARY})
-
-find_path(R_Rcpp_INCLUDE_DIR Rcpp.h
-  HINTS ${RCPP_PATH}
-  PATH_SUFFIXES include)
-
-list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR})

CMake/POET_Scripts.cmake

@@ -9,11 +9,11 @@ macro(get_POET_version)
       OUTPUT_VARIABLE POET_GIT_BRANCH
       OUTPUT_STRIP_TRAILING_WHITESPACE)
     execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --always
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always
       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
       OUTPUT_VARIABLE POET_GIT_VERSION
       OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if(NOT POET_GIT_BRANCH STREQUAL "master")
+    if(NOT POET_GIT_BRANCH STREQUAL "main")
       set(POET_VERSION "${POET_GIT_BRANCH}/${POET_GIT_VERSION}")
     else()
       set(POET_VERSION "${POET_GIT_VERSION}")
@@ -21,7 +21,7 @@ macro(get_POET_version)
   elseif(EXISTS ${PROJECT_SOURCE_DIR}/.svn)
     file(STRINGS .gitversion POET_VERSION)
   else()
-    set(POET_VERSION "0.1")
+    set(POET_VERSION "not_versioned")
   endif()
 
   message(STATUS "Configuring POET version ${POET_VERSION}")

CMakeLists.txt

@@ -1,31 +1,54 @@
-# Version 3.9+ offers new MPI package variables
+cmake_minimum_required(VERSION 3.20)
-cmake_minimum_required(VERSION 3.9)
 
-project(POET CXX C)
+project(POET
+  LANGUAGES CXX C
+  DESCRIPTION "A coupled reactive transport simulator")
 
 # specify the C++ standard
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
+set(DEFAULT_BUILD_TYPE "Release")
+if(NOT CMAKE_BUILD_TYPE)
+  message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}'.")
+  set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE
+    STRING "Choose the type of build." FORCE)
+  # Set the possible values of build type for cmake-gui
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
+    "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
+
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 include("CMake/POET_Scripts.cmake")
 list(APPEND CMAKE_MODULE_PATH "${POET_SOURCE_DIR}/CMake")
 
-# set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") add_definitions(${GCC_CXX_FLAGS})
+get_poet_version()
 
 find_package(MPI REQUIRED)
 
-find_package(R REQUIRED)
+find_package(RRuntime REQUIRED)
-find_package(Rcpp REQUIRED)
-find_package(RInside REQUIRED)
 
 add_subdirectory(src)
-add_subdirectory(R_lib)
-add_subdirectory(data)
 
-option(BUILD_DOC "Build documentation with doxygen" ON)
+option(POET_PREPROCESS_BENCHS "Preprocess benchmarks" ON)
 
-if(BUILD_DOC)
+if (POET_PREPROCESS_BENCHS)
-  add_subdirectory(docs)
+  add_subdirectory(bench)
-endif(BUILD_DOC)
+endif()
+
+# as tug will also pull in doctest as a dependency
+set(TUG_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
+
+add_subdirectory(ext/tug EXCLUDE_FROM_ALL)
+add_subdirectory(ext/litephreeqc EXCLUDE_FROM_ALL)
+
+option(POET_ENABLE_TESTING "Build test suite for POET" OFF)
+
+if (POET_ENABLE_TESTING)
+  add_subdirectory(test)
+endif()
+
+option(BUILD_DOC "Build documentation with doxygen" OFF)
+
+add_subdirectory(docs)

LICENSE

@@ -1,8 +1,8 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -278,3 +278,61 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
                      END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Moe Ghoul>, 1 April 1989
+  Moe Ghoul, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

README.md

@@ -1,86 +1,121 @@
-
-<!--
-    Time-stamp: "Last modified 2021-02-08 13:46:00 mluebke"
--->
-
 # POET
 
-POET is a coupled reactive transport simulator implementing a parallel
+**NOTE: GFZ is migrating its domain from <gfz-potsdam.de> to <gfz.de>.
-architecture and a fast, original MPI-based Distributed Hash Table.
+This should be finalized by the end of 2025. We adopt the NEW domain
+in all the links given below. If you encounter 'unreachable address'
+try the OLD domain.**
+
+[POET](https://doi.org/10.5281/zenodo.4757913) is a coupled reactive
+transport simulator implementing a parallel architecture and a fast,
+original MPI-based Distributed Hash Table.
+
+![POET's Coupling Scheme](./docs/POET_scheme.svg)
+
+## Parsed code documentiation
+
+A parsed version of POET's documentation can be found at [Gitlab
+pages](https://naaice.git-pages.gfz.de/poet).
 
 ## External Libraries
 
-The following external header library is shipped with POET:
+The following external libraries are shipped with POET:
 
-- **argh** - https://github.com/adishavit/argh (BSD license)
+- **CLI11** - <https://github.com/CLIUtils/CLI11>
+- **litephreeqc**: IPhreeqc
+   (<https://github.com/usgs-coupled/iphreeqc>) with patches from
+   GFZ/UP: <https://git.gfz.de/naaice/litephreeqc>
+- **tug** - <https://git.gfz.de/naaice/tug>
 
 ## Installation
 
 ### Requirements
 
-To compile POET you need several software to be installed:
+To compile POET you need following software to be installed:
 
 - C/C++ compiler (tested with GCC)
 - MPI-Implementation (tested with OpenMPI and MVAPICH)
-- R language and environment
+- CMake 3.20+
-- CMake 3.9+
+- Eigen3 3.4+ (required by `tug`)
+- *optional*: `doxygen` with `dot` bindings for documentation
+- R language and environment including headers or `-dev` packages
+  (distro dependent)
 
-If you want to build documentation during compilation, `doxygen`and `graphviz` must be provided too.
+The following R packages (and their dependencies) must also be
+installed:
 
-The following R libraries must then be installed, which will get the needed dependencies automatically:
-
-- [devtools](https://www.r-project.org/nosvn/pandoc/devtools.html)
 - [Rcpp](https://cran.r-project.org/web/packages/Rcpp/index.html)
 - [RInside](https://cran.r-project.org/web/packages/RInside/index.html)
-- [RedModRphree](https://git.gfz-potsdam.de/delucia/RedModRphree)
+- [qs](https://cran.r-project.org/web/packages/qs/index.html)
-- [Rmufits](https://git.gfz-potsdam.de/delucia/Rmufits)
+- [qs2](https://cran.r-project.org/web/packages/qs2/index.html)
+This can be simply achieved by issuing the following commands:
+
+```sh
+# start R environment
+$ R
+
+# install R dependencies (case sensitive!)
+> install.packages(c("Rcpp", "RInside","qs","qs2"))
+> q(save="no")
+```
+
+### Clone the repository
+
+POET can be anonimously cloned from this repo over https. Make sure to 
+also download the submodules:
+
+```sh
+git clone --recurse-submodules https://git.gfz.de/naaice/poet.git
+```
+The `--recurse-submodules` option is a shorthand for:
+```sh
+cd poet
+git submodule init && git submodule update
+```
 
 ### Compiling source code
 
-The generation of makefiles is done with CMake. If you obtained POET from git, you should be able to generate Makefiles by running
+POET is built with CMake. You can generate Makefiles by running the
+usual:
 
 ```sh
 mkdir build && cd build
-cmake ..
+cmake -DCMAKE_BUILD_TYPE=Release ..
 ```
 
-This will create the directory `build` and processes the CMake files and generate Makefiles from it. You're now able to run `make` to start build
+This will create the directory `build` and processes the CMake files
-process.
+and generate Makefiles from it. You're now able to run `make` to start
+build process.
 
-If POET was obtained from the official SVN repository or the redmine at <https://redmine.cs.uni-potsdam.de/projects/poet> the branch or tag to be used have to be set via
+If everything went well you'll find the executables at
+`build/src/poet`, but it is recommended to install the POET project
+structure to a desired `CMAKE_INSTALL_PREFIX` with `make install`.
 
-```sh
+During the generation of Makefiles, various options can be specified
-mkdir build && cd build
+via `cmake -D <option>=<value> [...]`. Currently, there are the
-cmake -D POET_SET_BRANCH="<BRANCH>" ..
+following available options:
-```
 
-where currently available branches/tags are:
+- **POET_DHT_Debug**=_boolean_ - toggles the output of detailed
-
+  statistics about DHT usage. Defaults to _OFF_.
-- dev
+- **POET_ENABLE_TESTING**=_boolean_ - enables small set of unit tests
-
+  (more to come). Defaults to _OFF_.
-If everything went well you'll find the executable at `build/src/poet`, but it is recommended to install the POET project structure to a desired `CMAKE_INSTALL_PREFIX` with `make install`.
+- **POET_PHT_ADDITIONAL_INFO**=_boolean_ - enabling the count of
-
+  accesses to one PHT bucket. Use with caution, as things will get
-During the generation of Makefiles, various options can be specified via `cmake -D <option>=<value> [...]`. Currently there are the following available options:
+  slowed down significantly. Defaults to _OFF_.
-
+- **POET_PREPROCESS_BENCHS**=*boolean* - enables the preprocessing of
-- **DHT_Debug**=_boolean_ - toggles the output of detailed statistics about DHT
+  predefined models/benchmarks. Defaults to *ON*.
-  usage (`cmake -D DHT_Debug=ON`). Defaults to _OFF_.
-- **BUILD_DOC**=_boolean_ - toggles the generation of documantiation during
-  compilation process. Defaults to _ON_.
-- _only from svn version:_ **POET_SET_BRANCH**=_string_ - set branch or tag whose code is used
   
 ### Example: Build from scratch
 
-Assuming that only the C/C++ compiler, MPI libraries, R runtime environment and
+Assuming that only the C/C++ compiler, MPI libraries, R runtime
-CMake have been installed, POET can be installed as follows:
+environment and CMake have been installed, POET can be installed as
+follows:
 
 ```sh
 # start R environment
 $ R
 
 # install R dependencies
-> install.packages(c("devtools", "Rcpp", "RInside"))
+> install.packages(c("Rcpp", "RInside","qs","qs2"))
-> devtools::install_gitlab("delucia/RedModRphree", host="https://git.gfz-potsdam.de")
-> devtools::install_gitlab("delucia/Rmufits", host="https://git.gfz-potsdam.de")
 > q(save="no")
 
 # cd into POET project root
@@ -88,75 +123,79 @@ $ cd <POET_dir>
 
 # Build process
 $ mkdir build && cd build
-$ cmake -DCMAKE_INSTALL_PREFIX=/home/<user>/poet ..
+$ cmake -DCMAKE_INSTALL_PREFIX=/home/<user>/poet -DCMAKE_BUILD_TYPE=Release ..
 $ make -j<max_numprocs>
 $ make install
 ```
 
-This will install a POET project structure into `/home/<user>/poet` which is
+This will install a POET project structure into `/home/<user>/poet`
-called hereinafter `<POET_INSTALL_DIR>`. With this version of POET we **do not
+which is called hereinafter `<POET_INSTALL_DIR>`. With this version of
-recommend** to install to hierarchies like `/usr/local/` etc.
+POET we **do not recommend** to install to hierarchies like
+`/usr/local/` etc.
 
 The correspondending directory tree would look like this:
 
 ```sh
-.
+poet
-└── poet/
+├── bin
-    ├── bin/
+│   ├── poet
-    │   └── poet
+│   └── poet_init
-    ├── data/
+└── share
-    │   └── SimDol2D.R
+    └── poet
-    ├── docs/
+        ├── barite
-    │    └── html/
+        │   ├── barite_200.qs2
-    │       ├── index.html
+        │   ├── barite_200_rt.R
-    │       └── ...
+        │   ├── barite_het.qs2
-    └── R_lib/
+        │   └── barite_het_rt.R
-        ├── kin_r_library.R
+        ├── dolo
-        └── parallel_r_library.R
+        │   ├── dolo_inner_large.qs2
+        │   ├── dolo_inner_large_rt.R
+        │   ├── dolo_interp.qs2
+        │   └── dolo_interp_rt.R
+        └── surfex
+            ├── PoetEGU_surfex_500.qs2
+            └── PoetEGU_surfex_500_rt.R
 ```
 
-The R libraries will be loaded at runtime and the paths are hardcoded
+With the installation of POET, two executables are provided: 
-absolute paths inside `poet.cpp`. So, if you consider to move `bin/poet` either
+  - `poet` - the main executable to run simulations
-change paths of the R source files and recompile POET or also move `R_lib/*`
+  - `poet_init` - a preprocessor to generate input files for POET from
-according to the binary.
+    R scripts
 
-To display the generated html documentation just open `docs/html/index.html`
+Preprocessed benchmarks can be found in the `share/poet` directory
-with the browser of your choice.
+with an according *runtime* setup. More on those files and how to
+create them later.
 
 ## Running
 
-Before POET is ready to run, a working directory must be created. In this
+Run POET by `mpirun ./poet [OPTIONS] <RUNFILE> <SIMFILE>
-directory you should find the executable file, the R scripts
+<OUTPUT_DIRECTORY>` where:
-`<POET_ROOT>/R_lib/kin_r_library.R` and `<POET_ROOT>/R_lib/parallel_r_library.R`
-and the simulation description e.g. `<POET_ROOT>/data/chem_problems/SimDol2D.R`.
 
-Run POET by `mpirun ./poet <OPTIONS> <SIMFILE> <OUTPUT_DIRECTORY>` where:
+- **OPTIONS** - POET options (explained below)
+- **RUNFILE** - Runtime parameters described as R script
+- **SIMFILE** - Simulation input prepared by `poet_init`
+- **OUTPUT_DIRECTORY** - path, where all output of POET should be
+  stored
 
-- **OPTIONS** - runtime parameters (explained below)
+### POET command line arguments
-- **SIMFILE** - simulation described as R script (currently supported:
-  `<POET_INSTALL_DIR>/data/SimDol2D.R`)
-- **OUTPUT_DIRECTORY** - path, where all output of POET should be stored
-
-### Runtime options
 
 The following parameters can be set:
 
-| Option                   | Value        | Description                                                    |
+| Option                      | Value        | Description                                                                      |
-| ------------------------ | ------------ | -------------------------------------------------------------- |
+|-----------------------------|--------------|----------------------------------------------------------------------------------|
-| **--work-package-size=** | _1..n_       | size of work packages (defaults to _5_)                        |
+| **--work-package-size=**    | _1..n_       | size of work packages (defaults to _5_)                                          |
-| **--ignore-result**      |              | disables store of simulation resuls                            |
+| **-P, --progress**          |              | show progress bar                                                                |
-| **--dht**                |              | enabling DHT usage (defaults to _OFF_)                         |
+| **--ai-surrogate**          |              | activates the AI surrogate chemistry model (defaults to _OFF_)                   |
-| **--dht-nolog**          |              | disabling applying of logarithm before rounding                |
+| **--dht**                   |              | enabling DHT usage (defaults to _OFF_)                                           |
-| **--dht-signif=**        | _1..n_       | set rounding to number of significant digits (defaults to _5_) |
+| **--qs**                    |              | store results using qs::qsave() (.qs extension) instead of default qs2 (.qs2)    |
-| **--dht-strategy=**      | _0-1_        | change DHT strategy. **NOT IMPLEMENTED YET** (Defaults to _0_) |
+| **--rds**                   |              | store results using saveRDS() (.rds extension) instead of default qs2 (.qs2)    |
-| **--dht-size=**          | _1-n_        | size of DHT per process involved in byte (defaults to _1 GiB_) |
+| **--dht-strategy=**         | _0-1_        | change DHT strategy. **NOT IMPLEMENTED YET** (Defaults to _0_)                   |
-| **--dht-snaps=**         | _0-2_        | disable or enable storage of DHT snapshots                     |
+| **--dht-size=**             | _1-n_        | size of DHT per process involved in megabyte (defaults to _1000 MByte_)          |
-| **--dht-file=**          | `<SNAPSHOT>` | initializes DHT with the given snapshot file                   |
+| **--dht-snaps=**            | _0-2_        | disable or enable storage of DHT snapshots                                       |
-
+| **--dht-file=**             | `<SNAPSHOT>` | initializes DHT with the given snapshot file                                     |
-#### Additions to `dht-signif`
+| **--interp-size**           | _1-n_        | size of PHT (interpolation) per process in megabyte                              |
-
+| **--interp-bucket-entries** | _1-n_        | number of entries to store at maximum in one PHT bucket                          |
-Only used if no vector is given in setup file. For individual values per column
+| **--interp-min**            | _1-n_        | number of entries in PHT bucket needed to start interpolation                    |
-use R vector `signif_vector` in `SIMFILE`.
 
 #### Additions to `dht-snaps`
 
@@ -170,32 +209,153 @@ Following values can be set:
 
 ### Example: Running from scratch
 
-We will continue the above example and start a simulation with `SimDol2D.R`,
+We will continue the above example and start a simulation with
-which is the only simulation supported at this moment. The required flow velocities 
+*barite_het*, which simulation files can be found in
-snapshots are included in the R package Rmufits. It's a 2D, 50x50 grid, with 20 time 
+`<INSTALL_DIR>/share/poet/barite/barite_het*`. As transport a
-steps. To start the simulation with 4 processes `cd` into your previously installed 
+heterogeneous diffusion is used. It's a small 2D grid, 2x5 grid,
-POET-dir `<POET_INSTALL_DIR>/bin` and run:
+simulating 50 time steps with a time step size of 100 seconds. To
+start the simulation with 4 processes `cd` into your previously
+installed POET-dir `<POET_INSTALL_DIR>/bin` and run:
 
 ```sh
-mpirun -n 4 ./poet ../data/SimDol2D.R output
+cp ../share/poet/barite/barite_het* .
+mpirun -n 4 ./poet barite_het_rt.R barite_het.qs2 output
 ```
 
-After a finished simulation all data generated by POET will be found in the
+After a finished simulation all data generated by POET will be found
-directory `output`.
+in the directory `output`.
 
 You might want to use the DHT to cache previously simulated data and
-reuse them in further time-steps. Just append `--dht` to the options of POET to
+reuse them in further time-steps. Just append `--dht` to the options
-activate the usage of the DHT. The resulting call would look like this:
+of POET to activate the usage of the DHT. Also, after each iteration a
+DHT snapshot shall be produced. This is done by appending the
+`--dht-snaps=<value>` option. The resulting call would look like this:
 
 ```sh
-mpirun -n 4 ./poet --dht SimDol2D.R output
+mpirun -n 4 ./poet --dht --dht-snaps=2 barite_het_rt.R barite_het.qs2 output
 ```
 
+### Example: Preparing Environment and Running with AI surrogate
+
+To run the AI surrogate, you need to install the R package `keras3`. The
+compilation process of POET remains the same as shown above.
+
+In the following code block, the installation process on the Turing Cluster is
+shown. `miniconda` is used to create a virtual environment to install
+tensorflow/keras. Please adapt the installation process to your needs.
+
+<!-- Start an R interactive session and install the required packages: -->
+
+```sh
+# First, install the required R packages
+R -e "install.packages('keras3', repos='https://cloud.r-project.org/')"
+
+# manually create a virtual environment to install keras/python using conda, 
+# as this is somehow broken on the Turing Cluster when using the `keras::install_keras()` function
+cd poet
+
+# create a virtual environment in the .ai directory with python 3.11
+conda create -p ./.ai python=3.11
+conda activate ./.ai
+
+# install tensorflow and keras
+pip install keras tensorflow[and-cuda]
+
+# add conda's python path to the R environment
+# make sure to have the conda environment activated
+echo -e "RETICULATE_PYTHON=$(which python)\n" >> ~/.Renviron
+```
+
+After setup the R environment, recompile POET and you're ready to run the AI
+surrogate.
+
+```sh
+cd <installation_dir>/bin
+
+# copy the benchmark files to the installation directory
+cp <project_root_dir>/bench/barite/{barite_50ai*,db_barite.dat,barite.pqi} .
+
+# preprocess the benchmark
+./poet_init barite_50ai.R
+
+# run POET with AI surrogate and GPU utilization
+srun --gres=gpu -N 1 -n 12 ./poet --ai-surrogate barite_50ai_rt.R barite_50ai.qs2 output
+```
+
+Keep in mind that the AI surrogate is currently not stable or might also not
+produce any valid predictions.
+
+## Defining a model
+
+In order to provide a model to POET, you need to setup a R script
+which can then be used by `poet_init` to generate the simulation
+input. Which parameters are required can be found in the
+[Wiki](https://git.gfz.de/naaice/poet/-/wikis/Initialization).
+We try to keep the document up-to-date. However, if you encounter
+missing information or need help, please get in touch with us via the
+issue tracker or E-Mail.
+
+`poet_init` can be used as follows:
+
+```sh
+./poet_init [-o, --output output_file] [-s, --setwd]  <script.R>
+```
+
+where: 
+
+- **output** - name of the output file (defaults to the input file
+  name with the extension `.qs2`)
+- **setwd** - set the working directory to the directory of the input
+  file (e.g. to allow relative paths in the input script). However,
+  the output file will be stored in the directory from which
+  `poet_init` was called.
+
 ## About the usage of MPI_Wtime()
 
-Implemented time measurement functions uses `MPI_Wtime()`. Some important
+Implemented time measurement functions uses `MPI_Wtime()`. Some
-information from the OpenMPI Man Page:
+important information from the OpenMPI Man Page:
 
-For example, on platforms that support it, the clock_gettime() function will be
+For example, on platforms that support it, the clock_gettime()
-used to obtain a monotonic clock value with whatever precision is supported on
+function will be used to obtain a monotonic clock value with whatever
-that platform (e.g., nanoseconds).
+precision is supported on that platform (e.g., nanoseconds).
+
+## Additional functions for the AI surrogate
+
+The AI surrogate can be activated for any benchmark and is by default
+initiated as a sequential keras model with three hidden layer of depth
+48, 96, 24 with relu activation and adam optimizer. All functions in
+`ai_surrogate_model.R` can be overridden by adding custom definitions
+via an R file in the input script. This is done by adding the path to
+this file in the input script. Simply add the path as an element
+called `ai_surrogate_input_script` to the `chemistry_setup` list.
+Please use the global variable `ai_surrogate_base_path` as a base path
+when relative filepaths are used in custom funtions.
+
+**There is currently no default implementation to determine the
+validity of predicted values.** This means, that every input script
+must include an R source file with a custom function
+`validate_predictions(predictors, prediction)`. Examples for custom
+functions can be found for the barite_200 benchmark
+
+The functions can be defined as follows:
+
+`validate_predictions(predictors, prediction)`: Returns a boolean
+index vector that signals for each row in the predictions if the
+values are considered valid. Can eg. be implemented as a mass balance
+threshold between the predictors and the prediction.
+
+`initiate_model()`: Returns a keras model. Can be used to load
+pretrained models.
+
+`preprocess(df, backtransform = FALSE, outputs = FALSE)`: Returns the
+scaled/transformed/backtransformed dataframe. The `backtransform` flag
+signals if the current processing step is applied to data that's
+assumed to be scaled and expects backtransformed values. The `outputs`
+flag signals if the current processing step is applied to the output
+or tatget of the model. This can be used to eg. skip these processing
+steps and only scale the model input.
+
+`training_step (model, predictor, target, validity)`: Trains the model
+after each iteration. `validity` is the bool index vector given by
+`validate_predictions` and can eg. be used to only train on values
+that have not been valid predictions.

R_lib/CMakeLists.txt

@@ -1 +0,0 @@
-install(FILES kin_r_library.R parallel_r_library.R DESTINATION R_lib)

R_lib/ai_surrogate_model.R

@@ -0,0 +1,75 @@
+## This file contains default function implementations for the ai surrogate.
+## To load pretrained models, use pre-/postprocessing or change hyperparameters
+## it is recommended to override these functions with custom implementations via
+## the input script. The path to the R-file containing the functions mus be set
+## in the variable "ai_surrogate_input_script". See the barite_200.R file as an
+## example and the general README for more information.
+
+require(keras3)
+require(tensorflow)
+
+initiate_model <- function() {
+  hidden_layers <- c(48, 96, 24)
+  activation <- "relu"
+  loss <- "mean_squared_error"
+
+  input_length <- length(ai_surrogate_species)
+  output_length <- length(ai_surrogate_species)
+  ## Creates a new sequential model from scratch
+  model <- keras_model_sequential()
+
+  ## Input layer defined by input data shape
+  model %>% layer_dense(units = input_length,
+                        activation = activation,
+                        input_shape = input_length,
+                        dtype = "float32")
+
+  for (layer_size in hidden_layers) {
+    model %>% layer_dense(units = layer_size,
+                          activation = activation,
+                          dtype = "float32")
+  }
+
+  ## Output data defined by output data shape
+  model %>% layer_dense(units = output_length,
+                        activation = activation,
+                        dtype = "float32")
+
+  model %>% compile(loss = loss,
+                    optimizer = "adam")
+  return(model)
+}
+
+gpu_info <- function() {
+  msgm(tf_gpu_configured())
+}
+
+prediction_step <- function(model, predictors) {
+  prediction <- predict(model, as.matrix(predictors))
+  colnames(prediction) <- colnames(predictors)
+  return(as.data.frame(prediction))
+}
+
+preprocess <- function(df, backtransform = FALSE, outputs = FALSE) {
+  return(df)
+}
+
+postprocess <- function(df, backtransform = TRUE, outputs = TRUE) {
+  return(df)
+}
+
+set_valid_predictions <- function(temp_field, prediction, validity) {
+  temp_field[validity == 1, ] <- prediction[validity == 1, ]
+  return(temp_field)
+}
+
+training_step <- function(model, predictor, target, validity) {
+  msgm("Training:")
+
+  x <- as.matrix(predictor)
+  y <- as.matrix(target[colnames(x)])
+
+  model %>% fit(x, y)
+
+  model %>% save_model_tf(paste0(out_dir, "/current_model.keras"))
+}

R_lib/init_r_lib.R

@@ -0,0 +1,112 @@
+### Copyright (C) 2018-2024 Marco De Lucia, Max Luebke (GFZ Potsdam, University of Potsdam)
+###
+### POET is free software; you can redistribute it and/or modify it under the
+### terms of the GNU General Public License as published by the Free Software
+### Foundation; either version 2 of the License, or (at your option) any later
+### version.
+###
+### POET is distributed in the hope that it will be useful, but WITHOUT ANY
+### WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+### A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License along with
+### this program; if not, write to the Free Software Foundation, Inc., 51
+### Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+##' @param pqc_mat matrix, containing IDs and PHREEQC outputs
+##' @param grid matrix, zonation referring to pqc_mat$ID
+##' @return a data.frame
+# pqc_to_grid <- function(pqc_mat, grid) {
+#     # Convert the input DataFrame to a matrix
+#     pqc_mat <- as.matrix(pqc_mat)
+
+#     # Flatten the matrix into a vector
+#     id_vector <- as.integer(t(grid))
+
+#     # Find the matching rows in the matrix
+#     row_indices <- match(id_vector, pqc_mat[, "ID"])
+
+#     # Extract the matching rows from pqc_mat to size of grid matrix
+#     result_mat <- pqc_mat[row_indices, ]
+
+#     # Convert the result matrix to a data frame
+#     res_df <- as.data.frame(result_mat)
+
+#     # Remove all columns which only contain NaN
+#     res_df <- res_df[, colSums(is.na(res_df)) != nrow(res_df)]
+
+#     # Remove row names
+#     rownames(res_df) <- NULL
+
+#     return(res_df)
+# }
+
+##' @param pqc_mat matrix, containing IDs and PHREEQC outputs 
+##' @param grid matrix, zonation referring to pqc_mat$ID 
+##' @return a data.frame
+pqc_to_grid <- function(pqc_mat, grid) {
+    # Convert the input DataFrame to a matrix
+    pqc_mat <- as.matrix(pqc_mat)
+
+    # Flatten the matrix into a vector
+    id_vector <- as.integer(t(grid))
+
+    # Find the matching rows in the matrix
+    row_indices <- match(id_vector, pqc_mat[, "ID"])
+
+    # Extract the matching rows from pqc_mat to size of grid matrix
+    result_mat <- pqc_mat[row_indices, ]
+
+    # Convert the result matrix to a data frame
+    res_df <- as.data.frame(result_mat)
+
+    # Remove all columns which only contain NaN
+    # res_df <- res_df[, colSums(is.na(res_df)) != nrow(res_df)]
+
+    # Remove row names
+    rownames(res_df) <- NULL
+
+    return(res_df)
+}
+
+
+##' @param pqc_mat matrix, 
+##' @param transport_spec column name of species in pqc_mat
+##' @param id
+##' @title 
+##' @return 
+resolve_pqc_bound <- function(pqc_mat, transport_spec, id) {
+    df <- as.data.frame(pqc_mat, check.names = FALSE)
+    value <- df[df$ID == id, transport_spec]
+
+    if (is.nan(value)) {
+        value <- 0
+    }
+
+    return(value)
+}
+
+##' @title 
+##' @param init_grid 
+##' @param new_names 
+##' @return 
+add_missing_transport_species <- function(init_grid, new_names) {
+    # add 'ID' to new_names front, as it is not a transport species but required
+    new_names <- c("ID", new_names)
+    sol_length <- length(new_names)
+
+    new_grid <- data.frame(matrix(0, nrow = nrow(init_grid), ncol = sol_length))
+    names(new_grid) <- new_names
+
+    matching_cols <- intersect(names(init_grid), new_names)
+
+    # Copy matching columns from init_grid to new_grid
+    new_grid[, matching_cols] <- init_grid[, matching_cols]
+
+
+    # Add missing columns to new_grid
+    append_df <- init_grid[, !(names(init_grid) %in% new_names)]
+    new_grid <- cbind(new_grid, append_df)
+
+    return(new_grid)
+}

R_lib/kin_r_library.R

@@ -1,4 +1,4 @@
-### Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
+### Copyright (C) 2018-2023 Marco De Lucia, Max Luebke (GFZ Potsdam)
 ###
 ### POET is free software; you can redistribute it and/or modify it under the
 ### terms of the GNU General Public License as published by the Free Software
@@ -13,665 +13,199 @@
 ### this program; if not, write to the Free Software Foundation, Inc., 51
 ### Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
-## Simple function to check file extension. It is needed to check if
+master_init <- function(setup, out_dir, init_field) {
-## the GridFile is SUM (MUFITS format) or rds/RData
-FileExt <- function (x)
-{
-    pos <- regexpr("\\.([[:alnum:]]+)$", x)
-    ifelse(pos > -1L, substring(x, pos + 1L), "")
-}
-
-### This function is only called by the master process. It sets up
-### the data structures for the coupled simulations and performs the
-### first timestep or "iteration zero", possibly involving the phreeqc
-### calculation of the initial state (if it is not already provided as
-### matrix) and the first Advection iteration
-master_init <- function(setup)
-{
-    msgm("Process with rank 0 reading GRID and MUFITS_sims")
-
-    ## MDL TODO: actually grid and all the MUFITS snapshots should be
-    ## read and stored only by the master process!!
-
     ## Setup the directory where we will store the results
-    verb <- FALSE
+    if (!dir.exists(out_dir)) {
-    if (local_rank==0) {
+        dir.create(out_dir)
-        verb <- TRUE ## verbosity loading MUFITS results
+        msgm("created directory ", out_dir)
-        if (!dir.exists(fileout)) {
-            dir.create(fileout)
-            msgm("created directory ", fileout)
-        } else {
-            msgm("dir ", fileout," already exists, I will overwrite!")
-        }
-        if (!exists("store_result"))
-            msgm("store_result doesn't exist!")
-        else
-            msgm("store_result is ", store_result)
     } else {
-        
+        msgm("dir ", out_dir, " already exists, I will overwrite!")
     }
-
+    if (is.null(setup$store_result)) {
-    ## to enhance flexibility, gridfile can be now given in SUM or,
+        msgm("store_result doesn't exist!")
-    ## already processed, in rds format. We check for extension first. 
-    gridext <- FileExt(setup$gridfile)
-    if (gridext=="SUM") {
-        msgm("Generating grid from MUFITS SUM file...")
-        mufits_grid <- ReadGrid(setup$gridfile, verbose=verb)
     } else {
-        msgm("Reading grid from rds file...")
+        msgm("store_result is ", setup$store_result)
-        mufits_grid <- readRDS(setup$gridfile)
-    }
-
-    ## load all the snapshots at once. The setup argument "snapshots"
-    ## now can be either a *directory* where the .SUM files are stored
-    ## or a rds file.
-    if (dir.exists(setup$snapshots)) {
-        msgm("<snapshots> points to a directory; reading from SUM files in there...")
-        mufits_sims <- LoadMufitsSumRes(dir=setup$snapshots, verbose=verb)
-    } else {
-        msgm("<snapshots> points to a file. Reading as rds...")
-        mufits_sims <- readRDS(setup$snapshots)
-    }
-    
-    ## Since this function is evaluated by the R process called from
-    ## the C++ program, we need to make available all these variables
-    ## to the R parent frame!
-    assign("mufits_grid", mufits_grid, pos=parent.frame())
-    assign("mufits_sims", mufits_sims, pos=parent.frame())
-    ## cat(local_rank, "assignement complete\n")
-
-
-    ## we calculate the *coupling* iterations
-    nstep <- length(mufits_sims)
-    msgm("using", nstep,"flow snapshots")
-    ## cat(local_rank, "nstep:", nstep, "\n")
-    ## cat(local_rank, "names:", paste(names(mufits_sims), collate=","), "\n")
-
-    ## we have these snapshots; we output the results of the coupling
-    ## after each timesteps
-    timesteps <- diff(sapply(mufits_sims, function(x) {return(x$info)}))
-    ## cat(local_rank, "timesteps:", paste0(timesteps, collate=" "), "\n")
-
-    dt_differ <- abs(max(timesteps) - min(timesteps)) != 0
-
-    maxiter <- length(timesteps)
-
-
-    ## steady state after last flow snapshot? It is controlled by
-    ## "setup$prolong", containing an integer which represents the
-    ## number of further iterations
-    setup$steady <- FALSE
-    if ("prolong" %in% names(setup)) {
-        
-        last_dt <- timesteps[length(timesteps)]
-        timesteps <- c(timesteps, rep(last_dt, setup$prolong))
-        msgm("simulation prolonged for", setup$prolong, "additional iterations")
-        ## we set this flag to TRUE to check afterwards which snapshot we need to use at each iteration 
-        setup$steady <- TRUE
-        setup$last_snapshot <- maxiter
-        maxiter <- maxiter + setup$prolong
-
-    }
-
-    ## now that we know how many iterations we're gonna have, we can
-    ## setup the optional output
-    if (local_rank==0) {
-        if (is.null(setup$iter_output)) {
-            ## "iter_output" is not specified: store all iterations
-            setup$out_save <- seq(1,maxiter)
-            msgm("setup$iter_output unspecified, storing all iterations")
-        } else if (setup$iter_output=="all") {
-            ## "iter_output" is "all": store all iterations
-            setup$out_save <- seq(1,maxiter)
-            msgm("storing all iterations")
-        } else if (is.numeric(setup$iter_output)) {
-            msgm("storing iterations:", paste(setup$iter_output, collapse=", "))
-            setup$out_save <- as.integer(setup$iter_output)
-        } else if (setup$iter_output=="last") {
-            msgm("storing only the last iteration")
-            setup$out_save <- maxiter
-        } else {## fallback to "all"
-            setup$out_save <- seq(1,maxiter)
-            msgm("invalid setup$iter_output: storing all iterations")
-        }
     }
 
     setup$iter <- 1
-    setup$maxiter <- maxiter
+    setup$timesteps <- setup$timesteps
-    setup$timesteps <- timesteps
+    setup$maxiter <- length(setup$timesteps)
+    setup$iterations <- setup$maxiter
     setup$simulation_time <- 0
-    setup$dt_differ <- dt_differ
 
-    if (nrow(setup$bound)==1) {
+    dgts <- as.integer(ceiling(log10(setup$maxiter)))
-        boundmatAct <- t(RedModRphree::pH2Act(setup$bound))
+    ## string format to use in sprintf
-        msg("formed correct matrix from setup$bound:")
+    fmt <- paste0("%0", dgts, "d")
-        print(boundmatAct)
+
-    } else {
+    if (is.null(setup[["store_result"]])) {
-        boundmatAct <- RedModRphree::pH2Act(setup$bound)
+        setup$store_result <- TRUE
     }
 
-    setup$boundmatAct <- boundmatAct
+    if (setup$store_result) {
-    return(setup)
+        init_field_out <- paste0(out_dir, "/iter_", sprintf(fmt = fmt, 0), ".", setup$out_ext)
-}
+        init_field <- data.frame(init_field, check.names = FALSE)
-
+        SaveRObj(x = init_field, path = init_field_out)
-## This function is called by all processes.
+        msgm("Stored initial field in ", init_field_out)
-## Since the worker processes need state_C in worker.cpp -> worker_function()
+        if (is.null(setup[["out_save"]])) {
-## even the worker need to run this code.
+            setup$out_save <- seq(1, setup$iterations)
-## TODO: worker.cpp -> worker_function() --> state_C is only needed to obtain headers 
-## and size of dataframe ... if this will be adjusted, this function will be 
-## unnecessary for workers
-init_chemistry <- function(setup) 
-{
-    ## setup the chemistry
-    if (!is.matrix(setup$initsim)) {
-        msgm("initial state defined through PHREEQC simulation, assuming homogeneous medium!")
-        tmpfirstrun <- RunPQC(setup$initsim, second=TRUE)
-        
-        ## if (local_rank==0){
-        ##     print(tmpfirstrun)
-        ## }
-        remcolnames <- colnames(tmpfirstrun)
-        if (nrow(tmpfirstrun) > 1) {
-            ## msgm("Iter 0 selecting second row")
-            firstrun <- matrix(tmpfirstrun[2,], nrow=1, byrow=TRUE)
-            colnames(firstrun) <- remcolnames
-        } else {
-            firstrun  <- tmpfirstrun
         }
-        state_C <- matrix(rep(firstrun,setup$n), byrow=TRUE, ncol=length(setup$prop))
-        colnames(state_C) <- colnames(firstrun)
-
-        ## if (local_rank==0)
-        ##     saveRDS(state_C, "initstate.rds")
-    } else {
-        msgm("given initial state")
-        state_C <- setup$initsim
     }
-    ## save state_T and state_C
+
-    setup$state_C <- state_C
+    setup$out_dir <- out_dir
 
     return(setup)
 }
 
-## relic code, may be useful in future
-finit <- function(setup)
-{
-    ## saved <- setup$saved
-    ## state_C <- setup$state_C
-    ## timesteps <- setup$timesteps
-    ## out_save <- setup$out_save
-    ## to_save <- setup$to_save
-
-    ## if (out_save) {
-    ##     msgm("saving <saved>")
-    ##     attr(saved,"savedtimesteps") <- timesteps[save]
-    ##     attr(saved,"timesteps") <- timesteps
-    ##     return(saved)
-    ## } else {    
-    ##     attr(saved,"timesteps") <- timesteps
-    ##     return(state_C)
-    ## }
-}
-
-## This function, called only by the master, computes the *inner*
-## timesteps for transport given the requested simulation timestep and
-## the current snapshot (through the Courant Fredrich Levy condition)
-## NB: we always iterate: 1)T 2)C
-master_iteration_setup <- function(setup)
-{
-    ## in this version, "grid" and "mufits_sims" are variables already
-    ## living in the environment of the R process with local_rank 0
-
-    ## if (local_rank == 0) {
-    ##     cat("Process ", local_rank, "iteration_start\n")
-    ## } else {
-    ##     cat("Process ", local_rank, "SHOULD NOT call iteration_start!\n")
-    ## }
-
-
-    ## in C++, "iter" starts from 1
-    iter <- setup$iter
-    
-    next_dt <- setup$timesteps[iter]
-
-    ## setting the current flow snapshot - we have to check if
-    ## we need prolongation or not
-    if (setup$steady) {
-        if (iter > setup$last_snapshot)
-            snap <- mufits_sims[[setup$last_snapshot]]
-        else
-            snap <- mufits_sims[[iter]]
-    } else
-        snap <- mufits_sims[[iter]]
-
-    msgm("Current simulation time:", round(setup$simulation_time),"[s] or ", round(setup$simulation_time/3600/24,2), "[day]")
-    msgm("Requested time step for current iteration:", round(next_dt),"[s] or ", round(next_dt/3600/24,2), "[day]")
-    setup$simulation_time <- setup$simulation_time+next_dt
-    msgm("Target simulation time:", round(setup$simulation_time),"[s] or ", round((setup$simulation_time)/3600/24,2), "[day]")
-
-    
-    ## since the phase and density may change in MUFITS
-    ## simulations/snapshots, we MUST tell their name at setup. Here
-    ## we match the right column for both variables
-    nphase   <- match(setup$phase,   colnames(snap$conn))
-    ndensity <- match(setup$density, colnames(snap$cell))
-
-    ## the "new cfl"
-    flux <- snap$conn[, nphase]
-    cellporvol <- mufits_grid$cell$PORO * mufits_grid$cell$CELLVOL
-    
-    ## in MUFITS, positive flux means from CELLID1 to CELLID2
-    pos_flux <- ifelse(flux>0, snap$conn$CONNID1, snap$conn$CONNID2)
-    poro <- mufits_grid$cell$PORO[pos_flux]
-    vol <- mufits_grid$cell$CELLVOL[snap$conn$CONNID1]
-
-    ## extract the right density for the right fluxes
-    density <- snap$cell[pos_flux, ndensity]
-    
-    ## transform flux from Mufits ton/day to m3/s. This is a VOLUMETRIC FLUX
-    fluxv <- flux/3600/24*1000/density
-    
-    ## now we want the darcy velocity
-    excl0 <- which(abs(fluxv)<.Machine$double.eps)
-    ## msgm("excl0"); print(excl0)
-    ## msgm("length(excl0)"); print(length(excl0))
-
-    ## The CFL condition is expressed in terms of total flux and total
-    ## pore volume
-    cfl <- as.numeric(min(abs(vol*poro/fluxv)[-excl0]))
-
-    if (!is.finite(cfl))
-        stop(msgm("CFL is ", cfl,"; quitting"))
-             
-    allowed_dt <- setup$Cf*cfl
-    requested_dt <- next_dt ## target_time - setup$simulation_time
-    msgm("CFL allows dt of <", round(allowed_dt, 2)," [s]; multiplicator is ", setup$Cf,
-         "; requested_dt is: ", round(requested_dt, 2))
-        
-    if (requested_dt > allowed_dt) {
-        inniter <- requested_dt%/%allowed_dt ## integer division
-        inner_timesteps <- c(rep(allowed_dt, inniter), requested_dt%%allowed_dt)
-        ## was: inner_timesteps <- c(rep(allowed_dt, inniter), requested_dt - allowed_dt * inniter)
-    } else {
-        inner_timesteps <- requested_dt
-        inniter <- 1
-    }
-
-    msgm("Performing ", inniter, " inner iterations")
-
-    setup$inner_timesteps <- inner_timesteps
-    setup$requested_dt <- requested_dt
-    setup$allowed_dt <- allowed_dt
-    setup$inniter <- inniter
-    setup$iter <- iter
-
-    ## TODO these 3 can be actually spared
-    setup$fluxv <- fluxv
-    ## setup$no_transport_conn <- excl0
-    setup$cellporvol <- cellporvol
-    
-    return(setup)
-}
-
-
 ## This function, called only by master, stores on disk the last
 ## calculated time step if store_result is TRUE and increments the
 ## iteration counter
-master_iteration_end <- function(setup) {
+master_iteration_end <- function(setup, state_T, state_C) {
-    iter    <- setup$iter
+    iter <- setup$iter
-    ## MDL Write on disk state_T and state_C after every iteration
+    # print(iter)
+    ## max digits for iterations
+    dgts <- as.integer(ceiling(log10(setup$maxiter + 1)))
+    ## string format to use in sprintf
+    fmt <- paste0("%0", dgts, "d")
+
+    ## Write on disk state_T and state_C after every iteration
     ## comprised in setup$out_save
-    if (store_result) {
+    if (setup$store_result) {
         if (iter %in% setup$out_save) {
-            nameout <- paste0(fileout, '/iter_', sprintf('%03d', iter), '.rds')
+            nameout <- paste0(setup$out_dir, "/iter_", sprintf(fmt = fmt, iter), ".", setup$out_ext)
-            info <- list(tr_req_dt     = as.integer(setup$requested_dt),
+            state_T <- data.frame(state_T, check.names = FALSE)
-                         tr_allow_dt   = setup$allowed_dt,
+            state_C <- data.frame(state_C, check.names = FALSE)
-                         tr_inniter    = as.integer(setup$inniter)
+
-                         )
+            ai_surrogate_info <- list(
-            saveRDS(list(T=setup$state_T, C=setup$state_C,
+                prediction_time = if (exists("ai_prediction_time")) as.integer(ai_prediction_time) else NULL,
-                         simtime=as.integer(setup$simulation_time),
+                training_time = if (exists("ai_training_time")) as.integer(ai_training_time) else NULL,
-                         tr_info=info), file=nameout)
+                valid_predictions = if (exists("validity_vector")) validity_vector else NULL
+            )
+
+            SaveRObj(x = list(
+                T = state_T,
+                C = state_C,
+                simtime = as.integer(setup$simulation_time),
+                totaltime = as.integer(totaltime),
+                ai_surrogate_info = ai_surrogate_info
+            ), path = nameout)
             msgm("results stored in <", nameout, ">")
         }
     }
-    msgm("done iteration", iter, "/", setup$maxiter)
+    ## Add last time step to simulation time
+    setup$simulation_time <- setup$simulation_time + setup$timesteps[iter]
+
+    ## msgm("done iteration", iter, "/", length(setup$timesteps))
     setup$iter <- setup$iter + 1
     return(setup)
 }
 
-## master: compute advection
-master_advection <- function(setup) {
-    msgm("requested dt=", setup$requested_dt)
-    concmat <- RedModRphree::pH2Act(setup$state_C)
-    inner_timesteps <- setup$inner_timesteps
-    Cf <- setup$Cf 
-    iter <- setup$iter
-
-    ## MDL: not used at the moment, so commenting it out
-    ## excl <- setup$no_transport_conn
-    ## msgm("excl"); print(excl)
-    immobile <- setup$immobile
-    boundmat <- setup$boundmatAct
-    
-    ## setting the current flow snapshot - we have to check if
-    ## we are in "prolongation" or not
-    if (setup$steady) {
-        if (iter > setup$last_snapshot)
-            snap <- mufits_sims[[setup$last_snapshot]]
-        else
-            snap <- mufits_sims[[iter]]
-    } else
-        snap <- mufits_sims[[iter]]
-
-    ## conc is a matrix with colnames 
-    if (is.matrix(concmat)) {
-        if (length(immobile)>0)
-            val <- concmat[,-immobile]
-        else
-            val <- concmat
-        
-        ## sort the columns of the matrix matching the names of boundary 
-        sptr <- colnames(val)
-        inflow <- boundmat[, colnames(boundmat)!="cbound", drop=FALSE]
-        spin <- colnames(inflow) 
-        ind <- match(spin,sptr)
-        if (TRUE %in% is.na(ind)) {
-            msgm("Missing boundary conditions for some species")
-            val <- cbind(val,matrix(0,ncol=sum(is.na(ind)),nrow=nrow(val) ))
-            colnames(val) <- c(sptr,spin[is.na(ind)])
-        }
-        
-        sptr <- colnames(val)
-        ind <- match(sptr,spin)
-        ## msgm("ind"); print(ind)
-        
-        cnew <- val
-
-        msgm("Computing transport of ", ncol(val), " species")
-        ## if (local_rank==0) {
-        ##     saveRDS(list(conc=newconc, times=times, activeCells=grid$cell$CELLID[-exclude_cell], fluxv=fluxv),
-        ##             file=paste0("TranspUpwind_", local_rank, ".RData"))
-        ## }
-        ## cnew[ boundmat[, 1] ] <- boundmat[, 2]
-
-        ## MDL 20200227: Here was the bug: "excl" refers to
-        ## CONNECTIONS and not GRID_CELLS!!
-        
-        ## cells where we won't update the concentrations: union of
-        ## boundary and no-flow cells
-        ## if (length(excl) == 0)
-        ##     exclude_cell <- sort(c(boundmat[, 1]))
-        ## else
-        ##     exclude_cell <- sort(c(boundmat[, 1], excl))
-        exclude_cell <- sort(c(boundmat[, 1]))
-
-        ## msgm("mufits_grid$cell$CELLID[-exclude_cell]:")
-        ## print(mufits_grid$cell$CELLID[-exclude_cell])
-
-        for (i in seq(1, ncol(val))) {
-            ## form a 2-column matrix with cell id and boundary
-            ## concentration for those elements
-            newboundmat <- boundmat[,c(1, ind[i]+1)]
-            ## vector with the old concentrations
-            concv <- val[,i]
-            ## apply boundary conditions to the concentration vector
-            ## (they should stay constant but better safe than sorry)
-            concv[newboundmat[, 1]] <- newboundmat[, 2]
-
-            ## call the function
-            cnew[,i] <- CppTransportUpwindIrr(concv = concv,
-                                              times = inner_timesteps, 
-                                              activeCells = mufits_grid$cell$CELLID[-exclude_cell],
-                                              fluxv = setup$fluxv, 
-                                              listconn = mufits_grid$listconn,
-                                              porVol = setup$cellporvol)
-        }
-        
-        colnames(cnew) <- colnames(val)
-        
-        if ( length(immobile) > 0)  { 
-            res <- cbind(cnew, concmat[,immobile])
-        } else {
-            res <- cnew
-        }
-
-        ## check for negative values. This SHOULD NOT OCCUR and may be
-        ## the result of numerical dispersion (or bug in my transport
-        ## code!)
-        if (any(res <0 )) {
-            rem_neg <- which(res<0, arr.ind=TRUE)
-            a <- nrow(rem_neg)
-            res[res < 0 ] <- 0
-            msgm("-> ", a, "concentrations were negative")
-            print(rem_neg)
-        }
-        ## msgm("state_T after iteration", setup$iter, ":")
-        ## print(head(res))
-    } else {
-        msgm("state_C at iteration", setup$iter, " is not a Matrix, doing nothing!")
-    }
-
-    ## retransform concentrations H+ and e- into pH and pe
-    state_T <- RedModRphree::Act2pH(res)
-
-    setup$state_T <- state_T
-    
-    return(setup)
-}
-
-
-## function for the workers to compute chemistry through PHREEQC
-slave_chemistry <- function(setup, data)
-{
-    base     <- setup$base
-    first    <- setup$first
-    prop     <- setup$prop
-    immobile <- setup$immobile
-    kin      <- setup$kin
-    ann      <- setup$ann
-
-    if (local_rank == 0) {
-        iter     <- setup$iter
-        timesteps <- setup$timesteps
-        dt       <- timesteps[iter]
-    }
-
-    state_T <- data ## not the global field, but the work-package
-
-    ## treat special H+/pH, e-/pe cases
-    state_T <- RedModRphree::Act2pH(state_T)
-    
-    ## reduction of the problem
-    if(setup$reduce)
-        reduced <- ReduceStateOmit(state_T, omit=setup$ann)
-    else
-        reduced <- state_T
-
-    ## if (local_rank==1) {
-    ##     msg("worker", local_rank,"; iter=", iter, "dt=", dt)
-    ##     msg("reduce is", setup$reduce)
-    ##     msg("data:")
-    ##     print(reduced)
-    ##     msg("base:")
-    ##     print(base)
-    ##     msg("first:")
-    ##     print(first)
-    ##     msg("ann:")
-    ##     print(ann)
-    ##     msg("prop:")
-    ##     print(prop)
-    ##     msg("immobile:")
-    ##     print(immobile)
-    ##     msg("kin:")
-    ##     print(kin)
-    ## }
-
-    ## form the PHREEQC input script for the current work package
-    inplist <- splitMultiKin(data=reduced, procs=1, base=base, first=first,
-                             ann=ann, prop=prop, minerals=immobile, kin=kin, dt=dt)
-
-    ## if (local_rank==1 & iter==1) 
-    ##         RPhreeWriteInp("FirstInp", inplist)
-
-    tmpC <- RunPQC(inplist, procs=1, second=TRUE)
-
-    ## recompose after the reduction
-    if (setup$reduce)
-        state_C <- RecomposeState(tmpC, reduced)
-    else {
-        state_C <- tmpC
-    }
-
-    ## the next line is needed since we don't need all columns of
-    ## PHREEQC output
-    return(state_C[, prop])
-} 
-
-
-
-## This function, called by master
-master_chemistry <- function(setup, data)
-{
-    state_T <- setup$state_T 
-
-    msgm(" chemistry iteration", setup$iter)
-
-    ## treat special H+/pH, e-/pe cases
-    state_T <- RedModRphree::Act2pH(state_T)
-    
-    ## reduction of the problem
-    if(setup$reduce)
-        reduced <- ReduceStateOmit(state_T, omit=setup$ann)
-    else
-        reduced <- state_T
-    
-    ### inject data from workers
-    res_C <- data
-
-    rownames(res_C) <- NULL
-    
-    ## print(res_C)
-    
-    if (nrow(res_C) > nrow(reduced)) {
-        res_C <- res_C[seq(2,nrow(res_C), by=2),]
-    }
-    
-    ## recompose after the reduction
-    if (setup$reduce)
-        state_C <- RecomposeState(res_C, reduced)
-    else {
-        state_C <- res_C
-    }
-    
-    setup$state_C <- state_C
-    setup$reduced <- reduced
-
-    return(setup)
-} 
-    
-
-## Adapted version for "reduction"
-ReduceStateOmit <- function (data, omit=NULL, sign=6) 
-{
-    require(mgcv)
-
-    rem <- colnames(data)
-    if (is.list(omit)) {
-        indomi <- match(names(omit), colnames(data))
-        datao <- data[, -indomi]
-    } else datao <- data
-
-    datao <- signif(datao, sign)
-    red <- mgcv::uniquecombs(datao)
-    inds <- attr(red, "index")
-    now <- ncol(red)
-
-    
-    ## reattach the omitted column(s)
-    ## FIXME: control if more than one ann is present
-    if (is.list(omit)) {
-        red <- cbind(red, rep( data[ 1, indomi], nrow(red)))
-    
-        colnames(red)[now+1] <- names(omit)
-
-        ret <- red[, colnames(data)]
-    } else {
-        ret <- red
-    }
-    rownames(ret) <- NULL
-    attr(ret, "index") <- inds
-    return(ret)
-}
-
-
 
 ## Attach the name of the calling function to the message displayed on
 ## R's stdout
-msgm <- function (...)
+msgm <- function(...) {
-{
+    prefix <- paste0("R: ")
-    
+    cat(paste(prefix, ..., "\n"))
-    if (local_rank==0) {
-        fname <- as.list(sys.call(-1))[[1]]
-        prefix <- paste0("R: ", fname, " ::")
-        cat(paste(prefix, ..., "\n"))
-    }
     invisible()
 }
 
 
-## Function called by master R process to store on disk all relevant
+GetWorkPackageSizesVector <- function(n_packages, package_size, len) {
-## parameters for the simulation
+    ids <- rep(1:n_packages, times = package_size, each = 1)[1:len]
-StoreSetup <- function(setup) {
+    return(as.integer(table(ids)))
-
-    to_store <- vector(mode="list", length=5)
-    names(to_store) <- c("Sim", "Flow", "Transport", "Chemistry", "DHT")
-
-    ## read the setup R file, which is sourced in kin.cpp
-    tmpbuff <- file(filesim, "r")
-    setupfile <- readLines(tmpbuff)
-    close.connection(tmpbuff) 
-
-    to_store$Sim <- setupfile
-    
-    to_store$Flow <- list(
-        snapshots  = setup$snapshots,
-        gridfile   = setup$gridfile,
-        phase      = setup$phase,
-        density    = setup$density,
-        dt_differ  = setup$dt_differ,
-        prolong    = setup$prolong,
-        maxiter    = setup$maxiter,
-        saved_iter = setup$iter_output,
-        out_save   = setup$out_save )
-
-    to_store$Transport <- list(
-        boundary = setup$bound,
-        Cf       = setup$Cf,
-        prop     = setup$prop,
-        immobile = setup$immobile,
-        reduce   = setup$reduce )
-    
-    to_store$Chemistry <- list(
-        nprocs   = n_procs,
-        wp_size  = work_package_size,
-        base     = setup$base,
-        first    = setup$first,
-        init     = setup$initsim,
-        db       = db,
-        kin      = setup$kin,
-        ann      = setup$ann)
-    
-    if (dht_enabled) {
-        to_store$DHT <- list(
-            enabled   = dht_enabled,
-            log       = dht_log,
-            signif    = dht_final_signif,
-            proptype  = dht_final_proptype)
-        
-    } else {
-        to_store$DHT <- FALSE
-    }
-
-    saveRDS(to_store, file=paste0(fileout,'/setup.rds'))
-    msgm("initialization stored in ", paste0(fileout,'/setup.rds'))
 }
+
+
+## Handler to read R objs from binary files using either builtin
+## readRDS(), qs::qread() or qs2::qs_read() based on file extension
+ReadRObj <- function(path) {
+    ## code borrowed from tools::file_ext()
+    pos <- regexpr("\\.([[:alnum:]]+)$", path)
+    extension <- ifelse(pos > -1L, substring(path, pos + 1L), "")
+
+    switch(extension,
+        rds = readRDS(path),
+        qs  = qs::qread(path),
+        qs2 = qs2::qs_read(path)
+    )
+}
+
+## Handler to store R objs to binary files using either builtin
+## saveRDS() or qs::qsave() based on file extension
+SaveRObj <- function(x, path) {
+    ## msgm("Storing to", path)
+    ## code borrowed from tools::file_ext()
+    pos <- regexpr("\\.([[:alnum:]]+)$", path)
+    extension <- ifelse(pos > -1L, substring(path, pos + 1L), "")
+
+    switch(extension,
+        rds = saveRDS(object = x, file = path),
+        qs  = qs::qsave(x = x, file = path),
+        qs2 = qs2::qs_save(object = x, file = path)
+    )
+}
+
+
+######## Old relic code
+
+## ## Function called by master R process to store on disk all relevant
+## ## parameters for the simulation
+## StoreSetup <- function(setup, filesim, out_dir) {
+##     to_store <- vector(mode = "list", length = 4)
+##     ## names(to_store) <- c("Sim", "Flow", "Transport", "Chemistry", "DHT")
+##     names(to_store) <- c("Sim", "Transport", "DHT", "Cmdline")
+
+##     ## read the setup R file, which is sourced in kin.cpp
+##     tmpbuff <- file(filesim, "r")
+##     setupfile <- readLines(tmpbuff)
+##     close.connection(tmpbuff)
+
+##     to_store$Sim <- setupfile
+
+##     ## to_store$Flow <- list(
+##     ##     snapshots  = setup$snapshots,
+##     ##     gridfile   = setup$gridfile,
+##     ##     phase      = setup$phase,
+##     ##     density    = setup$density,
+##     ##     dt_differ  = setup$dt_differ,
+##     ##     prolong    = setup$prolong,
+##     ##     maxiter    = setup$maxiter,
+##     ##     saved_iter = setup$iter_output,
+##     ##     out_save   = setup$out_save )
+
+##     to_store$Transport <- setup$diffusion
+
+##     ## to_store$Chemistry <- list(
+##     ##    nprocs   = n_procs,
+##     ##    wp_size  = work_package_size,
+##     ##    base     = setup$base,
+##     ##    first    = setup$first,
+##     ##    init     = setup$initsim,
+##     ##    db       = db,
+##     ##    kin      = setup$kin,
+##     ##    ann      = setup$ann)
+
+##     if (dht_enabled) {
+##         to_store$DHT <- list(
+##             enabled   = dht_enabled,
+##             log       = dht_log
+##             ## signif    = dht_final_signif,
+##             ## proptype  = dht_final_proptype
+##         )
+##     } else {
+##         to_store$DHT <- FALSE
+##     }
+
+##     if (dht_enabled) {
+##         to_store$DHT <- list(
+##             enabled   = dht_enabled,
+##             log       = dht_log
+##             # signif    = dht_final_signif,
+##             # proptype  = dht_final_proptype
+##         )
+##     } else {
+##         to_store$DHT <- FALSE
+##     }
+
+##     saveRDS(to_store, file = paste0(fileout, "/setup.rds"))
+##     msgm("initialization stored in ", paste0(fileout, "/setup.rds"))
+## }

R_lib/parallel_r_library.R

@@ -1,58 +0,0 @@
-### Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
-###
-### POET is free software; you can redistribute it and/or modify it under the
-### terms of the GNU General Public License as published by the Free Software
-### Foundation; either version 2 of the License, or (at your option) any later
-### version.
-###
-### POET is distributed in the hope that it will be useful, but WITHOUT ANY
-### WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-### A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-###
-### You should have received a copy of the GNU General Public License along with
-### this program; if not, write to the Free Software Foundation, Inc., 51
-### Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-distribute_work_packages <- function(len, package_size) 
-{
-    ## Check if work_package is a divisor of grid length and act
-    ## accordingly
-    if ((len %% package_size) == 0) {
-        n_packages <- len/package_size
-    } else {
-        n_packages <- floor(len/package_size) + 1
-    }
-
-    ids <- rep(1:n_packages, times=package_size, each = 1)[1:len]
-    return(ids)
-}
-
-compute_wp_sizes <- function(ids)
-{
-    as.integer(table(ids))
-}
-
-shuffle_field <- function(data, send_ord) 
-{
-    shf <- data[send_ord,]
-    return(shf)
-}
-
-unshuffle_field <- function(data, send_ord) 
-{
-    data[send_ord,] <- data
-    rownames(data) <- NULL
-    return(data)
-}
-
-stat_wp_sizes <- function(sizes)
-{
-    res <- as.data.frame(table(sizes))
-    if (nrow(res)>1) {
-        msgm("Chosen work_package_size is not a divisor of grid length. ")
-        colnames(res) <- c("Size", "N")
-        print(res)
-    } else {
-        msgm("All work packages of length ", sizes[1])
-    }
-}

apps/CMakeLists.txt

@@ -0,0 +1,4 @@
+file(GLOB INIT_SRCS CONFIGURE_DEPENDS "initializer/*.cpp")
+
+add_executable(poet_initializer ${INIT_SRCS})
+target_link_libraries(poet_initializer RRuntime tug)

apps/initializer/main.cpp

@@ -0,0 +1,3 @@
+#include <Rcpp.h>
+
+int main(int argc, char **argv) {}

bench/CMakeLists.txt

@@ -0,0 +1,43 @@
+function(ADD_BENCH_TARGET TARGET POET_BENCH_LIST RT_FILES OUT_PATH)
+    set(bench_install_dir share/poet/${OUT_PATH})
+
+    # create empty list 
+    set(OUT_FILES_LIST "")
+
+    foreach(BENCH_FILE ${${POET_BENCH_LIST}})
+        get_filename_component(BENCH_NAME ${BENCH_FILE} NAME_WE)
+        set(OUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/${BENCH_NAME})
+        set(OUT_FILE_EXT ${OUT_FILE}.qs2)
+
+        add_custom_command(
+            OUTPUT ${OUT_FILE_EXT}
+            COMMAND $<TARGET_FILE:poet_init> -n ${OUT_FILE} -s ${CMAKE_CURRENT_SOURCE_DIR}/${BENCH_FILE}
+            COMMENT "Running poet_init on ${BENCH_FILE}"
+            DEPENDS poet_init ${CMAKE_CURRENT_SOURCE_DIR}/${BENCH_FILE}
+            VERBATIM
+            COMMAND_EXPAND_LISTS
+        )
+
+        list(APPEND OUT_FILES_LIST ${OUT_FILE_EXT})
+
+    endforeach(BENCH_FILE ${${POET_BENCH_LIST}})
+        
+    add_custom_target(
+        ${TARGET}
+        DEPENDS ${OUT_FILES_LIST})
+            
+    install(FILES ${OUT_FILES_LIST} DESTINATION ${bench_install_dir})
+
+    # install all ADD_FILES to the same location
+    install(FILES ${${RT_FILES}} DESTINATION ${bench_install_dir})
+endfunction()
+
+
+# define target name 
+set(BENCHTARGET benchmarks)
+
+add_custom_target(${BENCHTARGET} ALL)
+
+add_subdirectory(barite)
+add_subdirectory(dolo)
+add_subdirectory(surfex)

bench/barite/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Create a list of files 
+set(bench_files
+    barite_200.R
+    barite_het.R
+)
+
+set(runtime_files
+    barite_200_rt.R
+    barite_het_rt.R
+)
+
+# add_custom_target(barite_bench DEPENDS ${bench_files} ${runtime_files})
+
+ADD_BENCH_TARGET(barite_bench 
+    bench_files 
+    runtime_files 
+    "barite"
+)
+
+add_dependencies(${BENCHTARGET} barite_bench)

bench/barite/README.org

@@ -0,0 +1,134 @@
+#+TITLE: Description of =barite= benchmark
+#+AUTHOR: MDL <delucia@gfz-potsdam.de>
+#+DATE: 2023-08-26
+#+STARTUP: inlineimages
+#+LATEX_CLASS_OPTIONS: [a4paper,9pt]
+#+LATEX_HEADER: \usepackage{fullpage}
+#+LATEX_HEADER: \usepackage{amsmath, systeme}
+#+LATEX_HEADER: \usepackage{graphicx}
+#+LATEX_HEADER: \usepackage{charter}
+#+OPTIONS: toc:nil
+
+* Quick start
+
+#+begin_src sh :language sh :frame single
+mpirun -np 4 ./poet barite.R barite_results
+mpirun -np 4 ./poet --interp  barite_interp_eval.R barite_results
+#+end_src
+
+* List of Files
+
+- =barite_het.R=: POET input script with homogeneous zones for a 5x2 simulation
+  grid 
+- =barite_200.R=: POET input script for a 200x200 simulation
+  grid
+- =barite_200ai_surrogate_input_script.R=: Defines the ai surrogate functions 
+  to load a pretrained model and apply min-max-feature scaling on the model inputs
+  and target. Prediction validity is assessed with a threshold of 3e-5 on the mass 
+  balance of Ba and Sr.
+- =barite_200min_max_bounds=: Minimum and maximum values from 50 iterations of the
+  barite_200 benchmark. Used for feature scaling in the ai surrogate.
+- =barite_200model_min_max.keras=: A sequential keras model that has been trained 
+  on 50 iterations of the barite_200 benchmark with min-max-scaled inputs
+  and targets/outputs.
+- =db_barite.dat=: PHREEQC database containing the kinetic expressions
+  for barite and celestite, stripped down from =phreeqc.dat=
+- =barite.pqi=: PHREEQC input script defining the chemical system
+
+* Chemical system
+
+The benchmark depicts an isotherm porous system at *25 °C* where pure
+water is initially at equilibrium with *celestite* (strontium sulfate;
+brute formula: SrSO_{4}). A solution containing only dissolved Ba^{2+}
+and Cl^- diffuses into the system causing celestite dissolution. The
+increased concentration of dissolved sulfate SO_{4}^{2-} induces
+precipitation of *barite* (barium sulfate; brute formula:
+BaSO_{4}^{2-}). The overall reaction can be written:
+
+Ba^{2+} + celestite \rightarrow barite + Sr^{2+}
+
+Both celestite dissolution and barite precipitation are calculated
+using a kinetics rate law based on transition state theory:
+
+rate = -S_{m} k_{barite} (1-SR_{m})
+
+where the reaction rate has units mol/s, S_{m} (m^{2}) is the reactive
+surface area, k (mol/m^{2}/s) is the kinetic coefficient, and SR is
+the saturation ratio, i.e., the ratio of the ion activity product of
+the reacting species and the solubility constant, calculated
+internally by PHREEQC from the speciated solution.
+
+For barite, the reaction rate is computed as sum of two mechanisms,
+r_{/acid/} and r_{/neutral/}:
+
+rate_{barite} = S_{barite} (k_{/acid/} + k_{/neutral/}) * (1 - SR_{barite})
+
+where:
+
+k_{/acid/} = 10^{-6.9} e^{-30800 / R} \cdot act(H^{+})^{0.22}
+
+k_{/neutral/} = 10^{-7.9} e^{-30800 / R}
+
+R (8.314462 J K^{-1} mol^{-1}) is the gas constant.
+
+For celestite the kinetic law considers only the acidic mechanism and
+reads:
+
+rate_{celestite} = S_{celestite} 10^{-5.66} e^{-23800 / R} \cdot
+act(H^{+})^{0.109} \cdot (1 - SR_{celestite})
+
+The kinetic rates as implemented in the =db_barite.dat= file accepts
+one parameter which represents reactive surface area in m^{2}. For the
+benchmarks the surface areas are set to
+
+- S_{barite}: 50 m^{2}
+- S_{celestite}: 10 m^{2}
+
+A starting seed for barite is given at 0.001 mol in each domain
+element.
+
+* Nucleation (TODO)
+
+Geochemical benchmark inspired by Tranter et al. (2021) without
+nucleation. 
+
+* POET simulations
+
+Currently these benchmarks are pure diffusion simulations. There are 7
+transported species: H, O, Charge, Ba, Cl, S(6), Sr.
+
+** =barite.R=
+
+- Grid discretization: square domain of 1 \cdot 1 m^{2} discretized in
+  20x20 cells
+- Boundary conditions: E, S and W sides of the domain are closed; the
+  N boundary has a *fixed concentration* (Dirichlet) of 0.1 molal
+  BaCl_{2}
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 20 iteration with \Delta t = 250 s
+- *DHT* parameters:
+  |  H |  O | Charge | Ba | Cl | S(6) | Sr |
+  | 10 | 10 |      3 |  5 |  5 |    5 |  5 |
+
+
+
+** =barite_interp_eval.R=
+- Grid discretization: rectangular domain of 40 \cdot 20 m^{2}
+  discretized in 400 \cdot 200 cells
+- Boundary conditions: all boundaries are closed. The center of the
+  domain at indeces (200, 100) has fixed concentration of 0.1 molal of
+  BaCl_{2}
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 200 iterations with \Delta t = 250 s
+- *DHT* parameters:
+  |  H |  O | Charge | Ba | Cl | S(6) | Sr |
+  | 10 | 10 |      3 |  5 |  5 |    5 |  5 |
+
+* References
+
+- Tranter, Wetzel, De Lucia and Kühn (2021): Reactive transport model
+  of kinetically controlled celestite to barite replacement, Advances
+  in Geosciences, 56, 57-–65, 2021.
+  https://doi.org/10.5194/adgeo-56-57-20211
+
+  

bench/barite/barite.pqi

@@ -0,0 +1,32 @@
+SOLUTION 1
+  units	mol/kgw
+  water	1
+  temperature	25
+  pH	7
+PURE 1 
+  Celestite 0.0 1
+END
+
+RUN_CELLS
+  -cells 1
+
+COPY solution 1 2
+
+KINETICS 2
+  Barite
+    -m 0.001
+    -parms 50.  # reactive surface area
+    -tol 1e-9
+  Celestite
+    -m 1
+    -parms 10.0  # reactive surface area
+    -tol 1e-9
+END
+
+SOLUTION 3
+  units mol/kgw
+  water 1
+  temperature 25
+  Ba 0.1
+  Cl 0.2
+END

bench/barite/barite_200.R

@@ -0,0 +1,59 @@
+cols <- 200
+rows <- 200
+
+s_cols <- 1
+s_rows <- 1
+
+grid_def <- matrix(2, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file = "./barite.pqi",
+  pqc_db_file = "./db_barite.dat", # Path to the database file for Phreeqc
+  grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+  grid_size = c(s_rows, s_cols), # Size of the grid in meters
+  constant_cells = c() # IDs of cells with constant concentration
+)
+
+bound_length <- 2
+
+bound_def <- list(
+  "type" = rep("constant", bound_length),
+  "sol_id" = rep(3, bound_length),
+  "cell" = seq(1, bound_length)
+)
+
+homogenous_alpha <- 1e-6
+
+diffusion_setup <- list(
+  boundaries = list(
+    "W" = bound_def,
+    "N" = bound_def
+  ),
+  alpha_x = homogenous_alpha,
+  alpha_y = homogenous_alpha
+)
+
+dht_species <- c(
+  "H"         = 3,
+  "O"         = 3,
+  "Charge"    = 6,
+  "Ba"        = 6,
+  "Cl"        = 6,
+  "S"         = 6,
+  "Sr"        = 6,
+  "Barite"    = 5,
+  "Celestite" = 5
+)
+
+chemistry_setup <- list(
+  dht_species = dht_species,
+  ai_surrogate_input_script = "./barite_200ai_surrogate_input_script.R"
+)
+
+# Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, # Parameters related to the grid structure
+  Diffusion = diffusion_setup, # Parameters related to the diffusion process
+  Chemistry = chemistry_setup
+)

bench/barite/barite_200_rt.R

@@ -0,0 +1,7 @@
+iterations <- 50
+dt <- 100
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE
+)

bench/barite/barite_200ai_surrogate_input_script.R

@@ -0,0 +1,48 @@
+## load a pretrained model from tensorflow file
+## Use the global variable "ai_surrogate_base_path" when using file paths
+## relative to the input script
+initiate_model <- function() {
+  init_model <- normalizePath(paste0(ai_surrogate_base_path,
+                                     "model_min_max_float64.keras"))
+  return(load_model_tf(init_model))
+}
+
+scale_min_max <- function(x, min, max, backtransform) {
+  if (backtransform) {
+    return((x * (max - min)) + min)
+  } else {
+    return((x - min) / (max - min))
+  }
+}
+
+preprocess <- function(df, backtransform = FALSE, outputs = FALSE) {
+  minmax_file <- normalizePath(paste0(ai_surrogate_base_path,
+                                      "min_max_bounds.rds"))
+  global_minmax <- readRDS(minmax_file)
+  for (column in colnames(df)) {
+    df[column] <- lapply(df[column],
+                         scale_min_max,
+                         global_minmax$min[column],
+                         global_minmax$max[column],
+                         backtransform)
+  }
+  return(df)
+}
+
+mass_balance <- function(predictors, prediction) {
+  dBa <- abs(prediction$Ba + prediction$Barite -
+             predictors$Ba - predictors$Barite)
+  dSr <- abs(prediction$Sr + prediction$Celestite -
+             predictors$Sr - predictors$Celestite)
+  return(dBa + dSr)
+}
+
+validate_predictions <- function(predictors, prediction) {
+  epsilon <- 3e-5
+  mb <- mass_balance(predictors, prediction)
+  msgm("Mass balance mean:", mean(mb))
+  msgm("Mass balance variance:", var(mb))
+  msgm("Rows where mass balance meets threshold", epsilon, ":",
+       sum(mb < epsilon))
+  return(mb < epsilon)
+}

bench/barite/barite_50ai.R

@@ -0,0 +1,60 @@
+## Time-stamp: "Last modified 2024-05-30 13:34:14 delucia"
+cols <- 50
+rows <- 50
+
+s_cols <- 0.25
+s_rows <- 0.25
+
+grid_def <- matrix(2, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file = "./barite.pqi",
+  pqc_db_file = "./db_barite.dat", ## Path to the database file for Phreeqc
+  grid_def = grid_def, ## Definition of the grid, containing IDs according to the Phreeqc input script
+  grid_size = c(s_rows, s_cols), ## Size of the grid in meters
+  constant_cells = c() ## IDs of cells with constant concentration
+)
+
+bound_length <- 2
+
+bound_def <- list(
+  "type" = rep("constant", bound_length),
+  "sol_id" = rep(3, bound_length),
+  "cell" = seq(1, bound_length)
+)
+
+homogenous_alpha <- 1e-8
+
+diffusion_setup <- list(
+  boundaries = list(
+    "W" = bound_def,
+    "N" = bound_def
+  ),
+  alpha_x = homogenous_alpha,
+  alpha_y = homogenous_alpha
+)
+
+dht_species <- c(
+  "H"         = 3,
+  "O"         = 3,
+  "Charge"    = 3,
+  "Ba"        = 6,
+  "Cl"        = 6,
+  "S"         = 6,
+  "Sr"        = 6,
+  "Barite"    = 5,
+  "Celestite" = 5
+)
+
+chemistry_setup <- list(
+  dht_species = dht_species,
+  ai_surrogate_input_script = "./barite_50ai_surr_mdl.R"
+)
+
+# Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, # Parameters related to the grid structure
+  Diffusion = diffusion_setup, # Parameters related to the diffusion process
+  Chemistry = chemistry_setup
+)

bench/barite/barite_50ai_rt.R

@@ -0,0 +1,9 @@
+iterations <- 1000
+
+dt <- 200
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE,
+    out_save = c(1, 5, seq(20, iterations, by=20))
+)

bench/barite/barite_50ai_surr_mdl.R

@@ -0,0 +1,90 @@
+## Time-stamp: "Last modified 2024-05-30 13:27:06 delucia"
+
+## load a pretrained model from tensorflow file
+## Use the global variable "ai_surrogate_base_path" when using file paths
+## relative to the input script
+initiate_model <- function() {
+    require(keras3)
+    require(tensorflow)
+    init_model <- normalizePath(paste0(ai_surrogate_base_path,
+                                       "barite_50ai_all.keras"))
+    Model <- keras3::load_model(init_model)
+    msgm("Loaded model:")
+    print(str(Model))
+    return(Model)
+}
+
+scale_min_max <- function(x, min, max, backtransform) {
+    if (backtransform) {
+        return((x * (max - min)) + min)
+    } else {
+        return((x - min) / (max - min))
+    }
+}
+
+minmax <- list(min = c(H = 111.012433592824, O = 55.5062185549492, Charge = -3.1028354471876e-08, 
+                       Ba = 1.87312878574393e-141, Cl = 0, `S(6)` = 4.24227510643685e-07, 
+                       Sr = 0.00049382996130541, Barite = 0.000999542409828586, Celestite = 0.244801877115968),
+               max = c(H = 111.012433679682, O = 55.5087003521685, Charge = 5.27666636082035e-07, 
+                       Ba = 0.0908849779513762, Cl = 0.195697626449355, `S(6)` = 0.000620774752665846, 
+                       Sr = 0.0558680070692722, Barite = 0.756779139057097, Celestite = 1.00075422160624
+                       ))
+
+preprocess <- function(df) {
+    if (!is.data.frame(df))
+        df <- as.data.frame(df, check.names = FALSE)
+    
+    as.data.frame(lapply(colnames(df),
+                         function(x) scale_min_max(x=df[x],
+                                                   min=minmax$min[x],
+                                                   max=minmax$max[x],
+                                                   backtransform=FALSE)),
+                  check.names = FALSE)
+}
+
+postprocess <- function(df) {
+    if (!is.data.frame(df))
+        df <- as.data.frame(df, check.names = FALSE)
+    
+    as.data.frame(lapply(colnames(df),
+                         function(x) scale_min_max(x=df[x],
+                                                   min=minmax$min[x],
+                                                   max=minmax$max[x],
+                                                   backtransform=TRUE)),
+                  check.names = FALSE)
+}
+
+mass_balance <- function(predictors, prediction) {
+    dBa <- abs(prediction$Ba + prediction$Barite -
+               predictors$Ba - predictors$Barite)
+    dSr <- abs(prediction$Sr + prediction$Celestite -
+               predictors$Sr - predictors$Celestite)
+    return(dBa + dSr)
+}
+
+validate_predictions <- function(predictors, prediction) {
+    epsilon <- 1E-7
+    mb <- mass_balance(predictors, prediction)
+    msgm("Mass balance mean:", mean(mb))
+    msgm("Mass balance variance:", var(mb))
+    ret <- mb < epsilon
+    msgm("Rows where mass balance meets threshold", epsilon, ":",
+         sum(ret))
+    return(ret)
+}
+
+training_step <- function(model, predictor, target, validity) {
+  msgm("Starting incremental training:")
+
+  ## x <- as.matrix(predictor)
+  ## y <- as.matrix(target[colnames(x)])
+
+  history <- model %>% keras3::fit(x = data.matrix(predictor),
+                                   y = data.matrix(target),
+                                   epochs = 10, verbose=1)
+
+  keras3::save_model(model,
+                     filepath = paste0(out_dir, "/current_model.keras"),
+                     overwrite=TRUE)
+  return(model)
+}

bench/barite/barite_het.R

@@ -0,0 +1,32 @@
+grid_def <- matrix(c(2, 3), nrow = 2, ncol = 5)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+    pqc_in_file = "./barite_het.pqi",
+    pqc_db_file = "./db_barite.dat", # Path to the database file for Phreeqc
+    grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+    grid_size = c(ncol(grid_def), nrow(grid_def)), # Size of the grid in meters
+    constant_cells = c() # IDs of cells with constant concentration
+)
+
+diffusion_setup <- list(
+    boundaries = list(
+        "W" = list(
+            "type" = rep("constant", nrow(grid_def)),
+            "sol_id" = rep(4, nrow(grid_def)),
+            "cell" = seq_len(nrow(grid_def))
+        )
+    ),
+    alpha_x = 1e-6,
+    alpha_y = matrix(runif(10, 1e-8, 1e-7),
+        nrow = nrow(grid_def),
+        ncol = ncol(grid_def)
+    )
+)
+
+# Define a setup list for simulation configuration
+setup <- list(
+    Grid = grid_setup, # Parameters related to the grid structure
+    Diffusion = diffusion_setup, # Parameters related to the diffusion process
+    Chemistry = list()
+)

bench/barite/barite_het.pqi

@@ -0,0 +1,80 @@
+## Initial: everywhere equilibrium with Celestite NB: The aqueous
+## solution *resulting* from this calculation is to be used as initial
+## state everywhere in the domain
+SOLUTION 1
+units	mol/kgw
+water	1
+temperature	25
+pH	7
+pe	4
+S(6)	1e-12
+Sr	1e-12
+Ba      1e-12
+Cl      1e-12
+PURE 1
+Celestite 0.0 1
+
+SAVE SOLUTION 2 # <- phreeqc keyword to store and later reuse these results
+END
+
+RUN_CELLS 
+   -cells 1
+
+COPY solution 1 2-3
+
+## Here a 5x2 domain:
+
+       |---+---+---+---+---|
+    -> | 2 | 2 | 2 | 2 | 2 |
+     4 |---+---+---+---+---|
+    -> | 3 | 3 | 3 | 3 | 3 |
+       |---+---+---+---+---|
+
+## East boundary: "injection" of solution 4. North, W, S boundaries: closed
+
+## Here the two distinct zones: nr 2 with kinetics Celestite (initial
+## amount is 0, is then allowed to precipitate) and nr 3 with kinetic
+## Celestite and Barite (both initially > 0) where the actual
+## replacement takes place
+
+#USE SOLUTION 2  <- PHREEQC keyword to reuse the results from previous calculation
+KINETICS 2
+Celestite
+-m  0    # Allowed to precipitate
+-parms 10.0
+-tol 1e-9
+
+END
+
+#USE SOLUTION 2  <- PHREEQC keyword to reuse the results from previous calculation
+KINETICS 3
+Barite
+-m 0.001
+-parms 50. 
+-tol 1e-9
+Celestite
+-m 1
+-parms 10.0 
+-tol 1e-9
+END
+
+## A BaCl2 solution (nr 4) is "injected" from the left boundary:
+SOLUTION 4
+units	mol/kgw
+pH 7
+water 1
+temp 25
+Ba  0.1
+Cl  0.2
+END
+## NB: again, the *result* of the SOLUTION 4 script defines the
+## concentration of all elements (+charge, tot H, tot O)
+
+## Ideally, in the initial state SOLUTION 1 we should not have to
+## define the 4 elemental concentrations (S(6), Sr, Ba and Cl) but
+## obtain them having run once the scripts with the aqueous solution
+## resulting from SOLUTION 1 once with KINETICS 2 and once with
+## KINETICS 3.
+
+RUN_CELLS
+ -cells 2-4

bench/barite/barite_het_rt.R

@@ -0,0 +1,4 @@
+list(
+    timesteps = rep(50, 100),
+    store_result = TRUE
+)

bench/barite/db_barite.dat

@@ -0,0 +1,195 @@
+DATABASE
+	###########################
+SOLUTION_MASTER_SPECIES
+	H      H+     -1            H         1.008  # phreeqc/
+	H(0)   H2      0            H                # phreeqc/
+	H(1)   H+     -1     0.0                     # phreeqc/
+	E      e-      0     0.0                0.0  # phreeqc/
+	O      H2O     0            O          16.0  # phreeqc/
+	O(0)   O2      0            O                # phreeqc/
+	O(-2)  H2O     0     0.0                     # phreeqc/
+	Na     Na+     0            Na      22.9898  # phreeqc/
+	Ba     Ba+2    0            Ba       137.34  # phreeqc/
+	Sr     Sr+2    0            Sr        87.62  # phreeqc/
+	Cl     Cl-     0            Cl       35.453  # phreeqc/
+	S      SO4-2   0            SO4      32.064  # phreeqc/
+	S(6)   SO4-2   0            SO4              # phreeqc/
+	S(-2)  HS-     1            S                # phreeqc/
+SOLUTION_SPECIES
+	H+ = H+
+		-gamma 9 0
+		-dw 9.31e-09
+		# source: phreeqc
+	e- = e-
+		# source: phreeqc
+	H2O = H2O
+		# source: phreeqc
+	Na+ = Na+
+		-gamma 4.08 0.082
+		-dw 1.33e-09
+		-Vm 2.28 -4.38 -4.1 -0.586 0.09 4 0.3 52 -0.00333 0.566
+		# source: phreeqc
+	Ba+2 = Ba+2
+		-gamma 4 0.153
+		-dw 8.48e-10
+		-Vm 2.063 -10.06 1.9534 -2.36 0.4218 5 1.58 -12.03 -0.00835 1
+		# source: phreeqc
+	Sr+2 = Sr+2
+		-gamma 5.26 0.121
+		-dw 7.94e-10
+		-Vm -0.0157 -10.15 10.18 -2.36 0.86 5.26 0.859 -27 -0.0041 1.97
+		# source: phreeqc
+	Cl- = Cl-
+		-gamma 3.63 0.017
+		-dw 2.03e-09
+		-Vm 4.465 4.801 4.325 -2.847 1.748 0 -0.331 20.16 0 1
+		# source: phreeqc
+	SO4-2 = SO4-2
+		-gamma 5 -0.04
+		-dw 1.07e-09
+		-Vm 8 2.3 -46.04 6.245 3.82 0 0 0 0 1
+		# source: phreeqc
+	H2O = OH- + H+
+		-analytical_expression 293.29227 0.1360833 -10576.913 -123.73158 0 -6.996455e-05
+		-gamma 3.5 0
+		-dw 5.27e-09
+		-Vm -9.66 28.5 80 -22.9 1.89 0 1.09 0 0 1
+		# source: phreeqc
+	2 H2O = O2 + 4 H+ + 4 e-
+		-log_k -86.08
+		-delta_h 134.79 kcal
+		-dw 2.35e-09
+		-Vm 5.7889 6.3536 3.2528 -3.0417 -0.3943     
+		# source: phreeqc
+	2 H+ + 2 e- = H2
+		-log_k -3.15
+		-delta_h -1.759 kcal
+		-dw 5.13e-09
+		-Vm 6.52 0.78 0.12       
+		# source: phreeqc
+	SO4-2 + H+ = HSO4-
+		-log_k 1.988
+		-delta_h 3.85 kcal
+		-analytical_expression -56.889 0.006473 2307.9 19.8858  
+		-dw 1.33e-09
+		-Vm 8.2 9.259 2.1108 -3.1618 1.1748 0 -0.3 15 0 1
+		# source: phreeqc
+	HS- = S-2 + H+
+		-log_k -12.918
+		-delta_h 12.1 kcal
+		-gamma 5 0
+		-dw 7.31e-10
+		# source: phreeqc
+	SO4-2 + 9 H+ + 8 e- = HS- + 4 H2O
+		-log_k 33.65
+		-delta_h -60.140 kcal
+		-gamma 3.5 0
+		-dw 1.73e-09
+		-Vm 5.0119 4.9799 3.4765 -2.9849 1.441     
+		# source: phreeqc
+	HS- + H+ = H2S
+		-log_k 6.994
+		-delta_h -5.30 kcal
+		-analytical_expression -11.17 0.02386 3279   
+		-dw 2.1e-09
+		-Vm 7.81 2.96 -0.46       
+		# source: phreeqc
+	Na+ + OH- = NaOH
+		-log_k -10
+		# source: phreeqc
+	Na+ + SO4-2 = NaSO4-
+		-log_k 0.7
+		-delta_h 1.120 kcal
+		-gamma 5.4 0
+		-dw 1.33e-09
+		-Vm 1e-05 16.4 -0.0678 -1.05 4.14 0 6.86 0 0.0242 0.53
+		# source: phreeqc
+	Ba+2 + H2O = BaOH+ + H+
+		-log_k -13.47
+		-gamma 5 0
+		# source: phreeqc
+	Ba+2 + SO4-2 = BaSO4
+		-log_k 2.7
+		# source: phreeqc
+	Sr+2 + H2O = SrOH+ + H+
+		-log_k -13.29
+		-gamma 5 0
+		# source: phreeqc
+	Sr+2 + SO4-2 = SrSO4
+		-log_k 2.29
+		-delta_h 2.08 kcal
+		-Vm 6.791 -0.9666 6.13 -2.739 -0.001     
+		# source: phreeqc
+PHASES
+	Barite
+		BaSO4 = Ba+2 + SO4-2
+		-log_k -9.97
+		-delta_h 6.35 kcal
+		-analytical_expression -282.43 -0.08972 5822 113.08  
+		-Vm 52.9
+		# source: phreeqc
+		# comment: 
+	Celestite
+		SrSO4 = Sr+2 + SO4-2
+		-log_k -6.63
+		-delta_h -4.037 kcal
+		-analytical_expression -7.14 0.00611 75 0 0 -1.79e-05
+		-Vm 46.4
+		# source: phreeqc
+		# comment: 
+RATES
+	Celestite  # Palandri & Kharaka 2004<--------------------------------change me
+# PARM(1): reactive surface area
+# am: acid mechanism, nm: neutral mechanism, bm: base mechanism
+    -start
+        10 sr_i = SR("Celestite")  # saturation ratio, (-)<----------change me
+        20 moles = 0  # init target variable, (mol)
+        30 IF ((M <= 0) AND (sr_i < 1)) OR (sr_i = 1.0) THEN GOTO 310
+        40 sa = PARM(1)  # reactive surface area, (m2)
+
+       100 r = 8.314462  # gas constant, (J K-1 mol-1)
+       110 dTi = (1 / TK) - (1 / 298.15)  # (K-1)
+       120 ea_am = 23800  # activation energy am, (J mol-1)<-----------change me
+       130 ea_nm = 0  # activation energy nm, (J mol-1)<-----------change me
+       140 ea_bm = 0  # activation energy bm, (J mol-1)<-----------change me
+       150 log_k_am = -5.66  # reaction constant am<-------------------change me
+       rem log_k_nm = -99  # reaction constant nm<-------------------change me
+       rem log_k_bm = -99  # reaction constant bm<-------------------change me
+       180 n_am = 0.109  # H+ reaction order am<-----------------------change me
+       rem n_bm = 0  # H+ reaction order bm<-----------------------change me
+       200 am = (10 ^ log_k_am) * EXP(-ea_am * dTi / r) * ACT("H+") ^ n_am
+       rem nm = (10 ^ log_k_nm) * EXP(-ea_nm * dTi / r)
+       rem bm = (10 ^ log_k_bm) * EXP(-ea_bm * dTi / r) * ACT("H+") ^ n_bm
+
+       300 moles = sa * (am) * (1 - sr_i)
+       310 save moles * time
+    -end
+
+	Barite  # Palandri & Kharaka 2004<-----------------------------------change me
+# PARM(1): reactive surface area
+# am: acid mechanism, nm: neutral mechanism, bm: base mechanism
+    -start
+        10 sr_i = SR("Barite")  # saturation ratio, (-)<----------change me
+        20 moles = 0  # init target variable, (mol)
+        30 IF ((M <= 0) AND (sr_i < 1)) OR (sr_i = 1.0) THEN GOTO 310
+        40 sa = PARM(1)  # reactive surface area, (m2)
+
+       100 r = 8.314462  # gas constant, (J K-1 mol-1)
+       110 dTi = (1 / TK) - (1 / 298.15)  # (K-1)
+       120 ea_am = 30800  # activation energy am, (J mol-1)<---------change me
+       130 ea_nm = 30800  # activation energy nm, (J mol-1)<---------change me
+       rem ea_bm = 0  # activation energy bm, (J mol-1)<---------change me
+       150 log_k_am = -6.90  # reaction constant am<-----------------change me
+       160 log_k_nm = -7.90  # reaction constant nm<-----------------change me
+       rem log_k_bm = -99  # reaction constant bm<-------------------change me
+       180 n_am = 0.22  # H+ reaction order am<----------------------change me
+       rem n_bm = 0  # H+ reaction order bm<-----------------------change me
+       200 am = (10 ^ log_k_am) * EXP(-ea_am * dTi / r) * ACT("H+") ^ n_am
+       210 nm = (10 ^ log_k_nm) * EXP(-ea_nm * dTi / r)
+       rem bm = (10 ^ log_k_bm) * EXP(-ea_bm * dTi / r) * ACT("H+") ^ n_bm
+
+       300 moles = sa * (am + nm) * (1 - sr_i)
+       310 save moles * time
+    -end
+
+END

bench/dolo/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(bench_files
+    dolo_inner_large.R
+    dolo_interp.R
+)
+
+set(runtime_files
+    dolo_inner_large_rt.R
+    dolo_interp_rt.R
+)
+
+ADD_BENCH_TARGET(
+    dolo_bench
+    bench_files 
+    runtime_files
+    "dolo"
+)
+
+add_dependencies(${BENCHTARGET} dolo_bench)

bench/dolo/README.org

@@ -0,0 +1,159 @@
+#+TITLE: Description of =dolo= benchmark
+#+AUTHOR: MDL <delucia@gfz-potsdam.de>
+#+DATE: 2023-08-26
+#+STARTUP: inlineimages
+#+LATEX_CLASS_OPTIONS: [a4paper,9pt]
+#+LATEX_HEADER: \usepackage{fullpage}
+#+LATEX_HEADER: \usepackage{amsmath, systeme}
+#+LATEX_HEADER: \usepackage{graphicx}
+#+LATEX_HEADER: \usepackage{charter}
+#+OPTIONS: toc:nil
+
+* Quick start
+
+#+begin_src sh :language sh :frame single
+mpirun -np 4 ./poet dolo_diffu_inner.R dolo_diffu_inner_res
+mpirun -np 4 ./poet --dht --interp dolo_interp_long.R dolo_interp_long_res
+#+end_src
+
+* List of Files
+
+- =dolo_interp.R=: POET input script for a 400x200 simulation
+  grid
+- =dolo_diffu_inner_large.R=: POET input script for a 400x200
+  simulation grid
+- =phreeqc_kin.dat=: PHREEQC database containing the kinetic expressions
+  for dolomite and celestite, stripped down from =phreeqc.dat=
+- =dol.pqi=: PHREEQC input script for the chemical system
+
+* Chemical system
+
+This model describes a simplified version of /dolomitization/ of
+calcite, itself a complex and not yet fully understood natural process
+which is observed naturally at higher temperatures (Möller and De
+Lucia, 2020). Variants of such model have been widely used in many
+instances especially for the purpose of benchmarking numerical codes
+(De Lucia et al., 2021 and references therein).
+
+We consider an isotherm porous system at *25 °C* in which pure water
+is initially at equilibrium with *calcite* (calcium carbonate; brute
+formula: CaCO_{3}). An MgCl_{2} solution enters the system causing
+calcite dissolution. The released excess concentration of dissolved
+calcium Ca^{2+} and carbonate (CO_{3}^{2-}) induces after a while
+supersaturation and hence precipitation of *dolomite*
+(calcium-magnesium carbonate; brute formula: CaMg(CO_{3})_{2}). The
+overall /dolomitization/ reaction can be written:
+
+Mg^{2+} + 2 \cdot calcite \rightarrow dolomite + 2 \cdot Ca^{2+}
+
+The precipitation of dolomite continues until enough Ca^{2+} is
+present in solution. Further injection of MgCl_{2} changes its
+saturation state causing its dissolution too. After enough time, the
+whole system has depleted all minerals and the injected MgCl_{2}
+solution fills up the domain.
+
+Both calcite dissolution and dolomite precipitation/dissolution follow
+a kinetics rate law based on transition state theory (Palandri and
+Karhaka, 2004; De Lucia et al., 2021).
+
+rate = -S_{m} k_{m} (1-SR_{m})
+
+where the reaction rate has units mol/s, S_{m} (m^{2}) is the reactive
+surface area, k_{m} (mol/m^{2}/s) is the kinetic coefficient, and SR
+is the saturation ratio, i.e., the ratio of the ion activity product
+of the reacting species and the solubility constant, calculated
+internally by PHREEQC from the speciated solution.
+
+For dolomite, the kinetic coefficient results from the sum of two
+mechanisms, r_{/acid/} and r_{/neutral/}:
+
+rate_{dolomite} = S_{dolomite} (k_{/acid/} + k_{/neutral/}) * (1 - SR_{dolomite})
+
+where:
+
+k_{/acid/} = 10^{-3.19} e^{-36100 / R} \cdot act(H^{+})^{0.5}
+
+k_{/neutral/} = 10^{-7.53} e^{-52200 / R}
+
+R (8.314462 J K^{-1} mol^{-1}) is the gas constant.
+
+Similarly, the kinetic law for calcite reads:
+
+k_{/acid/} = 10^{-0.3} e^{-14400 / R} \cdot act(H^{+})^{0.5}
+
+k_{/neutral/} = 10^{-5.81} e^{-23500 / R}
+
+The kinetic laws as implemented in the =phreeqc_kin.dat= file accepts
+one parameter which represents reactive surface area in m^{2}. For the
+benchmarks the surface areas are set to
+
+- S_{dolomite}: 0.005 m^{2}
+- S_{calcite}: 0.05 m^{2}
+
+The initial content of calcite in the domain is of 0.0002 mol per kg
+of water. A constant partial pressure of 10^{-1675} atm of O_{2(g)} is
+maintained at any time in the domain in order to fix the redox
+potential of the solution to an oxidizing state (pe around 9).
+
+Note that Cl is unreactive in this system and only effects the
+computation of the activities in solution.
+
+* POET simulations
+
+Several benchmarks based on the same chemical system are defined here
+with different grid sizes, resolution and boundary conditions. The
+transported elemental concentrations are 7: C(4), Ca, Cl, Mg and the
+implicit total H, total O and Charge as required by PHREEQC_RM.
+
+** =dolo_diffu_inner.R=
+
+- Grid discretization: square domain of 1 \cdot 1 m^{2} discretized in
+  100x100 cells
+- Boundary conditions: All sides of the domain are closed. *Fixed
+  concentration* of 0.001 molal of MgCl_{2} is defined in the domain
+  cell (20, 20) and of 0.002 molal MgCl_{2} at cells (60, 60) and
+  (80, 80)
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 10 iterations with \Delta t of 200 s
+- *DHT* parameters:
+  |  H |  O | Charge | C(4) | Ca | Cl | Mg | Calcite | Dolomite |
+  | 10 | 10 |      3 |    5 |  5 |  5 |  5 |       5 |        5 |
+- Hooks: the following hooks are defined:
+  1. =dht_fill=: 
+  2. =dht_fuzz=:
+  3. =interp_pre_func=:
+  4. =interp_post_func=:
+
+
+** =dolo_interp_long.R=
+
+- Grid discretization: rectangular domain of 5 \cdot 2.5 m^{2}
+  discretized in 400 \times 200 cells
+- Boundary conditions: *Fixed concentrations* equal to the initial
+  state are imposed at all four sides of the domain. *Fixed
+  concentration* of 0.001 molal of MgCl_{2} is defined in the domain
+  center at cell (100, 50)
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 20000 iterations with \Delta t of 200 s
+- *DHT* parameters:
+  |  H |  O | Charge | C(4) | Ca | Cl | Mg | Calcite | Dolomite |
+  | 10 | 10 |      3 |    5 |  5 |  5 |  5 |       5 |        5 |
+- Hooks: the following hooks are defined:
+  1. =dht_fill=: 
+  2. =dht_fuzz=:
+  3. =interp_pre_func=:
+  4. =interp_post_func=:
+
+     
+* References
+
+- De Lucia, Kühn, Lindemann, Lübke, Schnor: POET (v0.1): speedup of
+  many-core parallel reactive transport simulations with fast DHT
+  lookups, Geosci. Model Dev., 14, 7391–7409, 2021.
+  https://doi.org/10.5194/gmd-14-7391-2021
+- Möller, Marco De Lucia: The impact of Mg^{2+} ions on equilibration
+  of Mg-Ca carbonates in groundwater and brines, Geochemistry
+  80, 2020. https://doi.org/10.1016/j.chemer.2020.125611
+- Palandri, Kharaka: A Compilation of Rate Parameters of Water-Mineral
+  Interaction Kinetics for Application to Geochemical Modeling, Report
+  2004-1068, USGS, 2004.

bench/dolo/dol.pqi

@@ -0,0 +1,43 @@
+SOLUTION 1
+    units	mol/kgw
+    water	1
+    temperature	25
+    pH	7
+    pe	4
+PURE 1
+    Calcite 0.0 1
+END
+
+RUN_CELLS 
+    -cells 1
+
+COPY solution 1 2
+
+PURE 2
+    O2g -0.1675 10
+KINETICS 2
+    Calcite
+    -m      0.000207
+    -parms  0.05
+    -tol    1e-10
+    Dolomite
+    -m      0.0
+    -parms  0.005
+    -tol    1e-10
+END
+
+SOLUTION 3
+    units	mol/kgw
+    water 1
+    temp 25 
+    Mg 0.001
+    Cl 0.002
+END
+
+SOLUTION 4
+    units mol/kgw 
+    water 1
+    temp 25
+    Mg 0.002
+    Cl 0.004
+END

bench/dolo/dolo_inner_large.R

@@ -0,0 +1,115 @@
+rows <- 2000
+cols <- 1000
+
+grid_def <- matrix(2, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+    pqc_in_file = "./dol.pqi",
+    pqc_db_file = "./phreeqc_kin.dat", # Path to the database file for Phreeqc
+    grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+    grid_size = c(cols, rows) / 100, # Size of the grid in meters
+    constant_cells = c() # IDs of cells with constant concentration
+)
+
+bound_size <- 2
+
+diffusion_setup <- list(
+    inner_boundaries = list(
+        "row" = c(400, 1400, 1600),
+        "col" = c(200, 800, 800),
+        "sol_id" = c(3, 4, 4)
+    ),
+    alpha_x = 1e-6,
+    alpha_y = 1e-6
+)
+
+check_sign_cal_dol_dht <- function(old, new) {
+    if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+        return(TRUE)
+    }
+    if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+        return(TRUE)
+    }
+    return(FALSE)
+}
+
+fuzz_input_dht_keys <- function(input) {
+    dht_species <- c(
+        "H" = 3,
+        "O" = 3,
+        "Charge" = 3,
+        "C(4)" = 6,
+        "Ca" = 6,
+        "Cl" = 3,
+        "Mg" = 5,
+        "Calcite" = 4,
+        "Dolomite" = 4
+    )
+    return(input[names(dht_species)])
+}
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+    dht_species <- c(
+        "H" = 3,
+        "O" = 3,
+        "Charge" = 3,
+        "C(4)" = 6,
+        "Ca" = 6,
+        "Cl" = 3,
+        "Mg" = 5,
+        "Calcite" = 4,
+        "Dolomite" = 4
+    )
+    data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+    names(data_set) <- names(dht_species)
+    cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+    dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+    cal_dol_same_sig <- cal == dol
+    return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+    neg_sign <- (result["Calcite"] < 0) || (result["Dolomite"] < 0)
+    return(neg_sign)
+}
+
+# Optional when using Interpolation (example with less key species and custom
+# significant digits)
+
+pht_species <- c(
+    "C(4)" = 3,
+    "Ca" = 3,
+    "Mg" = 2,
+    "Calcite" = 2,
+    "Dolomite" = 2
+)
+
+chemistry_setup <- list(
+    dht_species = c(
+        "H" = 3,
+        "O" = 3,
+        "Charge" = 3,
+        "C(4)" = 6,
+        "Ca" = 6,
+        "Cl" = 3,
+        "Mg" = 5,
+        "Calcite" = 4,
+        "Dolomite" = 4
+    ),
+    pht_species = pht_species,
+    hooks = list(
+        dht_fill = check_sign_cal_dol_dht,
+        dht_fuzz = fuzz_input_dht_keys,
+        interp_pre = check_sign_cal_dol_interp,
+        interp_post = check_neg_cal_dol
+    )
+)
+
+# Define a setup list for simulation configuration
+setup <- list(
+    Grid = grid_setup, # Parameters related to the grid structure
+    Diffusion = diffusion_setup, # Parameters related to the diffusion process
+    Chemistry = chemistry_setup # Parameters related to the chemistry process
+)

bench/dolo/dolo_inner_large_rt.R

@@ -0,0 +1,10 @@
+iterations <- 500
+dt <- 50
+
+out_save <- seq(5, iterations, by = 5)
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE,
+    out_save = out_save
+)

bench/dolo/dolo_interp.R

@@ -0,0 +1,131 @@
+rows <- 400
+cols <- 200
+
+grid_def <- matrix(2, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+    pqc_in_file = "./dol.pqi",
+    pqc_db_file = "./phreeqc_kin.dat", # Path to the database file for Phreeqc
+    grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+    grid_size = c(2.5, 5), # Size of the grid in meters
+    constant_cells = c() # IDs of cells with constant concentration
+)
+
+bound_def_we <- list(
+    "type" = rep("constant", rows),
+    "sol_id" = rep(1, rows),
+    "cell" = seq(1, rows)
+)
+
+bound_def_ns <- list(
+    "type" = rep("constant", cols),
+    "sol_id" = rep(1, cols),
+    "cell" = seq(1, cols)
+)
+
+diffusion_setup <- list(
+    boundaries = list(
+        "W" = bound_def_we,
+        "E" = bound_def_we,
+        "N" = bound_def_ns,
+        "S" = bound_def_ns
+    ),
+    inner_boundaries = list(
+        "row" = floor(rows / 2),
+        "col" = floor(cols / 2),
+        "sol_id" = c(3)
+    ),
+    alpha_x = 1e-6,
+    alpha_y = 1e-6
+)
+
+check_sign_cal_dol_dht <- function(old, new) {
+    # if ((old["Calcite"] == 0) != (new["Calcite"] == 0)) {
+    #     return(TRUE)
+    # }
+    # if ((old["Dolomite"] == 0) != (new["Dolomite"] == 0)) {
+    #     return(TRUE)
+    # }
+    return(FALSE)
+}
+
+# fuzz_input_dht_keys <- function(input) {
+#     dht_species <- c(
+#         "H" = 3,
+#         "O" = 3,
+#         "Charge" = 3,
+#         "C" = 6,
+#         "Ca" = 6,
+#         "Cl" = 3,
+#         "Mg" = 5,
+#         "Calcite" = 4,
+#         "Dolomite" = 4
+#     )
+#     return(input[names(dht_species)])
+# }
+
+check_sign_cal_dol_interp <- function(to_interp, data_set) {
+    dht_species <- c(
+        "H" = 3,
+        "O" = 3,
+        "Charge" = 3,
+        "C" = 6,
+        "Ca" = 6,
+        "Cl" = 3,
+        "Mg" = 5,
+        "Calcite" = 4,
+        "Dolomite" = 4
+    )
+    data_set <- as.data.frame(do.call(rbind, data_set), check.names = FALSE, optional = TRUE)
+    names(data_set) <- names(dht_species)
+    cal <- (data_set$Calcite == 0) == (to_interp["Calcite"] == 0)
+    dol <- (data_set$Dolomite == 0) == (to_interp["Dolomite"] == 0)
+
+    cal_dol_same_sig <- cal == dol
+    return(rev(which(!cal_dol_same_sig)))
+}
+
+check_neg_cal_dol <- function(result) {
+    neg_sign <- (result["Calcite"] < 0) || (result["Dolomite"] < 0)
+    return(neg_sign)
+}
+
+# Optional when using Interpolation (example with less key species and custom
+# significant digits)
+
+pht_species <- c(
+    "C" = 3,
+    "Ca" = 3,
+    "Mg" = 2,
+    "Calcite" = 2,
+    "Dolomite" = 2
+)
+
+chemistry_setup <- list(
+    dht_species = c(
+        "H" = 3,
+        "O" = 3,
+        "Charge" = 3,
+        "C" = 6,
+        "Ca" = 6,
+        "Cl" = 3,
+        "Mg" = 5,
+        "Calcite" = 4,
+        "Dolomite" = 4
+    ),
+    pht_species = pht_species,
+    hooks = list(
+        dht_fill = check_sign_cal_dol_dht,
+        # dht_fuzz = fuzz_input_dht_keys,
+        interp_pre = check_sign_cal_dol_interp,
+        interp_post = check_neg_cal_dol
+    )
+)
+
+# Define a setup list for simulation configuration
+setup <- list(
+    Grid = grid_setup, # Parameters related to the grid structure
+    Diffusion = diffusion_setup, # Parameters related to the diffusion process
+    Chemistry = chemistry_setup # Parameters related to the chemistry process
+)

bench/dolo/dolo_interp_rt.R

@@ -0,0 +1,10 @@
+iterations <- 20000
+dt <- 200
+
+out_save <- seq(50, iterations, by = 50)
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE,
+    out_save = out_save
+)

bench/fgcs/20241211_README.tex

@@ -0,0 +1,102 @@
+% Created 2024-12-11 mer 23:24
+% Intended LaTeX compiler: pdflatex
+\documentclass[a4paper, 9pt]{article}
+\usepackage[utf8]{inputenc}
+\usepackage[T1]{fontenc}
+\usepackage{graphicx}
+\usepackage{longtable}
+\usepackage{wrapfig}
+\usepackage{rotating}
+\usepackage[normalem]{ulem}
+\usepackage{amsmath}
+\usepackage{amssymb}
+\usepackage{capt-of}
+\usepackage{hyperref}
+\usepackage{fullpage}
+\usepackage{amsmath}
+\usepackage{graphicx}
+\usepackage{charter}
+\usepackage{listings}
+\lstloadlanguages{R}
+\author{MDL <delucia@gfz.de>}
+\date{2024-12-11}
+\title{A \texttt{barite}-based benchmark for FGCS interpolation paper}
+
+
+
+\begin{document}
+
+\maketitle
+
+\section{Description}
+\label{sec:org739879a}
+
+\begin{itemize}
+\item \texttt{barite\_fgcs\_2.R}: POET input script with circular
+  "crystals" on a 200x200 nodes grid
+
+\item \(\alpha\): isotropic 10\textsuperscript{-5}
+  m\textsuperscript{2}/s outside of the crystals,
+  10\textsuperscript{-7} inside
+\item 200 iterations, dt = 1000 
+\item \texttt{barite\_fgcs\_2.pqi}: PHREEQC input, 4 SOLUTIONS
+  (basically the same as in \texttt{barite} benchmark):
+  \begin{enumerate}
+  \item Equilibrium with Celestite, no mineral \(Rightarrow\)
+  \item Equilibrium with Celestite, KINETICS Celestite (1 mol) and
+    Barite (0 mol)
+  \item Injection of 0.1 BaCl2 from NW corner
+  \item Injection of 0.2 BaCl2 from SE corner
+\end{enumerate}
+
+\item \texttt{db\_barite.dat}: PHREEQC database containing the kinetic
+  expressions for barite and celestite, stripped down from
+  \texttt{phreeqc.dat}
+\end{itemize}
+
+\begin{figure}[htbp]
+  \centering
+  \includegraphics[width=0.48\textwidth]{./fgcs_Celestite_init.pdf}
+  \includegraphics[width=0.48\textwidth]{./fgcs_Barite_200.pdf}
+  \caption{\textbf{Left:} Initial distribution of Celestite
+    "crystals". \textbf{Right:} precipitated Barite}
+\end{figure}
+
+\section{Interpolation}
+\label{sec:org2a09431}
+
+Using the following parametrization:
+
+\begin{lstlisting}
+dht_species <- c("H"         = 7,
+                 "O"         = 7,
+                 "Ba"        = 7,
+                 "Cl"        = 7,
+                 "S(6)"      = 7,
+                 "Sr"        = 7,
+                 "Barite"    = 4,
+                 "Celestite" = 4)
+
+pht_species <- c("Ba"        = 4,
+                 "Cl"        = 3,
+                 "S(6)"      = 3,
+                 "Sr"        = 3,
+                 "Barite"    = 2,
+                 "Celestite" = 2 )
+\end{lstlisting}
+
+Runtime goes from 1800 to 600 s (21 CPUs) but there are "suspect"
+errors especially in O and H, where "suspect" means some values appear
+to be multiplied by 2:
+\begin{figure}[htbp]
+  \centering
+  \includegraphics[width=0.9\textwidth]{./fgcs_interp_1.pdf}
+  \caption{Scatterplots reference vs interpolated after 1 coupling
+    iteration}
+\end{figure}
+\end{document}
+
+%%% Local Variables:
+%%% mode: LaTeX
+%%% TeX-master: t
+%%% End:

bench/fgcs/EvalFGCS.R

@@ -0,0 +1,90 @@
+## Time-stamp: "Last modified 2024-12-11 23:21:25 delucia"
+
+library(PoetUtils)
+library(viridis)
+
+
+res <- ReadPOETSims("./res_fgcs2_96/")
+
+pp <- PlotField(res$iter_200$C$Barite, rows = 200, cols = 200, contour = FALSE,
+                nlevels=12, palette=terrain.colors)
+
+cairo_pdf("fgcs_Celestite_init.pdf", family="serif")
+par(mar=c(0,0,0,0))
+pp <- PlotField((res$iter_000$Celestite), rows = 200, cols = 200,
+                contour = FALSE, breaks=c(-0.5,0.5,1.5),
+                palette = grey.colors, plot.axes = FALSE, scale = FALSE,
+                main="Initial Celestite crystals")
+dev.off()
+
+
+cairo_pdf("fgcs_Ba_init.pdf", family="serif")
+par(mar=c(0,0,0,0))
+pp <- PlotField(log10(res$iter_001$C$Cl), rows = 200, cols = 200,
+                contour = FALSE, 
+                palette = terrain.colors, plot.axes = FALSE, scale = FALSE,
+                main="log10(Ba)")
+dev.off()
+
+
+
+pp <- PlotField(log10(res$iter_002$C$Ba), rows = 200, cols = 200,
+                contour = FALSE, palette = viridis, rev.palette = FALSE,
+                main = "log10(Ba) after 5 iterations")
+
+pp <- PlotField(log10(res$iter_200$C$`S(6)`), rows = 200, cols = 200, contour = FALSE)
+
+
+str(res$iter_00)
+
+res$iter_178$C$Barite
+
+pp <- res$iter_043$C$Barite
+
+breaks <- pretty(pp, n = 5)
+
+br <- c(0, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1)
+
+pp <- PlotField(res$iter_200$C$Barite, rows = 200, cols = 200, contour = FALSE,
+                breaks = br, palette=terrain.colors)
+
+
+
+cairo_pdf("fgcs_Barite_200.pdf", family="serif")
+pp <- PlotField(log10(res$iter_200$C$Barite), rows = 200, cols = 200,
+                contour = FALSE, palette = terrain.colors, plot.axes = FALSE,
+                rev.palette = FALSE, main = "log10(Barite) after 200 iter")
+dev.off()
+
+ref <- ReadPOETSims("./res_fgcs_2_ref")
+
+rei <- ReadPOETSims("./res_fgcs_2_interp1/")
+
+
+timref <- ReadRObj("./res_fgcs_2_ref/timings.qs")
+timint <- ReadRObj("./res_fgcs_2_interp1/timings.qs")
+
+timref
+
+timint
+
+wch <- c("H","O", "Ba", "Sr","Cl", "S(6)")
+
+rf <- data.matrix(ref$iter_001$C[, wch])
+r1 <- data.matrix(rei$iter_001$C[, wch])
+
+r1[is.nan(r1)] <- NA
+rf[is.nan(rf)] <- NA
+
+cairo_pdf("fgcs_interp_1.pdf", family="serif", width = 10, height = 7)
+PlotScatter(rf, r1, which = wch, labs = c("ref", "interp"), cols = 3, log="", las = 1, pch=4)
+dev.off()
+
+
+
+head(r1)
+
+head(rf)
+
+rf$O
+r1$O

bench/fgcs/README.org

@@ -0,0 +1,2 @@
+* Refer to the LaTeX file (and pdf) for more information
+

bench/fgcs/barite_fgcs_2.R

@@ -0,0 +1,105 @@
+## Time-stamp: "Last modified 2024-12-11 16:08:11 delucia"
+
+cols <- 1000
+rows <- 1000
+
+dim_cols <- 50
+dim_rows <- 50
+
+ncirc <- 20 ## number of crystals
+rmax <- cols / 10 ## max radius (in nodes)
+
+set.seed(22933)
+
+centers <- cbind(sample(seq_len(cols), ncirc), sample(seq_len(rows), ncirc))
+radii <- sample(seq_len(rmax), ncirc, replace = TRUE)
+mi <- matrix(rep(seq_len(cols), rows), byrow = TRUE, nrow = rows)
+mj <- matrix(rep(seq_len(cols), each = rows), byrow = TRUE, nrow = rows)
+
+tmpl <- lapply(seq_len(ncirc), function(x) which((mi - centers[x, 1])^2 + (mj - centers[x, 2])^2 < radii[x]^2, arr.ind = TRUE))
+
+inds <- do.call(rbind, tmpl)
+grid <- matrix(1, nrow = rows, ncol = cols)
+grid[inds] <- 2
+
+alpha <- matrix(1e-5, ncol = cols, nrow = rows)
+alpha[inds] <- 1e-7
+
+## image(grid, asp=1)
+
+## Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file    = "./barite_fgcs_2.pqi",
+  pqc_db_file    = "../barite/db_barite.dat", ## database file
+  grid_def       = grid, ## grid definition, IDs according to the Phreeqc input
+  grid_size      = c(dim_cols, dim_rows), ## grid size in meters
+  constant_cells = c() ## IDs of cells with constant concentration
+)
+
+bound_length <- cols / 10
+
+bound_N <- list(
+  "type"   = rep("constant", bound_length),
+  "sol_id" = rep(3, bound_length),
+  "cell"   = seq(1, bound_length)
+)
+
+bound_W <- list(
+  "type"   = rep("constant", bound_length),
+  "sol_id" = rep(3, bound_length),
+  "cell"   = seq(1, bound_length)
+)
+bound_E <- list(
+  "type"   = rep("constant", bound_length),
+  "sol_id" = rep(4, bound_length),
+  "cell"   = seq(rows - bound_length + 1, rows)
+)
+
+bound_S <- list(
+  "type"   = rep("constant", bound_length),
+  "sol_id" = rep(4, bound_length),
+  "cell"   = seq(cols - bound_length + 1, cols)
+)
+
+diffusion_setup <- list(
+  boundaries = list(
+    "W" = bound_W,
+    "N" = bound_N,
+    "E" = bound_E,
+    "S" = bound_S
+  ),
+  alpha_x = alpha,
+  alpha_y = alpha
+)
+
+dht_species <- c(
+  "H"         = 7,
+  "O"         = 7,
+  "Ba"        = 7,
+  "Cl"        = 7,
+  "S"         = 7,
+  "Sr"        = 7,
+  "Barite"    = 4,
+  "Celestite" = 4
+)
+
+pht_species <- c(
+  "Ba"        = 4,
+  "Cl"        = 3,
+  "S"         = 3,
+  "Sr"        = 3,
+  "Barite"    = 0,
+  "Celestite" = 0
+)
+
+chemistry_setup <- list(
+  dht_species = dht_species,
+  pht_species = pht_species
+)
+
+## Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, ## Parameters related to the grid structure
+  Diffusion = diffusion_setup, ## Parameters related to the diffusion process
+  Chemistry = chemistry_setup
+)

bench/fgcs/barite_fgcs_2.pqi

@@ -0,0 +1,49 @@
+SOLUTION 1
+  units	mol/kgw
+  water	1
+  temperature	25
+  pH     7.008 
+  pe    10.798
+  S      6.205e-04
+  Sr     6.205e-04
+END
+
+SOLUTION 2
+  units	mol/kgw
+  water	1
+  temperature	25
+  pH     7.008 
+  pe    10.798
+  S      6.205e-04
+  Sr     6.205e-04
+KINETICS 2
+  Barite
+    -m 0.00
+    -parms 50.  # reactive surface area
+    -tol 1e-9
+  Celestite
+    -m 1
+    -parms 10.0  # reactive surface area
+    -tol 1e-9
+END
+
+SOLUTION 3
+  units mol/kgw
+  water 1
+  temperature 25
+  Ba 0.1
+  Cl 0.2
+END
+
+SOLUTION 4
+  units mol/kgw
+  water 1
+  temperature 25
+  Ba 0.2
+  Cl 0.4
+END
+
+
+RUN_CELLS
+  -cells 1 2 3 4
+END

bench/fgcs/barite_fgcs_2_rt.R

@@ -0,0 +1,7 @@
+iterations <- 200
+dt <- 1000
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE
+)

bench/surfex/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(bench_files
+    # surfex.R
+    # ex.R
+    PoetEGU_surfex_500.R
+)
+
+set(runtime_files
+    # surfex_rt.R 
+    # ex_rt.R
+    PoetEGU_surfex_500_rt.R
+)
+
+ADD_BENCH_TARGET(
+    surfex_bench
+    bench_files 
+    runtime_files
+    "surfex"
+)
+
+add_dependencies(${BENCHTARGET} surfex_bench)

bench/surfex/ExBase.pqi

@@ -0,0 +1,63 @@
+##  Time-stamp: "Last modified 2023-03-21 11:49:43 mluebke"
+KNOBS
+ -logfile            false
+ -iterations         10000
+ -convergence_tolerance  1E-12
+ -step_size          2
+ -pe_step_size       2
+SELECTED_OUTPUT
+ -reset false
+ -high_precision true
+ -solution true
+ -state true
+ -step true 
+ -pH true 
+ -pe true
+ -ionic_strength true 
+ -water true
+SOLUTION 1
+temp         13
+units        mol/kgw
+pH           7.06355
+pe          -2.626517
+C(4)         0.001990694
+Ca           0.02172649
+Cl           0.3227673 charge
+Fe           0.0001434717
+K            0.001902357
+Mg           0.01739704
+Na           0.2762882
+S(6)         0.01652701
+Sr           0.0004520361
+U(4)         8.147792e-12
+U(6)         2.237946e-09
+-water       0.00133
+EXCHANGE 1
+ -equil 1
+  Z     0.0012585
+  Y     0.0009418
+END
+
+SOLUTION 2
+temp 13 
+units mol/kgw
+
+C(-4)  2.92438561098248e-21
+C(4)  2.65160558871092e-06
+Ca  2.89001071336443e-05
+Cl  0.000429291158114428
+Fe(2)  1.90823391198114e-07
+Fe(3)  3.10832423034763e-12
+H(0)  2.7888235127385e-15
+K  2.5301787e-06
+Mg  2.31391999937907e-05
+Na  0.00036746969
+S(-2)  1.01376078438546e-14
+S(2)  1.42247026981542e-19
+S(4)  9.49422092568557e-18
+S(6)  2.19812504654191e-05
+Sr  6.01218519999999e-07
+U(4)  4.82255946569383e-12
+U(5)  5.49050615347901e-13
+U(6)  1.32462838991902e-09
+END

bench/surfex/PoetEGU_surfex_500.R

@@ -0,0 +1,40 @@
+rows <- 500
+cols <- 200
+
+grid_left <- matrix(1, nrow = rows, ncol = cols/2)
+grid_rght <- matrix(2, nrow = rows, ncol = cols/2)
+grid_def <- cbind(grid_left, grid_rght)
+
+
+# Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file = "./SurfexEGU.pqi",
+  pqc_db_file = "./SMILE_2021_11_01_TH.dat", # Path to the database file for Phreeqc
+  grid_def = grid_def,    # Definition of the grid, containing IDs according to the Phreeqc input script
+  grid_size = c(10, 4),   # Size of the grid in meters
+  constant_cells = c()    # IDs of cells with constant concentration
+)
+
+bound_def <- list(
+  "type" = rep("constant", cols),
+  "sol_id" = rep(3, cols),
+  "cell" = seq(1, cols)
+)
+
+diffusion_setup <- list(
+  boundaries = list(
+    "N" = bound_def
+  ),
+  alpha_x = matrix(runif(rows*cols))*1e-8, 
+  alpha_y = matrix(runif(rows*cols))*1e-9## ,1e-10
+)
+
+
+chemistry_setup <- list()
+
+# Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, # Parameters related to the grid structure
+  Diffusion = diffusion_setup, # Parameters related to the diffusion process
+  Chemistry = chemistry_setup # Parameters related to the chemistry process
+)

bench/surfex/PoetEGU_surfex_500_rt.R

@@ -0,0 +1,11 @@
+iterations <- 200
+dt <- 1000
+
+out_save <- c(1, 2, seq(5, iterations, by=5))
+## out_save <- seq(1, iterations)
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE,
+    out_save = out_save
+)

bench/surfex/README.org

@@ -0,0 +1,100 @@
+#+TITLE: Description of =surfex= benchmark
+#+AUTHOR: MDL <delucia@gfz-potsdam.de>
+#+DATE: 2023-08-26
+#+STARTUP: inlineimages
+#+LATEX_CLASS_OPTIONS: [a4paper,9pt]
+#+LATEX_HEADER: \usepackage{fullpage}
+#+LATEX_HEADER: \usepackage{amsmath, systeme}
+#+LATEX_HEADER: \usepackage{graphicx}
+#+LATEX_HEADER: \usepackage{charter}
+#+OPTIONS: toc:nil
+
+* Quick start
+
+#+begin_src sh :language sh :frame single
+mpirun -np 4 ./poet ex.R ex_res
+mpirun -np 4 ./poet surfex.R surfex_res
+#+end_src
+
+* List of Files
+- =ex.R=: POET input script for a 100x100 simulation grid, only
+  exchange
+- =ExBase.pqi=: PHREEQC input script for the =ex.R= model
+- =surfex.R=: POET input script for a 1000x1000 simulation grid
+  considering both cation exchange and surface complexation
+- =SurfExBase.pqi=: PHREEQC input script for the =surfex.R= model
+- =SMILE_2021_11_01_TH.dat=: PHREEQC database containing the
+  parametrized data for Surface and Exchange, based on the SMILE
+  Thermodynamic Database (Version 01-November-2021)
+
+* Chemical system
+
+This model describes migration of Uranium radionuclide in Opalinus
+clay subject to surface complexation and cation exchange on the
+surface of clay minerals. These two processes account for the binding
+of aqueous complexes to the surfaces of minerals, which may have a
+significant impact on safety of underground nuclear waste repository.
+Namely, they can act as retardation buffer for uranium complexes
+entering into a natural system. The system is kindly provided by Dr.
+T. Hennig and is inspired to the sandy facies BWS-A3 sample from the
+Mont Terri underground lab (Hennig and Kühn, 2021).
+
+This chemical system is highly redox-sensitive, and several elements
+are defined in significant amounts in different valence states. In
+total, 20 elemental concentrations and valences are transported:
+C(-4), C(4), Ca, Cl, Fe(2), Fe(3), K, Mg, Na, S(-2), S(2), S(4), S(6),
+Sr , U(4), U(5), U(6); plus the total H, total O and Charge implicitly
+required by PHREEQC_RM.
+
+** Exchange
+
+The SMILE database defines thermodynamical data for exchange of all
+major cations and uranyl-ions on Illite and Montmorillonite. In
+PHREEQC terms:
+- *Y* for Montmorillonite, with a total amount of 1.2585
+  milliequivalents and
+- *Z* for Illite, with a total amount of 0.9418 meq
+
+** Surface
+
+Here we consider a Donnan diffuse double layer of 0.49 nm. Six
+distinct sorption sites are defined:
+- Kln_aOH (aluminol site) and Kln_siOH (silanol) for Kaolinite
+- For Illite, strong and weak sites Ill_sOH and Ill_wOH respectively
+- For Montmorillonite, strong and weak sites Mll_sOH and Mll_wOH
+  respectively
+  
+Refer to the =SurfExBase.pqi= script for the actual numerical values
+of the parameters.
+
+* POET simulations
+
+** =ex.R=
+
+This benchmark only considers EXCHANGE, no mineral or SURFACE
+complexation is involved.
+
+- Grid discretization: square domain of 1 \cdot 1 m^{2} discretized in
+  100x100 cells
+- Boundary conditions: E, S and W sides of the domain are closed.
+  *Fixed concentrations* are fixed at the N boundary.
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 10 iterations with \Delta t of 200 s
+- *DHT* is not implemented as of yet for models including SURFACE and
+  EXCHANGE geochemical processes *TODO*
+- Hooks: no hooks defined *TODO*
+
+** =surfex.R=
+
+- Grid discretization: rectangular domain of 1 \cdot 1 m^{2}
+  discretized in 10 \times 10 cells
+- Boundary conditions: E, S and W sides of the domain are closed.
+  *Fixed concentrations* are fixed at the N boundary.
+- Diffusion coefficients: isotropic homogeneous \alpha = 1E-06
+- Time steps & iterations: 10 iterations with \Delta t of 200 s
+     
+* References
+
+- Hennig, T.; Kühn, M.Surrogate Model for Multi-Component Diffusion of
+  Uranium through Opalinus Clay on the Host Rock Scale. Appl. Sci.
+  2021, 11, 786. https://doi.org/10.3390/app11020786

bench/surfex/SurfExBase.pqi

@@ -0,0 +1,80 @@
+##  Time-stamp: "Last modified 2023-02-27 14:31:11 delucia"
+KNOBS
+ -logfile            false
+ -iterations         10000
+ -convergence_tolerance  1E-12
+ -step_size          2
+ -pe_step_size       2
+SELECTED_OUTPUT
+ -reset false
+ -high_precision true
+ -solution true
+ -state true
+ -step true 
+ -pH true 
+ -pe true
+ -ionic_strength true 
+ -water true
+USER_PUNCH
+-head total_o total_h cb  C(-4) C(4) Ca Cl Fe(2) Fe(3) H(0) K Mg Na S(-2) S(2) S(4) S(6) Sr U(4) U(5) U(6) UO2(am,hyd) KdU
+-start
+5 w=TOT("water")
+10 PUNCH TOTMOLE("O"), TOTMOLE("H"), CHARGE_BALANCE, w*TOT("C(-4)"), w*TOT("C(4)"), w*TOT("Ca"), w*TOT("Cl"), w*TOT("Fe(2)"), w*TOT("Fe(3)"), w*TOT("H(0)"), w*TOT("K"), w*TOT("Mg"), w*TOT("Na"), w*TOT("S(-2)"), w*TOT("S(2)"), w*TOT("S(4)"), w*TOT("S(6)"), w*TOT("Sr"), w*TOT("U(4)"), w*TOT("U(5)"), w*TOT("U(6)"), EQUI("UO2(am,hyd)")
+20 PUNCH ((SURF("U, Ill")+SURF("U, Mll")+SURF("U, Kln")+EDL("U, Ill")+EDL("U, Mll")+EDL("U, Kln"))/((TOT("U")*1.01583)))/(0.002251896406*1000)
+-end
+SOLUTION 1
+temp         13
+units        mol/kgw
+pH           7.06355
+pe          -2.626517
+C(4)         0.001990694
+Ca           0.02172649
+Cl           0.3227673 charge
+Fe           0.0001434717
+K            0.001902357
+Mg           0.01739704
+Na           0.2762882
+S(6)         0.01652701
+Sr           0.0004520361
+U(4)         8.147792e-12
+U(6)         2.237946e-09
+-water       0.00133
+SURFACE 1
+-equil 1
+-sites_units density
+-donnan 4.9e-10
+  Kln_aOH  1.155    11.  5.0518
+  Kln_siOH 1.155
+  Ill_sOH  0.05    100.  5.5931
+  Ill_wOH  2.26
+  Mll_sOH  0.05    100.  1.0825
+  Mll_wOH  2.26
+EXCHANGE 1
+ -equil 1
+  Z     0.0012585
+  Y     0.0009418
+END
+
+SOLUTION 2
+temp 13 
+units mol/kgw
+
+C(-4)  2.92438561098248e-21
+C(4)  2.65160558871092e-06
+Ca  2.89001071336443e-05
+Cl  0.000429291158114428
+Fe(2)  1.90823391198114e-07
+Fe(3)  3.10832423034763e-12
+H(0)  2.7888235127385e-15
+K  2.5301787e-06
+Mg  2.31391999937907e-05
+Na  0.00036746969
+S(-2)  1.01376078438546e-14
+S(2)  1.42247026981542e-19
+S(4)  9.49422092568557e-18
+S(6)  2.19812504654191e-05
+Sr  6.01218519999999e-07
+U(4)  4.82255946569383e-12
+U(5)  5.49050615347901e-13
+U(6)  1.32462838991902e-09
+END

bench/surfex/SurfexEGU.pqi

@@ -0,0 +1,108 @@
+##  Time-stamp: "Last modified 2024-04-12 10:59:59 delucia"
+## KNOBS
+##  -logfile            false
+##  -iterations         10000
+##  -convergence_tolerance  1E-12
+##  -step_size          2
+##  -pe_step_size       2
+
+SOLUTION 1                     ## Porewater composition Opalinus Clay, WITHOUT radionuclides, AFTER EQUI_PHASES
+  pe           -2.627          ## Eh = -227 mV, Value from Bossart & Thury (2008)-> PC borehole measurement 2003, Eh still decreasing  
+  density      1.01583         ## kg/dm³ = g/cm³
+  temp         13              ## mean temperature Mont Terri, Bossart & Thury (2008), calculations performed for 25°C
+  units        mol/kgw
+  ## Mean composition   
+  pH           7.064                    
+  Na           2.763e-01
+  Cl           3.228e-01  charge 
+  S(6)         1.653e-02  as SO4
+  Ca           2.173e-02 
+  Mg           1.740e-02
+  K            1.902e-03 
+  Sr           4.520e-04 
+  Fe           1.435e-04                 
+  U            2.247e-09                
+
+SURFACE 1 Opalinus Clay, clay minerals 
+  ## calculated with rho_b=2.2903 kg/dm³, poro=0.1662
+  ## 1 dm³ = 13.565641 kg_sed/kg_pw
+  -equil 1                ## equilibrate with solution 1 
+  -sites_units density    ## set unit for binding site density to sites/nm2
+  -donnan 4.9e-10         ## calculated after Wigger & Van Loon (2018) for ionic strength after equilibration with minerales for pCO2=2.2 log10 bar    
+
+  # surface  density SSA (m2/g)  mass (g/kgw) 
+  Kln_aOH  1.155   11.         3798.4     ## Kaolinite 28 wt% (aluminol and silanol sites) 
+  Kln_siOH 1.155               
+  Ill_sOH  0.05    100.        4205.35    ## Illite 31 wt% (weak und strong binding sites) 
+  Ill_wOH  2.26                           ## 2 % strong binding sites  
+  Mll_sOH  0.05    100.        813.94     ## Montmorillonite = smektite = 6 wt% (weak und strong binding sites)            
+  Mll_wOH  2.26                           ## 2 % strong binding sites
+
+EXCHANGE 1 Exchanger, only illite+montmorillonite 
+  ## Illite = 0.225 eq/kg_rock, Montmorillonit = 0.87 eq/kg_rock 
+ -equil 1         ## equilibrate with solution 1
+  Z     0.9462    ## = Illite
+  Y     0.70813   ## = Montmorillonite
+END
+
+SOLUTION 2                     ##  Porewater composition Opalinus Clay, WITHOUT radionuclides, AFTER EQUI_PHASES
+  pe           -2.627          ## Eh = -227 mV, Value from Bossart & Thury (2008)-> PC borehole measurement 2003, Eh still decreasing  
+  density      1.01583         ## kg/dm³ = g/cm³
+  temp         13              ## mean temperature Mont Terri, Bossart & Thury (2008), calculations performed for 25°C
+  units        mol/kgw
+  ## Mean composition   
+  pH           7.064                    
+  Na           2.763e-01
+  Cl           3.228e-01  charge 
+  S(6)         1.653e-02  as SO4
+  Ca           2.173e-02 
+  Mg           1.740e-02
+  K            1.902e-03 
+  Sr           4.520e-04 
+  Fe           1.435e-04                 
+  U            2.247e-09                
+
+SURFACE 2 Opalinus Clay, clay minerals 
+  -equil 2                ## equilibrate with solution 2 
+  -sites_units density    ## set unit for binding site density to
+			  ## sites/nm2
+  -donnan 4.9e-10         ## calculated after Wigger & Van Loon (2018)
+			  ## for ionic strength after equilibration
+			  ## with minerales for pCO2=2.2 log10 bar
+
+  ## surface  density SSA (m2/g)  mass (g/kgw) 
+  Kln_aOH  1.155   11.         2798.4     ## Kaolinite 28 wt% (aluminol and silanol sites) 
+  Kln_siOH 1.155               
+  Ill_sOH  0.05    100.        1205.35    ## Illite 31 wt% (weak und strong binding sites) 
+  Ill_wOH  2.26                           ## 2 % strong binding sites  
+  Mll_sOH  0.05    100.        113.94     ## Montmorillonite = smektite = 6 wt% (weak und strong binding sites)            
+  Mll_wOH  2.26                           ## 2 % strong binding sites
+
+EXCHANGE 2 Exchanger, only illite+montmorillonite 
+  ## Illite = 0.225 eq/kg_rock, Montmorillonit = 0.87 eq/kg_rock 
+ -equil 2         ## equilibrate with solution 1
+  Z     0.5       ## = Illite
+  Y     0.2       ## = Montmorillonite
+END
+
+SOLUTION 3
+  pe           -2.627          ## Eh = -227 mV, Value from Bossart & Thury (2008)-> PC borehole measurement 2003, Eh still decreasing  
+  density      1.01583         ## kg/dm³ = g/cm³
+  temp         13              ## mean temperature Mont Terri, Bossart & Thury (2008), calculations performed for 25°C
+  units        mol/kgw
+  ## Mean composition   
+  pH           7.064                    
+  Na           3.763e-01
+  Cl           4.228e-01  charge 
+  S(6)         1.653e-02  as SO4
+  Ca           2.173e-02 
+  Mg           1.740e-02
+  K            1.902e-03 
+  Sr           4.520e-04 
+  Fe           1.435e-04                 
+  U            1e-6            
+  C            1.991e-03 
+END
+
+RUN_CELLS
+END

bench/surfex/ex.R

@@ -0,0 +1,37 @@
+rows <- 100
+cols <- 100
+
+grid_def <- matrix(1, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file = "./SurfExBase.pqi",
+  pqc_db_file = "./SMILE_2021_11_01_TH.dat", # Path to the database file for Phreeqc
+  grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+  grid_size = c(1, 1), # Size of the grid in meters
+  constant_cells = c() # IDs of cells with constant concentration
+)
+
+bound_def <- list(
+  "type" = rep("constant", cols),
+  "sol_id" = rep(2, cols),
+  "cell" = seq(1, cols)
+)
+
+diffusion_setup <- list(
+  boundaries = list(
+    "N" = bound_def
+  ),
+  alpha_x = 1e-6,
+  alpha_y = 1e-6
+)
+
+
+chemistry_setup <- list()
+
+# Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, # Parameters related to the grid structure
+  Diffusion = diffusion_setup, # Parameters related to the diffusion process
+  Chemistry = chemistry_setup # Parameters related to the chemistry process
+)

bench/surfex/ex_rt.R

@@ -0,0 +1,7 @@
+iterations <- 10
+dt <- 200
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE
+)

bench/surfex/surfex.R

@@ -0,0 +1,37 @@
+rows <- 1000
+cols <- 1000
+
+grid_def <- matrix(1, nrow = rows, ncol = cols)
+
+# Define grid configuration for POET model
+grid_setup <- list(
+  pqc_in_file = "./SurfExBase.pqi",
+  pqc_db_file = "./SMILE_2021_11_01_TH.dat", # Path to the database file for Phreeqc
+  grid_def = grid_def, # Definition of the grid, containing IDs according to the Phreeqc input script
+  grid_size = c(rows, cols) / 10, # Size of the grid in meters
+  constant_cells = c() # IDs of cells with constant concentration
+)
+
+bound_def <- list(
+  "type" = rep("constant", cols),
+  "sol_id" = rep(2, cols),
+  "cell" = seq(1, cols)
+)
+
+diffusion_setup <- list(
+  boundaries = list(
+    "N" = bound_def
+  ),
+  alpha_x = 1e-6,
+  alpha_y = 1e-6
+)
+
+
+chemistry_setup <- list()
+
+# Define a setup list for simulation configuration
+setup <- list(
+  Grid = grid_setup, # Parameters related to the grid structure
+  Diffusion = diffusion_setup, # Parameters related to the diffusion process
+  Chemistry = chemistry_setup # Parameters related to the chemistry process
+)

bench/surfex/surfex_rt.R

@@ -0,0 +1,10 @@
+iterations <- 100
+dt <- 200
+
+out_save <- seq(5, iterations, by = 5)
+
+list(
+    timesteps = rep(dt, iterations),
+    store_result = TRUE,
+    out_save = out_save
+)

data/CMakeLists.txt

@@ -1 +0,0 @@
-install(FILES SimDol2D.R DESTINATION data)

data/SimComp2D.R

@@ -1,128 +0,0 @@
-## chemical database
-db <- RPhreeFile("mdl_quint_kin.dat", is.db=TRUE)
-phreeqc::phrLoadDatabaseString(db)
-
-## only the directory
-demodir <- system.file("extdata", "demo_rtwithmufits", package="Rmufits")
-
-prop <- c("Al", "C","Ca","Cl","Fe", "K", "Mg","Na", "Si", "pH", ## "pe",
-          "Albite", "Calcite", "Chlorite", "Illite", "Kaolinite")
-
-signif_vector <- c(7,7,7,7,7,7,7,7,7,6, 5,5,5,5,5)
-prop_type <- rep("normal", length(signif_vector))
-
-base <- c("SOLUTION 1",
-          "units  mol/kgw",
-          "pH     6.77",
-          "temp  35",
-          "-water 1",
-          "Al  8.06386e-09",
-          "C   0.0006108294",
-          "Ca  0.09709463",
-          "Cl  4.340042",
-          "Fe  1.234357e-05",
-          "K   0.01117434",
-          "Mg  0.0406959",
-          "Na  4.189209",
-          "Si  0.0001935754",
-          "INCREMENTAL_REACTIONS true",
-          "KINETICS 1 ",
-          "-steps 86400",
-          "-bad_step_max 10000",
-          "-cvode  true",
-          "Albite",
-          "-m      8.432165", ## 1540.0",
-          "-parms  01.54  100",
-          "Calcite",
-          "-m   0.0",
-          "-parms  10  100",
-          "Chlorite",
-          "-m      1.106585", ## 202.100",
-          "-parms  64.84  100",
-          "Illite",
-          "-m      0.9549153", ## 174.400",
-          "-parms  43.38  100",
-          "Kaolinite",
-          "-m      0.0",
-          "-parms  29.17  100",
-          "END")
-
-selout <- c("KNOBS",
-            "-convergence_tolerance 1E-6",
-            "SELECTED_OUTPUT",
-            "-reset            false",
-            "USER_PUNCH",
-            "-head  Al C Ca Cl Fe K Mg Na Si pH Albite Calcite Chlorite Illite Kaolinite", ## pe
-            "10 PUNCH TOT(\"Al\"), TOT(\"C\"), TOT(\"Ca\"), TOT(\"Cl\"), TOT(\"Fe\"), TOT(\"K\"), TOT(\"Mg\"), TOT(\"Na\"), TOT(\"Si\"), -LA(\"H+\"), KIN(\"Albite\"), KIN(\"Calcite\"), KIN(\"Chlorite\"), KIN(\"Illite\"), KIN(\"Kaolinite\")" )
-
-## Define initial conditions as equilibrium with primary minerals
-ipr <- c(Al  = 8.689e-10,
-         C   = 0.0006108,
-         Ca  = 0.09709,
-         Cl  = 4.34,
-         Fe  = 1.802e-06,
-         K   = 0.01131,
-         Mg  = 0.04074,
-         Na  = 4.189,
-         Si  = 7.653e-05,
-         pH  = 6.889,
-         Albite    =  5.0,
-         Calcite   =  0.0,
-         Chlorite  = 10.0,
-         Illite    =  2.0,
-         Kaolinite =  0.0
-         )
-
-initstate <- matrix(rep(ipr, 2500), byrow=TRUE, ncol=length(ipr))
-colnames(initstate) <- names(ipr)
-
-vecinj <- c(Al= 8.694e-10,
-            C = 8.182e-01,
-            Ca= 9.710e-02,
-            Cl= 4.340e+00,
-            Fe= 1.778e-06,
-            K = 1.131e-02,
-            Mg= 4.074e-02,
-            Na= 4.189e+00,
-            Si= 7.652e-05,
-            pH= 2.556228)
-
-
-## setup boundary conditions for transport - we have already read the
-## GRID with the following code:
-## grid <- Rmufits::ReadGrid(paste0(demodir,"/d2ascii.run.GRID.SUM"))
-## cbound <- which(grid$cell$ACTNUM == 2)
-## dput(cbound)
-cbound <- c(1L, 50L, 100L, 150L, 200L, 250L, 300L, 350L, 400L, 450L, 500L,
-            550L, 600L, 650L, 700L, 750L, 800L, 850L, 900L, 950L, 1000L,
-            1050L, 1100L, 1150L, 1200L, 1250L, 1300L, 1350L, 1400L, 1450L,
-            1500L, 1550L, 1600L, 1650L, 1700L, 1750L, 1800L, 1850L, 1900L,
-            1950L, 2000L, 2050L, 2100L, 2150L, 2200L, 2250L, 2300L, 2350L,
-            2400L, 2450L, 2451L, 2452L, 2453L, 2454L, 2455L, 2456L, 2457L,
-            2458L, 2459L, 2460L, 2461L, 2462L, 2463L, 2464L, 2465L, 2466L,
-            2467L, 2468L, 2469L, 2470L, 2471L, 2472L, 2473L, 2474L, 2475L,
-            2476L, 2477L, 2478L, 2479L, 2480L, 2481L, 2482L, 2483L, 2484L,
-            2485L, 2486L, 2487L, 2488L, 2489L, 2490L, 2491L, 2492L, 2493L,
-            2494L, 2495L, 2496L, 2497L, 2498L, 2499L, 2500L)
-
-boundary_matrix <- matrix(rep(ipr[1:10], length(cbound)), byrow=TRUE, nrow=length(cbound))
-colnames(boundary_matrix) <- names(ipr[1:10])
-boundary_matrix[1, ] <- vecinj
-
-boundary_matrix <- cbind(cbound,boundary_matrix)
-
-setup <- list(n = 2500,
-              bound = boundary_matrix,
-              base  = base,
-              first = selout,
-              initsim = initstate,
-              Cf    = 1,
-              prop  = prop,
-              immobile = seq(11,15),
-              kin      = seq(11,15),
-              phase   = "FLUX1",
-              density = "DEN1",
-              reduce = FALSE,
-              snapshots = demodir, ## directory where we will read MUFITS SUM files
-              gridfile  = paste0(demodir,"/d2ascii.run.GRID.SUM")
-              )

data/SimCompKtz.R

@@ -1,131 +0,0 @@
-db <- RPhreeFile("mdl_quint_kin.dat", is.db=TRUE)
-phreeqc::phrLoadDatabaseString(db)
-
-## only the directory
-demodir <- "./snaps/"
-         
-base <- c("SOLUTION 1",
-          "units  mol/kgw",
-          "pH     6.77",
-          "temp  35",
-          "-water 1",
-          "Al  8.06386e-09",
-          "C   0.0006108294",
-          "Ca  0.09709463",
-          "Cl  4.340042",
-          "Fe  1.234357e-05",
-          "K   0.01117434", 
-          "Mg  0.0406959",
-          "Na  4.189209",
-          "Si  0.0001935754",
-          "INCREMENTAL_REACTIONS true",
-          "KINETICS 1 ",
-          "-steps 86400", 
-          "-bad_step_max 10000",
-          "-cvode  true",
-          "Albite",
-          "-m      8.432165", ## 1540.0",
-          "-parms  01.54  100",
-          "Calcite",
-          "-m   0.0",
-          "-parms  10  100",
-          "Chlorite",
-          "-m      1.106585", ## 202.100",
-          "-parms  64.84  100",
-          "Illite",
-          "-m      0.9549153", ## 174.400",
-          "-parms  43.38  100",
-          "Kaolinite",
-          "-m      0.0",
-          "-parms  29.17  100",
-          "END")
-
-selout <- c("KNOBS",
-            "-convergence_tolerance 1E-6",
-            "SELECTED_OUTPUT",
-            "-reset            false",
-            "USER_PUNCH",
-            "-head  Al C Ca Cl Fe K Mg Na Si pH Albite Calcite Chlorite Illite Kaolinite", ## pe 
-            "10 PUNCH TOT(\"Al\"), TOT(\"C\"), TOT(\"Ca\"), TOT(\"Cl\"), TOT(\"Fe\"), TOT(\"K\"), TOT(\"Mg\"), TOT(\"Na\"), TOT(\"Si\"), -LA(\"H+\"), KIN(\"Albite\"), KIN(\"Calcite\"), KIN(\"Chlorite\"), KIN(\"Illite\"), KIN(\"Kaolinite\")" )
-
-## Define initial conditions as equilibrium with primary minerals
-ipr <- c(Al  = 8.689e-10,
-         C   = 0.0006108,
-         Ca  = 0.09709,
-         Cl  = 4.34,
-         Fe  = 1.802e-06,
-         K   = 0.01131,
-         Mg  = 0.04074,
-         Na  = 4.189,
-         Si  = 7.653e-05,
-         pH  = 6.889,
-         Albite    =  5.0,
-         Calcite   =  0.0,
-         Chlorite  = 10.0,
-         Illite    =  2.0,
-         Kaolinite =  0.0
-         )
-
-initstate <- matrix(rep(ipr, 648420), byrow=TRUE, ncol=length(ipr))
-colnames(initstate) <- names(ipr)
-
-vecinj <- c(Al= 8.694e-10,
-            C = 8.182e-01,
-            Ca= 9.710e-02,
-            Cl= 4.340e+00,
-            Fe= 1.778e-06,
-            K = 1.131e-02,
-            Mg= 4.074e-02,
-            Na= 4.189e+00,
-            Si= 7.652e-05,
-            pH= 2.556228)
-
-            
-            
-
-prop <- c("Al", "C","Ca","Cl","Fe", "K", "Mg","Na", "Si", "pH", ## "pe",
-          "Albite", "Calcite", "Chlorite", "Illite", "Kaolinite")
-
-bound_elm <- c(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 
-               15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 
-               28L, 29L, 30L, 31L, 32L, 33L, 34L, 35L, 36L, 37L, 38L, 39L, 40L, 
-               41L, 42L, 43L, 44L, 45L, 46L, 47L, 48L, 49L, 50L, 51L, 52L, 53L, 
-               54L, 55L, 56L, 57L, 58L, 59L, 60L, 61L, 62L, 63L, 64L, 65L, 66L, 
-               67L, 68L, 69L, 70L, 71L, 72L, 73L, 74L, 75L, 76L, 77L, 78L, 79L, 
-               80L, 81L, 82L, 83L, 84L, 85L, 86L, 87L, 88L, 89L, 90L, 91L, 92L, 
-               93L, 94L, 95L, 96L, 97L, 98L, 99L, 100L, 101L, 102L, 103L, 104L, 
-               105L, 106L, 107L, 108L, 214L, 215L)
-
-inj_elm <- c(7426L, 18233L, 29040L, 39847L, 
-             50654L, 61461L, 72268L, 83075L, 93882L, 104689L, 115496L, 126303L, 
-             137110L, 147917L, 158724L, 169531L, 180338L, 191145L, 201952L, 
-             212759L, 223566L, 234373L, 245180L, 255987L, 266794L, 277601L, 
-             288408L, 299215L, 310022L, 320829L, 331636L, 342443L, 353250L, 
-             364057L, 374864L, 385671L, 396478L, 407285L, 418092L, 428899L, 
-             439706L, 450513L, 461320L, 472127L, 482934L, 493741L, 504548L, 
-             515355L)
-
-cbound <- inj_elm
-
-boundinit <- matrix(rep(vecinj, length(cbound)), ncol=length(vecinj), byrow=TRUE) 
-myboundmat <- cbind(cbound,boundinit)
-
-## distinguish between injection and real boundaries
-colnames(myboundmat) <- c("cbound", names(vecinj))
-
-setup <- list(n=648420,
-              base=base, 
-              bound=myboundmat, 
-              first=selout, 
-              initsim=initstate,  
-              Cf=1, 
-              prop=prop, 
-              immobile=seq(11,15),
-              kin= seq(11,15),
-              phase="FLUX1",
-              density="DENS",
-              reduce=TRUE,
-              snapshots="snaps/AllSnaps_cmp_v3.rds",
-              gridfile ="snaps/GridKtz_cmp_v3.rds")
-
-         

data/SimDol2D.R

@@ -1,118 +0,0 @@
-## chemical database
-db <- RPhreeFile(system.file("extdata", "phreeqc_kin.dat",
-                             package="RedModRphree"), is.db=TRUE)
-
-phreeqc::phrLoadDatabaseString(db)
-
-## only the directory
-demodir <- system.file("extdata", "demo_rtwithmufits", package="Rmufits")
-
-prop <- c("C","Ca","Cl","Mg","pH","pe","O2g", "Calcite","Dolomite")
-
-signif_vector <- c(7,7,7,7,7,7,7,5,5)
-prop_type <- c("act","act","act","act","logact","logact","ignore","act","act")
-
-
-base <- c("SOLUTION 1",
-          "units mol/kgw",
-          "temp 25.0",
-          "water 1",
-          "pH 9.91 charge",
-          "pe 4.0",
-          "C   1.2279E-04",
-          "Ca  1.2279E-04",
-          "Mg 0.001",
-          "Cl 0.002",
-          "PURE 1",
-          "O2g -0.1675 10",
-          "KINETICS 1",
-          "-steps 100",
-          "-step_divide 100",
-          "-bad_step_max 2000",
-          "Calcite", "-m      0.000207",
-          "-parms  0.0032",
-          "Dolomite",
-          "-m      0.0",
-          "-parms  0.00032",
-          "END")
-
-selout <- c("SELECTED_OUTPUT", "-high_precision true", "-reset false",
-            "-time", "-soln", "-temperature true", "-water true",
-            "-pH", "-pe", "-totals C Ca Cl Mg",
-            "-kinetic_reactants Calcite Dolomite", "-equilibrium O2g")
-
-initsim <- c("SELECTED_OUTPUT", "-high_precision true",
-             "-reset false",
-             "USER_PUNCH",
-             "-head C Ca Cl Mg pH pe O2g Calcite Dolomite",
-             "10 PUNCH TOT(\"C\"), TOT(\"Ca\"), TOT(\"Cl\"), TOT(\"Mg\"), -LA(\"H+\"), -LA(\"e-\"), EQUI(\"O2g\"), EQUI(\"Calcite\"), EQUI(\"Dolomite\")",
-             "SOLUTION 1",
-             "units mol/kgw",
-             "temp 25.0", "water 1",
-             "pH 9.91 charge",
-             "pe 4.0",
-             "C   1.2279E-04",
-             "Ca  1.2279E-04",
-             "Cl 1E-12",
-             "Mg 1E-12",
-             "PURE 1",
-             "O2g     -0.6788 10.0",
-             "Calcite  0.0 2.07E-3",
-             "Dolomite 0.0 0.0",
-             "END")
-
-vecinj <- c("C"= 0,
-            "Ca"  = 0,
-            "Cl"  = 0.002,
-            "Mg"  = 0.001,
-            "pe"   = 4,
-            "pH"   = 7)
-
-init <- c("C(4)"= 1.2279E-4,
-          "Ca"  =1.2279E-4,
-          "Cl"  =0,        
-          "Mg"  =0,
-          "pe"  =4,
-          "pH"  =7,
-          "Calcite"= 2.07e-4,
-          "Dolomite"= 0)
-
-
-## setup boundary conditions for transport - we have already read the
-## GRID with the following code:
-## grid <- Rmufits::ReadGrid(paste0(demodir,"/d2ascii.run.GRID.SUM"))
-## cbound <- which(grid$cell$ACTNUM == 2)
-## dput(cbound)
-cbound <- c(1L, 50L, 100L, 150L, 200L, 250L, 300L, 350L, 400L, 450L, 500L, 
-            550L, 600L, 650L, 700L, 750L, 800L, 850L, 900L, 950L, 1000L, 
-            1050L, 1100L, 1150L, 1200L, 1250L, 1300L, 1350L, 1400L, 1450L, 
-            1500L, 1550L, 1600L, 1650L, 1700L, 1750L, 1800L, 1850L, 1900L, 
-            1950L, 2000L, 2050L, 2100L, 2150L, 2200L, 2250L, 2300L, 2350L, 
-            2400L, 2450L, 2451L, 2452L, 2453L, 2454L, 2455L, 2456L, 2457L, 
-            2458L, 2459L, 2460L, 2461L, 2462L, 2463L, 2464L, 2465L, 2466L, 
-            2467L, 2468L, 2469L, 2470L, 2471L, 2472L, 2473L, 2474L, 2475L, 
-            2476L, 2477L, 2478L, 2479L, 2480L, 2481L, 2482L, 2483L, 2484L, 
-            2485L, 2486L, 2487L, 2488L, 2489L, 2490L, 2491L, 2492L, 2493L, 
-            2494L, 2495L, 2496L, 2497L, 2498L, 2499L, 2500L)
-
-boundinit <- matrix(rep(init[-c(7,8)], length(cbound)), byrow=TRUE, nrow=length(cbound))
-myboundmat <- cbind(cbound,boundinit)
-myboundmat[cbound==1, c(2:7)] <- vecinj
-colnames(myboundmat) <- c("cbound", names(vecinj))
-
-setup <- list(n=2500,
-              bound=myboundmat,   
-              base=base,
-              first=selout,
-              initsim=initsim,
-              Cf=1,
-              prop=prop,
-              immobile=c(7,8,9),
-              kin= c(8,9),
-              ann=list(O2g=-0.1675), 
-              phase="FLUX1",
-              density="DEN1",
-              reduce=FALSE,
-              snapshots=demodir, ## directory where we will read MUFITS SUM files
-              gridfile=paste0(demodir,"/d2ascii.run.GRID.SUM")
-              )

data/SimDolKtz.R

@@ -1,130 +0,0 @@
-# library(RedModRphree)
-# library(Rmufits)
-# library(RcppVTK)
-
-db <- RPhreeFile(system.file("extdata", "phreeqc_kin.dat",
-                             package="RedModRphree"), is.db=TRUE)
-
-phreeqc::phrLoadDatabaseString(db)
-
-prop <- c("C","Ca","Cl","Mg","pH","pe","O2g", "Calcite","Dolomite")
-signif_vector <- c(7,7,7,7,7,7,7,5,5)
-prop_type <- c("act","act","act","act","logact","logact","ignore","act","act")
-
-
-base <- c("SOLUTION 1",
-          "units mol/kgw",
-          "temp 25.0",
-          "water 1",
-          "pH 9.91 charge",
-          "pe 4.0",
-          "C   1.2279E-04",
-          "Ca  1.2279E-04",
-          "Mg 0.001",
-          "Cl 0.002",
-          "PURE 1",
-          "O2g -0.1675 10",
-          "KINETICS 1",
-          "-steps 100",
-          "-step_divide 100",
-          "-bad_step_max 2000",
-          "Calcite", "-m      0.000207",
-          "-parms  0.0032",
-          "Dolomite",
-          "-m      0.0",
-          "-parms  0.00032",
-          "END")
-
-selout <- c("SELECTED_OUTPUT",
-            "-high_precision true",
-            "-reset false",
-            "-time",
-            "-soln",
-            "-temperature true",
-            "-water true",
-            "-pH",
-            "-pe",
-            "-totals C Ca Cl Mg",
-            "-kinetic_reactants Calcite Dolomite",
-            "-equilibrium O2g")
-
-initsim <- c("SELECTED_OUTPUT",
-             "-high_precision true",
-             "-reset false",
-             "USER_PUNCH",
-             "-head C Ca Cl Mg pH pe O2g Calcite Dolomite",
-             "10 PUNCH TOT(\"C\"), TOT(\"Ca\"), TOT(\"Cl\"), TOT(\"Mg\"), -LA(\"H+\"), -LA(\"e-\"), EQUI(\"O2g\"), EQUI(\"Calcite\"), EQUI(\"Dolomite\")",
-             "SOLUTION 1",
-             "units mol/kgw",
-             "temp 25.0", "water 1",
-             "pH 9.91 charge",
-             "pe 4.0",
-             "C   1.2279E-04",
-             "Ca  1.2279E-04",
-             "Cl 1E-12",
-             "Mg 1E-12",
-             "PURE 1",
-             "O2g     -0.6788 10.0",
-             "Calcite  0.0 2.07E-3",
-             "Dolomite 0.0 0.0",
-             "END")
-
-vecinj <- c("C"= 0,
-            "Ca"  = 0,
-            "Cl"  = 0.002,
-            "Mg"  = 0.001,
-            "pe"   = 4,
-            "pH"   = 7)
-
-init <- c("C(4)"= 1.2279E-4,
-          "Ca"  =1.2279E-4,
-          "Cl"  =0,        
-          "Mg"  =0,
-          "pe"  =4,
-          "pH"  =7,
-          "Calcite"= 2.07e-4,
-          "Dolomite"= 0)
-
-bound_elm <- c(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L, 14L, 
-               15L, 16L, 17L, 18L, 19L, 20L, 21L, 22L, 23L, 24L, 25L, 26L, 27L, 
-               28L, 29L, 30L, 31L, 32L, 33L, 34L, 35L, 36L, 37L, 38L, 39L, 40L, 
-               41L, 42L, 43L, 44L, 45L, 46L, 47L, 48L, 49L, 50L, 51L, 52L, 53L, 
-               54L, 55L, 56L, 57L, 58L, 59L, 60L, 61L, 62L, 63L, 64L, 65L, 66L, 
-               67L, 68L, 69L, 70L, 71L, 72L, 73L, 74L, 75L, 76L, 77L, 78L, 79L, 
-               80L, 81L, 82L, 83L, 84L, 85L, 86L, 87L, 88L, 89L, 90L, 91L, 92L, 
-               93L, 94L, 95L, 96L, 97L, 98L, 99L, 100L, 101L, 102L, 103L, 104L, 
-               105L, 106L, 107L, 108L, 214L, 215L)
-
-inj_elm <- c(7426L, 18233L, 29040L, 39847L, 
-             50654L, 61461L, 72268L, 83075L, 93882L, 104689L, 115496L, 126303L, 
-             137110L, 147917L, 158724L, 169531L, 180338L, 191145L, 201952L, 
-             212759L, 223566L, 234373L, 245180L, 255987L, 266794L, 277601L, 
-             288408L, 299215L, 310022L, 320829L, 331636L, 342443L, 353250L, 
-             364057L, 374864L, 385671L, 396478L, 407285L, 418092L, 428899L, 
-             439706L, 450513L, 461320L, 472127L, 482934L, 493741L, 504548L, 
-             515355L)
-
-cbound <- inj_elm
-
-boundinit <- matrix(rep(vecinj, length(cbound)), ncol=length(vecinj), byrow=TRUE) 
-myboundmat <- cbind(cbound,boundinit)
-
-## distinguish between injection and real boundaries
-colnames(myboundmat) <- c("cbound", names(vecinj))
-
-setup <- list(n=648420,
-              bound=myboundmat,
-              base=base,
-              first=selout,
-              initsim=initsim,
-              Cf=1,
-              prop=prop,
-              immobile=c(7,8,9),
-              kin= c(8,9),
-              ann=list(O2g=-0.1675),
-              phase="FLUX1",
-              density="DENS",
-              reduce=FALSE,
-              snapshots="snaps/AllSnaps_cmp_v3.rds",
-              gridfile ="snaps/GridKtz_cmp_v3.rds"
-              )

docs/CMakeLists.txt

@@ -1,18 +1,19 @@
 find_package(Doxygen)
+
 if(DOXYGEN_FOUND)
+  set(DOXYGEN_GENERATE_HTML YES)
+  set(DOXYGEN_EXCLUDE_PATTERNS ${PROJECT_SOURCE_DIR}/*/argh.hpp)
+  set(DOXYGEN_USE_MDFILE_AS_MAINPAGE ${PROJECT_SOURCE_DIR}/README.md)
 
-  set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/./Doxyfile.in)
+  # See https://www.doxygen.nl/manual/customize.html
-  set(DOXYGEN_OUT ${CMAKE_BINARY_DIR}/Doxyfile)
+  set(DOXYGEN_DISABLE_INDEX NO)
+  set(DOXYGEN_GENERATE_TREEVIEW YES)
+  set(DOXYGEN_FULL_SIDEBAR YES)
+  set(DOXYGEN_PROJECT_NUMBER ${POET_VERSION})
 
-  configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
+  doxygen_add_docs(doxygen
-
+    ${PROJECT_SOURCE_DIR}/src
-  add_custom_target(doc_doxygen ALL
+    ${PROJECT_SOURCE_DIR}/README.md
-    COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
+    ${PROJECT_SOURCE_DIR}/docs/Output.md
-    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generate html pages")
-    COMMENT "Generating documantation with doxygen"
+endif()
-    VERBATIM)
-
-  install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/ DESTINATION docs)
-else(DOXYGEN_FOUND)
-  message("Doxygen not found. Please install.")
-endif(DOXYGEN_FOUND)

docs/Output.md

@@ -35,32 +35,50 @@ corresponding values can be found in `<OUTPUT_DIRECTORY>/timings.rds`
 and possible to read out within a R runtime with
 `readRDS("timings.rds")`. There you will find the following values:
 
-| Value              | Description                                                                |
+| Value     | Description                                                                |
-|--------------------|----------------------------------------------------------------------------|
+| --------- | -------------------------------------------------------------------------- |
-| simtime            | time spent in whole simulation loop without any initialization and cleanup |
+| simtime   | time spent in whole simulation loop without any initialization and cleanup |
-| simtime\_transport | measured time in *transport* subroutine                                    |
+| chemistry | measured time in *chemistry* subroutine                                    |
-| simtime\_chemistry | measured time in *chemistry* subroutine (actual parallelized part)         |
+| diffusion | measured time in *diffusion* subroutine                                    |
 
-### chemistry subsetting
+### Chemistry subsetting
 
-If running parallel there are also measured timings which are subsets of
+| Value         | Description                                               |
-*simtime\_​chemistry*.
+| ------------- | --------------------------------------------------------- |
+| simtime       | overall runtime of chemistry                              |
+| loop          | time spent in send/recv loop of master                    |
+| sequential    | sequential part of the master (e.g. shuffling field)      |
+| idle\_master  | idling time of the master waiting for workers             |
+| idle\_worker  | idling time (waiting for work from master) of the workers |
+| phreeqc\_time | accumulated times for Phreeqc calls of every worker       |
 
-| Value                      | Description                                        |
+#### DHT usage
-|----------------------------|----------------------------------------------------|
-| simtime\_workers           | time spent in send/​recv loop of master             |
-| simtime\_chemistry\_master | sequential part of master chemistry                |
-| phreeqc                    | measured time of each worker in PHREEQC subroutine |
-
-### DHT usage {#DHT-usage}
 
 If running in parallel and with activated DHT, two more timings and also
 some profiling about the DHT usage are given:
 
 | Value           | Description                                             |
-|-----------------|---------------------------------------------------------|
+| --------------- | ------------------------------------------------------- |
-| dht\_fill\_time | time to write data to DHT                               |
+| dht\_hits       | count of data points retrieved from DHT                 |
-| dht\_get\_time  | time to retreive data from DHT                          |
-| dh\_hits        | count of data points retrieved from DHT                 |
-| dht\_miss       | count of misses/count of data points written to DHT     |
 | dht\_evictions  | count of data points evicted by another write operation |
+| dht\_get\_time  | time to retreive data from DHT                          |
+| dht\_fill\_time | time to write data to DHT                               |
+
+#### Interpolation
+
+If using interpolation, the following values are given:
+
+| Value          | Description                                                           |
+| -------------- | --------------------------------------------------------------------- |
+| interp\_w      | time spent to write to PHT                                            |
+| interp\_r      | time spent to read from DHT/PHT/Cache                                 |
+| interp\_g      | time spent to gather results from DHT                                 |
+| interp\_fc     | accumulated time spent in interpolation function call                 |
+| interp\_calls  | count of interpolations                                               |
+| interp\_cached | count of interpolation data sets, which where cached in the local map |
+
+### Diffusion subsetting
+
+| Value     | Description                                |
+| --------- | ------------------------------------------ |
+| simtime   | overall runtime of diffusion               |

ext/litephreeqc

@@ -0,0 +1 @@
+Subproject commit 953c752431d2b2758268083f407f943843efc7ad

ext/tug

@@ -0,0 +1 @@
+Subproject commit 9c4aeee410c71d064f7567143d4f8d6451ade75a

src/Base/Macros.hpp

@@ -0,0 +1,16 @@
+#ifndef MACROS_H
+#define MACROS_H
+
+#include <iostream>
+#include <string>
+
+// Prepend "msg" with name of calling function 
+#define MSG(msg)    std::cout << "CPP: " << __func__ << ": " << std::string(msg) << std::endl;
+
+#define MSG_NOENDL(msg)    std::cout << "CPP: " << __func__ << ": " << std::string(msg);
+
+#define ERRMSG(msg) std::cerr << "CPP_ERROR: " << __func__ << ": " << std::string(msg) << std::endl;
+
+#define BOOL_PRINT(bool) std::string(bool ? "ON" : "OFF")
+
+#endif // MACROS_H

src/Base/RInsidePOET.hpp

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <RInside.h>
+#include <Rcpp.h>
+#include <Rinternals.h>
+#include <exception>
+#include <memory>
+#include <string>
+
+namespace poet {
+class RInsidePOET : public RInside {
+public:
+  static RInsidePOET &getInstance() {
+    static RInsidePOET instance;
+
+    return instance;
+  }
+
+  RInsidePOET(RInsidePOET const &) = delete;
+  void operator=(RInsidePOET const &) = delete;
+
+  inline bool checkIfExists(const std::string &R_name,
+                            const std::string &where) {
+    return Rcpp::as<bool>(
+        this->parseEval("'" + R_name + "' %in% names(" + where + ")"));
+  }
+
+private:
+  RInsidePOET() : RInside(){};
+};
+
+/**
+ * @brief Deferred evaluation function
+ *
+ * The class is intended to call R functions within an existing RInside
+ * instance. The problem with "original" Rcpp::Function is that they require:
+ * 1. RInside instance already present, restricting the declaration of
+ *     Rcpp::Functions in global scope
+ * 2. Require the function to be present. Otherwise, they will throw an
+ *     exception.
+ * This class solves both problems by deferring the evaluation of the function
+ *  until the constructor is called and evaluating whether the function is
+ *  present or not, wihout throwing an exception.
+ *
+ * @tparam T Return type of the function
+ */
+class DEFunc {
+public:
+  DEFunc() {}
+  DEFunc(const std::string &f_name) {
+    try {
+      this->func = std::make_shared<Rcpp::Function>(f_name);
+    } catch (const std::exception &e) {
+    }
+  }
+
+  DEFunc(SEXP f) {
+    try {
+      this->func = std::make_shared<Rcpp::Function>(f);
+    } catch (const std::exception &e) {
+    }
+  }
+
+  template <typename... Args> SEXP operator()(Args... args) const {
+    if (func) {
+      return (*this->func)(args...);
+    } else {
+      throw std::exception();
+    }
+  }
+
+  DEFunc &operator=(const DEFunc &rhs) {
+    this->func = rhs.func;
+    return *this;
+  }
+
+  DEFunc(const DEFunc &rhs) { this->func = rhs.func; }
+
+  bool isValid() const { return static_cast<bool>(func); }
+
+  SEXP asSEXP() const {
+    if (!func) {
+      return R_NilValue;
+    }
+    return Rcpp::as<SEXP>(*this->func.get());
+  }
+
+private:
+  std::shared_ptr<Rcpp::Function> func;
+};
+
+} // namespace poet

src/CMakeLists.txt

@@ -1,18 +1,101 @@
-set(SRC_CODE_DIR
+add_library(POETLib
-    ${CMAKE_CURRENT_SOURCE_DIR}
+  Init/InitialList.cpp 
-    CACHE INTERNAL "directory indicating which source code version is used")
+  Init/GridInit.cpp 
+  Init/DiffusionInit.cpp
+  Init/ChemistryInit.cpp
+  DataStructures/Field.cpp
+  Transport/DiffusionModule.cpp
+  Chemistry/ChemistryModule.cpp
+  Chemistry/MasterFunctions.cpp
+  Chemistry/WorkerFunctions.cpp
+  Chemistry/SurrogateModels/DHT_Wrapper.cpp
+  Chemistry/SurrogateModels/DHT.c
+  Chemistry/SurrogateModels/HashFunctions.cpp
+  Chemistry/SurrogateModels/InterpolationModule.cpp
+  Chemistry/SurrogateModels/ProximityHashTable.cpp
+)
 
-get_poet_version()
+set(POET_TUG_APPROACH "Implicit" CACHE STRING "tug numerical approach to use")
+set_property(CACHE POET_TUG_APPROACH PROPERTY STRINGS "Implicit" "Explicit")
 
-configure_file(poet.h.in poet.h)
+if (POET_TUG_APPROACH STREQUAL "Implicit")
+  target_compile_definitions(POETLib PRIVATE POET_TUG_BTCS)
+elseif (POET_TUG_APPROACH STREQUAL "Explicit")
+  target_compile_definitions(POETLib PRIVATE POET_TUG_FTCS)
+endif()
+
+target_include_directories(POETLib PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
+target_link_libraries(
+  POETLib 
+  PUBLIC RRuntime
+  PUBLIC IPhreeqcPOET
+  PUBLIC tug
+  PUBLIC MPI::MPI_C
+)
+
+include(FetchContent)
+FetchContent_Declare(
+  cli11
+  QUIET 
+  GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git
+  GIT_TAG v2.5.0
+)
+
+FetchContent_MakeAvailable(cli11)
+
+# add_library(poetlib
+#   Base/Grid.cpp
+#   Base/SimParams.cpp
+#   Chemistry/ChemistryModule.cpp
+#   Chemistry/MasterFunctions.cpp 
+#   Chemistry/WorkerFunctions.cpp 
+#   Chemistry/SurrogateModels/DHT_Wrapper.cpp 
+#   Chemistry/SurrogateModels/DHT.c 
+#   Chemistry/SurrogateModels/HashFunctions.cpp 
+#   Chemistry/SurrogateModels/InterpolationModule.cpp 
+#   Chemistry/SurrogateModels/ProximityHashTable.cpp
+#   Transport/DiffusionModule.cpp
+# )
+
+# target_link_libraries(poetlib PUBLIC
+#   DataStructures
+#   Init
+#   MPI::MPI_C 
+#   ${MATH_LIBRARY} 
+#   RRuntime 
+#   PhreeqcRM
+#   tug
+# )
+
+target_compile_definitions(POETLib PUBLIC STRICT_R_HEADERS OMPI_SKIP_MPICXX)
+
+# mark_as_advanced(PHREEQCRM_BUILD_MPI PHREEQCRM_DISABLE_OPENMP)
+# set(PHREEQCRM_DISABLE_OPENMP ON CACHE BOOL "" FORCE)
+
+# option(POET_DHT_DEBUG "Build with DHT debug info" OFF)
+
+# if(POET_DHT_DEBUG)
+#   target_compile_definitions(poetlib PRIVATE DHT_STATISTICS)
+# endif()
+
+# option(POET_PHT_ADDITIONAL_INFO "Enables additional information in the PHT" OFF)
+
+# if (POET_PHT_ADDITIONAL_INFO)
+#   target_compile_definitions(poetlib PRIVATE POET_PHT_ADD)
+# endif()
+
+file(READ "${PROJECT_SOURCE_DIR}/R_lib/kin_r_library.R" R_KIN_LIB )
+file(READ "${PROJECT_SOURCE_DIR}/R_lib/init_r_lib.R" R_INIT_LIB)
+file(READ "${PROJECT_SOURCE_DIR}/R_lib/ai_surrogate_model.R" R_AI_SURROGATE_LIB)
+
+configure_file(poet.hpp.in poet.hpp @ONLY)
 
 add_executable(poet poet.cpp)
-target_include_directories(poet PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
+target_link_libraries(poet PRIVATE POETLib MPI::MPI_C RRuntime CLI11::CLI11)
-target_link_libraries(poet PUBLIC POET_Model POET_Util MPI::MPI_C)
+target_include_directories(poet PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
-target_compile_definitions(poet PRIVATE OMPI_SKIP_MPICXX)
 
-install(TARGETS poet DESTINATION bin)
+add_executable(poet_init initializer.cpp)
+target_link_libraries(poet_init PRIVATE POETLib RRuntime CLI11::CLI11)
+target_include_directories(poet_init PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
 
-add_subdirectory(DHT)
+install(TARGETS poet poet_init DESTINATION bin)
-add_subdirectory(model)
-add_subdirectory(util)

src/Chemistry/ChemistryDefs.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+namespace poet {
+
+enum DHT_PROP_TYPES { DHT_TYPE_DEFAULT, DHT_TYPE_CHARGE, DHT_TYPE_TOTAL };
+enum CHEMISTRY_OUT_SOURCE { CHEM_PQC, CHEM_DHT, CHEM_INTERP, CHEM_AISURR };
+
+struct WorkPackage {
+  std::size_t size;
+  std::vector<std::vector<double>> input;
+  std::vector<std::vector<double>> output;
+  std::vector<std::uint8_t> mapping;
+
+  WorkPackage(std::size_t _size)
+      : size(_size), input(size), output(size), mapping(size, CHEM_PQC) {}
+};
+} // namespace poet

src/Chemistry/ChemistryModule.cpp

@@ -0,0 +1,324 @@
+#include "ChemistryModule.hpp"
+
+#include "PhreeqcEngine.hpp"
+#include "PhreeqcMatrix.hpp"
+#include "PhreeqcRunner.hpp"
+#include "SurrogateModels/DHT_Wrapper.hpp"
+#include "SurrogateModels/Interpolation.hpp"
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <cstdint>
+#include <memory>
+#include <mpi.h>
+#include <string>
+#include <vector>
+
+std::vector<double>
+inverseDistanceWeighting(const std::vector<std::int32_t> &to_calc,
+                         const std::vector<double> &from,
+                         const std::vector<std::vector<double>> &input,
+                         const std::vector<std::vector<double>> &output) {
+  std::vector<double> results = from;
+
+  // const std::uint32_t buffer_size = input.size() + 1;
+  // double buffer[buffer_size];
+  // double from_rescaled;
+
+  const std::uint32_t data_set_n = input.size();
+  double rescaled[to_calc.size()][data_set_n + 1];
+  double weights[data_set_n];
+
+  // rescaling over all key elements
+  for (std::uint32_t key_comp_i = 0; key_comp_i < to_calc.size();
+       key_comp_i++) {
+    const auto output_comp_i = to_calc[key_comp_i];
+
+    // rescale input between 0 and 1
+    for (std::uint32_t point_i = 0; point_i < data_set_n; point_i++) {
+      rescaled[key_comp_i][point_i] = input[point_i][key_comp_i];
+    }
+
+    rescaled[key_comp_i][data_set_n] = from[output_comp_i];
+
+    const double min = *std::min_element(rescaled[key_comp_i],
+                                         rescaled[key_comp_i] + data_set_n + 1);
+    const double max = *std::max_element(rescaled[key_comp_i],
+                                         rescaled[key_comp_i] + data_set_n + 1);
+
+    for (std::uint32_t point_i = 0; point_i < data_set_n; point_i++) {
+      rescaled[key_comp_i][point_i] =
+          ((max - min) != 0
+               ? (rescaled[key_comp_i][point_i] - min) / (max - min)
+               : 0);
+    }
+    rescaled[key_comp_i][data_set_n] =
+        ((max - min) != 0 ? (from[output_comp_i] - min) / (max - min) : 0);
+  }
+
+  // calculate distances for each data set
+  double inv_sum = 0;
+  for (std::uint32_t point_i = 0; point_i < data_set_n; point_i++) {
+    double distance = 0;
+    for (std::uint32_t key_comp_i = 0; key_comp_i < to_calc.size();
+         key_comp_i++) {
+      distance += std::pow(
+          rescaled[key_comp_i][point_i] - rescaled[key_comp_i][data_set_n], 2);
+    }
+
+    weights[point_i] = distance != 0 ? 1 / std::sqrt(distance) : 0;
+    assert(!std::isnan(weights[point_i]));
+    inv_sum += weights[point_i];
+  }
+
+  assert(!std::isnan(inv_sum));
+
+  // actual interpolation
+  // bool has_h = false;
+  // bool has_o = false;
+
+  for (std::uint32_t key_comp_i = 0; key_comp_i < to_calc.size();
+       key_comp_i++) {
+    const auto output_comp_i = to_calc[key_comp_i];
+    double key_delta = 0;
+
+    // if (interp_i == 0) {
+    //   has_h = true;
+    // }
+
+    // if (interp_i == 1) {
+    //   has_o = true;
+    // }
+
+    for (std::uint32_t j = 0; j < data_set_n; j++) {
+      key_delta += weights[j] * output[j][output_comp_i];
+    }
+
+    key_delta /= inv_sum;
+
+    results[output_comp_i] = from[output_comp_i] + key_delta;
+    assert(!std::isnan(results[output_comp_i]));
+  }
+
+  return results;
+}
+
+poet::ChemistryModule::ChemistryModule(
+    uint32_t wp_size_, const InitialList::ChemistryInit chem_params,
+    MPI_Comm communicator)
+    : group_comm(communicator), wp_size(wp_size_), params(chem_params) {
+  MPI_Comm_rank(communicator, &comm_rank);
+  MPI_Comm_size(communicator, &comm_size);
+
+  this->is_sequential = comm_size == 1;
+  this->is_master = comm_rank == 0;
+
+  this->n_cells = chem_params.total_grid_cells;
+
+  if (!is_master) {
+    PhreeqcMatrix pqc_mat =
+        PhreeqcMatrix(chem_params.database, chem_params.pqc_script,
+                      chem_params.with_h0_o0, chem_params.with_redox);
+
+    this->pqc_runner =
+        std::make_unique<PhreeqcRunner>(pqc_mat.subset(chem_params.pqc_ids));
+  }
+}
+
+poet::ChemistryModule::~ChemistryModule() {
+  if (dht) {
+    delete dht;
+  }
+}
+
+void poet::ChemistryModule::initializeDHT(
+    uint32_t size_mb, const NamedVector<std::uint32_t> &key_species,
+    bool has_het_ids) {
+  constexpr uint32_t MB_FACTOR = 1E6;
+
+  MPI_Comm dht_comm;
+
+  if (this->is_master) {
+    MPI_Comm_split(this->group_comm, MPI_UNDEFINED, this->comm_rank, &dht_comm);
+    return;
+  }
+
+  if (!this->is_master) {
+
+    MPI_Comm_split(this->group_comm, 1, this->comm_rank, &dht_comm);
+
+    auto map_copy = key_species;
+
+    if (key_species.empty()) {
+      std::vector<std::uint32_t> default_signif(
+          this->prop_count, DHT_Wrapper::DHT_KEY_SIGNIF_DEFAULT);
+      map_copy = NamedVector<std::uint32_t>(this->prop_names, default_signif);
+    }
+
+    auto key_indices = parseDHTSpeciesVec(key_species, this->prop_names);
+
+    if (this->dht) {
+      delete this->dht;
+    }
+
+    const std::uint64_t dht_size = size_mb * MB_FACTOR;
+
+    this->dht = new DHT_Wrapper(dht_comm, dht_size, map_copy, key_indices,
+                                this->prop_names, params.hooks,
+                                this->prop_count, interp_enabled, has_het_ids);
+    this->dht->setBaseTotals(base_totals.at(0), base_totals.at(1));
+  }
+}
+
+inline std::vector<std::int32_t> poet::ChemistryModule::parseDHTSpeciesVec(
+    const NamedVector<std::uint32_t> &key_species,
+    const std::vector<std::string> &to_compare) const {
+  std::vector<int32_t> species_indices;
+  species_indices.reserve(key_species.size());
+
+  const auto test = key_species.getNames();
+
+  for (const auto &species : key_species.getNames()) {
+    auto it = std::find(to_compare.begin(), to_compare.end(), species);
+    if (it == to_compare.end()) {
+      species_indices.push_back(DHT_Wrapper::DHT_KEY_INPUT_CUSTOM);
+      continue;
+    }
+    const std::uint32_t index = it - to_compare.begin();
+    species_indices.push_back(index);
+  }
+
+  return species_indices;
+}
+
+void poet::ChemistryModule::BCastStringVec(std::vector<std::string> &io) {
+
+  if (this->is_master) {
+    int vec_size = io.size();
+    ChemBCast(&vec_size, 1, MPI_INT);
+
+    for (const auto &value : io) {
+      int buf_size = value.size() + 1;
+      ChemBCast(&buf_size, 1, MPI_INT);
+      ChemBCast(const_cast<char *>(value.c_str()), buf_size, MPI_CHAR);
+    }
+  } else {
+    int vec_size;
+    ChemBCast(&vec_size, 1, MPI_INT);
+
+    io.resize(vec_size);
+
+    for (int i = 0; i < vec_size; i++) {
+      int buf_size;
+      ChemBCast(&buf_size, 1, MPI_INT);
+      char buf[buf_size];
+      ChemBCast(buf, buf_size, MPI_CHAR);
+      io[i] = std::string{buf};
+    }
+  }
+}
+
+void poet::ChemistryModule::setDHTSnapshots(int type,
+                                            const std::string &out_dir) {
+  if (this->is_master) {
+    return;
+  }
+
+  this->dht_file_out_dir = out_dir;
+  this->dht_snaps_type = type;
+}
+
+void poet::ChemistryModule::setDHTReadFile(const std::string &input_file) {
+  if (this->is_master) {
+    return;
+  }
+
+  if (!input_file.empty()) {
+    WorkerReadDHTDump(input_file);
+  }
+}
+
+void poet::ChemistryModule::initializeInterp(
+    std::uint32_t bucket_size, std::uint32_t size_mb, std::uint32_t min_entries,
+    const NamedVector<std::uint32_t> &key_species) {
+
+  if (!this->is_master) {
+
+    constexpr uint32_t MB_FACTOR = 1E6;
+
+    assert(this->dht);
+
+    this->interp_enabled = true;
+
+    auto map_copy = key_species;
+
+    if (key_species.empty()) {
+      map_copy = this->dht->getKeySpecies();
+      for (auto i = 0; i < map_copy.size(); i++) {
+        const std::uint32_t signif =
+            static_cast<std::uint32_t>(map_copy[i]) -
+            (map_copy[i] > InterpolationModule::COARSE_DIFF
+                 ? InterpolationModule::COARSE_DIFF
+                 : 0);
+        map_copy[i] = signif;
+      }
+    }
+
+    auto key_indices =
+        parseDHTSpeciesVec(map_copy, dht->getKeySpecies().getNames());
+
+    if (this->interp) {
+      this->interp.reset();
+    }
+
+    const uint64_t pht_size = size_mb * MB_FACTOR;
+
+    interp = std::make_unique<poet::InterpolationModule>(
+        bucket_size, pht_size, min_entries, *(this->dht), map_copy, key_indices,
+        this->prop_names, this->params.hooks);
+
+    interp->setInterpolationFunction(inverseDistanceWeighting);
+  }
+}
+
+std::vector<double>
+poet::ChemistryModule::shuffleField(const std::vector<double> &in_field,
+                                    uint32_t size_per_prop, uint32_t prop_count,
+                                    uint32_t wp_count) {
+  std::vector<double> out_buffer(in_field.size());
+  uint32_t write_i = 0;
+  for (uint32_t i = 0; i < wp_count; i++) {
+    for (uint32_t j = i; j < size_per_prop; j += wp_count) {
+      for (uint32_t k = 0; k < prop_count; k++) {
+        out_buffer[(write_i * prop_count) + k] =
+            in_field[(k * size_per_prop) + j];
+      }
+      write_i++;
+    }
+  }
+  return out_buffer;
+}
+
+void poet::ChemistryModule::unshuffleField(const std::vector<double> &in_buffer,
+                                           uint32_t size_per_prop,
+                                           uint32_t prop_count,
+                                           uint32_t wp_count,
+                                           std::vector<double> &out_field) {
+  uint32_t read_i = 0;
+
+  for (uint32_t i = 0; i < wp_count; i++) {
+    for (uint32_t j = i; j < size_per_prop; j += wp_count) {
+      for (uint32_t k = 0; k < prop_count; k++) {
+        out_field[(k * size_per_prop) + j] =
+            in_buffer[(read_i * prop_count) + k];
+      }
+      read_i++;
+    }
+  }
+}
+
+void poet::ChemistryModule::set_ai_surrogate_validity_vector(
+    std::vector<int> r_vector) {
+  this->ai_surrogate_validity_vector = r_vector;
+}

src/Chemistry/ChemistryModule.hpp

@@ -0,0 +1,410 @@
+
+#ifndef CHEMISTRYMODULE_H_
+#define CHEMISTRYMODULE_H_
+
+#include "DataStructures/Field.hpp"
+#include "DataStructures/NamedVector.hpp"
+
+#include "ChemistryDefs.hpp"
+
+#include "Init/InitialList.hpp"
+#include "NameDouble.h"
+#include "SurrogateModels/DHT_Wrapper.hpp"
+#include "SurrogateModels/Interpolation.hpp"
+
+#include "PhreeqcRunner.hpp"
+#include <array>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mpi.h>
+#include <string>
+#include <vector>
+
+namespace poet {
+/**
+ * \brief Wrapper around PhreeqcRM to provide POET specific parallelization with
+ * easy access.
+ */
+class ChemistryModule {
+public:
+  /**
+   * Creates a new instance of Chemistry module with given grid cell count, work
+   * package size and communicator.
+   *
+   * This constructor shall only be called by the master. To create workers, see
+   * ChemistryModule::createWorker .
+   *
+   * When the use of parallelization is intended, the nxyz value shall be set to
+   * 1 to save memory and only one node is needed for initialization.
+   *
+   * \param nxyz Count of grid cells to allocate and initialize for each
+   * process. For parellel use set to 1 at the master.
+   * \param wp_size Count of grid cells to fill each work package at maximum.
+   * \param communicator MPI communicator to distribute work in.
+   */
+  ChemistryModule(uint32_t wp_size,
+                  const InitialList::ChemistryInit chem_params,
+                  MPI_Comm communicator);
+
+  /**
+   * Deconstructor, which frees DHT data structure if used.
+   */
+  ~ChemistryModule();
+
+  void masterSetField(Field field);
+  /**
+   * Run the chemical simulation with parameters set.
+   */
+  void simulate(double dt);
+
+  /**
+   * Returns all known species names, including not only aqueous species, but
+   * also equilibrium, exchange, surface and kinetic reactants.
+   */
+  // auto GetPropNames() const { return this->prop_names; }
+
+  /**
+   * Return the accumulated runtime in seconds for chemical simulation.
+   */
+  auto GetChemistryTime() const { return this->chem_t; }
+
+  void setFilePadding(std::uint32_t maxiter) {
+    this->file_pad =
+        static_cast<std::uint8_t>(std::ceil(std::log10(maxiter + 1)));
+  }
+
+  struct SurrogateSetup {
+    std::vector<std::string> prop_names;
+    std::array<double, 2> base_totals;
+    bool has_het_ids;
+
+    bool dht_enabled;
+    std::uint32_t dht_size_mb;
+    int dht_snaps;
+    std::string dht_out_dir;
+
+    bool interp_enabled;
+    std::uint32_t interp_bucket_size;
+    std::uint32_t interp_size_mb;
+    std::uint32_t interp_min_entries;
+    bool ai_surrogate_enabled;
+  };
+
+  void masterEnableSurrogates(const SurrogateSetup &setup) {
+    // FIXME: This is a hack to get the prop_names and prop_count from the setup
+    this->prop_names = setup.prop_names;
+    this->prop_count = setup.prop_names.size();
+
+    this->dht_enabled = setup.dht_enabled;
+    this->interp_enabled = setup.interp_enabled;
+    this->ai_surrogate_enabled = setup.ai_surrogate_enabled;
+
+    this->base_totals = setup.base_totals;
+
+    if (this->dht_enabled || this->interp_enabled) {
+      this->initializeDHT(setup.dht_size_mb, this->params.dht_species,
+                          setup.has_het_ids);
+
+      if (setup.dht_snaps != DHT_SNAPS_DISABLED) {
+        this->setDHTSnapshots(setup.dht_snaps, setup.dht_out_dir);
+      }
+    }
+
+    if (this->interp_enabled) {
+      this->initializeInterp(setup.interp_bucket_size, setup.interp_size_mb,
+                             setup.interp_min_entries,
+                             this->params.interp_species);
+    }
+  }
+
+  /**
+   * Intended to alias input parameters for grid initialization with a single
+   * value per species.
+   */
+  using SingleCMap = std::unordered_map<std::string, double>;
+
+  /**
+   * Intended to alias input parameters for grid initialization with mutlitple
+   * values per species.
+   */
+  using VectorCMap = std::unordered_map<std::string, std::vector<double>>;
+
+  /**
+   * Enumerating DHT file options
+   */
+  enum {
+    DHT_SNAPS_DISABLED = 0, //!< disabled file output
+    DHT_SNAPS_SIMEND,       //!< only output of snapshot after simulation
+    DHT_SNAPS_ITEREND       //!< output snapshots after each iteration
+  };
+
+  /**
+   * **Only called by workers!** Start the worker listening loop.
+   */
+  void WorkerLoop();
+
+  /**
+   * **Called by master** Advise the workers to break the loop.
+   */
+  void MasterLoopBreak();
+
+  /**
+   * **Master only** Return count of grid cells.
+   */
+  auto GetNCells() const { return this->n_cells; }
+  /**
+   * **Master only** Return work package size.
+   */
+  auto GetWPSize() const { return this->wp_size; }
+  /**
+   * **Master only** Return the time in seconds the master spent waiting for any
+   * free worker.
+   */
+  auto GetMasterIdleTime() const { return this->idle_t; }
+  /**
+   * **Master only** Return the time in seconds the master spent in sequential
+   * part of the simulation, including times for shuffling/unshuffling field
+   * etc.
+   */
+  auto GetMasterSequentialTime() const { return this->seq_t; }
+  /**
+   * **Master only** Return the time in seconds the master spent in the
+   * send/receive loop.
+   */
+  auto GetMasterLoopTime() const { return this->send_recv_t; }
+
+  /**
+   * **Master only** Collect and return all accumulated timings recorded by
+   * workers to run Phreeqc simulation.
+   *
+   * \return Vector of all accumulated Phreeqc timings.
+   */
+  std::vector<double> GetWorkerPhreeqcTimings() const;
+  /**
+   * **Master only** Collect and return all accumulated timings recorded by
+   * workers to get values from the DHT.
+   *
+   * \return Vector of all accumulated DHT get times.
+   */
+  std::vector<double> GetWorkerDHTGetTimings() const;
+  /**
+   * **Master only** Collect and return all accumulated timings recorded by
+   * workers to write values to the DHT.
+   *
+   * \return Vector of all accumulated DHT fill times.
+   */
+  std::vector<double> GetWorkerDHTFillTimings() const;
+  /**
+   * **Master only** Collect and return all accumulated timings recorded by
+   * workers waiting for work packages from the master.
+   *
+   * \return Vector of all accumulated waiting times.
+   */
+  std::vector<double> GetWorkerIdleTimings() const;
+
+  /**
+   * **Master only** Collect and return DHT hits of all workers.
+   *
+   * \return Vector of all count of DHT hits.
+   */
+  std::vector<uint32_t> GetWorkerDHTHits() const;
+
+  /**
+   * **Master only** Collect and return DHT evictions of all workers.
+   *
+   * \return Vector of all count of DHT evictions.
+   */
+  std::vector<uint32_t> GetWorkerDHTEvictions() const;
+
+  /**
+   * **Master only** Returns the current state of the chemical field.
+   *
+   * \return Reference to the chemical field.
+   */
+  Field &getField() { return this->chem_field; }
+
+  /**
+   * **Master only** Enable/disable progress bar.
+   *
+   * \param enabled True if print progressbar, false if not.
+   */
+  void setProgressBarPrintout(bool enabled) {
+    this->print_progessbar = enabled;
+  };
+
+  /**
+   *  **Master only** Set the ai surrogate validity vector from R
+   */
+  void set_ai_surrogate_validity_vector(std::vector<int> r_vector);
+
+  std::vector<uint32_t> GetWorkerInterpolationCalls() const;
+
+  std::vector<double> GetWorkerInterpolationWriteTimings() const;
+  std::vector<double> GetWorkerInterpolationReadTimings() const;
+  std::vector<double> GetWorkerInterpolationGatherTimings() const;
+  std::vector<double> GetWorkerInterpolationFunctionCallTimings() const;
+
+  std::vector<uint32_t> GetWorkerPHTCacheHits() const;
+
+  std::vector<int> ai_surrogate_validity_vector;
+
+protected:
+  void initializeDHT(uint32_t size_mb,
+                     const NamedVector<std::uint32_t> &key_species,
+                     bool has_het_ids);
+  void setDHTSnapshots(int type, const std::string &out_dir);
+  void setDHTReadFile(const std::string &input_file);
+
+  void initializeInterp(std::uint32_t bucket_size, std::uint32_t size_mb,
+                        std::uint32_t min_entries,
+                        const NamedVector<std::uint32_t> &key_species);
+
+  enum {
+    CHEM_FIELD_INIT,
+    CHEM_DHT_ENABLE,
+    CHEM_DHT_SIGNIF_VEC,
+    CHEM_DHT_SNAPS,
+    CHEM_DHT_READ_FILE,
+    CHEM_IP_ENABLE,
+    CHEM_IP_MIN_ENTRIES,
+    CHEM_IP_SIGNIF_VEC,
+    CHEM_WORK_LOOP,
+    CHEM_PERF,
+    CHEM_BREAK_MAIN_LOOP,
+    CHEM_AI_BCAST_VALIDITY
+  };
+
+  enum { LOOP_WORK, LOOP_END };
+
+  enum {
+    WORKER_PHREEQC,
+    WORKER_DHT_GET,
+    WORKER_DHT_FILL,
+    WORKER_IDLE,
+    WORKER_IP_WRITE,
+    WORKER_IP_READ,
+    WORKER_IP_GATHER,
+    WORKER_IP_FC,
+    WORKER_DHT_HITS,
+    WORKER_DHT_EVICTIONS,
+    WORKER_PHT_CACHE_HITS,
+    WORKER_IP_CALLS
+  };
+
+  std::vector<uint32_t> interp_calls;
+  std::vector<uint32_t> dht_hits;
+  std::vector<uint32_t> dht_evictions;
+
+  struct worker_s {
+    double phreeqc_t = 0.;
+    double dht_get = 0.;
+    double dht_fill = 0.;
+    double idle_t = 0.;
+  };
+
+  struct worker_info_s {
+    char has_work = 0;
+    double *send_addr;
+  };
+
+  using worker_list_t = std::vector<struct worker_info_s>;
+  using workpointer_t = std::vector<double>::iterator;
+
+  void MasterRunParallel(double dt);
+  void MasterRunSequential();
+
+  void MasterSendPkgs(worker_list_t &w_list, workpointer_t &work_pointer,
+                      int &pkg_to_send, int &count_pkgs, int &free_workers,
+                      double dt, uint32_t iteration,
+                      const std::vector<uint32_t> &wp_sizes_vector);
+  void MasterRecvPkgs(worker_list_t &w_list, int &pkg_to_recv, bool to_send,
+                      int &free_workers);
+
+  std::vector<double> MasterGatherWorkerTimings(int type) const;
+  std::vector<uint32_t> MasterGatherWorkerMetrics(int type) const;
+
+  void WorkerProcessPkgs(struct worker_s &timings, uint32_t &iteration);
+
+  void WorkerDoWork(MPI_Status &probe_status, int double_count,
+                    struct worker_s &timings);
+  void WorkerPostIter(MPI_Status &prope_status, uint32_t iteration);
+  void WorkerPostSim(uint32_t iteration);
+
+  void WorkerWriteDHTDump(uint32_t iteration);
+  void WorkerReadDHTDump(const std::string &dht_input_file);
+
+  void WorkerPerfToMaster(int type, const struct worker_s &timings);
+  void WorkerMetricsToMaster(int type);
+
+  void WorkerRunWorkPackage(WorkPackage &work_package, double dSimTime,
+                            double dTimestep);
+
+  std::vector<uint32_t> CalculateWPSizesVector(uint32_t n_cells,
+                                               uint32_t wp_size) const;
+
+  std::vector<double> shuffleField(const std::vector<double> &in_field,
+                                   uint32_t size_per_prop, uint32_t prop_count,
+                                   uint32_t wp_count);
+  void unshuffleField(const std::vector<double> &in_buffer,
+                      uint32_t size_per_prop, uint32_t prop_count,
+                      uint32_t wp_count, std::vector<double> &out_field);
+  std::vector<std::int32_t>
+  parseDHTSpeciesVec(const NamedVector<std::uint32_t> &key_species,
+                     const std::vector<std::string> &to_compare) const;
+
+  void BCastStringVec(std::vector<std::string> &io);
+
+  int comm_size, comm_rank;
+  MPI_Comm group_comm;
+
+  bool is_sequential;
+  bool is_master;
+
+  uint32_t wp_size;
+  bool dht_enabled{false};
+  int dht_snaps_type{DHT_SNAPS_DISABLED};
+  std::string dht_file_out_dir;
+
+  poet::DHT_Wrapper *dht = nullptr;
+
+  bool interp_enabled{false};
+  std::unique_ptr<poet::InterpolationModule> interp;
+
+  bool ai_surrogate_enabled{false};
+
+  static constexpr uint32_t BUFFER_OFFSET = 5;
+
+  inline void ChemBCast(void *buf, int count, MPI_Datatype datatype) const {
+    MPI_Bcast(buf, count, datatype, 0, this->group_comm);
+  }
+
+  inline void PropagateFunctionType(int &type) const {
+    ChemBCast(&type, 1, MPI_INT);
+  }
+  double simtime = 0.;
+  double idle_t = 0.;
+  double seq_t = 0.;
+  double send_recv_t = 0.;
+
+  std::array<double, 2> base_totals{0};
+
+  bool print_progessbar{false};
+
+  std::uint8_t file_pad{1};
+
+  double chem_t{0.};
+
+  uint32_t n_cells = 0;
+  uint32_t prop_count = 0;
+  std::vector<std::string> prop_names;
+
+  Field chem_field;
+
+  const InitialList::ChemistryInit params;
+
+  std::unique_ptr<PhreeqcRunner> pqc_runner;
+};
+} // namespace poet
+
+#endif // CHEMISTRYMODULE_H_

src/Chemistry/MasterFunctions.cpp

@@ -0,0 +1,501 @@
+#include "ChemistryModule.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <mpi.h>
+#include <vector>
+
+std::vector<uint32_t>
+poet::ChemistryModule::MasterGatherWorkerMetrics(int type) const {
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  uint32_t dummy;
+  std::vector<uint32_t> metrics(this->comm_size);
+
+  MPI_Gather(&dummy, 1, MPI_UINT32_T, metrics.data(), 1, MPI_UINT32_T, 0,
+             this->group_comm);
+
+  metrics.erase(metrics.begin());
+  return metrics;
+}
+
+std::vector<double>
+poet::ChemistryModule::MasterGatherWorkerTimings(int type) const {
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  double dummy;
+  std::vector<double> timings(this->comm_size);
+
+  MPI_Gather(&dummy, 1, MPI_DOUBLE, timings.data(), 1, MPI_DOUBLE, 0,
+             this->group_comm);
+
+  timings.erase(timings.begin());
+  return timings;
+}
+
+std::vector<double> poet::ChemistryModule::GetWorkerPhreeqcTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_PHREEQC);
+}
+
+std::vector<double> poet::ChemistryModule::GetWorkerDHTGetTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_DHT_GET);
+}
+
+std::vector<double> poet::ChemistryModule::GetWorkerDHTFillTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_DHT_FILL);
+}
+
+std::vector<double> poet::ChemistryModule::GetWorkerIdleTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_IDLE);
+}
+
+std::vector<uint32_t> poet::ChemistryModule::GetWorkerDHTHits() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  type = WORKER_DHT_HITS;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  MPI_Status probe;
+  MPI_Probe(MPI_ANY_SOURCE, WORKER_DHT_HITS, this->group_comm, &probe);
+  int count;
+  MPI_Get_count(&probe, MPI_UINT32_T, &count);
+
+  std::vector<uint32_t> ret(count);
+  MPI_Recv(ret.data(), count, MPI_UINT32_T, probe.MPI_SOURCE, WORKER_DHT_HITS,
+           this->group_comm, NULL);
+
+  return ret;
+}
+
+std::vector<uint32_t> poet::ChemistryModule::GetWorkerDHTEvictions() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  type = WORKER_DHT_EVICTIONS;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  MPI_Status probe;
+  MPI_Probe(MPI_ANY_SOURCE, WORKER_DHT_EVICTIONS, this->group_comm, &probe);
+  int count;
+  MPI_Get_count(&probe, MPI_UINT32_T, &count);
+
+  std::vector<uint32_t> ret(count);
+  MPI_Recv(ret.data(), count, MPI_UINT32_T, probe.MPI_SOURCE,
+           WORKER_DHT_EVICTIONS, this->group_comm, NULL);
+
+  return ret;
+}
+
+std::vector<double>
+poet::ChemistryModule::GetWorkerInterpolationWriteTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_IP_WRITE);
+}
+
+std::vector<double>
+poet::ChemistryModule::GetWorkerInterpolationReadTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_IP_READ);
+}
+
+std::vector<double>
+poet::ChemistryModule::GetWorkerInterpolationGatherTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_IP_GATHER);
+}
+
+std::vector<double>
+poet::ChemistryModule::GetWorkerInterpolationFunctionCallTimings() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  return MasterGatherWorkerTimings(WORKER_IP_FC);
+}
+
+std::vector<uint32_t>
+poet::ChemistryModule::GetWorkerInterpolationCalls() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  type = WORKER_IP_CALLS;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  MPI_Status probe;
+  MPI_Probe(MPI_ANY_SOURCE, WORKER_IP_CALLS, this->group_comm, &probe);
+  int count;
+  MPI_Get_count(&probe, MPI_UINT32_T, &count);
+
+  std::vector<uint32_t> ret(count);
+  MPI_Recv(ret.data(), count, MPI_UINT32_T, probe.MPI_SOURCE, WORKER_IP_CALLS,
+           this->group_comm, NULL);
+
+  return ret;
+}
+
+std::vector<uint32_t> poet::ChemistryModule::GetWorkerPHTCacheHits() const {
+  int type = CHEM_PERF;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+  type = WORKER_PHT_CACHE_HITS;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+
+  MPI_Status probe;
+  MPI_Probe(MPI_ANY_SOURCE, type, this->group_comm, &probe);
+  int count;
+  MPI_Get_count(&probe, MPI_UINT32_T, &count);
+
+  std::vector<uint32_t> ret(count);
+  MPI_Recv(ret.data(), count, MPI_UINT32_T, probe.MPI_SOURCE, type,
+           this->group_comm, NULL);
+
+  return ret;
+}
+
+inline std::vector<int> shuffleVector(const std::vector<int> &in_vector,
+                                      uint32_t size_per_prop,
+                                      uint32_t wp_count) {
+  std::vector<int> out_buffer(in_vector.size());
+  uint32_t write_i = 0;
+  for (uint32_t i = 0; i < wp_count; i++) {
+    for (uint32_t j = i; j < size_per_prop; j += wp_count) {
+      out_buffer[write_i] = in_vector[j];
+      write_i++;
+    }
+  }
+  return out_buffer;
+}
+
+inline std::vector<double> shuffleField(const std::vector<double> &in_field,
+                                        uint32_t size_per_prop,
+                                        uint32_t prop_count,
+                                        uint32_t wp_count) {
+  std::vector<double> out_buffer(in_field.size());
+  uint32_t write_i = 0;
+  for (uint32_t i = 0; i < wp_count; i++) {
+    for (uint32_t j = i; j < size_per_prop; j += wp_count) {
+      for (uint32_t k = 0; k < prop_count; k++) {
+        out_buffer[(write_i * prop_count) + k] =
+            in_field[(k * size_per_prop) + j];
+      }
+      write_i++;
+    }
+  }
+  return out_buffer;
+}
+
+inline void unshuffleField(const std::vector<double> &in_buffer,
+                           uint32_t size_per_prop, uint32_t prop_count,
+                           uint32_t wp_count, std::vector<double> &out_field) {
+  uint32_t read_i = 0;
+
+  for (uint32_t i = 0; i < wp_count; i++) {
+    for (uint32_t j = i; j < size_per_prop; j += wp_count) {
+      for (uint32_t k = 0; k < prop_count; k++) {
+        out_field[(k * size_per_prop) + j] =
+            in_buffer[(read_i * prop_count) + k];
+      }
+      read_i++;
+    }
+  }
+}
+
+inline void printProgressbar(int count_pkgs, int n_wp, int barWidth = 70) {
+  /* visual progress */
+  double progress = (float)(count_pkgs + 1) / n_wp;
+
+  std::cout << "[";
+  int pos = barWidth * progress;
+  for (int iprog = 0; iprog < barWidth; ++iprog) {
+    if (iprog < pos)
+      std::cout << "=";
+    else if (iprog == pos)
+      std::cout << ">";
+    else
+      std::cout << " ";
+  }
+  std::cout << "] " << int(progress * 100.0) << " %\r";
+  std::cout.flush();
+  /* end visual progress */
+}
+
+inline void poet::ChemistryModule::MasterSendPkgs(
+    worker_list_t &w_list, workpointer_t &work_pointer, int &pkg_to_send,
+    int &count_pkgs, int &free_workers, double dt, uint32_t iteration,
+    const std::vector<uint32_t> &wp_sizes_vector) {
+  /* declare variables */
+  int local_work_package_size;
+
+  /* search for free workers and send work */
+  for (int p = 0; p < this->comm_size - 1; p++) {
+    if (w_list[p].has_work == 0 && pkg_to_send > 0) /* worker is free */ {
+      /* to enable different work_package_size, set local copy of
+       * work_package_size to pre-calculated work package size vector */
+
+      local_work_package_size = (int)wp_sizes_vector[count_pkgs];
+      count_pkgs++;
+
+      /* note current processed work package in workerlist */
+      w_list[p].send_addr = work_pointer.base();
+
+      /* push work pointer to next work package */
+      const uint32_t end_of_wp = local_work_package_size * this->prop_count;
+      std::vector<double> send_buffer(end_of_wp + this->BUFFER_OFFSET);
+      std::copy(work_pointer, work_pointer + end_of_wp, send_buffer.begin());
+
+      work_pointer += end_of_wp;
+
+      // fill send buffer starting with work_package ...
+      // followed by: work_package_size
+      send_buffer[end_of_wp] = (double)local_work_package_size;
+      // current iteration of simulation
+      send_buffer[end_of_wp + 1] = (double)iteration;
+      // size of timestep in seconds
+      send_buffer[end_of_wp + 2] = dt;
+      // current time of simulation (age) in seconds
+      send_buffer[end_of_wp + 3] = this->simtime;
+      // current work package start location in field
+      uint32_t wp_start_index = std::accumulate(wp_sizes_vector.begin(), std::next(wp_sizes_vector.begin(), count_pkgs), 0);
+      send_buffer[end_of_wp + 4] = wp_start_index;
+
+
+      /* ATTENTION Worker p has rank p+1 */
+      // MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, p + 1,
+      //          LOOP_WORK, this->group_comm);
+      MPI_Send(send_buffer.data(), send_buffer.size(), MPI_DOUBLE, p + 1,
+               LOOP_WORK, this->group_comm);
+
+      /* Mark that worker has work to do */
+      w_list[p].has_work = 1;
+      free_workers--;
+      pkg_to_send -= 1;
+    }
+  }
+}
+
+inline void poet::ChemistryModule::MasterRecvPkgs(worker_list_t &w_list,
+                                                  int &pkg_to_recv,
+                                                  bool to_send,
+                                                  int &free_workers) {
+  /* declare most of the variables here */
+  int need_to_receive = 1;
+  double idle_a, idle_b;
+  int p, size;
+
+  MPI_Status probe_status;
+  // master_recv_a = MPI_Wtime();
+  /* start to loop as long there are packages to recv and the need to receive
+   */
+  while (need_to_receive && pkg_to_recv > 0) {
+    // only of there are still packages to send and free workers are available
+    if (to_send && free_workers > 0)
+      // non blocking probing
+      MPI_Iprobe(MPI_ANY_SOURCE, LOOP_WORK, MPI_COMM_WORLD, &need_to_receive,
+                 &probe_status);
+    else {
+      idle_a = MPI_Wtime();
+      // blocking probing
+      MPI_Probe(MPI_ANY_SOURCE, LOOP_WORK, MPI_COMM_WORLD, &probe_status);
+      idle_b = MPI_Wtime();
+      this->idle_t += idle_b - idle_a;
+    }
+
+    /* if need_to_receive was set to true above, so there is a message to
+     * receive */
+    if (need_to_receive) {
+      p = probe_status.MPI_SOURCE;
+      MPI_Get_count(&probe_status, MPI_DOUBLE, &size);
+      MPI_Recv(w_list[p - 1].send_addr, size, MPI_DOUBLE, p, LOOP_WORK,
+               this->group_comm, MPI_STATUS_IGNORE);
+      w_list[p - 1].has_work = 0;
+      pkg_to_recv -= 1;
+      free_workers++;
+    }
+  }
+}
+
+void poet::ChemistryModule::simulate(double dt) {
+  double start_t{MPI_Wtime()};
+  if (this->is_sequential) {
+    MasterRunSequential();
+    return;
+  }
+
+  MasterRunParallel(dt);
+  double end_t{MPI_Wtime()};
+  this->chem_t += end_t - start_t;
+}
+
+void poet::ChemistryModule::MasterRunSequential() {
+  // std::vector<double> shuffled_field =
+  //     shuffleField(chem_field.AsVector(), n_cells, prop_count, 1);
+
+  // std::vector<std::vector<double>> input;
+  // for (std::size_t i = 0; i < n_cells; i++) {
+  //   input.push_back(
+  //       std::vector<double>(shuffled_field.begin() + (i * prop_count),
+  //                           shuffled_field.begin() + ((i + 1) *
+  //                           prop_count)));
+  // }
+
+  // this->setDumpedField(input);
+  // PhreeqcRM::RunCells();
+  // this->getDumpedField(input);
+
+  // shuffled_field.clear();
+  // for (std::size_t i = 0; i < n_cells; i++) {
+  //   shuffled_field.insert(shuffled_field.end(), input[i].begin(),
+  //                         input[i].end());
+  // }
+
+  // std::vector<double> out_vec{shuffled_field};
+  // unshuffleField(shuffled_field, n_cells, prop_count, 1, out_vec);
+  // chem_field = out_vec;
+}
+
+void poet::ChemistryModule::MasterRunParallel(double dt) {
+  /* declare most of the needed variables here */
+  double seq_a, seq_b, seq_c, seq_d;
+  double worker_chemistry_a, worker_chemistry_b;
+  double sim_e_chemistry, sim_f_chemistry;
+  int pkg_to_send, pkg_to_recv;
+  int free_workers;
+  int i_pkgs;
+  int ftype;
+
+  const std::vector<uint32_t> wp_sizes_vector =
+      CalculateWPSizesVector(this->n_cells, this->wp_size);
+
+  if (this->ai_surrogate_enabled) {
+    ftype = CHEM_AI_BCAST_VALIDITY;
+    PropagateFunctionType(ftype);
+    this->ai_surrogate_validity_vector = shuffleVector(this->ai_surrogate_validity_vector,
+                                                       this->n_cells, 
+                                                       wp_sizes_vector.size());
+    ChemBCast(&this->ai_surrogate_validity_vector.front(), this->n_cells, MPI_INT);
+  }  
+
+  ftype = CHEM_WORK_LOOP;
+  PropagateFunctionType(ftype);
+
+  MPI_Barrier(this->group_comm);
+
+  static uint32_t iteration = 0;
+
+  /* start time measurement of sequential part */
+  seq_a = MPI_Wtime();
+
+  /* shuffle grid */
+  // grid.shuffleAndExport(mpi_buffer);
+  std::vector<double> mpi_buffer =
+      shuffleField(chem_field.AsVector(), this->n_cells, this->prop_count,
+                   wp_sizes_vector.size());
+
+  /* setup local variables */
+  pkg_to_send = wp_sizes_vector.size();
+  pkg_to_recv = wp_sizes_vector.size();
+
+  workpointer_t work_pointer = mpi_buffer.begin();
+  worker_list_t worker_list(this->comm_size - 1);
+
+  free_workers = this->comm_size - 1;
+  i_pkgs = 0;
+
+  /* end time measurement of sequential part */
+  seq_b = MPI_Wtime();
+  seq_t += seq_b - seq_a;
+
+  /* start time measurement of chemistry time needed for send/recv loop */
+  worker_chemistry_a = MPI_Wtime();
+
+  /* start send/recv loop */
+  // while there are still packages to recv
+  while (pkg_to_recv > 0) {
+    // print a progressbar to stdout
+    if (print_progessbar) {
+      printProgressbar((int)i_pkgs, (int)wp_sizes_vector.size());
+    }
+    // while there are still packages to send
+    if (pkg_to_send > 0) {
+      // send packages to all free workers ...
+      MasterSendPkgs(worker_list, work_pointer, pkg_to_send, i_pkgs,
+                     free_workers, dt, iteration, wp_sizes_vector);
+    }
+    // ... and try to receive them from workers who has finished their work
+    MasterRecvPkgs(worker_list, pkg_to_recv, pkg_to_send > 0, free_workers);
+  }
+
+  // Just to complete the progressbar
+  std::cout << std::endl;
+
+  /* stop time measurement of chemistry time needed for send/recv loop */
+  worker_chemistry_b = MPI_Wtime();
+  this->send_recv_t += worker_chemistry_b - worker_chemistry_a;
+
+  /* start time measurement of sequential part */
+  seq_c = MPI_Wtime();
+
+  /* unshuffle grid */
+  // grid.importAndUnshuffle(mpi_buffer);
+  std::vector<double> out_vec{mpi_buffer};
+  unshuffleField(mpi_buffer, this->n_cells, this->prop_count,
+                 wp_sizes_vector.size(), out_vec);
+  chem_field = out_vec;
+
+  /* do master stuff */
+
+  /* start time measurement of master chemistry */
+  sim_e_chemistry = MPI_Wtime();
+
+  /* end time measurement of sequential part */
+  seq_d = MPI_Wtime();
+  this->seq_t += seq_d - seq_c;
+
+  /* end time measurement of whole chemistry simulation */
+
+  /* advise workers to end chemistry iteration */
+  for (int i = 1; i < this->comm_size; i++) {
+    MPI_Send(NULL, 0, MPI_DOUBLE, i, LOOP_END, this->group_comm);
+  }
+
+  this->simtime += dt;
+  iteration++;
+}
+
+void poet::ChemistryModule::MasterLoopBreak() {
+  int type = CHEM_BREAK_MAIN_LOOP;
+  MPI_Bcast(&type, 1, MPI_INT, 0, this->group_comm);
+}
+
+std::vector<uint32_t>
+poet::ChemistryModule::CalculateWPSizesVector(uint32_t n_cells,
+                                              uint32_t wp_size) const {
+  bool mod_pkgs = (n_cells % wp_size) != 0;
+  uint32_t n_packages =
+      (uint32_t)(n_cells / wp_size) + static_cast<int>(mod_pkgs);
+
+  std::vector<uint32_t> wp_sizes_vector(n_packages, 0);
+
+  for (int i = 0; i < n_cells; i++) {
+    wp_sizes_vector[i % n_packages] += 1;
+  }
+
+  return wp_sizes_vector;
+}
+
+void poet::ChemistryModule::masterSetField(Field field) {
+  this->chem_field = field;
+  this->prop_count = field.GetProps().size();
+
+  int ftype = CHEM_FIELD_INIT;
+  PropagateFunctionType(ftype);
+
+  ChemBCast(&this->prop_count, 1, MPI_UINT32_T);
+}

src/Chemistry/SurrogateModels/DHT.c

@@ -17,8 +17,11 @@
 
 #include "DHT.h"
 
+#include <mpi.h>
+
 #include <inttypes.h>
 #include <math.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,8 +35,8 @@ static void determine_dest(uint64_t hash, int comm_size,
   /** how many bytes do we need for one index? */
   int index_size = sizeof(double) - (index_count - 1);
   for (int i = 0; i < index_count; i++) {
-    tmp_index = 0;
+    tmp_index = (hash >> (i * 8)) & ((1ULL << (index_size * 8)) - 1);
-    memcpy(&tmp_index, (char *)&hash + i, index_size);
+    /* memcpy(&tmp_index, (char *)&hash + i, index_size); */
     index[i] = (uint64_t)(tmp_index % table_size);
   }
   *dest_rank = (unsigned int)(hash % comm_size);
@@ -52,7 +55,8 @@ static int read_flag(char flag_byte) {
 }
 
 DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size,
-                unsigned int key_size, uint64_t (*hash_func)(int, void *)) {
+                unsigned int key_size,
+                uint64_t (*hash_func)(int, const void *)) {
   DHT *object;
   MPI_Win window;
   void *mem_alloc;
@@ -61,17 +65,20 @@ DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size,
   // calculate how much bytes for the index are needed to address count of
   // buckets per process
   index_bytes = (int)ceil(log2(size));
-  if (index_bytes % 8 != 0) index_bytes = index_bytes + (8 - (index_bytes % 8));
+  if (index_bytes % 8 != 0)
+    index_bytes = index_bytes + (8 - (index_bytes % 8));
 
   // allocate memory for dht-object
   object = (DHT *)malloc(sizeof(DHT));
-  if (object == NULL) return NULL;
+  if (object == NULL)
+    return NULL;
 
   // every memory allocation has 1 additional byte for flags etc.
   if (MPI_Alloc_mem(size * (1 + data_size + key_size), MPI_INFO_NULL,
                     &mem_alloc) != 0)
     return NULL;
-  if (MPI_Comm_size(comm, &comm_size) != 0) return NULL;
+  if (MPI_Comm_size(comm, &comm_size) != 0)
+    return NULL;
 
   // since MPI_Alloc_mem doesn't provide memory allocation with the memory set
   // to zero, we're doing this here
@@ -98,13 +105,16 @@ DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size,
   object->index_count = 9 - (index_bytes / 8);
   object->index = (uint64_t *)malloc((object->index_count) * sizeof(uint64_t));
   object->mem_alloc = mem_alloc;
+  object->sum_idx = 0;
+  object->cnt_idx = 0;
 
   // if set, initialize dht_stats
 #ifdef DHT_STATISTICS
   DHT_stats *stats;
 
   stats = (DHT_stats *)malloc(sizeof(DHT_stats));
-  if (stats == NULL) return NULL;
+  if (stats == NULL)
+    return NULL;
 
   object->stats = stats;
   object->stats->writes_local = (int *)calloc(comm_size, sizeof(int));
@@ -118,7 +128,109 @@ DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size,
   return object;
 }
 
-int DHT_write(DHT *table, void *send_key, void *send_data) {
+void DHT_set_accumulate_callback(DHT *table,
+                                 int (*callback_func)(int, void *, int,
+                                                      void *)) {
+  table->accumulate_callback = callback_func;
+}
+
+int DHT_write_accumulate(DHT *table, const void *send_key, int data_size,
+                         void *send_data, uint32_t *proc, uint32_t *index,
+                         int *callback_ret) {
+  unsigned int dest_rank, i;
+  int result = DHT_SUCCESS;
+
+#ifdef DHT_STATISTICS
+  table->stats->w_access++;
+#endif
+
+  // determine destination rank and index by hash of key
+  determine_dest(table->hash_func(table->key_size, send_key), table->comm_size,
+                 table->table_size, &dest_rank, table->index,
+                 table->index_count);
+
+  // concatenating key with data to write entry to DHT
+  set_flag((char *)table->send_entry);
+  memcpy((char *)table->send_entry + 1, (char *)send_key, table->key_size);
+  /* memcpy((char *)table->send_entry + table->key_size + 1, (char *)send_data,
+   */
+  /*        table->data_size); */
+
+  // locking window of target rank with exclusive lock
+  if (MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dest_rank, 0, table->window) != 0)
+    return DHT_MPI_ERROR;
+  for (i = 0; i < table->index_count; i++) {
+    if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size,
+                MPI_BYTE, dest_rank, table->index[i],
+                1 + table->data_size + table->key_size, MPI_BYTE,
+                table->window) != 0)
+      return DHT_MPI_ERROR;
+    if (MPI_Win_flush(dest_rank, table->window) != 0)
+      return DHT_MPI_ERROR;
+
+    // increment eviction counter if receiving key doesn't match sending key
+    // entry has write flag and last index is reached.
+    if (read_flag(*(char *)table->recv_entry)) {
+      if (memcmp(send_key, (char *)table->recv_entry + 1, table->key_size) !=
+          0) {
+        if (i == (table->index_count) - 1) {
+          table->evictions += 1;
+#ifdef DHT_STATISTICS
+          table->stats->evictions += 1;
+#endif
+          result = DHT_WRITE_SUCCESS_WITH_EVICTION;
+          break;
+        }
+      } else
+        break;
+    } else {
+#ifdef DHT_STATISTICS
+      table->stats->writes_local[dest_rank]++;
+#endif
+      break;
+    }
+  }
+
+  table->cnt_idx += 1;
+  table->sum_idx += (i + 1);
+
+  if (result == DHT_WRITE_SUCCESS_WITH_EVICTION) {
+    memset((char *)table->send_entry + 1 + table->key_size, '\0',
+           table->data_size);
+  } else {
+    memcpy((char *)table->send_entry + 1 + table->key_size,
+           (char *)table->recv_entry + 1 + table->key_size, table->data_size);
+  }
+
+  *callback_ret = table->accumulate_callback(
+      data_size, (char *)send_data, table->data_size,
+      (char *)table->send_entry + 1 + table->key_size);
+
+  // put data to DHT (with last selected index by value i)
+  if (*callback_ret == 0) {
+    if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size,
+                MPI_BYTE, dest_rank, table->index[i],
+                1 + table->data_size + table->key_size, MPI_BYTE,
+                table->window) != 0)
+      return DHT_MPI_ERROR;
+  }
+  // unlock window of target rank
+  if (MPI_Win_unlock(dest_rank, table->window) != 0)
+    return DHT_MPI_ERROR;
+
+  if (proc) {
+    *proc = dest_rank;
+  }
+
+  if (index) {
+    *index = table->index[i];
+  }
+
+  return result;
+}
+
+int DHT_write(DHT *table, void *send_key, void *send_data, uint32_t *proc,
+              uint32_t *index) {
   unsigned int dest_rank, i;
   int result = DHT_SUCCESS;
 
@@ -146,7 +258,8 @@ int DHT_write(DHT *table, void *send_key, void *send_data) {
                 1 + table->data_size + table->key_size, MPI_BYTE,
                 table->window) != 0)
       return DHT_MPI_ERROR;
-    if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+    if (MPI_Win_flush(dest_rank, table->window) != 0)
+      return DHT_MPI_ERROR;
 
     // increment eviction counter if receiving key doesn't match sending key
     // entry has write flag and last index is reached.
@@ -171,6 +284,9 @@ int DHT_write(DHT *table, void *send_key, void *send_data) {
     }
   }
 
+  table->cnt_idx += 1;
+  table->sum_idx += (i + 1);
+
   // put data to DHT (with last selected index by value i)
   if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size,
               MPI_BYTE, dest_rank, table->index[i],
@@ -178,12 +294,21 @@ int DHT_write(DHT *table, void *send_key, void *send_data) {
               table->window) != 0)
     return DHT_MPI_ERROR;
   // unlock window of target rank
-  if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+  if (MPI_Win_unlock(dest_rank, table->window) != 0)
+    return DHT_MPI_ERROR;
+
+  if (proc) {
+    *proc = dest_rank;
+  }
+
+  if (index) {
+    *index = table->index[i];
+  }
 
   return result;
 }
 
-int DHT_read(DHT *table, void *send_key, void *destination) {
+int DHT_read(DHT *table, const void *send_key, void *destination) {
   unsigned int dest_rank, i;
 
 #ifdef DHT_STATISTICS
@@ -205,7 +330,8 @@ int DHT_read(DHT *table, void *send_key, void *destination) {
                 1 + table->data_size + table->key_size, MPI_BYTE,
                 table->window) != 0)
       return DHT_MPI_ERROR;
-    if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+    if (MPI_Win_flush(dest_rank, table->window) != 0)
+      return DHT_MPI_ERROR;
 
     // increment read error counter if write flag isn't set ...
     if ((read_flag(*(char *)table->recv_entry)) == 0) {
@@ -214,7 +340,8 @@ int DHT_read(DHT *table, void *send_key, void *destination) {
       table->stats->read_misses += 1;
 #endif
       // unlock window and return
-      if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+      if (MPI_Win_unlock(dest_rank, table->window) != 0)
+        return DHT_MPI_ERROR;
       return DHT_READ_MISS;
     }
 
@@ -227,7 +354,8 @@ int DHT_read(DHT *table, void *send_key, void *destination) {
         table->stats->read_misses += 1;
 #endif
         // unlock window an return
-        if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+        if (MPI_Win_unlock(dest_rank, table->window) != 0)
+          return DHT_MPI_ERROR;
         return DHT_READ_MISS;
       }
     } else
@@ -235,7 +363,8 @@ int DHT_read(DHT *table, void *send_key, void *destination) {
   }
 
   // unlock window of target rank
-  if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR;
+  if (MPI_Win_unlock(dest_rank, table->window) != 0)
+    return DHT_MPI_ERROR;
 
   // if matching key was found copy data into memory of passed pointer
   memcpy((char *)destination, (char *)table->recv_entry + table->key_size + 1,
@@ -244,6 +373,34 @@ int DHT_read(DHT *table, void *send_key, void *destination) {
   return DHT_SUCCESS;
 }
 
+int DHT_read_location(DHT *table, uint32_t proc, uint32_t index,
+                      void *destination) {
+  const uint32_t bucket_size = table->data_size + table->key_size + 1;
+
+#ifdef DHT_STATISTICS
+  table->stats->r_access++;
+#endif
+
+  // locking window of target rank with shared lock
+  if (MPI_Win_lock(MPI_LOCK_SHARED, proc, 0, table->window) != 0)
+    return DHT_MPI_ERROR;
+  // receive data
+  if (MPI_Get(table->recv_entry, bucket_size, MPI_BYTE, proc, index,
+              bucket_size, MPI_BYTE, table->window) != 0) {
+    return DHT_MPI_ERROR;
+  }
+
+  // unlock window of target rank
+  if (MPI_Win_unlock(proc, table->window) != 0)
+    return DHT_MPI_ERROR;
+
+  // if matching key was found copy data into memory of passed pointer
+  memcpy((char *)destination, (char *)table->recv_entry + 1 + table->key_size,
+         table->data_size);
+
+  return DHT_SUCCESS;
+}
+
 int DHT_to_file(DHT *table, const char *filename) {
   // open file
   MPI_File file;
@@ -257,17 +414,15 @@ int DHT_to_file(DHT *table, const char *filename) {
 
   // write header (key_size and data_size)
   if (rank == 0) {
-    if (MPI_File_write(file, &table->key_size, 1, MPI_INT, MPI_STATUS_IGNORE) !=
+    if (MPI_File_write_shared(file, &table->key_size, 1, MPI_INT,
-        0)
+                              MPI_STATUS_IGNORE) != 0)
       return DHT_FILE_WRITE_ERROR;
-    if (MPI_File_write(file, &table->data_size, 1, MPI_INT,
+    if (MPI_File_write_shared(file, &table->data_size, 1, MPI_INT,
-                       MPI_STATUS_IGNORE) != 0)
+                              MPI_STATUS_IGNORE) != 0)
       return DHT_FILE_WRITE_ERROR;
   }
 
-  // seek file pointer behind header for all processes
+  MPI_Barrier(table->communicator);
-  if (MPI_File_seek_shared(file, DHT_FILEHEADER_SIZE, MPI_SEEK_SET) != 0)
-    return DHT_FILE_IO_ERROR;
 
   char *ptr;
   int bucket_size = table->key_size + table->data_size + 1;
@@ -283,8 +438,12 @@ int DHT_to_file(DHT *table, const char *filename) {
         return DHT_FILE_WRITE_ERROR;
     }
   }
+
+  MPI_Barrier(table->communicator);
+
   // close file
-  if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR;
+  if (MPI_File_close(&file) != 0)
+    return DHT_FILE_IO_ERROR;
 
   return DHT_SUCCESS;
 }
@@ -303,7 +462,8 @@ int DHT_from_file(DHT *table, const char *filename) {
     return DHT_FILE_IO_ERROR;
 
   // get file size
-  if (MPI_File_get_size(file, &f_size) != 0) return DHT_FILE_IO_ERROR;
+  if (MPI_File_get_size(file, &f_size) != 0)
+    return DHT_FILE_IO_ERROR;
 
   MPI_Comm_rank(table->communicator, &rank);
 
@@ -322,8 +482,10 @@ int DHT_from_file(DHT *table, const char *filename) {
     return DHT_FILE_READ_ERROR;
 
   // compare if written header data and key size matches current sizes
-  if (*(int *)buffer != table->key_size) return DHT_WRONG_FILE;
+  if (*(int *)buffer != table->key_size)
-  if (*(int *)(buffer + 4) != table->data_size) return DHT_WRONG_FILE;
+    return DHT_WRONG_FILE;
+  if (*(int *)(buffer + 4) != table->data_size)
+    return DHT_WRONG_FILE;
 
   // set offset for each process
   offset = bucket_size * table->comm_size;
@@ -348,14 +510,16 @@ int DHT_from_file(DHT *table, const char *filename) {
     // extract key and data and write to DHT
     key = buffer;
     data = (buffer + table->key_size);
-    if (DHT_write(table, key, data) == DHT_MPI_ERROR) return DHT_MPI_ERROR;
+    if (DHT_write(table, key, data, NULL, NULL) == DHT_MPI_ERROR)
+      return DHT_MPI_ERROR;
 
     // increment current position
     cur_pos += offset;
   }
 
   free(buffer);
-  if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR;
+  if (MPI_File_close(&file) != 0)
+    return DHT_FILE_IO_ERROR;
 
   return DHT_SUCCESS;
 }
@@ -377,8 +541,10 @@ int DHT_free(DHT *table, int *eviction_counter, int *readerror_counter) {
       return DHT_MPI_ERROR;
     *readerror_counter = buf;
   }
-  if (MPI_Win_free(&(table->window)) != 0) return DHT_MPI_ERROR;
+  if (MPI_Win_free(&(table->window)) != 0)
-  if (MPI_Free_mem(table->mem_alloc) != 0) return DHT_MPI_ERROR;
+    return DHT_MPI_ERROR;
+  if (MPI_Free_mem(table->mem_alloc) != 0)
+    return DHT_MPI_ERROR;
   free(table->recv_entry);
   free(table->send_entry);
   free(table->index);
@@ -392,6 +558,41 @@ int DHT_free(DHT *table, int *eviction_counter, int *readerror_counter) {
   return DHT_SUCCESS;
 }
 
+float DHT_get_used_idx_factor(DHT *table, int with_reset) {
+  int rank;
+  MPI_Comm_rank(table->communicator, &rank);
+
+  float my_avg_idx = (float)table->sum_idx / (float)table->cnt_idx;
+
+  float max_mean_index;
+
+  MPI_Reduce(&my_avg_idx, &max_mean_index, 1, MPI_FLOAT, MPI_MAX, 0,
+             table->communicator);
+
+  MPI_Bcast(&max_mean_index, 1, MPI_FLOAT, 0, table->communicator);
+
+  if (!!with_reset) {
+    table->sum_idx = 0;
+    table->cnt_idx = 0;
+  }
+
+  return max_mean_index;
+}
+
+int DHT_flush(DHT *table) {
+  // make sure all processes are synchronized
+  MPI_Barrier(table->communicator);
+
+  // wipe local memory with zeros
+  memset(table->mem_alloc, '\0',
+         table->table_size * (1 + table->data_size + table->key_size));
+
+  table->sum_idx = 0;
+  table->cnt_idx = 0;
+
+  return DHT_SUCCESS;
+}
+
 int DHT_print_statistics(DHT *table) {
 #ifdef DHT_STATISTICS
   int *written_buckets;
@@ -407,7 +608,8 @@ int DHT_print_statistics(DHT *table) {
 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
 
   // obtaining all values from all processes in the communicator
-  if (rank == 0) read_misses = (int *)malloc(table->comm_size * sizeof(int));
+  if (rank == 0)
+    read_misses = (int *)malloc(table->comm_size * sizeof(int));
   if (MPI_Gather(&table->stats->read_misses, 1, MPI_INT, read_misses, 1,
                  MPI_INT, 0, table->communicator) != 0)
     return DHT_MPI_ERROR;
@@ -416,7 +618,8 @@ int DHT_print_statistics(DHT *table) {
     return DHT_MPI_ERROR;
   table->stats->read_misses = 0;
 
-  if (rank == 0) evictions = (int *)malloc(table->comm_size * sizeof(int));
+  if (rank == 0)
+    evictions = (int *)malloc(table->comm_size * sizeof(int));
   if (MPI_Gather(&table->stats->evictions, 1, MPI_INT, evictions, 1, MPI_INT, 0,
                  table->communicator) != 0)
     return DHT_MPI_ERROR;
@@ -425,7 +628,8 @@ int DHT_print_statistics(DHT *table) {
     return DHT_MPI_ERROR;
   table->stats->evictions = 0;
 
-  if (rank == 0) w_access = (int *)malloc(table->comm_size * sizeof(int));
+  if (rank == 0)
+    w_access = (int *)malloc(table->comm_size * sizeof(int));
   if (MPI_Gather(&table->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0,
                  table->communicator) != 0)
     return DHT_MPI_ERROR;
@@ -434,7 +638,8 @@ int DHT_print_statistics(DHT *table) {
     return DHT_MPI_ERROR;
   table->stats->w_access = 0;
 
-  if (rank == 0) r_access = (int *)malloc(table->comm_size * sizeof(int));
+  if (rank == 0)
+    r_access = (int *)malloc(table->comm_size * sizeof(int));
   if (MPI_Gather(&table->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0,
                  table->communicator) != 0)
     return DHT_MPI_ERROR;
@@ -443,13 +648,14 @@ int DHT_print_statistics(DHT *table) {
     return DHT_MPI_ERROR;
   table->stats->r_access = 0;
 
-  if (rank == 0) written_buckets = (int *)calloc(table->comm_size, sizeof(int));
+  if (rank == 0)
+    written_buckets = (int *)calloc(table->comm_size, sizeof(int));
   if (MPI_Reduce(table->stats->writes_local, written_buckets, table->comm_size,
                  MPI_INT, MPI_SUM, 0, table->communicator) != 0)
     return DHT_MPI_ERROR;
 
-  if (rank == 0) {  // only process with rank 0 will print out results as a
+  if (rank == 0) { // only process with rank 0 will print out results as a
-                    // table
+                   // table
     int sum_written_buckets = 0;
 
     for (int i = 0; i < table->comm_size; i++) {

src/Chemistry/SurrogateModels/DHT.h

@@ -67,7 +67,7 @@
  */
 typedef struct {
   /** Count of writes to specific process this process did. */
-  int* writes_local;
+  int *writes_local;
   /** Writes after last call of DHT_print_statistics. */
   int old_writes;
   /** How many read misses occur? */
@@ -100,27 +100,40 @@ typedef struct {
   /** Size of the MPI communicator respectively all participating processes. */
   int comm_size;
   /** Pointer to a hashfunction. */
-  uint64_t (*hash_func)(int, void*);
+  uint64_t (*hash_func)(int, const void *);
   /** Pre-allocated memory where a bucket can be received. */
-  void* recv_entry;
+  void *recv_entry;
   /** Pre-allocated memory where a bucket to send can be stored. */
-  void* send_entry;
+  void *send_entry;
   /** Allocated memory on which the MPI window was created. */
-  void* mem_alloc;
+  void *mem_alloc;
   /** Count of read misses over all time. */
   int read_misses;
   /** Count of evictions over all time. */
   int evictions;
   /** Array of indeces where a bucket can be stored. */
-  uint64_t* index;
+  uint64_t *index;
   /** Count of possible indeces. */
   unsigned int index_count;
+
+  int (*accumulate_callback)(int, void *, int, void *);
+
+  size_t sum_idx;
+  size_t cnt_idx;
 #ifdef DHT_STATISTICS
   /** Detailed statistics of the usage of the DHT. */
-  DHT_stats* stats;
+  DHT_stats *stats;
 #endif
 } DHT;
 
+extern void DHT_set_accumulate_callback(DHT *table,
+                                        int (*callback_func)(int, void *, int,
+                                                             void *));
+
+extern int DHT_write_accumulate(DHT *table, const void *key, int send_size,
+                                void *data, uint32_t *proc, uint32_t *index,
+                                int *callback_ret);
+
 /**
  * @brief Create a DHT.
  *
@@ -141,9 +154,9 @@ typedef struct {
  * @return DHT* The returned value is the \a DHT-object which serves as a handle
  * for all DHT operations. If an error occured NULL is returned.
  */
-extern DHT* DHT_create(MPI_Comm comm, uint64_t size_per_process,
+extern DHT *DHT_create(MPI_Comm comm, uint64_t size_per_process,
                        unsigned int data_size, unsigned int key_size,
-                       uint64_t (*hash_func)(int, void*));
+                       uint64_t (*hash_func)(int, const void *));
 
 /**
  * @brief Write data into DHT.
@@ -161,10 +174,14 @@ extern DHT* DHT_create(MPI_Comm comm, uint64_t size_per_process,
  * @param table Pointer to the \a DHT-object.
  * @param key Pointer to the key.
  * @param data Pointer to the data.
+ * @param proc If not NULL, returns the process number written to.
+ * @param index If not NULL, returns the index of the bucket where the data was
+ * written to.
  * @return int Returns either DHT_SUCCESS on success or correspondending error
  * value on eviction or error.
  */
-extern int DHT_write(DHT* table, void* key, void* data);
+extern int DHT_write(DHT *table, void *key, void *data, uint32_t *proc,
+                     uint32_t *index);
 
 /**
  * @brief Read data from DHT.
@@ -187,8 +204,10 @@ extern int DHT_write(DHT* table, void* key, void* data);
  * @return int Returns either DHT_SUCCESS on success or correspondending error
  * value on read miss or error.
  */
-extern int DHT_read(DHT* table, void* key, void* destination);
+extern int DHT_read(DHT *table, const void *key, void *destination);
 
+extern int DHT_read_location(DHT *table, uint32_t proc, uint32_t index,
+                             void *destination);
 /**
  * @brief Write current state of DHT to file.
  *
@@ -203,7 +222,7 @@ extern int DHT_read(DHT* table, void* key, void* destination);
  * @return int Returns DHT_SUCCESS on succes, DHT_FILE_IO_ERROR if file can't be
  * opened/closed or DHT_WRITE_ERROR if file is not writable.
  */
-extern int DHT_to_file(DHT* table, const char* filename);
+extern int DHT_to_file(DHT *table, const char *filename);
 
 /**
  * @brief Read state of DHT from file.
@@ -223,7 +242,7 @@ extern int DHT_to_file(DHT* table, const char* filename);
  * file doesn't match expectation. This is possible if the data size or key size
  * is different.
  */
-extern int DHT_from_file(DHT* table, const char* filename);
+extern int DHT_from_file(DHT *table, const char *filename);
 
 /**
  * @brief Free ressources of DHT.
@@ -241,7 +260,7 @@ extern int DHT_from_file(DHT* table, const char* filename);
  * @return int Returns either DHT_SUCCESS on success or DHT_MPI_ERROR on
  * internal MPI error.
  */
-extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter);
+extern int DHT_free(DHT *table, int *eviction_counter, int *readerror_counter);
 
 /**
  * @brief Prints a table with statistics about current use of DHT.
@@ -267,46 +286,10 @@ extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter);
  * @return int Returns DHT_SUCCESS on success or DHT_MPI_ERROR on internal MPI
  * error.
  */
-extern int DHT_print_statistics(DHT* table);
+extern int DHT_print_statistics(DHT *table);
 
-/**
+extern float DHT_get_used_idx_factor(DHT *table, int with_reset);
- * @brief Determine destination rank and index.
- *
- * This is done by looping over all possbile indices. First of all, set a
- * temporary index to zero and copy count of bytes for each index into the
- * memory area of the temporary index. After that the current index is
- * calculated by the temporary index modulo the table size. The destination rank
- * of the process is simply determined by hash modulo the communicator size.
- *
- * @param hash Calculated 64 bit hash.
- * @param comm_size Communicator size.
- * @param table_size Count of buckets per process.
- * @param dest_rank Reference to the destination rank variable.
- * @param index Pointer to the array index.
- * @param index_count Count of possible indeces.
- */
-static void determine_dest(uint64_t hash, int comm_size,
-                           unsigned int table_size, unsigned int* dest_rank,
-                           uint64_t* index, unsigned int index_count);
 
-/**
+extern int DHT_flush(DHT *table);
- * @brief Set the occupied flag.
- *
- * This will set the first bit of a bucket to 1.
- *
- * @param flag_byte First byte of a bucket.
- */
-static void set_flag(char* flag_byte);
-
-/**
- * @brief Get the occupied flag.
- *
- * This function determines whether the occupied flag of a bucket was set or
- * not.
- *
- * @param flag_byte First byte of a bucket.
- * @return int Returns 1 for true or 0 for false.
- */
-static int read_flag(char flag_byte);
 
 #endif /* DHT_H */

src/Chemistry/SurrogateModels/DHT_Wrapper.cpp

@@ -0,0 +1,350 @@
+
+/*
+** Copyright (C) 2018-2021 Alexander Lindemann, Max Luebke (University of
+** Potsdam)
+**
+** Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include "DHT_Wrapper.hpp"
+
+#include "Init/InitialList.hpp"
+#include "Rounding.hpp"
+
+#include <Rcpp/proxy/ProtectedProxy.h>
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <stdexcept>
+#include <vector>
+
+using namespace std;
+
+namespace poet {
+
+DHT_Wrapper::DHT_Wrapper(MPI_Comm dht_comm, std::uint64_t dht_size,
+                         const NamedVector<std::uint32_t> &key_species,
+                         const std::vector<std::int32_t> &key_indices,
+                         const std::vector<std::string> &_output_names,
+                         const InitialList::ChemistryHookFunctions &_hooks,
+                         uint32_t data_count, bool _with_interp,
+                         bool _has_het_ids)
+    : key_count(key_indices.size()), data_count(data_count),
+      input_key_elements(key_indices), communicator(dht_comm),
+      key_species(key_species), output_names(_output_names), hooks(_hooks),
+      with_interp(_with_interp), has_het_ids(_has_het_ids) {
+  // initialize DHT object
+  // key size = count of key elements + timestep
+  uint32_t key_size = (key_count + 1) * sizeof(Lookup_Keyelement);
+  uint32_t data_size =
+      (data_count + (with_interp ? input_key_elements.size() : 0)) *
+      sizeof(double);
+  uint32_t buckets_per_process =
+      static_cast<std::uint32_t>(dht_size / (data_size + key_size));
+  dht_object = DHT_create(dht_comm, buckets_per_process, data_size, key_size,
+                          &poet::Murmur2_64A);
+
+  dht_signif_vector = key_species.getValues();
+
+  // this->dht_signif_vector.resize(key_size, DHT_KEY_SIGNIF_DEFAULT);
+
+  this->dht_prop_type_vector.resize(key_count, DHT_TYPE_DEFAULT);
+
+  const auto key_names = key_species.getNames();
+
+  auto tot_h = std::find(key_names.begin(), key_names.end(), "H");
+  if (tot_h != key_names.end()) {
+    this->dht_prop_type_vector[tot_h - key_names.begin()] = DHT_TYPE_TOTAL;
+  }
+  auto tot_o = std::find(key_names.begin(), key_names.end(), "O");
+  if (tot_o != key_names.end()) {
+    this->dht_prop_type_vector[tot_o - key_names.begin()] = DHT_TYPE_TOTAL;
+  }
+  auto charge = std::find(key_names.begin(), key_names.end(), "Charge");
+  if (charge != key_names.end()) {
+    this->dht_prop_type_vector[charge - key_names.begin()] = DHT_TYPE_CHARGE;
+  }
+}
+
+DHT_Wrapper::~DHT_Wrapper() {
+  // free DHT
+  DHT_free(dht_object, NULL, NULL);
+}
+auto DHT_Wrapper::checkDHT(WorkPackage &work_package)
+    -> const DHT_ResultObject & {
+
+  const auto length = work_package.size;
+
+  std::vector<double> bucket_writer(
+      this->data_count + (with_interp ? input_key_elements.size() : 0));
+
+  // loop over every grid cell contained in work package
+  for (int i = 0; i < length; i++) {
+    // point to current grid cell
+    auto &key_vector = dht_results.keys[i];
+
+    // overwrite input with data from DHT, IF value is found in DHT
+    int res =
+        DHT_read(this->dht_object, key_vector.data(), bucket_writer.data());
+
+    switch (res) {
+    case DHT_SUCCESS:
+      work_package.output[i] =
+          (with_interp
+               ? inputAndRatesToOutput(bucket_writer, work_package.input[i])
+               : bucket_writer);
+      work_package.mapping[i] = CHEM_DHT;
+      this->dht_hits++;
+      break;
+    case DHT_READ_MISS:
+      break;
+    }
+  }
+
+  return dht_results;
+}
+
+void DHT_Wrapper::fillDHT(const WorkPackage &work_package) {
+
+  const auto length = work_package.size;
+
+  // loop over every grid cell contained in work package
+  dht_results.locations.resize(length);
+  dht_results.filledDHT = std::vector<bool>(length, false);
+  for (int i = 0; i < length; i++) {
+    // If true grid cell was simulated, needs to be inserted into dht
+    if (work_package.mapping[i] != CHEM_PQC) {
+      continue;
+    }
+
+    if (work_package.input[i][0] != 2) {
+      continue;
+    }
+
+    // check if calcite or dolomite is absent and present, resp.n and vice
+    // versa in input/output. If this is the case -> Do not write to DHT!
+    // HACK: hardcoded, should be fixed!
+    if (hooks.dht_fill.isValid()) {
+      NamedVector<double> old_values(output_names, work_package.input[i]);
+      NamedVector<double> new_values(output_names, work_package.output[i]);
+
+      if (Rcpp::as<bool>(hooks.dht_fill(old_values, new_values))) {
+        continue;
+      }
+    }
+
+    uint32_t proc, index;
+    auto &key = dht_results.keys[i];
+    const auto data =
+        (with_interp ? outputToInputAndRates(work_package.input[i],
+                                             work_package.output[i])
+                     : work_package.output[i]);
+    // void *data = (void *)&(work_package[i * this->data_count]);
+    // fuzz data (round, logarithm etc.)
+
+    // insert simulated data with fuzzed key into DHT
+    int res = DHT_write(this->dht_object, key.data(),
+                        const_cast<double *>(data.data()), &proc, &index);
+
+    dht_results.locations[i] = {proc, index};
+
+    // if data was successfully written ...
+    if ((res != DHT_SUCCESS) && (res == DHT_WRITE_SUCCESS_WITH_EVICTION)) {
+      dht_evictions++;
+    }
+
+    dht_results.filledDHT[i] = true;
+  }
+}
+
+inline std::vector<double>
+DHT_Wrapper::outputToInputAndRates(const std::vector<double> &old_results,
+                                   const std::vector<double> &new_results) {
+  const int prefix_size = this->input_key_elements.size();
+
+  std::vector<double> output(prefix_size + this->data_count);
+  std::copy(new_results.begin(), new_results.end(),
+            output.begin() + prefix_size);
+
+  for (int i = 0; i < prefix_size; i++) {
+    const int data_elem_i = input_key_elements[i];
+    output[i] = old_results[data_elem_i];
+    output[prefix_size + data_elem_i] -= old_results[data_elem_i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::inputAndRatesToOutput(const std::vector<double> &dht_data,
+                                   const std::vector<double> &input_values) {
+  const int prefix_size = this->input_key_elements.size();
+
+  std::vector<double> output(input_values);
+
+  for (int i = 0; i < prefix_size; i++) {
+    const int data_elem_i = input_key_elements[i];
+    output[data_elem_i] += dht_data[i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::outputToRates(const std::vector<double> &old_results,
+                           const std::vector<double> &new_results) {
+  std::vector<double> output(new_results);
+
+  for (const auto &data_elem_i : input_key_elements) {
+    output[data_elem_i] -= old_results[data_elem_i];
+  }
+
+  return output;
+}
+
+inline std::vector<double>
+DHT_Wrapper::ratesToOutput(const std::vector<double> &dht_data,
+                           const std::vector<double> &input_values) {
+  std::vector<double> output(input_values);
+
+  for (const auto &data_elem_i : input_key_elements) {
+    output[data_elem_i] += dht_data[data_elem_i];
+  }
+
+  return output;
+}
+
+// void DHT_Wrapper::resultsToWP(std::vector<double> &work_package) {
+//   for (int i = 0; i < dht_results.length; i++) {
+//     if (!dht_results.needPhreeqc[i]) {
+//       std::copy(dht_results.results[i].begin(), dht_results.results[i].end(),
+//                 work_package.begin() + (data_count * i));
+//     }
+//   }
+// }
+
+int DHT_Wrapper::tableToFile(const char *filename) {
+  int res = DHT_to_file(dht_object, filename);
+  return res;
+}
+
+int DHT_Wrapper::fileToTable(const char *filename) {
+  int res = DHT_from_file(dht_object, filename);
+  if (res != DHT_SUCCESS)
+    return res;
+
+#ifdef DHT_STATISTICS
+  DHT_print_statistics(dht_object);
+#endif
+
+  return DHT_SUCCESS;
+}
+
+void DHT_Wrapper::printStatistics() {
+  int res;
+
+  res = DHT_print_statistics(dht_object);
+
+  if (res != DHT_SUCCESS) {
+    // MPI ERROR ... WHAT TO DO NOW?
+    // RUNNING CIRCLES WHILE SCREAMING
+  }
+}
+
+LookupKey DHT_Wrapper::fuzzForDHT_R(const std::vector<double> &cell,
+                                    double dt) {
+  const auto c_zero_val = std::pow(10, AQUEOUS_EXP);
+
+  NamedVector<double> input_nv(this->output_names, cell);
+
+  const std::vector<double> eval_vec =
+      Rcpp::as<std::vector<double>>(hooks.dht_fuzz(input_nv));
+  assert(eval_vec.size() == this->key_count);
+  LookupKey vecFuzz(this->key_count + 1 + has_het_ids, {.0});
+
+  DHT_Rounder rounder;
+
+  int totals_i = 0;
+  // introduce fuzzing to allow more hits in DHT
+  // loop over every variable of grid cell
+  for (std::uint32_t i = 0; i < eval_vec.size(); i++) {
+    double curr_key = eval_vec[i];
+    if (curr_key != 0) {
+      if (this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL) {
+        curr_key -= base_totals[totals_i++];
+      }
+      vecFuzz[i] =
+          rounder.round(curr_key, dht_signif_vector[i],
+                        this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL);
+    }
+  }
+  // add timestep to the end of the key as double value
+  vecFuzz[this->key_count].fp_element = dt;
+  if (has_het_ids) {
+    vecFuzz[this->key_count + 1].fp_element = cell[0];
+  }
+
+  return vecFuzz;
+}
+
+LookupKey DHT_Wrapper::fuzzForDHT(const std::vector<double> &cell, double dt) {
+  const auto c_zero_val = std::pow(10, AQUEOUS_EXP);
+
+  LookupKey vecFuzz(this->key_count + 1 + has_het_ids, {.0});
+  DHT_Rounder rounder;
+
+  int totals_i = 0;
+  // introduce fuzzing to allow more hits in DHT
+  // loop over every variable of grid cell
+  for (std::uint32_t i = 0; i < input_key_elements.size(); i++) {
+    if (input_key_elements[i] == DHT_KEY_INPUT_CUSTOM) {
+      continue;
+    }
+    double curr_key = cell[input_key_elements[i]];
+    if (curr_key != 0) {
+      if (curr_key < c_zero_val &&
+          this->dht_prop_type_vector[i] == DHT_TYPE_DEFAULT) {
+        continue;
+      }
+      if (this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL) {
+        curr_key -= base_totals[totals_i++];
+      }
+      vecFuzz[i] =
+          rounder.round(curr_key, dht_signif_vector[i],
+                        this->dht_prop_type_vector[i] == DHT_TYPE_TOTAL);
+    }
+  }
+  // add timestep to the end of the key as double value
+  vecFuzz[this->key_count].fp_element = dt;
+  if (has_het_ids) {
+    vecFuzz[this->key_count + 1].fp_element = cell[0];
+  }
+
+  return vecFuzz;
+}
+
+void poet::DHT_Wrapper::SetSignifVector(std::vector<uint32_t> signif_vec) {
+  if (signif_vec.size() != this->key_count) {
+    throw std::runtime_error(
+        "Significant vector size mismatches count of key elements.");
+  }
+
+  this->dht_signif_vector = signif_vec;
+}
+} // namespace poet

src/Chemistry/SurrogateModels/DHT_Wrapper.hpp

@@ -0,0 +1,271 @@
+
+/*
+** Copyright (C) 2018-2021 Alexander Lindemann, Max Luebke (University of
+** Potsdam)
+**
+** Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#ifndef DHT_WRAPPER_H
+#define DHT_WRAPPER_H
+
+#include "Base/RInsidePOET.hpp"
+#include "DataStructures/NamedVector.hpp"
+
+#include "Chemistry/ChemistryDefs.hpp"
+
+#include "Init/InitialList.hpp"
+#include "LookupKey.hpp"
+
+#include <array>
+#include <cstdint>
+#include <limits>
+#include <string>
+#include <utility>
+#include <vector>
+
+extern "C" {
+#include "DHT.h"
+}
+
+#include <mpi.h>
+
+namespace poet {
+
+using DHT_Location = std::pair<std::uint32_t, std::uint32_t>;
+
+/**
+ * @brief C++-Wrapper around DHT implementation
+ *
+ * Provides an API to interact with the current DHT implentation. This class is
+ * POET specific and can't be used outside the POET application.
+ *
+ */
+class DHT_Wrapper {
+public:
+  using DHT_ResultObject = struct DHTResobj {
+    std::vector<LookupKey> keys;
+    std::vector<DHT_Location> locations;
+    std::vector<bool> filledDHT;
+  };
+
+  static constexpr std::int32_t DHT_KEY_INPUT_CUSTOM =
+      std::numeric_limits<std::int32_t>::min();
+
+  static constexpr int DHT_KEY_SIGNIF_DEFAULT = 5;
+
+  /**
+   * @brief Construct a new dht wrapper object
+   *
+   * The constructor will initialize the private dht_object of this class by
+   * calling DHT_create with all given parameters. Also the fuzzing buffer will
+   * be allocated and all needed parameters extracted from simparams struct.
+   *
+   * @param dht_comm Communicator which addresses all participating DHT
+   * processes
+   * @param buckets_per_process Count of buckets to allocate for each
+   * process
+   * @param key_indices Vector indexing elements of one grid cell used
+   * for key creation.
+   * @param data_count Count of data entries
+   */
+  DHT_Wrapper(MPI_Comm dht_comm, std::uint64_t dht_size,
+              const NamedVector<std::uint32_t> &key_species,
+              const std::vector<std::int32_t> &key_indices,
+              const std::vector<std::string> &output_names,
+              const InitialList::ChemistryHookFunctions &hooks,
+              uint32_t data_count, bool with_interp, bool has_het_ids);
+  /**
+   * @brief Destroy the dht wrapper object
+   *
+   * By destroying this object the DHT will also be freed. Since all statistics
+   * are stored inside this object, no statistics will be retrieved during the
+   * call of DHT_free. After freeing the DHT the fuzzing buffer will be also
+   * freed.
+   *
+   */
+  ~DHT_Wrapper();
+
+  DHT_Wrapper &operator=(const DHT_Wrapper &) = delete;
+  DHT_Wrapper(const DHT_Wrapper &) = delete;
+
+  /**
+   * @brief Check if values of workpackage are stored in DHT
+   *
+   * Call DHT_read for all grid cells of the given workpackage and if a
+   * previously simulated grid cell was found mark this grid cell as 'not be
+   * simulated'. Therefore all values of a grid cell are fuzzed by fuzzForDHT
+   * and used as input key. The correspondending retrieved value might be stored
+   * directly into the memory area of the work_package and out_result_index is
+   * marked with false ('not to be simulated').
+   *
+   * @param length Count of grid cells inside work package
+   * @param[out] out_result_index Indexing work packages which should be
+   * simulated
+   * @param[in,out] work_package Pointer to current work package
+   * @param dt Current timestep of simulation
+   */
+  auto checkDHT(WorkPackage &work_package) -> const DHT_ResultObject &;
+
+  /**
+   * @brief Write simulated values into DHT
+   *
+   * Call DHT_write for all grid cells of the given workpackage which was
+   * simulated shortly before by the worker. Whether the grid cell was simulated
+   * is given by result_index. For every grid cell indicated with true inside
+   * result_index write the simulated value into the DHT.
+   *
+   * @param length Count of grid cells inside work package
+   * @param result_index Indexing work packages which was simulated
+   * @param work_package Pointer to current work package which was used as input
+   * of PHREEQC
+   * @param results Pointer to current work package which are the resulting
+   * outputs of the PHREEQC simulation
+   * @param dt Current timestep of simulation
+   */
+  void fillDHT(const WorkPackage &work_package);
+
+  void resultsToWP(std::vector<double> &work_package);
+
+  /**
+   * @brief Dump current DHT state into file.
+   *
+   * This function will simply execute DHT_to_file with given file name (see
+   * DHT.h for more info).
+   *
+   * @param filename Name of the dump file
+   * @return int Returns 0 on success, otherwise an error value
+   */
+  int tableToFile(const char *filename);
+
+  /**
+   * @brief Load dump file into DHT.
+   *
+   * This function will simply execute DHT_from_file with given file name (see
+   * DHT.h for more info).
+   *
+   * @param filename Name of the dump file
+   * @return int Returns 0 on success, otherwise an error value
+   */
+  int fileToTable(const char *filename);
+
+  /**
+   * @brief Print a detailed statistic of DHT usage.
+   *
+   * This function will simply execute DHT_print_statistics with given file name
+   * (see DHT.h for more info).
+   *
+   */
+  void printStatistics();
+
+  /**
+   * @brief Get the Hits object
+   *
+   * @return uint64_t Count of hits
+   */
+  auto getHits() { return this->dht_hits; };
+
+  /**
+   * @brief Get the Evictions object
+   *
+   * @return uint64_t Count of evictions
+   */
+  auto getEvictions() { return this->dht_evictions; };
+
+  void resetCounter() {
+    this->dht_hits = 0;
+    this->dht_evictions = 0;
+  }
+
+  void SetSignifVector(std::vector<uint32_t> signif_vec);
+
+  auto getDataCount() { return this->data_count; }
+  auto getCommunicator() { return this->communicator; }
+  DHT *getDHT() { return this->dht_object; };
+
+  DHT_ResultObject &getDHTResults() { return this->dht_results; }
+
+  const auto &getKeyElements() const { return this->input_key_elements; }
+  const auto &getKeySpecies() const { return this->key_species; }
+
+  void setBaseTotals(double tot_h, double tot_o) {
+    this->base_totals = {tot_h, tot_o};
+  }
+
+  std::uint32_t getInputCount() const {
+    return this->input_key_elements.size();
+  }
+
+  std::uint32_t getOutputCount() const { return this->data_count; }
+
+  inline void prepareKeys(const std::vector<std::vector<double>> &input_values,
+                          double dt) {
+    dht_results.keys.resize(input_values.size());
+    for (std::size_t i = 0; i < input_values.size(); i++) {
+      if (this->hooks.dht_fuzz.isValid()) {
+        dht_results.keys[i] = fuzzForDHT_R(input_values[i], dt);
+      } else {
+        dht_results.keys[i] = fuzzForDHT(input_values[i], dt);
+      }
+    }
+  }
+
+private:
+  uint32_t key_count;
+  uint32_t data_count;
+
+  DHT *dht_object;
+  MPI_Comm communicator;
+
+  LookupKey fuzzForDHT(const std::vector<double> &cell, double dt);
+  LookupKey fuzzForDHT_R(const std::vector<double> &cell, double dt);
+
+  std::vector<double>
+  outputToInputAndRates(const std::vector<double> &old_results,
+                        const std::vector<double> &new_results);
+
+  std::vector<double>
+  inputAndRatesToOutput(const std::vector<double> &dht_data,
+                        const std::vector<double> &input_values);
+
+  std::vector<double> outputToRates(const std::vector<double> &old_results,
+                                    const std::vector<double> &new_results);
+
+  std::vector<double> ratesToOutput(const std::vector<double> &dht_data,
+                                    const std::vector<double> &input_values);
+
+  uint32_t dht_hits = 0;
+  uint32_t dht_evictions = 0;
+
+  NamedVector<std::uint32_t> key_species;
+
+  std::vector<std::uint32_t> dht_signif_vector;
+  std::vector<std::uint32_t> dht_prop_type_vector;
+  std::vector<std::int32_t> input_key_elements;
+
+  const std::vector<std::string> &output_names;
+
+  const InitialList::ChemistryHookFunctions &hooks;
+  const bool with_interp;
+
+  DHT_ResultObject dht_results;
+
+  std::array<double, 2> base_totals{0};
+  bool has_het_ids{false};
+};
+} // namespace poet
+
+#endif // DHT_WRAPPER_H

src/Chemistry/SurrogateModels/HashFunctions.cpp

@@ -0,0 +1,94 @@
+/*
+**-----------------------------------------------------------------------------
+** MurmurHash2 was written by Austin Appleby, and is placed in the public
+** domain. The author hereby disclaims copyright to this source code.
+**-----------------------------------------------------------------------------
+**
+** Copyright (C) 2018-2021 Alexander Lindemann, Max Luebke (University of
+** Potsdam)
+**
+** Copyright (C) 2018-2022 Marco De Lucia, Max Luebke (GFZ Potsdam)
+**
+** POET is free software; you can redistribute it and/or modify it under the
+** terms of the GNU General Public License as published by the Free Software
+** Foundation; either version 2 of the License, or (at your option) any later
+** version.
+**
+** POET is distributed in the hope that it will be useful, but WITHOUT ANY
+** WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+** A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License along with
+** this program; if not, write to the Free Software Foundation, Inc., 51
+** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+*/
+
+#include "HashFunctions.hpp"
+
+#if defined(_MSC_VER)
+
+#define BIG_CONSTANT(x) (x)
+
+// Other compilers
+
+#else // defined(_MSC_VER)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+// MurmurHash2, 64-bit versions, by Austin Appleby
+
+// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment
+// and endian-ness issues if used across multiple platforms.
+
+// 64-bit hash for 64-bit platforms
+// objsize: 0x170-0x321: 433
+
+uint64_t poet::Murmur2_64A(int len, const void *key) {
+  const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
+  const int r = 47;
+
+  uint64_t h = HASH_SEED ^ (len * m);
+
+  const uint64_t *data = (const uint64_t *)key;
+  const uint64_t *end = data + (len / 8);
+
+  while (data != end) {
+    uint64_t k = *data++;
+
+    k *= m;
+    k ^= k >> r;
+    k *= m;
+
+    h ^= k;
+    h *= m;
+  }
+
+  const unsigned char *data2 = (const unsigned char *)data;
+
+  switch (len & 7) {
+  case 7:
+    h ^= uint64_t(data2[6]) << 48;
+  case 6:
+    h ^= uint64_t(data2[5]) << 40;
+  case 5:
+    h ^= uint64_t(data2[4]) << 32;
+  case 4:
+    h ^= uint64_t(data2[3]) << 24;
+  case 3:
+    h ^= uint64_t(data2[2]) << 16;
+  case 2:
+    h ^= uint64_t(data2[1]) << 8;
+  case 1:
+    h ^= uint64_t(data2[0]);
+    h *= m;
+  };
+
+  h ^= h >> r;
+  h *= m;
+  h ^= h >> r;
+
+  return h;
+}

src/Chemistry/SurrogateModels/HashFunctions.hpp

@@ -1,8 +1,9 @@
+
 /*
 ** Copyright (C) 2018-2021 Alexander Lindemann, Max Luebke (University of
 ** Potsdam)
 **
-** Copyright (C) 2018-2021 Marco De Lucia (GFZ Potsdam)
+** Copyright (C) 2018-2022 Marco De Lucia, Max Luebke (GFZ Potsdam)
 **
 ** POET is free software; you can redistribute it and/or modify it under the
 ** terms of the GNU General Public License as published by the Free Software
@@ -18,27 +19,18 @@
 ** Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */
 
-#include "TransportSim.h"
+#ifndef HASHFUNCTIONS_H_
+#define HASHFUNCTIONS_H_
 
-#include <mpi.h>
+#include <cstdint>
 
-using namespace poet;
+namespace poet {
 
-TransportSim::TransportSim(RRuntime &R_) : R(R_) {}
+// Sum of POET interpreted as ASCII
+constexpr uint32_t HASH_SEED = 80 + 79 + 69 + 84;
 
-void TransportSim::run() {
+uint64_t Murmur2_64A(int len, const void *key);
-  double sim_a_transport, sim_b_transport;
 
-  sim_b_transport = MPI_Wtime();
+} // namespace poet
-  R.parseEvalQ("mysetup <- master_advection(setup=mysetup)");
-  sim_a_transport = MPI_Wtime();
 
-  transport_t += sim_a_transport - sim_b_transport;
+#endif // HASHFUNCTIONS_H_
-}
-
-void TransportSim::end() {
-  R["simtime_transport"] = transport_t;
-  R.parseEvalQ("profiling$simtime_transport <- simtime_transport");
-}
-
-double TransportSim::getTransportTime() { return this->transport_t; }

src/Chemistry/SurrogateModels/Interpolation.hpp

@@ -0,0 +1,271 @@
+
+#ifndef INTERPOLATION_H_
+#define INTERPOLATION_H_
+
+#include "DataStructures/NamedVector.hpp"
+
+#include "DHT_Wrapper.hpp"
+#include "Init/InitialList.hpp"
+#include "LookupKey.hpp"
+#include "Rounding.hpp"
+
+#include <list>
+#include <memory>
+#include <mpi.h>
+#include <string>
+#include <utility>
+
+extern "C" {
+#include "DHT.h"
+}
+
+#include <cstdint>
+#include <unordered_map>
+#include <vector>
+
+namespace poet {
+
+class ProximityHashTable {
+public:
+  using bucket_indicator = std::uint64_t;
+
+  ProximityHashTable(uint32_t key_size, uint32_t data_size,
+                     uint32_t entry_count, uint32_t size_per_process,
+                     MPI_Comm communicator);
+  ~ProximityHashTable();
+
+  // delete assign and copy operator
+  ProximityHashTable &operator=(const ProximityHashTable *) = delete;
+  ProximityHashTable(const ProximityHashTable &) = delete;
+
+  struct PHT_Result {
+    std::uint32_t size;
+    std::vector<std::vector<double>> in_values;
+    std::vector<std::vector<double>> out_values;
+
+    std::uint32_t getMemSize() const {
+      std::uint32_t sum{0};
+      for (const auto &results : out_values) {
+        sum += results.size() * sizeof(double);
+      }
+      return sum;
+    }
+  };
+
+  void setSourceDHT(DHT *src) {
+    this->source_dht = src;
+    this->dht_key_count = src->key_size / sizeof(Lookup_Keyelement);
+    this->dht_data_count = src->data_size / sizeof(double);
+    this->dht_buffer.resize(src->data_size + src->key_size);
+  }
+
+  void writeLocationToPHT(LookupKey key, DHT_Location location);
+
+  const PHT_Result &query(const LookupKey &key,
+                          std::uint32_t min_entries_needed,
+                          std::uint32_t input_count,
+                          std::uint32_t output_count);
+
+  std::uint64_t getLocations(const LookupKey &key);
+
+  void getEntriesFromLocation(const PHT_Result &locations,
+                              const std::vector<uint32_t> &signif);
+
+  void writeStats() { DHT_print_statistics(this->prox_ht); }
+
+  DHT *getDHTObject() { return this->prox_ht; }
+
+  auto getPHTWriteTime() const -> double { return this->pht_write_t; };
+  auto getPHTReadTime() const -> double { return this->pht_read_t; };
+  auto getDHTGatherTime() const -> double { return this->pht_gather_dht_t; };
+
+  auto getLocalCacheHits() const -> std::vector<std::uint32_t> {
+    return this->all_cache_hits;
+  }
+
+  void storeAndResetCounter() {
+    all_cache_hits.push_back(cache_hits);
+    cache_hits = 0;
+  }
+
+#if POET_PHT_ADD
+  void incrementReadCounter(const LookupKey &key);
+#endif
+
+private:
+  enum { INTERP_CB_OK, INTERP_CB_FULL, INTERP_CB_ALREADY_IN };
+
+  static int PHT_callback_function(int in_data_size, void *in_data,
+                                   int out_data_size, void *out_data);
+
+  static std::vector<double> convertKeysFromDHT(Lookup_Keyelement *keys_in,
+                                                std::uint32_t key_size);
+
+  static bool similarityCheck(const LookupKey &fine, const LookupKey &coarse,
+                              const std::vector<uint32_t> &signif);
+
+  char *bucket_store;
+
+  class Cache
+      : private std::unordered_map<LookupKey, PHT_Result, LookupKeyHasher> {
+  public:
+    void operator()(const LookupKey &key, const PHT_Result val);
+
+    std::pair<bool, PHT_Result> operator[](const LookupKey &key);
+    void flush() { this->clear(); }
+
+  protected:
+  private:
+    static constexpr std::int64_t MAX_CACHE_SIZE = 100E6;
+
+    std::int64_t free_mem{MAX_CACHE_SIZE};
+
+    std::list<LookupKey> lru_queue;
+    using lru_iterator = typename std::list<LookupKey>::iterator;
+
+    std::unordered_map<LookupKey, lru_iterator, LookupKeyHasher> keyfinder;
+  };
+
+  Cache localCache;
+  DHT *prox_ht;
+
+  std::uint32_t dht_evictions = 0;
+
+  DHT *source_dht = nullptr;
+
+  PHT_Result lookup_results;
+  std::vector<char> dht_buffer;
+
+  std::uint32_t dht_key_count;
+  std::uint32_t dht_data_count;
+
+  MPI_Comm communicator;
+
+  double pht_write_t = 0.;
+  double pht_read_t = 0.;
+  double pht_gather_dht_t = 0.;
+
+  std::uint32_t cache_hits{0};
+  std::vector<std::uint32_t> all_cache_hits{};
+};
+
+class InterpolationModule {
+public:
+  using InterpFunction = std::vector<double> (*)(
+      const std::vector<std::int32_t> &, const std::vector<double> &,
+      const std::vector<std::vector<double>> &,
+      const std::vector<std::vector<double>> &);
+
+  InterpolationModule(std::uint32_t entries_per_bucket,
+                      std::uint64_t size_per_process,
+                      std::uint32_t min_entries_needed, DHT_Wrapper &dht,
+                      const NamedVector<std::uint32_t> &interp_key_signifs,
+                      const std::vector<std::int32_t> &dht_key_indices,
+                      const std::vector<std::string> &out_names,
+                      const InitialList::ChemistryHookFunctions &hooks);
+
+  enum result_status { RES_OK, INSUFFICIENT_DATA, NOT_NEEDED };
+
+  DHT *getDHTObject() { return this->pht->getDHTObject(); }
+
+  struct InterpolationResult {
+    std::vector<std::vector<double>> results;
+    std::vector<result_status> status;
+
+    void ResultsToWP(std::vector<double> &currWP);
+  };
+
+  void setInterpolationFunction(InterpFunction func) {
+    this->f_interpolate = func;
+  }
+
+  void setMinEntriesNeeded(std::uint32_t entries) {
+    this->min_entries_needed = entries;
+  }
+
+  auto getMinEntriesNeeded() { return this->min_entries_needed; }
+
+  void writePairs();
+
+  void tryInterpolation(WorkPackage &work_package);
+
+  void resultsToWP(std::vector<double> &work_package) const;
+
+  auto getPHTWriteTime() const { return pht->getPHTWriteTime(); };
+  auto getPHTReadTime() const { return pht->getPHTReadTime(); };
+  auto getDHTGatherTime() const { return pht->getDHTGatherTime(); };
+  auto getInterpolationTime() const { return this->interp_t; };
+
+  auto getInterpolationCount() const -> std::uint32_t {
+    return this->interpolations;
+  }
+
+  auto getPHTLocalCacheHits() const -> std::vector<std::uint32_t> {
+    return this->pht->getLocalCacheHits();
+  }
+
+  void resetCounter() {
+    this->interpolations = 0;
+    this->pht->storeAndResetCounter();
+  }
+
+  void writePHTStats() { this->pht->writeStats(); }
+  void dumpPHTState(const std::string &filename) {
+    DHT_to_file(this->pht->getDHTObject(), filename.c_str());
+  }
+
+  static constexpr std::uint32_t COARSE_DIFF = 2;
+  static constexpr std::uint32_t COARSE_SIGNIF_DEFAULT =
+      DHT_Wrapper::DHT_KEY_SIGNIF_DEFAULT - COARSE_DIFF;
+
+private:
+  void initPHT(std::uint32_t key_count, std::uint32_t entries_per_bucket,
+               std::uint32_t size_per_process, MPI_Comm communicator);
+
+  static std::vector<double> dummy(const std::vector<std::int32_t> &,
+                                   const std::vector<double> &,
+                                   const std::vector<std::vector<double>> &,
+                                   const std::vector<std::vector<double>> &) {
+    return {};
+  }
+
+  double interp_t = 0.;
+
+  std::uint32_t interpolations{0};
+
+  InterpFunction f_interpolate = dummy;
+
+  std::uint32_t min_entries_needed = 5;
+
+  std::unique_ptr<ProximityHashTable> pht;
+
+  DHT_Wrapper &dht_instance;
+
+  NamedVector<std::uint32_t> key_signifs;
+  std::vector<std::int32_t> key_indices;
+
+  InterpolationResult interp_result;
+  PHT_Rounder rounder;
+
+  LookupKey roundKey(const LookupKey &in_key) {
+    LookupKey out_key;
+
+    for (std::uint32_t i = 0; i < key_indices.size(); i++) {
+      out_key.push_back(rounder.round(in_key[key_indices[i]], key_signifs[i]));
+    }
+
+    // timestep
+    out_key.push_back(in_key.back());
+
+    return out_key;
+  }
+
+  const InitialList::ChemistryHookFunctions &hooks;
+  const std::vector<std::string> &out_names;
+  const std::vector<std::string> dht_names;
+
+  std::unordered_map<int, std::vector<std::int32_t>> to_calc_cache;
+};
+} // namespace poet
+
+#endif // INTERPOLATION_H_

src/Chemistry/SurrogateModels/InterpolationModule.cpp

@@ -0,0 +1,176 @@
+#include "Init/InitialList.hpp"
+#include "Interpolation.hpp"
+
+#include "DHT_Wrapper.hpp"
+#include "DataStructures/NamedVector.hpp"
+#include "HashFunctions.hpp"
+#include "LookupKey.hpp"
+#include "Rounding.hpp"
+
+#include <Rcpp.h>
+#include <Rcpp/proxy/ProtectedProxy.h>
+#include <Rinternals.h>
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <iterator>
+#include <memory>
+#include <mpi.h>
+#include <string>
+#include <utility>
+#include <vector>
+
+extern "C" {
+#include "DHT.h"
+}
+
+namespace poet {
+
+InterpolationModule::InterpolationModule(
+    std::uint32_t entries_per_bucket, std::uint64_t size_per_process,
+    std::uint32_t min_entries_needed, DHT_Wrapper &dht,
+    const NamedVector<std::uint32_t> &interp_key_signifs,
+    const std::vector<std::int32_t> &dht_key_indices,
+    const std::vector<std::string> &_out_names,
+    const InitialList::ChemistryHookFunctions &_hooks)
+    : dht_instance(dht), key_signifs(interp_key_signifs),
+      key_indices(dht_key_indices), min_entries_needed(min_entries_needed),
+      dht_names(dht.getKeySpecies().getNames()), out_names(_out_names),
+      hooks(_hooks) {
+
+  initPHT(this->key_signifs.size(), entries_per_bucket, size_per_process,
+          dht.getCommunicator());
+
+  pht->setSourceDHT(dht.getDHT());
+}
+
+void InterpolationModule::initPHT(std::uint32_t key_count,
+                                  std::uint32_t entries_per_bucket,
+                                  std::uint32_t size_per_process,
+                                  MPI_Comm communicator) {
+  uint32_t key_size = key_count * sizeof(Lookup_Keyelement) + sizeof(double);
+  uint32_t data_size = sizeof(DHT_Location);
+
+  pht = std::make_unique<ProximityHashTable>(
+      key_size, data_size, entries_per_bucket, size_per_process, communicator);
+}
+
+void InterpolationModule::writePairs() {
+  const auto in = this->dht_instance.getDHTResults();
+  for (int i = 0; i < in.filledDHT.size(); i++) {
+    if (in.filledDHT[i]) {
+      const auto coarse_key = roundKey(in.keys[i]);
+      pht->writeLocationToPHT(coarse_key, in.locations[i]);
+    }
+  }
+}
+
+void InterpolationModule::tryInterpolation(WorkPackage &work_package) {
+  interp_result.status.resize(work_package.size, NOT_NEEDED);
+
+  const auto dht_results = this->dht_instance.getDHTResults();
+
+  for (int wp_i = 0; wp_i < work_package.size; wp_i++) {
+    if (work_package.input[wp_i][0] != 2) {
+      interp_result.status[wp_i] = INSUFFICIENT_DATA;
+      continue;
+    }
+
+    if (work_package.mapping[wp_i] != CHEM_PQC) {
+      interp_result.status[wp_i] = NOT_NEEDED;
+      continue;
+    }
+
+    const auto rounded_key = roundKey(dht_results.keys[wp_i]);
+
+    auto pht_result =
+        pht->query(rounded_key, this->min_entries_needed,
+                   dht_instance.getInputCount(), dht_instance.getOutputCount());
+
+    if (pht_result.size < this->min_entries_needed) {
+      interp_result.status[wp_i] = INSUFFICIENT_DATA;
+      continue;
+    }
+
+    if (hooks.interp_pre.isValid()) {
+      NamedVector<double> nv_in(this->out_names, work_package.input[wp_i]);
+
+      std::vector<int> rm_indices = Rcpp::as<std::vector<int>>(
+          hooks.interp_pre(nv_in, pht_result.in_values));
+
+      pht_result.size -= rm_indices.size();
+
+      if (pht_result.size < this->min_entries_needed) {
+        interp_result.status[wp_i] = INSUFFICIENT_DATA;
+        continue;
+      }
+
+      for (const auto &index : rm_indices) {
+        pht_result.in_values.erase(
+            std::next(pht_result.in_values.begin(), index - 1));
+        pht_result.out_values.erase(
+            std::next(pht_result.out_values.begin(), index - 1));
+      }
+    }
+
+#ifdef POET_PHT_ADD
+    this->pht->incrementReadCounter(roundKey(rounded_key));
+#endif
+
+    const int cell_id = static_cast<int>(work_package.input[wp_i][0]);
+
+    if (!to_calc_cache.contains(cell_id)) {
+      const std::vector<std::int32_t> &to_calc = dht_instance.getKeyElements();
+      std::vector<std::int32_t> keep_indices;
+
+      for (std::size_t i = 0; i < to_calc.size(); i++) {
+        if (!std::isnan(work_package.input[wp_i][to_calc[i]])) {
+          keep_indices.push_back(to_calc[i]);
+        }
+      }
+
+      to_calc_cache[cell_id] = keep_indices;
+    }
+
+    double start_fc = MPI_Wtime();
+
+    work_package.output[wp_i] =
+        f_interpolate(to_calc_cache[cell_id], work_package.input[wp_i],
+                      pht_result.in_values, pht_result.out_values);
+
+    if (hooks.interp_post.isValid()) {
+      NamedVector<double> nv_result(this->out_names, work_package.output[wp_i]);
+      if (Rcpp::as<bool>(hooks.interp_post(nv_result))) {
+        interp_result.status[wp_i] = INSUFFICIENT_DATA;
+        continue;
+      }
+    }
+
+    // interp_result.results[i][0] = mean_water;
+    this->interp_t += MPI_Wtime() - start_fc;
+
+    this->interpolations++;
+
+    work_package.mapping[wp_i] = CHEM_INTERP;
+    interp_result.status[wp_i] = RES_OK;
+  }
+}
+
+void InterpolationModule::resultsToWP(std::vector<double> &work_package) const {
+  for (uint32_t i = 0; i < interp_result.status.size(); i++) {
+    if (interp_result.status[i] == RES_OK) {
+      const std::size_t length =
+          interp_result.results[i].end() - interp_result.results[i].begin();
+      std::copy(interp_result.results[i].begin(),
+                interp_result.results[i].end(),
+                work_package.begin() + (length * i));
+    }
+  }
+}
+
+} // namespace poet

src/Chemistry/SurrogateModels/LookupKey.hpp

@@ -0,0 +1,64 @@
+
+#ifndef LOOKUPKEY_H_
+#define LOOKUPKEY_H_
+
+#include "HashFunctions.hpp"
+
+#include <cstdint>
+#include <cstring>
+#include <vector>
+
+namespace poet {
+
+constexpr std::int8_t SC_NOTATION_EXPONENT_MASK = -128;
+constexpr std::int64_t SC_NOTATION_SIGNIFICANT_MASK = 0xFFFFFFFFFFFF;
+
+struct Lookup_SC_notation {
+  std::int8_t exp : 8;
+  std::int64_t significant : 56;
+
+  constexpr static Lookup_SC_notation nan() {
+    return {SC_NOTATION_EXPONENT_MASK, SC_NOTATION_SIGNIFICANT_MASK};
+  }
+
+  constexpr bool isnan() const {
+    return !!(exp == SC_NOTATION_EXPONENT_MASK &&
+              significant == SC_NOTATION_SIGNIFICANT_MASK);
+  }
+};
+
+union Lookup_Keyelement {
+  double fp_element;
+  Lookup_SC_notation sc_notation;
+
+  bool operator==(const Lookup_Keyelement &other) const {
+    return std::memcmp(this, &other, sizeof(Lookup_Keyelement)) == 0 ? true
+                                                                     : false;
+  }
+
+  template <typename T> bool operator>(const T &other) const {
+    return this->sc_notation.significant > other;
+  }
+};
+
+class LookupKey : public std::vector<Lookup_Keyelement> {
+public:
+  using std::vector<Lookup_Keyelement>::vector;
+
+  std::vector<double> to_double() const;
+  static Lookup_SC_notation round_from_double(const double in,
+                                              std::uint32_t signif);
+  static double to_double(const Lookup_SC_notation in);
+};
+
+struct LookupKeyHasher {
+  std::uint64_t operator()(const LookupKey &k) const {
+    const uint32_t key_size = k.size() * sizeof(Lookup_Keyelement);
+
+    return poet::Murmur2_64A(key_size, k.data());
+  }
+};
+
+} // namespace poet
+
+#endif // LOOKUPKEY_H_

src/Chemistry/SurrogateModels/ProximityHashTable.cpp

@@ -0,0 +1,254 @@
+#include "Interpolation.hpp"
+
+#include "DHT_Wrapper.hpp"
+#include "HashFunctions.hpp"
+#include "LookupKey.hpp"
+#include "Rounding.hpp"
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <iostream>
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+extern "C" {
+#include "DHT.h"
+}
+
+namespace poet {
+
+ProximityHashTable::ProximityHashTable(uint32_t key_size, uint32_t data_size,
+                                       uint32_t entry_count,
+                                       uint32_t size_per_process,
+                                       MPI_Comm communicator_)
+    : communicator(communicator_) {
+
+  data_size *= entry_count;
+  data_size += sizeof(bucket_indicator);
+
+#ifdef POET_PHT_ADD
+  data_size += sizeof(std::uint64_t);
+#endif
+
+  bucket_store = new char[data_size];
+
+  uint32_t buckets_per_process =
+      static_cast<std::uint32_t>(size_per_process / (data_size + key_size));
+
+  this->prox_ht = DHT_create(communicator, buckets_per_process, data_size,
+                             key_size, &poet::Murmur2_64A);
+
+  DHT_set_accumulate_callback(this->prox_ht, PHT_callback_function);
+}
+
+ProximityHashTable::~ProximityHashTable() {
+  delete[] bucket_store;
+  if (prox_ht) {
+    DHT_free(prox_ht, NULL, NULL);
+  }
+}
+
+int ProximityHashTable::PHT_callback_function(int in_data_size, void *in_data,
+                                              int out_data_size,
+                                              void *out_data) {
+  const int max_elements_per_bucket =
+      static_cast<int>((out_data_size - sizeof(bucket_indicator)
+#ifdef POET_PHT_ADD
+                        - sizeof(std::uint64_t)
+#endif
+                            ) /
+                       in_data_size);
+  DHT_Location *input = reinterpret_cast<DHT_Location *>(in_data);
+
+  bucket_indicator *occupied_buckets =
+      reinterpret_cast<bucket_indicator *>(out_data);
+  DHT_Location *pairs = reinterpret_cast<DHT_Location *>(occupied_buckets + 1);
+
+  if (*occupied_buckets == max_elements_per_bucket) {
+    return INTERP_CB_FULL;
+  }
+
+  for (bucket_indicator i = 0; i < *occupied_buckets; i++) {
+    if (pairs[i] == *input) {
+      return INTERP_CB_ALREADY_IN;
+    }
+  }
+
+  pairs[(*occupied_buckets)++] = *input;
+
+  return INTERP_CB_OK;
+}
+
+void ProximityHashTable::writeLocationToPHT(LookupKey key,
+                                            DHT_Location location) {
+
+  double start = MPI_Wtime();
+
+  // if (localCache[key].first) {
+  //   return;
+  // }
+
+  int ret_val;
+
+  int status = DHT_write_accumulate(prox_ht, key.data(), sizeof(location),
+                                    &location, NULL, NULL, &ret_val);
+
+  if (status == DHT_WRITE_SUCCESS_WITH_EVICTION) {
+    this->dht_evictions++;
+  }
+
+  // if (ret_val == INTERP_CB_FULL) {
+  //   localCache(key, {});
+  // }
+
+  this->pht_write_t += MPI_Wtime() - start;
+}
+
+const ProximityHashTable::PHT_Result &ProximityHashTable::query(
+    const LookupKey &key, const std::uint32_t min_entries_needed,
+    const std::uint32_t input_count, const std::uint32_t output_count) {
+
+  double start_r = MPI_Wtime();
+  const auto cache_ret = localCache[key];
+  if (cache_ret.first) {
+    cache_hits++;
+    return (lookup_results = cache_ret.second);
+  }
+
+  int res = DHT_read(prox_ht, key.data(), bucket_store);
+  this->pht_read_t += MPI_Wtime() - start_r;
+
+  if (res != DHT_SUCCESS) {
+    this->lookup_results.size = 0;
+    return lookup_results;
+  }
+
+  auto *bucket_element_count =
+      reinterpret_cast<bucket_indicator *>(bucket_store);
+  auto *bucket_elements =
+      reinterpret_cast<DHT_Location *>(bucket_element_count + 1);
+
+  if (*bucket_element_count < min_entries_needed) {
+    this->lookup_results.size = 0;
+    return lookup_results;
+  }
+
+  lookup_results.size = *bucket_element_count;
+  auto locations = std::vector<DHT_Location>(
+      bucket_elements, bucket_elements + *(bucket_element_count));
+
+  lookup_results.in_values.clear();
+  lookup_results.in_values.reserve(*bucket_element_count);
+
+  lookup_results.out_values.clear();
+  lookup_results.out_values.reserve(*bucket_element_count);
+
+  for (const auto &loc : locations) {
+    double start_g = MPI_Wtime();
+    DHT_read_location(source_dht, loc.first, loc.second, dht_buffer.data());
+    this->pht_gather_dht_t += MPI_Wtime() - start_g;
+
+    auto *buffer = reinterpret_cast<double *>(dht_buffer.data());
+
+    lookup_results.in_values.push_back(
+        std::vector<double>(buffer, buffer + input_count));
+
+    buffer += input_count;
+    lookup_results.out_values.push_back(
+        std::vector<double>(buffer, buffer + output_count));
+  }
+
+  if (lookup_results.size != 0) {
+    localCache(key, lookup_results);
+  }
+
+  return lookup_results;
+}
+
+inline bool
+ProximityHashTable::similarityCheck(const LookupKey &fine,
+                                    const LookupKey &coarse,
+                                    const std::vector<uint32_t> &signif) {
+
+  PHT_Rounder rounder;
+
+  for (int i = 0; i < signif.size(); i++) {
+    if (!(rounder.round(fine[i], signif[i]) == coarse[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+inline std::vector<double>
+ProximityHashTable::convertKeysFromDHT(Lookup_Keyelement *keys_in,
+                                       std::uint32_t key_size) {
+  std::vector<double> output(key_size);
+  DHT_Rounder rounder;
+  for (int i = 0; i < key_size; i++) {
+    output[i] = rounder.convert(keys_in[i]);
+  }
+
+  return output;
+}
+
+void ProximityHashTable::Cache::operator()(const LookupKey &key,
+                                           const PHT_Result val) {
+  const auto elemIt = this->find(key);
+
+  if (elemIt == this->end()) {
+
+    if (this->free_mem < 0) {
+      const LookupKey &to_del = this->lru_queue.back();
+      const auto elem_d = this->find(to_del);
+      this->free_mem += elem_d->second.getMemSize();
+      this->erase(to_del);
+      this->keyfinder.erase(to_del);
+      this->lru_queue.pop_back();
+    }
+
+    this->insert({key, val});
+    this->lru_queue.emplace_front(key);
+    this->keyfinder[key] = lru_queue.begin();
+    this->free_mem -= val.getMemSize();
+    return;
+  }
+
+  elemIt->second = val;
+}
+
+std::pair<bool, ProximityHashTable::PHT_Result>
+ProximityHashTable::Cache::operator[](const LookupKey &key) {
+  const auto elemIt = this->find(key);
+
+  if (elemIt == this->end()) {
+    return {false, {}};
+  }
+
+  this->lru_queue.splice(lru_queue.begin(), lru_queue, this->keyfinder[key]);
+  return {true, elemIt->second};
+}
+
+#ifdef POET_PHT_ADD
+static int PHT_increment_counter(int in_data_size, void *in_data,
+                                 int out_data_size, void *out_data) {
+  char *start = reinterpret_cast<char *>(out_data);
+  std::uint64_t *counter = reinterpret_cast<std::uint64_t *>(
+      start + out_data_size - sizeof(std::uint64_t));
+  *counter += 1;
+
+  return 0;
+}
+
+void ProximityHashTable::incrementReadCounter(const LookupKey &key) {
+  auto *old_func_ptr = this->prox_ht->accumulate_callback;
+  DHT_set_accumulate_callback(prox_ht, PHT_increment_counter);
+  int ret, dummy;
+  DHT_write_accumulate(prox_ht, key.data(), 0, NULL, NULL, NULL, &ret);
+  DHT_set_accumulate_callback(prox_ht, old_func_ptr);
+}
+#endif
+} // namespace poet

src/Chemistry/SurrogateModels/Rounding.hpp

@@ -0,0 +1,105 @@
+#ifndef ROUNDING_H_
+#define ROUNDING_H_
+
+#include "LookupKey.hpp"
+#include <cmath>
+#include <cstdint>
+
+namespace poet {
+
+constexpr std::int8_t AQUEOUS_EXP = -13;
+
+template <typename Input, typename Output, typename ConvertTo = double>
+class IRounding {
+public:
+  virtual Output round(const Input &, std::uint32_t signif) = 0;
+  virtual ConvertTo convert(const Output &) = 0;
+};
+
+class DHT_Rounder {
+public:
+  Lookup_Keyelement round(const double &value, std::uint32_t signif,
+                          bool is_ho) {
+
+    if (std::isnan(value)) {
+      return {.sc_notation = Lookup_SC_notation::nan()};
+    }
+
+    std::int8_t exp =
+        static_cast<std::int8_t>(std::floor(std::log10(std::fabs(value))));
+
+    if (!is_ho) {
+      if (exp < AQUEOUS_EXP) {
+        return {.sc_notation = {0, 0}};
+      }
+
+      std::int8_t diff = exp - signif + 1;
+      if (diff < AQUEOUS_EXP) {
+        signif -= AQUEOUS_EXP - diff;
+      }
+    }
+
+    Lookup_Keyelement elem;
+    elem.sc_notation.exp = exp;
+    elem.sc_notation.significant =
+        static_cast<std::int64_t>(value * std::pow(10, signif - exp - 1));
+
+    return elem;
+  }
+
+  double convert(const Lookup_Keyelement &key_elem) {
+    std::int32_t normalized_exp = static_cast<std::int32_t>(
+        -std::log10(std::fabs(key_elem.sc_notation.significant)));
+
+    // add stored exponent to normalized exponent
+    normalized_exp += key_elem.sc_notation.exp;
+
+    // return significant times 10 to the power of exponent
+    return key_elem.sc_notation.significant * std::pow(10., normalized_exp);
+  }
+};
+
+class PHT_Rounder : public IRounding<Lookup_Keyelement, Lookup_Keyelement> {
+public:
+  Lookup_Keyelement round(const Lookup_Keyelement &value,
+                          std::uint32_t signif) {
+    Lookup_Keyelement new_val = value;
+
+    if (value.sc_notation.isnan()) {
+      return {.sc_notation = Lookup_SC_notation::nan()};
+    }
+
+    if (signif == 0) {
+      return {.sc_notation = {0, value > 0}};
+    }
+
+    std::uint32_t diff_signif =
+        static_cast<std::uint32_t>(
+            std::ceil(std::log10(std::abs(value.sc_notation.significant)))) -
+        signif;
+
+    new_val.sc_notation.significant = static_cast<int64_t>(
+        value.sc_notation.significant / std::pow(10., diff_signif));
+
+    if (new_val.sc_notation.significant == 0) {
+      new_val.sc_notation.exp = 0;
+    }
+
+    return new_val;
+  }
+
+  double convert(const Lookup_Keyelement &key_elem) {
+    std::int32_t normalized_exp = static_cast<std::int32_t>(
+        -std::log10(std::fabs(key_elem.sc_notation.significant)));
+
+    // add stored exponent to normalized exponent
+    normalized_exp += key_elem.sc_notation.exp;
+
+    // return significant times 10 to the power of exponent
+    return key_elem.sc_notation.significant * std::pow(10., normalized_exp);
+  }
+};
+
+} // namespace poet
+
+#endif // ROUNDING_H_