From e255ffa39c08f547e96d54f8b7649576db05a33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Sun, 29 Nov 2020 19:32:22 +0100 Subject: [PATCH 01/50] Translated old Makefile to CMake Logic --- src/CMakeLists.txt | 136 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 src/CMakeLists.txt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..9277ceb5c --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,136 @@ +# this is the current cmake version distributed by debian stable (buster) +cmake_minimum_required(VERSION 3.13) + +project(Poet VERSION 0.0) + +add_executable(poet kin.cpp) + +# specify the C++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(MPI) + +list(APPEND R_DEPS "Rcpp" "RInside") + +# prepare R environment (Rcpp + RInside) +find_program(R_EXE "R") + +if(R_EXE) + execute_process(COMMAND ${R_EXE} RHOME + OUTPUT_VARIABLE R_ROOT_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + find_path(R_INCLUDE_DIR R.h + HINTS ${R_ROOT_DIR} + PATHS /usr/inlcude /usr/local/include + PATH_SUFFIXES include/R R/include + ) + + find_library(R_LIBRARY R + HINTS ${R_ROOT_DIR}/lib + ) +else() + message(FATAL_ERROR "No R runtime found!") +endif() + +set(R_LIBRARIES ${R_LIBRARY}) +set(R_INCLUDE_DIRS ${R_INCLUDE_DIR}) + +#foreach(_r_deps ${R_DEPS}) +# execute_process(COMMAND echo "cat(find.package('${_r_deps}'))" +# COMMAND ${R_EXE} --vanilla --slave +# RESULT_VARIABLE _rd +# ERROR_QUIET +# OUTPUT_VARIABLE _r_deps_path +# OUTPUT_STRIP_TRAILING_WHITESPACE +# ) +# +# if(_rd) +# message(FATAL_ERROR "${_r_deps} not found!") +# endif() +# +# find_library(R_${_r_deps}_LIBRARY +# lib${_r_deps}.so +# HINTS ${_r_deps_path}/lib +# ) +# +# if(R_${_r_deps}_LIBRARY) +# list(APPEND R_LIBRARIES ${R_${_r_deps}_LIBRARY}) +# else() +# message(STATUS "lol") +# unset(${R_${_r_deps}_LIBRARY}) +# endif() +# +#endforeach() + + +execute_process(COMMAND echo "cat(find.package('Rcpp'))" + COMMAND ${R_EXE} --vanilla --slave + RESULT_VARIABLE RCPP_NOT_FOUND + ERROR_QUIET + OUTPUT_VARIABLE RCPP_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +if(RCPP_NOT_FOUND) + message(FATAL_ERROR "Rcpp not found!") +endif() + +find_path(R_Rcpp_INCLUDE_DIR Rcpp.h + HINTS ${RCPP_PATH} + PATH_SUFFIXES include) + +list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR}) + +execute_process(COMMAND echo "cat(find.package('RInside'))" + COMMAND ${R_EXE} --vanilla --slave + RESULT_VARIABLE RINSIDE_NOT_FOUND + ERROR_QUIET + OUTPUT_VARIABLE RINSIDE_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +if(RInside_NOT_FOUND) + message(FATAL_ERROR "RInside not found!") +endif() + +find_library(R_RInside_LIBRARY libRInside.so + HINTS ${RINSIDE_PATH}/lib) + +list(APPEND R_LIBRARIES ${R_RInside_LIBRARY}) + +find_path(R_RInside_INCLUDE_DIR RInside.h + HINTS ${RINSIDE_PATH} + PATH_SUFFIXES include) + +list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR}) + +#execute_process( +# COMMAND ${R_EXE} "--slave" "--vanilla" "-e" "stopifnot(require('Rcpp'));cat(Rcpp:::Rcpp.system.file('include'))" +# OUTPUT_VARIABLE LIBRCPP_INCLUDE_DIRS +#) +# +#execute_process( +# COMMAND ${R_EXE} "--slave" "--vanilla" "-e" "stopifnot(require('RInside'));cat(RInside:::RInsideCFlags())" +# OUTPUT_VARIABLE LIBRINSIDE_INCLUDE_DIRS +#) +# +#message(STATUS ${LIBRCPP_INCLUDE_DIRS}) +#message(STATUS ${LIBRINSIDE_INCLUDE_DIRS}) + +# may find a better way ... +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBRINSIDE_INCLUDE_DIRS}") + +include_directories(${R_INCLUDE_DIRS}) +include_directories(${MPI_INCLUDE_PATH}) + +add_library(Worker STATIC worker.cpp) + +add_library(DHT STATIC DHT.cpp dht_wrapper.cpp) +target_link_libraries(DHT crypto) + +add_library(R_UTILS STATIC r_utils.cpp) + +target_link_libraries(poet PUBLIC Worker DHT R_UTILS ${MPI_C_LIBRARIES} ${R_LIBRARIES}) From 89276a4e81cdb895c1a7eb6437b6a4cb4406e11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 2 Dec 2020 13:54:22 +0100 Subject: [PATCH 02/50] Added "STRICT_R_HEADERS" to CMake --- src/CMakeLists.txt | 73 ++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 54 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9277ceb5c..631342a07 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,21 +1,24 @@ -# this is the current cmake version distributed by debian stable (buster) -cmake_minimum_required(VERSION 3.13) +# Version 3.9+ offers new MPI package variables +cmake_minimum_required(VERSION 3.9) -project(Poet VERSION 0.0) +project(POET VERSION 0.1) add_executable(poet kin.cpp) +# Not needed until now # specify the C++ standard -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED True) +#set(CMAKE_CXX_STANDARD 11) +#set(CMAKE_CXX_STANDARD_REQUIRED True) -find_package(MPI) +find_package(MPI REQUIRED) -list(APPEND R_DEPS "Rcpp" "RInside") +set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS -Wall -O3") +add_definitions(${GCC_CXX_FLAGS}) # prepare R environment (Rcpp + RInside) find_program(R_EXE "R") +# search for R executable, R header file and library path if(R_EXE) execute_process(COMMAND ${R_EXE} RHOME OUTPUT_VARIABLE R_ROOT_DIR @@ -24,7 +27,7 @@ if(R_EXE) find_path(R_INCLUDE_DIR R.h HINTS ${R_ROOT_DIR} - PATHS /usr/inlcude /usr/local/include + PATHS /usr/inlcude /usr/local/include /usr/share PATH_SUFFIXES include/R R/include ) @@ -38,34 +41,8 @@ endif() set(R_LIBRARIES ${R_LIBRARY}) set(R_INCLUDE_DIRS ${R_INCLUDE_DIR}) -#foreach(_r_deps ${R_DEPS}) -# execute_process(COMMAND echo "cat(find.package('${_r_deps}'))" -# COMMAND ${R_EXE} --vanilla --slave -# RESULT_VARIABLE _rd -# ERROR_QUIET -# OUTPUT_VARIABLE _r_deps_path -# OUTPUT_STRIP_TRAILING_WHITESPACE -# ) -# -# if(_rd) -# message(FATAL_ERROR "${_r_deps} not found!") -# endif() -# -# find_library(R_${_r_deps}_LIBRARY -# lib${_r_deps}.so -# HINTS ${_r_deps_path}/lib -# ) -# -# if(R_${_r_deps}_LIBRARY) -# list(APPEND R_LIBRARIES ${R_${_r_deps}_LIBRARY}) -# else() -# message(STATUS "lol") -# unset(${R_${_r_deps}_LIBRARY}) -# endif() -# -#endforeach() - +# find Rcpp include directory execute_process(COMMAND echo "cat(find.package('Rcpp'))" COMMAND ${R_EXE} --vanilla --slave RESULT_VARIABLE RCPP_NOT_FOUND @@ -84,6 +61,8 @@ find_path(R_Rcpp_INCLUDE_DIR Rcpp.h list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR}) + +# find RInside libraries and include path execute_process(COMMAND echo "cat(find.package('RInside'))" COMMAND ${R_EXE} --vanilla --slave RESULT_VARIABLE RINSIDE_NOT_FOUND @@ -107,30 +86,16 @@ find_path(R_RInside_INCLUDE_DIR RInside.h list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR}) -#execute_process( -# COMMAND ${R_EXE} "--slave" "--vanilla" "-e" "stopifnot(require('Rcpp'));cat(Rcpp:::Rcpp.system.file('include'))" -# OUTPUT_VARIABLE LIBRCPP_INCLUDE_DIRS -#) -# -#execute_process( -# COMMAND ${R_EXE} "--slave" "--vanilla" "-e" "stopifnot(require('RInside'));cat(RInside:::RInsideCFlags())" -# OUTPUT_VARIABLE LIBRINSIDE_INCLUDE_DIRS -#) -# -#message(STATUS ${LIBRCPP_INCLUDE_DIRS}) -#message(STATUS ${LIBRINSIDE_INCLUDE_DIRS}) - -# may find a better way ... -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBRINSIDE_INCLUDE_DIRS}") - -include_directories(${R_INCLUDE_DIRS}) -include_directories(${MPI_INCLUDE_PATH}) +#include found directories for the whole scope (will be changed with modularization) +include_directories(${R_INCLUDE_DIRS} ${MPI_CXX_INCLUDE_DIRS}) +#define program libraries add_library(Worker STATIC worker.cpp) +target_link_libraries(Worker MPI::MPI_CXX) add_library(DHT STATIC DHT.cpp dht_wrapper.cpp) target_link_libraries(DHT crypto) add_library(R_UTILS STATIC r_utils.cpp) -target_link_libraries(poet PUBLIC Worker DHT R_UTILS ${MPI_C_LIBRARIES} ${R_LIBRARIES}) +target_link_libraries(poet PUBLIC Worker DHT R_UTILS ${R_LIBRARIES} MPI::MPI_CXX) From 3203bbe70eb63c1d37ad459cf5b144a36b0ccdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 15 Dec 2020 14:20:25 +0100 Subject: [PATCH 03/50] Substitute r_utils.* --- .gitignore | 140 ++++ src/CMakeLists.txt | 24 +- src/dht_wrapper.cpp | 10 +- src/dht_wrapper.h | 10 +- src/kin.cpp | 1425 +++++++++++++++++++++-------------------- src/util/RRuntime.cpp | 56 ++ src/util/RRuntime.h | 36 ++ src/worker.cpp | 515 ++++++++------- src/worker.h | 9 +- 9 files changed, 1252 insertions(+), 973 deletions(-) create mode 100644 .gitignore create mode 100644 src/util/RRuntime.cpp create mode 100644 src/util/RRuntime.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..801010414 --- /dev/null +++ b/.gitignore @@ -0,0 +1,140 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/c,c++,r,cmake +# Edit at https://www.toptal.com/developers/gitignore?templates=c,c++,r,cmake + +### C ### +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +### C++ ### +# Prerequisites + +# Compiled Object files +*.slo + +# Precompiled Headers + +# Compiled Dynamic libraries + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai + +# Executables + +### CMake ### +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +### CMake Patch ### +# External projects +*-prefix/ + +### R ### +# History files +.Rhistory +.Rapp.history + +# Session Data files +.RData + +# User-specific files +.Ruserdata + +# Example code in package build process +*-Ex.R + +# Output files from R CMD build +/*.tar.gz + +# Output files from R CMD check +/*.Rcheck/ + +# RStudio files +.Rproj.user/ + +# produced vignettes +vignettes/*.html +vignettes/*.pdf + +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth + +# knitr and R markdown default cache directories +*_cache/ +/cache/ + +# Temporary files created by R markdown +*.utf8.md +*.knit.md + +# R Environment Variables +.Renviron + +### R.Bookdown Stack ### +# R package: bookdown caching files +/*_files/ + +# End of https://www.toptal.com/developers/gitignore/api/c,c++,r,cmake diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 631342a07..35b93c905 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,12 +3,10 @@ cmake_minimum_required(VERSION 3.9) project(POET VERSION 0.1) -add_executable(poet kin.cpp) - # Not needed until now # specify the C++ standard -#set(CMAKE_CXX_STANDARD 11) -#set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(MPI REQUIRED) @@ -87,15 +85,19 @@ find_path(R_RInside_INCLUDE_DIR RInside.h list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR}) #include found directories for the whole scope (will be changed with modularization) -include_directories(${R_INCLUDE_DIRS} ${MPI_CXX_INCLUDE_DIRS}) +include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(Worker STATIC worker.cpp) -target_link_libraries(Worker MPI::MPI_CXX) -add_library(DHT STATIC DHT.cpp dht_wrapper.cpp) -target_link_libraries(DHT crypto) +add_library(POET_Libs OBJECT util/RRuntime.cpp dht_wrapper.cpp worker.cpp DHT.cpp) +target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) +target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) -add_library(R_UTILS STATIC r_utils.cpp) +#add_library(DHT OBJECT DHT.cpp dht_wrapper.cpp) +#target_link_libraries(DHT crypto R_Wrapper) -target_link_libraries(poet PUBLIC Worker DHT R_UTILS ${R_LIBRARIES} MPI::MPI_CXX) +#add_library(Worker OBJECT worker.cpp) +#target_link_libraries(Worker ${R_LIBRARIES} MPI::MPI_CXX R_Wrapper) + +add_executable(poet kin.cpp) +target_link_libraries(poet PUBLIC MPI::MPI_CXX POET_Libs) diff --git a/src/dht_wrapper.cpp b/src/dht_wrapper.cpp index 44f0a865c..5042cfb6e 100644 --- a/src/dht_wrapper.cpp +++ b/src/dht_wrapper.cpp @@ -1,6 +1,8 @@ #include "dht_wrapper.h" #include +using namespace poet; + /*init globals*/ bool dht_enabled; int dht_snaps; @@ -37,7 +39,7 @@ uint64_t get_md5(int key_size, void *key) { return retval; } -double Round_off(RInside &R, double N, double n) { +double Round_off(RRuntime R, double N, double n) { double result; R["roundsig"] = n; R["roundin"] = N; @@ -50,7 +52,7 @@ double Round_off(RInside &R, double N, double n) { /* * Stores fuzzed version of key in fuzzing_buffer */ -void fuzz_for_dht(RInside &R, int var_count, void *key, double dt) { +void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt) { unsigned int i = 0; //introduce fuzzing to allow more hits in DHT for (i = 0; i < (unsigned int)var_count; i++) { @@ -82,7 +84,7 @@ void fuzz_for_dht(RInside &R, int var_count, void *key, double dt) { fuzzing_buffer[var_count] = dt; } -void check_dht(RInside &R, int length, std::vector &out_result_index, double *work_package) { +void check_dht(RRuntime R, int length, std::vector &out_result_index, double *work_package) { void *key; int res; int var_count = prop_type_vector.size(); @@ -114,7 +116,7 @@ void check_dht(RInside &R, int length, std::vector &out_result_index, doub } } -void fill_dht(RInside &R, int length, std::vector &result_index, double *work_package, double *results) { +void fill_dht(RRuntime R, int length, std::vector &result_index, double *work_package, double *results) { void *key; void *data; int res; diff --git a/src/dht_wrapper.h b/src/dht_wrapper.h index 3557f7f5b..36aed4585 100644 --- a/src/dht_wrapper.h +++ b/src/dht_wrapper.h @@ -1,18 +1,18 @@ #pragma once -#include +#include "util/RRuntime.h" #include #include #include #include "DHT.h" using namespace std; -using namespace Rcpp; +using namespace poet; /*Functions*/ uint64_t get_md5(int key_size, void* key); -void fuzz_for_dht(RInside &R, int var_count, void *key, double dt); -void check_dht(RInside &R, int length, std::vector &out_result_index, double *work_package); -void fill_dht(RInside &R, int length, std::vector &result_index, double *work_package, double *results); +void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt); +void check_dht(RRuntime R, int length, std::vector &out_result_index, double *work_package); +void fill_dht(RRuntime R, int length, std::vector &result_index, double *work_package, double *results); void print_statistics(); int table_to_file(char* filename); int file_to_table(char* filename); diff --git a/src/kin.cpp b/src/kin.cpp index 4d19a72d8..7786f50ef 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -1,818 +1,845 @@ +#include +#include #include #include -#include -#include -#include // for the embedded R via RInside - -#include // mpi header file +#include -#include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced -#include "DHT.h" // MPI-DHT Implementation -#include "worker.h" -#include "r_utils.h" +#include // mpi header file + +#include "DHT.h" // MPI-DHT Implementation +#include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" +#include "worker.h" +#include "util/RRuntime.h" using namespace std; +using namespace poet; using namespace Rcpp; -double* mpi_buffer; -double* mpi_buffer_results; +double *mpi_buffer; +double *mpi_buffer_results; -uint32_t work_package_size; +uint32_t work_package_size; #define WORK_PACKAGE_SIZE_DEFAULT 5 bool store_result; std::set paramList() { - std::set options; - //global - options.insert("work-package-size"); - //only DHT - options.insert("dht-signif"); - options.insert("dht-strategy"); - options.insert("dht-size"); - options.insert("dht-snaps"); - options.insert("dht-file"); + std::set options; + // global + options.insert("work-package-size"); + // only DHT + options.insert("dht-signif"); + options.insert("dht-strategy"); + options.insert("dht-size"); + options.insert("dht-snaps"); + options.insert("dht-file"); - return options; + return options; } std::set flagList() { - std::set options; - //global - options.insert("ignore-result"); - //only DHT - options.insert("dht"); - options.insert("dht-log"); + std::set options; + // global + options.insert("ignore-result"); + // only DHT + options.insert("dht"); + options.insert("dht-log"); - return options; + return options; } std::list checkOptions(argh::parser cmdl) { - std::list retList; - std::set flist = flagList(); - std::set plist = paramList(); + std::list retList; + std::set flist = flagList(); + std::set plist = paramList(); - for (auto& flag: cmdl.flags()) { - if (!(flist.find(flag) != flist.end())) retList.push_back(flag); - } + for (auto &flag : cmdl.flags()) { + if (!(flist.find(flag) != flist.end())) + retList.push_back(flag); + } - for (auto& param: cmdl.params()) { - if (!(plist.find(param.first) != plist.end())) retList.push_back(param.first); - } + for (auto ¶m : cmdl.params()) { + if (!(plist.find(param.first) != plist.end())) + retList.push_back(param.first); + } - return retList; + return retList; } -typedef struct -{ - char has_work; - double* send_addr; +typedef struct { + char has_work; + double *send_addr; } worker_struct; - int main(int argc, char *argv[]) { - double sim_start, sim_b_transport, sim_a_transport, sim_b_chemistry, sim_a_chemistry, - sim_end; + double sim_start, sim_b_transport, sim_a_transport, sim_b_chemistry, + sim_a_chemistry, sim_end; - double cummul_transport = 0.f; - double cummul_chemistry = 0.f; - double cummul_workers = 0.f; - double cummul_chemistry_master = 0.f; + double cummul_transport = 0.f; + double cummul_chemistry = 0.f; + double cummul_workers = 0.f; + double cummul_chemistry_master = 0.f; - double cummul_master_seq_pre_loop = 0.f; - double cummul_master_seq_loop = 0.f; - double master_idle = 0.f; + double cummul_master_seq_pre_loop = 0.f; + double cummul_master_seq_loop = 0.f; + double master_idle = 0.f; - double master_send_a, master_send_b; - double cummul_master_send = 0.f; - double master_recv_a, master_recv_b; - double cummul_master_recv = 0.f; + double master_send_a, master_send_b; + double cummul_master_send = 0.f; + double master_recv_a, master_recv_b; + double cummul_master_recv = 0.f; - double sim_a_seq, sim_b_seq, sim_c_seq, sim_d_seq; - double idle_a, idle_b; + double sim_a_seq, sim_b_seq, sim_c_seq, sim_d_seq; + double idle_a, idle_b; - double sim_c_chemistry, sim_d_chemistry; - double sim_e_chemistry, sim_f_chemistry; + double sim_c_chemistry, sim_d_chemistry; + double sim_e_chemistry, sim_f_chemistry; - argh::parser cmdl(argv); + argh::parser cmdl(argv); - // cout << "CPP: Start Init (MPI)" << endl; + // cout << "CPP: Start Init (MPI)" << endl; - MPI_Init( &argc, &argv ); + MPI_Init(&argc, &argv); - int world_size; - MPI_Comm_size(MPI_COMM_WORLD, &world_size); + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - /*Create custom Communicator with all processes except 0 (the master) for DHT storage*/ - //only needed if strategy == 0, but done anyway - MPI_Group group_world; - MPI_Group dht_group; - MPI_Comm dht_comm; - int* process_ranks; - - // make a list of processes in the new communicator - process_ranks= (int*) malloc(world_size*sizeof(int)); - for(int I = 1; I < world_size; I++) - process_ranks[I-1] = I; - - //get the group under MPI_COMM_WORLD - MPI_Comm_group(MPI_COMM_WORLD, &group_world); - //create the new group - MPI_Group_incl(group_world, world_size-1, process_ranks, &dht_group); - // create the new communicator - MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); - free (process_ranks); //cleanup - // cout << "Done"; + /*Create custom Communicator with all processes except 0 (the master) for DHT + * storage*/ + // only needed if strategy == 0, but done anyway + MPI_Group group_world; + MPI_Group dht_group; + MPI_Comm dht_comm; + int *process_ranks; - if (cmdl[{"help", "h"}]) { - if (world_rank == 0) { - cout << "Todo" << endl << - "See README.md for further information." << endl; - } - MPI_Finalize(); - return EXIT_SUCCESS; + // make a list of processes in the new communicator + process_ranks = (int *)malloc(world_size * sizeof(int)); + for (int I = 1; I < world_size; I++) + process_ranks[I - 1] = I; + + // get the group under MPI_COMM_WORLD + MPI_Comm_group(MPI_COMM_WORLD, &group_world); + // create the new group + MPI_Group_incl(group_world, world_size - 1, process_ranks, &dht_group); + // create the new communicator + MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); + free(process_ranks); // cleanup + // cout << "Done"; + + if (cmdl[{"help", "h"}]) { + if (world_rank == 0) { + cout << "Todo" << endl + << "See README.md for further information." << endl; } + MPI_Finalize(); + return EXIT_SUCCESS; + } - /*INIT is now done separately in an R file provided here as argument!*/ - if (!cmdl(2)) { - if (world_rank == 0) { - cerr << "ERROR. Kin needs 2 positional arguments: " << endl << - "1) the R script defining your simulation and" << endl << - "2) the directory prefix where to save results and profiling" << endl; - } - MPI_Finalize(); - return EXIT_FAILURE; - } - - std::list optionsError = checkOptions(cmdl); - if (!optionsError.empty()) { - if (world_rank == 0) { - cerr << "Unrecognized option(s):\n" << endl; - for (auto option: optionsError) { - cerr << option << endl; - } - cerr << "\nMake sure to use available options. Exiting!" << endl; - } - MPI_Finalize(); - return EXIT_FAILURE; + /*INIT is now done separately in an R file provided here as argument!*/ + if (!cmdl(2)) { + if (world_rank == 0) { + cerr << "ERROR. Kin needs 2 positional arguments: " << endl + << "1) the R script defining your simulation and" << endl + << "2) the directory prefix where to save results and profiling" + << endl; } + MPI_Finalize(); + return EXIT_FAILURE; + } + + std::list optionsError = checkOptions(cmdl); + if (!optionsError.empty()) { + if (world_rank == 0) { + cerr << "Unrecognized option(s):\n" << endl; + for (auto option : optionsError) { + cerr << option << endl; + } + cerr << "\nMake sure to use available options. Exiting!" << endl; + } + MPI_Finalize(); + return EXIT_FAILURE; + } + + /*Parse DHT arguments*/ + dht_enabled = cmdl["dht"]; + // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; + + if (dht_enabled) { + cmdl("dht-strategy", 0) >> dht_strategy; + // cout << "CPP: DHT strategy is " << dht_strategy << endl; + + cmdl("dht-signif", 5) >> dht_significant_digits; + // cout << "CPP: DHT significant digits = " << dht_significant_digits << + // endl; + + dht_logarithm = cmdl["dht-log"]; + // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" + // : "OFF" ) << endl; + + cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> dht_size_per_process; + // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << + // endl; + + cmdl("dht-snaps", 0) >> dht_snaps; + + cmdl("dht-file") >> dht_file; + } + + /*Parse work package size*/ + cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> work_package_size; + + /*Parse output options*/ + store_result = !cmdl["ignore-result"]; + + if (world_rank == 0) { + cout << "CPP: Complete results storage is " << (store_result ? "ON" : "OFF") + << endl; + cout << "CPP: Work Package Size: " << work_package_size << endl; + cout << "CPP: DHT is " << (dht_enabled ? "ON" : "OFF") << '\n'; - /*Parse DHT arguments*/ - dht_enabled = cmdl["dht"]; - // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - if (dht_enabled) { - cmdl("dht-strategy", 0) >> dht_strategy; - // cout << "CPP: DHT strategy is " << dht_strategy << endl; - - cmdl("dht-signif", 5) >> dht_significant_digits; - // cout << "CPP: DHT significant digits = " << dht_significant_digits << endl; - - dht_logarithm = cmdl["dht-log"]; - // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" : "OFF" ) << endl; - - cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> dht_size_per_process; - // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << endl; + cout << "CPP: DHT strategy is " << dht_strategy << endl; + cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + "defined) = " + << dht_significant_digits << endl; + cout << "CPP: DHT logarithm before rounding: " + << (dht_logarithm ? "ON" : "OFF") << endl; + cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process + << endl; + cout << "CPP: DHT save snapshots is " << dht_snaps << endl; + cout << "CPP: DHT load file is " << dht_file << endl; + } + } - cmdl("dht-snaps", 0) >> dht_snaps; + cout << "CPP: R Init (RInside) on process " << world_rank << endl; + RRuntime R(argc, argv); - cmdl("dht-file") >> dht_file; + // if local_rank == 0 then master else worker + R["local_rank"] = world_rank; + + /*Loading Dependencies*/ + std::string r_load_dependencies = "suppressMessages(library(Rmufits));" + "suppressMessages(library(RedModRphree));" + "source('kin_r_library.R');" + "source('parallel_r_library.R');"; + R.parseEvalQ(r_load_dependencies); + + std::string filesim; + cmdl(1) >> filesim; // <- first positional argument + R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' + R.parseEvalQ("source(filesim)"); // eval the init string, ignoring any returns + + std::string out_dir; + if (world_rank == + 0) { // only rank 0 initializes goes through the whole initialization + cmdl(2) >> out_dir; // <- second positional argument + R["fileout"] = wrap(out_dir); // assign a char* (string) to 'fileout' + + // Note: R::sim_init() checks if the directory already exists, + // if not it makes it + + // pass the boolean "store_result" to the R process + R["store_result"] = store_result; + + // get timestep vector from grid_init function ... + std::string master_init_code = "mysetup <- master_init(setup=setup)"; + R.parseEval(master_init_code); + + dt_differ = R.parseEval("mysetup$dt_differ"); + MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + } else { // workers will only read the setup DataFrame defined by input file + R.parseEval("mysetup <- setup"); + MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + } + + if (world_rank == 0) { + cout << "CPP: R init done on process with rank " << world_rank << endl; + } + + // initialize chemistry on all processes + std::string init_chemistry_code = "mysetup <- init_chemistry(setup=mysetup)"; + R.parseEval(init_chemistry_code); + + /* Retrieve state_C from R context for MPI buffer generation */ + Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); + + /* Init Parallel helper functions */ + R["n_procs"] = world_size - 1; /* worker count */ + R["work_package_size"] = work_package_size; + + // Removed additional field for ID in previous versions + if (world_rank == 0) { + mpi_buffer = + (double *)calloc(state_C.nrow() * (state_C.ncol()), sizeof(double)); + } else { + mpi_buffer = (double *)calloc( + (work_package_size * (state_C.ncol())) + BUFFER_OFFSET, sizeof(double)); + mpi_buffer_results = + (double *)calloc(work_package_size * (state_C.ncol()), sizeof(double)); + } + + if (world_rank == 0) { + cout << "CPP: parallel init completed (buffers allocated)!" << endl; + } + + // MDL: pass to R the DHT stuff (basically, only for storing of + // simulation parameters). These 2 variables are always defined: + R["dht_enabled"] = dht_enabled; + R["dht_log"] = dht_logarithm; + + if (dht_enabled) { + // cout << "\nCreating DHT\n"; + // determine size of dht entries + int dht_data_size = state_C.ncol() * sizeof(double); + int dht_key_size = + state_C.ncol() * sizeof(double) + (dt_differ * sizeof(double)); + + // determine bucket count for preset memory usage + // bucket size is key + value + 1 byte for status + int dht_buckets_per_process = + dht_size_per_process / (1 + dht_data_size + dht_key_size); + + // MDL : following code moved here from worker.cpp + /*Load significance vector from R setup file (or set default)*/ + bool signif_vector_exists = R.parseEval("exists('signif_vector')"); + if (signif_vector_exists) { + dht_significant_digits_vector = as>(R["signif_vector"]); + } else { + dht_significant_digits_vector.assign( + dht_object->key_size / sizeof(double), dht_significant_digits); } - /*Parse work package size*/ - cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> work_package_size; - - - /*Parse output options*/ - store_result = !cmdl["ignore-result"]; - - - if (world_rank==0) { - cout << "CPP: Complete results storage is " << ( store_result ? "ON" : "OFF" ) << endl; - cout << "CPP: Work Package Size: " << work_package_size << endl; - cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - - if (dht_enabled) { - cout << "CPP: DHT strategy is " << dht_strategy << endl; - cout << "CPP: DHT key default digits (ignored if 'signif_vector' is defined) = " << dht_significant_digits << endl; - cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" : "OFF" ) << endl; - cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << endl; - cout << "CPP: DHT save snapshots is " << dht_snaps << endl; - cout << "CPP: DHT load file is " << dht_file << endl; - } + /*Load property type vector from R setup file (or set default)*/ + bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); + if (prop_type_vector_exists) { + prop_type_vector = as>(R["prop_type"]); + } else { + prop_type_vector.assign(dht_object->key_size / sizeof(double), "act"); } - cout << "CPP: R Init (RInside) on process " << world_rank << endl; - RInside R(argc, argv); + if (world_rank == 0) { + // print only on master, values are equal on all workes + cout << "CPP: dht_data_size: " << dht_data_size << "\n"; + cout << "CPP: dht_key_size: " << dht_key_size << "\n"; + cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process + << endl; - // if local_rank == 0 then master else worker - R["local_rank"] = world_rank; + // MDL: new output on signif_vector and prop_type + if (signif_vector_exists) { + cout << "CPP: using problem-specific rounding digits: " << endl; + R.parseEval("print(data.frame(prop=prop, type=prop_type, " + "digits=signif_vector))"); + } else { + cout << "CPP: using DHT default rounding digits = " + << dht_significant_digits << endl; + } - /*Loading Dependencies*/ - std::string r_load_dependencies = - "suppressMessages(library(Rmufits));" - "suppressMessages(library(RedModRphree));" - "source('kin_r_library.R');" - "source('parallel_r_library.R');"; - R.parseEvalQ(r_load_dependencies); - - std::string filesim; - cmdl(1) >> filesim; // <- first positional argument - R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' - R.parseEvalQ("source(filesim)"); // eval the init string, ignoring any returns - - std::string out_dir; - if (world_rank == 0) { // only rank 0 initializes goes through the whole initialization - cmdl(2) >> out_dir; // <- second positional argument - R["fileout"] = wrap(out_dir); // assign a char* (string) to 'fileout' - - // Note: R::sim_init() checks if the directory already exists, - // if not it makes it - - // pass the boolean "store_result" to the R process - R["store_result"] = store_result; - - //get timestep vector from grid_init function ... - std::string master_init_code = "mysetup <- master_init(setup=setup)"; - R.parseEval(master_init_code); - - dt_differ = R.parseEval("mysetup$dt_differ"); - MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); - } else { // workers will only read the setup DataFrame defined by input file - R.parseEval("mysetup <- setup"); - MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + // MDL: pass to R the DHT stuff. These variables exist + // only if dht_enabled is true + R["dht_final_signif"] = dht_significant_digits_vector; + R["dht_final_proptype"] = prop_type_vector; } - if (world_rank==0) { - cout << "CPP: R init done on process with rank " << world_rank << endl; + if (dht_strategy == 0) { + if (world_rank != 0) { + dht_object = DHT_create(dht_comm, dht_buckets_per_process, + dht_data_size, dht_key_size, get_md5); + + // storing for access from worker and callback functions + fuzzing_buffer = (double *)malloc(dht_key_size); + } + } else { + dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, + dht_data_size, dht_key_size, get_md5); } - //initialize chemistry on all processes - std::string init_chemistry_code = "mysetup <- init_chemistry(setup=mysetup)"; - R.parseEval(init_chemistry_code); - - /* Retrieve state_C from R context for MPI buffer generation */ - Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); - - /* Init Parallel helper functions */ - R["n_procs"] = world_size-1; /* worker count */ - R["work_package_size"] = work_package_size; - - // Removed additional field for ID in previous versions - if (world_rank == 0) - { - mpi_buffer = (double*) calloc(state_C.nrow() * (state_C.ncol()), sizeof(double)); - } else - { - mpi_buffer = (double*) calloc((work_package_size * (state_C.ncol())) + BUFFER_OFFSET, sizeof(double)); - mpi_buffer_results = (double*) calloc(work_package_size * (state_C.ncol()), sizeof(double)); + if (world_rank == 0) { + cout << "CPP: DHT successfully created!" << endl; } + } - if (world_rank==0) { - cout << "CPP: parallel init completed (buffers allocated)!" << endl; - } + // MDL: store all parameters + if (world_rank == 0) { + cout << "CPP: Calling R Function to store calling parameters" << endl; + R.parseEvalQ("StoreSetup(setup=mysetup)"); + } - // MDL: pass to R the DHT stuff (basically, only for storing of - // simulation parameters). These 2 variables are always defined: - R["dht_enabled"] = dht_enabled; - R["dht_log"] = dht_logarithm; + MPI_Barrier(MPI_COMM_WORLD); - if (dht_enabled) - { - //cout << "\nCreating DHT\n"; - //determine size of dht entries - int dht_data_size = state_C.ncol() * sizeof(double); - int dht_key_size = state_C.ncol() * sizeof(double) + (dt_differ * sizeof(double)); + if (world_rank == 0) { /* This is executed by the master */ - //determine bucket count for preset memory usage - //bucket size is key + value + 1 byte for status - int dht_buckets_per_process = dht_size_per_process / (1 + dht_data_size + dht_key_size); + Rcpp::NumericVector master_send; + Rcpp::NumericVector master_recv; - // MDL : following code moved here from worker.cpp - /*Load significance vector from R setup file (or set default)*/ - bool signif_vector_exists = R.parseEval("exists('signif_vector')"); - if (signif_vector_exists) - { - dht_significant_digits_vector = as>(R["signif_vector"]); - } else - { - dht_significant_digits_vector.assign(dht_object->key_size / sizeof(double), dht_significant_digits); + sim_a_seq = MPI_Wtime(); + + worker_struct *workerlist = + (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); + int need_to_receive; + MPI_Status probe_status; + double *timings; + uint64_t *dht_perfs = NULL; + + int local_work_package_size; + + // a temporary send buffer + double *send_buffer; + send_buffer = (double *)calloc( + (work_package_size * (state_C.ncol())) + BUFFER_OFFSET, sizeof(double)); + + // helper variables + int iteration; + double dt, current_sim_time; + + int n_wp = 1; // holds the actual number of wp which is + // computed later in R::distribute_work_packages() + std::vector wp_sizes_vector; // vector with the sizes of + // each package + + sim_start = MPI_Wtime(); + + // Iteration Count is dynamic, retrieving value from R (is only needed by + // master for the following loop) + uint32_t maxiter = R.parseEval("mysetup$maxiter"); + + sim_b_seq = MPI_Wtime(); + + cummul_master_seq_pre_loop += sim_b_seq - sim_a_seq; + + /*SIMULATION LOOP*/ + for (uint32_t iter = 1; iter < maxiter + 1; iter++) { + sim_a_seq = MPI_Wtime(); + + cummul_master_send = 0.f; + cummul_master_recv = 0.f; + + cout << "CPP: Evaluating next time step" << endl; + R.parseEvalQ("mysetup <- master_iteration_setup(mysetup)"); + + /*displaying iteration number, with C++ and R iterator*/ + cout << "CPP: Going through iteration " << iter << endl; + cout << "CPP: R's $iter: " << ((uint32_t)(R.parseEval("mysetup$iter"))) + << ". Iteration" << endl; + + cout << "CPP: Calling Advection" << endl; + + sim_b_transport = MPI_Wtime(); + R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); + sim_a_transport = MPI_Wtime(); + + cout << "CPP: Chemistry" << endl; + /*Fallback for sequential execution*/ + sim_b_chemistry = MPI_Wtime(); + + if (world_size == 1) { + // MDL : the transformation of values into pH and pe + // takes now place in master_advection() so the + // following line is unneeded + // R.parseEvalQ("mysetup$state_T <- + // RedModRphree::Act2pH(mysetup$state_T)"); + R.parseEvalQ( + "result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); + R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + } else { /*send work to workers*/ + + // NEW: only in the first iteration we call + // R::distribute_work_packages()!! + if (iter == 1) { + R.parseEvalQ( + "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_T), " + "package_size=work_package_size)"); + + // we only sort once the vector + R.parseEvalQ("ordered_ids <- order(wp_ids)"); + R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); + n_wp = (int)R.parseEval("length(wp_sizes_vector)"); + wp_sizes_vector = as>(R["wp_sizes_vector"]); + cout << "CPP: Total number of work packages: " << n_wp << endl; + R.parseEval("stat_wp_sizes(wp_sizes_vector)"); } - /*Load property type vector from R setup file (or set default)*/ - bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); - if (prop_type_vector_exists) - { - prop_type_vector = as>(R["prop_type"]); - } else - { - prop_type_vector.assign(dht_object->key_size / sizeof(double), "act"); - } + /* shuffle and extract data + MDL: we now apply :Act2pH directly in master_advection + */ + // R.parseEval("tmp <- + // shuffle_field(RedModRphree::Act2pH(mysetup$state_T), ordered_ids)"); + R.parseEval("tmp <- shuffle_field(mysetup$state_T, ordered_ids)"); + R.setBufferDataFrame("tmp"); + R.to_C_domain(mpi_buffer); + //Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); - if(world_rank == 0) - { - //print only on master, values are equal on all workes - cout << "CPP: dht_data_size: " << dht_data_size << "\n"; - cout << "CPP: dht_key_size: " << dht_key_size << "\n"; - cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process << endl; + //convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); + // cout << "CPP: shuffle_field() done" << endl; - // MDL: new output on signif_vector and prop_type - if (signif_vector_exists) { - cout << "CPP: using problem-specific rounding digits: " << endl; - R.parseEval("print(data.frame(prop=prop, type=prop_type, digits=signif_vector))"); - } else - { - cout << "CPP: using DHT default rounding digits = " << dht_significant_digits << endl; - } + /* send and receive work; this is done by counting + * the wp */ + int pkg_to_send = n_wp; + int pkg_to_recv = n_wp; + size_t colCount = R.getBufferNCol(); + int free_workers = world_size - 1; + double *work_pointer = mpi_buffer; + sim_c_chemistry = MPI_Wtime(); - // MDL: pass to R the DHT stuff. These variables exist - // only if dht_enabled is true - R["dht_final_signif"] = dht_significant_digits_vector; - R["dht_final_proptype"] = prop_type_vector; - } + /* visual progress */ + float progress = 0.0; + int barWidth = 70; - if (dht_strategy == 0) - { - if(world_rank != 0) { - dht_object = DHT_create(dht_comm, dht_buckets_per_process, dht_data_size, dht_key_size, get_md5); - - //storing for access from worker and callback functions - fuzzing_buffer = (double *) malloc (dht_key_size); - } - } else { - dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, dht_data_size, dht_key_size, get_md5); - } + // retrieve data from R runtime + iteration = (int)R.parseEval("mysetup$iter"); + dt = (double)R.parseEval("mysetup$requested_dt"); + current_sim_time = + (double)R.parseEval("mysetup$simulation_time-mysetup$requested_dt"); - if (world_rank==0) { - cout << "CPP: DHT successfully created!" << endl; - } - } - - - // MDL: store all parameters - if (world_rank==0) { - cout << "CPP: Calling R Function to store calling parameters" << endl; - R.parseEvalQ("StoreSetup(setup=mysetup)"); - - } - - MPI_Barrier(MPI_COMM_WORLD); - - if (world_rank == 0) - { /* This is executed by the master */ - - Rcpp::NumericVector master_send; - Rcpp::NumericVector master_recv; - - sim_a_seq = MPI_Wtime(); - - worker_struct* workerlist = (worker_struct*) calloc(world_size-1, sizeof(worker_struct)); - int need_to_receive; - MPI_Status probe_status; - double* timings; - uint64_t* dht_perfs = NULL; - - int local_work_package_size; - - // a temporary send buffer - double* send_buffer; - send_buffer = (double*) calloc((work_package_size * (state_C.ncol() )) + BUFFER_OFFSET, sizeof(double)); - - // helper variables - int iteration; - double dt, current_sim_time; - - - int n_wp = 1; // holds the actual number of wp which is - // computed later in R::distribute_work_packages() - std::vector wp_sizes_vector; // vector with the sizes of - // each package - - sim_start = MPI_Wtime(); - - //Iteration Count is dynamic, retrieving value from R (is only needed by master for the following loop) - uint32_t maxiter = R.parseEval("mysetup$maxiter"); + int count_pkgs = 0; sim_b_seq = MPI_Wtime(); - cummul_master_seq_pre_loop += sim_b_seq - sim_a_seq; + sim_c_chemistry = MPI_Wtime(); - /*SIMULATION LOOP*/ - for(uint32_t iter = 1; iter < maxiter+1; iter++) + while (pkg_to_recv > 0) // start dispatching work packages { - sim_a_seq = MPI_Wtime(); + /* visual progress */ + progress = (float)(count_pkgs + 1) / n_wp; - cummul_master_send = 0.f; - cummul_master_recv = 0.f; + cout << "["; + int pos = barWidth * progress; + for (int iprog = 0; iprog < barWidth; ++iprog) { + if (iprog < pos) + cout << "="; + else if (iprog == pos) + cout << ">"; + else + cout << " "; + } + std::cout << "] " << int(progress * 100.0) << " %\r"; + std::cout.flush(); + /* end visual progress */ + if (pkg_to_send > 0) { - cout << "CPP: Evaluating next time step" << endl; - R.parseEvalQ("mysetup <- master_iteration_setup(mysetup)"); + master_send_a = MPI_Wtime(); + /*search for free workers and send work*/ + for (int p = 0; p < world_size - 1; p++) { + if (workerlist[p].has_work == 0 && + pkg_to_send > 0) /* worker is free */ { - /*displaying iteration number, with C++ and R iterator*/ - cout << "CPP: Going through iteration " << iter << endl; - cout << "CPP: R's $iter: "<< ((uint32_t) (R.parseEval("mysetup$iter"))) <<". Iteration" << endl; + // to enable different work_package_size, set local copy of + // work_package_size to either global work_package size or + // remaining 'to_send' packages to_send >= work_package_size ? + // local_work_package_size = work_package_size : + // local_work_package_size = to_send; - cout << "CPP: Calling Advection" << endl; - - sim_b_transport = MPI_Wtime(); - R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); - sim_a_transport = MPI_Wtime(); - - - cout << "CPP: Chemistry" << endl; - /*Fallback for sequential execution*/ - sim_b_chemistry = MPI_Wtime(); + local_work_package_size = (int)wp_sizes_vector[count_pkgs]; + count_pkgs++; - if(world_size == 1) - { - // MDL : the transformation of values into pH and pe - // takes now place in master_advection() so the - // following line is unneeded - // R.parseEvalQ("mysetup$state_T <- RedModRphree::Act2pH(mysetup$state_T)"); - R.parseEvalQ("result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); - R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); - } else { /*send work to workers*/ + // cout << "CPP: sending pkg n. " << count_pkgs << " with size " + // << local_work_package_size << endl; - - // NEW: only in the first iteration we call - // R::distribute_work_packages()!! - if (iter==1) - { - R.parseEvalQ("wp_ids <- distribute_work_packages(len=nrow(mysetup$state_T), package_size=work_package_size)"); + /*push pointer forward to next work package, after taking the + * current one*/ + workerlist[p].send_addr = work_pointer; - // we only sort once the vector - R.parseEvalQ("ordered_ids <- order(wp_ids)"); - R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); - n_wp = (int) R.parseEval("length(wp_sizes_vector)"); - wp_sizes_vector = as>(R["wp_sizes_vector"]); - cout << "CPP: Total number of work packages: " << n_wp << endl; - R.parseEval("stat_wp_sizes(wp_sizes_vector)"); - - } + int end_of_wp = local_work_package_size * colCount; + work_pointer = &(work_pointer[end_of_wp]); - /* shuffle and extract data - MDL: we now apply :Act2pH directly in master_advection - */ - // R.parseEval("tmp <- shuffle_field(RedModRphree::Act2pH(mysetup$state_T), ordered_ids)"); - R.parseEval("tmp <- shuffle_field(mysetup$state_T, ordered_ids)"); - Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); + // fill send buffer starting with work_package ... + std::memcpy(send_buffer, workerlist[p].send_addr, + (end_of_wp) * sizeof(double)); + // followed by: work_package_size + send_buffer[end_of_wp] = (double)local_work_package_size; + // current iteration of simulation + send_buffer[end_of_wp + 1] = (double)iteration; + // size of timestep in seconds + send_buffer[end_of_wp + 2] = dt; + // current time of simulation (age) in seconds + send_buffer[end_of_wp + 3] = current_sim_time; + // placeholder for work_package_count + send_buffer[end_of_wp + 4] = 0.; - convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); - // cout << "CPP: shuffle_field() done" << endl; - - /* send and receive work; this is done by counting - * the wp */ - int pkg_to_send = n_wp; - int pkg_to_recv = n_wp; - size_t colCount = chemistry_data.ncol(); - int free_workers = world_size-1; - double* work_pointer = mpi_buffer; - sim_c_chemistry = MPI_Wtime(); - - /* visual progress */ - float progress = 0.0; - int barWidth = 70; - - //retrieve data from R runtime - iteration = (int) R.parseEval("mysetup$iter"); - dt = (double) R.parseEval("mysetup$requested_dt"); - current_sim_time = (double) R.parseEval("mysetup$simulation_time-mysetup$requested_dt"); - - int count_pkgs = 0; - - sim_b_seq = MPI_Wtime(); - - sim_c_chemistry = MPI_Wtime(); - - while (pkg_to_recv > 0) // start dispatching work packages - { - /* visual progress */ - progress = (float) (count_pkgs+1)/n_wp; - - cout << "["; - int pos = barWidth * progress; - for (int iprog = 0; iprog < barWidth; ++iprog) { - if (iprog < pos) - cout << "="; - else if (iprog == pos) - cout << ">"; - else - cout << " "; - } - std::cout << "] " << int(progress * 100.0) << " %\r"; - std::cout.flush(); - /* end visual progress */ - - - if (pkg_to_send > 0) { - - master_send_a = MPI_Wtime(); - /*search for free workers and send work*/ - for (int p = 0; p < world_size-1; p++) { - if (workerlist[p].has_work == 0 && pkg_to_send > 0) /* worker is free */ { - - // to enable different work_package_size, set local copy of work_package_size to - // either global work_package size or remaining 'to_send' packages - // to_send >= work_package_size ? local_work_package_size = work_package_size : local_work_package_size = to_send; - - local_work_package_size = (int) wp_sizes_vector[count_pkgs]; - count_pkgs++; - - // cout << "CPP: sending pkg n. " << count_pkgs << " with size " << local_work_package_size << endl; - - /*push pointer forward to next work package, after taking the current one*/ - workerlist[p].send_addr = work_pointer; - - int end_of_wp = local_work_package_size * colCount; - work_pointer = &(work_pointer[end_of_wp]); - - // fill send buffer starting with work_package ... - std::memcpy(send_buffer, workerlist[p].send_addr, (end_of_wp) * sizeof(double)); - // followed by: work_package_size - send_buffer[end_of_wp] = (double) local_work_package_size; - // current iteration of simulation - send_buffer[end_of_wp + 1] = (double) iteration; - // size of timestep in seconds - send_buffer[end_of_wp + 2] = dt; - // current time of simulation (age) in seconds - send_buffer[end_of_wp + 3] = current_sim_time; - // placeholder for work_package_count - send_buffer[end_of_wp + 4] = 0.; - - /* ATTENTION Worker p has rank p+1 */ - MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, p+1, TAG_WORK, MPI_COMM_WORLD); - - workerlist[p].has_work = 1; - free_workers--; - pkg_to_send -= 1; - } - } - master_send_b = MPI_Wtime(); - cummul_master_send += master_send_b - master_send_a; - } - - - - /*check if there are results to receive and receive them*/ - need_to_receive = 1; - master_recv_a = MPI_Wtime(); - while(need_to_receive && pkg_to_recv > 0) - { - - if (pkg_to_send > 0 && free_workers > 0) - MPI_Iprobe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &need_to_receive, &probe_status); - else { - idle_a = MPI_Wtime(); - MPI_Probe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &probe_status); - idle_b = MPI_Wtime(); - master_idle += idle_b - idle_a; - } - - if(need_to_receive) - { - int p = probe_status.MPI_SOURCE; - int size; - MPI_Get_count(&probe_status, MPI_DOUBLE, &size); - MPI_Recv(workerlist[p-1].send_addr, size, MPI_DOUBLE, p, TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - workerlist[p-1].has_work = 0; - pkg_to_recv -= 1; - free_workers++; - } - } - master_recv_b = MPI_Wtime(); - cummul_master_recv += master_recv_b - master_recv_a; - } - - sim_c_seq = MPI_Wtime(); - - // don't overwrite last progress - cout << endl; - - sim_d_chemistry = MPI_Wtime(); - cummul_workers += sim_d_chemistry - sim_c_chemistry; - - convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); - - R["chemistry_data"] = chemistry_data; - - /* unshuffle results */ - R.parseEval("result <- unshuffle_field(chemistry_data, ordered_ids)"); - - /* do master stuff */ - sim_e_chemistry = MPI_Wtime(); - R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); - sim_f_chemistry = MPI_Wtime(); - cummul_chemistry_master += sim_f_chemistry - sim_e_chemistry; + /* ATTENTION Worker p has rank p+1 */ + MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, + p + 1, TAG_WORK, MPI_COMM_WORLD); + workerlist[p].has_work = 1; + free_workers--; + pkg_to_send -= 1; + } } - sim_a_chemistry = MPI_Wtime(); + master_send_b = MPI_Wtime(); + cummul_master_send += master_send_b - master_send_a; + } - // MDL master_iteration_end just writes on disk state_T and - // state_C after every iteration if the cmdline option - // --ignore-results is not given (and thus the R variable - // store_result is TRUE) - R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); + /*check if there are results to receive and receive them*/ + need_to_receive = 1; + master_recv_a = MPI_Wtime(); + while (need_to_receive && pkg_to_recv > 0) { - cummul_transport += sim_a_transport - sim_b_transport; - cummul_chemistry += sim_a_chemistry - sim_b_chemistry; - - cout << endl << "CPP: End of *coupling* iteration "<< iter <<"/" << maxiter << endl << endl; - - if (dht_enabled) { - for (int i=1; i < world_size; i++) { - MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_STATS, MPI_COMM_WORLD); - } - - // MPI_Barrier(MPI_COMM_WORLD); - - if (dht_snaps == 2) { - std::stringstream outfile; - outfile << out_dir << "/iter_" << std::setfill('0') << std::setw(3) << iter << ".dht"; - for (int i=1; i < world_size; i++) { - MPI_Send(outfile.str().c_str(), outfile.str().size(), MPI_CHAR, i, TAG_DHT_STORE, MPI_COMM_WORLD); - } - MPI_Barrier(MPI_COMM_WORLD); - } + if (pkg_to_send > 0 && free_workers > 0) + MPI_Iprobe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, + &need_to_receive, &probe_status); + else { + idle_a = MPI_Wtime(); + MPI_Probe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, + &probe_status); + idle_b = MPI_Wtime(); + master_idle += idle_b - idle_a; } - - sim_d_seq = MPI_Wtime(); - - cummul_master_seq_loop += ((sim_b_seq - sim_a_seq) - (sim_a_transport - sim_b_transport)) + (sim_d_seq - sim_c_seq); - master_send.push_back(cummul_master_send, "it_" + to_string(iter)); - master_recv.push_back(cummul_master_recv, "it_" + to_string(iter)); - - } // END SIMULATION LOOP - - sim_end = MPI_Wtime(); - - if (dht_enabled && dht_snaps > 0) { - cout << "CPP: Master: Instruct workers to write DHT to file ..." << endl; - std::string outfile; - outfile = out_dir + ".dht"; - for (int i=1; i < world_size; i++) { - MPI_Send(outfile.c_str(), outfile.size(), MPI_CHAR, i, TAG_DHT_STORE, MPI_COMM_WORLD); + if (need_to_receive) { + int p = probe_status.MPI_SOURCE; + int size; + MPI_Get_count(&probe_status, MPI_DOUBLE, &size); + MPI_Recv(workerlist[p - 1].send_addr, size, MPI_DOUBLE, p, + TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + workerlist[p - 1].has_work = 0; + pkg_to_recv -= 1; + free_workers++; } - MPI_Barrier(MPI_COMM_WORLD); - cout << "CPP: Master: ... done" << endl; + } + master_recv_b = MPI_Wtime(); + cummul_master_recv += master_recv_b - master_recv_a; } - Rcpp::NumericVector phreeqc_time; - Rcpp::NumericVector dht_get_time; - Rcpp::NumericVector dht_fill_time; - Rcpp::IntegerVector phreeqc_counts; - Rcpp::NumericVector idle_worker; + sim_c_seq = MPI_Wtime(); - int phreeqc_tmp; + // don't overwrite last progress + cout << endl; - timings = (double*) calloc(3, sizeof(double)); + sim_d_chemistry = MPI_Wtime(); + cummul_workers += sim_d_chemistry - sim_c_chemistry; - if (dht_enabled) { - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; - dht_perfs = (uint64_t*) calloc(3, sizeof(uint64_t)); + //convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); + R.from_C_domain(mpi_buffer); + + R["chemistry_data"] = R.getBufferDataFrame(); + + /* unshuffle results */ + R.parseEval("result <- unshuffle_field(chemistry_data, ordered_ids)"); + + /* do master stuff */ + sim_e_chemistry = MPI_Wtime(); + R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + sim_f_chemistry = MPI_Wtime(); + cummul_chemistry_master += sim_f_chemistry - sim_e_chemistry; + } + sim_a_chemistry = MPI_Wtime(); + + // MDL master_iteration_end just writes on disk state_T and + // state_C after every iteration if the cmdline option + // --ignore-results is not given (and thus the R variable + // store_result is TRUE) + R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); + + cummul_transport += sim_a_transport - sim_b_transport; + cummul_chemistry += sim_a_chemistry - sim_b_chemistry; + + cout << endl + << "CPP: End of *coupling* iteration " << iter << "/" << maxiter + << endl + << endl; + + if (dht_enabled) { + for (int i = 1; i < world_size; i++) { + MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_STATS, MPI_COMM_WORLD); } - double idle_worker_tmp; + // MPI_Barrier(MPI_COMM_WORLD); - for (int p = 0; p < world_size-1; p++) - { - /* ATTENTION Worker p has rank p+1 */ - /* Send termination message to worker */ - MPI_Send(NULL, 0, MPI_DOUBLE, p+1, TAG_FINISH, MPI_COMM_WORLD); - - - MPI_Recv(timings, 3, MPI_DOUBLE, p+1, TAG_TIMING, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - phreeqc_time.push_back(timings[0], "w" + to_string(p+1)); - - MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p+1, TAG_TIMING, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p+1)); - - MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p+1, TAG_TIMING, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - idle_worker.push_back(idle_worker_tmp, "w" + to_string(p+1)); - - if (dht_enabled) - { - dht_get_time.push_back(timings[1], "w" + to_string(p+1)); - dht_fill_time.push_back(timings[2], "w" + to_string(p+1)); - - MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p+1, TAG_DHT_PERF, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - dht_hits += dht_perfs[0]; - dht_miss += dht_perfs[1]; - dht_collision += dht_perfs[2]; - } + if (dht_snaps == 2) { + std::stringstream outfile; + outfile << out_dir << "/iter_" << std::setfill('0') << std::setw(3) + << iter << ".dht"; + for (int i = 1; i < world_size; i++) { + MPI_Send(outfile.str().c_str(), outfile.str().size(), MPI_CHAR, i, + TAG_DHT_STORE, MPI_COMM_WORLD); + } + MPI_Barrier(MPI_COMM_WORLD); } + } - R.parseEvalQ("profiling <- list()"); + sim_d_seq = MPI_Wtime(); - R["simtime"] = sim_end - sim_start; - R.parseEvalQ("profiling$simtime <- simtime"); - R["simtime_transport"] = cummul_transport; - R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - R["simtime_chemistry"] = cummul_chemistry; - R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - R["simtime_workers"] = cummul_workers; - R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - R["simtime_chemistry_master"] = cummul_chemistry_master; - R.parseEvalQ("profiling$simtime_chemistry_master <- simtime_chemistry_master"); + cummul_master_seq_loop += + ((sim_b_seq - sim_a_seq) - (sim_a_transport - sim_b_transport)) + + (sim_d_seq - sim_c_seq); + master_send.push_back(cummul_master_send, "it_" + to_string(iter)); + master_recv.push_back(cummul_master_recv, "it_" + to_string(iter)); - R["seq_master_prep"] = cummul_master_seq_pre_loop; - R.parseEvalQ("profiling$seq_master_prep <- seq_master_prep"); - R["seq_master_loop"] = cummul_master_seq_loop; - R.parseEvalQ("profiling$seq_master_loop <- seq_master_loop"); + } // END SIMULATION LOOP - // R["master_send"] = master_send; - // R.parseEvalQ("profiling$master_send <- master_send"); - // R["master_recv"] = master_recv; - // R.parseEvalQ("profiling$master_recv <- master_recv"); + sim_end = MPI_Wtime(); - R["idle_master"] = master_idle; - R.parseEvalQ("profiling$idle_master <- idle_master"); - R["idle_worker"] = idle_worker; - R.parseEvalQ("profiling$idle_worker <- idle_worker"); - - R["phreeqc_time"] = phreeqc_time; - R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - - R["phreeqc_count"] = phreeqc_counts; - R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - - if (dht_enabled) - { - R["dht_hits"] = dht_hits; - R.parseEvalQ("profiling$dht_hits <- dht_hits"); - R["dht_miss"] = dht_miss; - R.parseEvalQ("profiling$dht_miss <- dht_miss"); - R["dht_collision"] = dht_collision; - R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - R["dht_get_time"] = dht_get_time; - R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - R["dht_fill_time"] = dht_fill_time; - R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - } - - - free(workerlist); - free(timings); - - if (dht_enabled) - free(dht_perfs); - - cout << "CPP: Done! Results are stored as R objects into <" << out_dir << "/timings.rds>" << endl; - /*exporting results and profiling data*/ - - std::string r_vis_code; - r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - R.parseEval(r_vis_code); - } else { /*This is executed by the workers*/ - if (!dht_file.empty()) { - int res = file_to_table((char *) dht_file.c_str()); - if (res != DHT_SUCCESS) { - if (res == DHT_WRONG_FILE) { - if (world_rank == 2) cerr << "CPP: Worker: Wrong File" << endl; - } else { - if (world_rank == 2) cerr << "CPP: Worker: Error in loading current state of DHT from file" << endl; - } - return EXIT_FAILURE; - } else { - if (world_rank == 2) cout << "CPP: Worker: Successfully loaded state of DHT from file " << dht_file << endl; - std::cout.flush(); - } - } - worker_function(R); - free(mpi_buffer_results); + if (dht_enabled && dht_snaps > 0) { + cout << "CPP: Master: Instruct workers to write DHT to file ..." << endl; + std::string outfile; + outfile = out_dir + ".dht"; + for (int i = 1; i < world_size; i++) { + MPI_Send(outfile.c_str(), outfile.size(), MPI_CHAR, i, TAG_DHT_STORE, + MPI_COMM_WORLD); + } + MPI_Barrier(MPI_COMM_WORLD); + cout << "CPP: Master: ... done" << endl; } - - cout << "CPP: finished, cleanup of process " << world_rank << endl; + + Rcpp::NumericVector phreeqc_time; + Rcpp::NumericVector dht_get_time; + Rcpp::NumericVector dht_fill_time; + Rcpp::IntegerVector phreeqc_counts; + Rcpp::NumericVector idle_worker; + + int phreeqc_tmp; + + timings = (double *)calloc(3, sizeof(double)); + + if (dht_enabled) { + dht_hits = 0; + dht_miss = 0; + dht_collision = 0; + dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); + } + + double idle_worker_tmp; + + for (int p = 0; p < world_size - 1; p++) { + /* ATTENTION Worker p has rank p+1 */ + /* Send termination message to worker */ + MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + + MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, + MPI_COMM_WORLD, MPI_STATUS_IGNORE); + idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + if (dht_enabled) { + dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, + MPI_COMM_WORLD, MPI_STATUS_IGNORE); + dht_hits += dht_perfs[0]; + dht_miss += dht_perfs[1]; + dht_collision += dht_perfs[2]; + } + } + + R.parseEvalQ("profiling <- list()"); + + R["simtime"] = sim_end - sim_start; + R.parseEvalQ("profiling$simtime <- simtime"); + R["simtime_transport"] = cummul_transport; + R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + R["simtime_chemistry"] = cummul_chemistry; + R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + R["simtime_workers"] = cummul_workers; + R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + R["simtime_chemistry_master"] = cummul_chemistry_master; + R.parseEvalQ( + "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + + R["seq_master_prep"] = cummul_master_seq_pre_loop; + R.parseEvalQ("profiling$seq_master_prep <- seq_master_prep"); + R["seq_master_loop"] = cummul_master_seq_loop; + R.parseEvalQ("profiling$seq_master_loop <- seq_master_loop"); + + // R["master_send"] = master_send; + // R.parseEvalQ("profiling$master_send <- master_send"); + // R["master_recv"] = master_recv; + // R.parseEvalQ("profiling$master_recv <- master_recv"); + + R["idle_master"] = master_idle; + R.parseEvalQ("profiling$idle_master <- idle_master"); + R["idle_worker"] = idle_worker; + R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + R["phreeqc_time"] = phreeqc_time; + R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + R["phreeqc_count"] = phreeqc_counts; + R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + if (dht_enabled) { + R["dht_hits"] = dht_hits; + R.parseEvalQ("profiling$dht_hits <- dht_hits"); + R["dht_miss"] = dht_miss; + R.parseEvalQ("profiling$dht_miss <- dht_miss"); + R["dht_collision"] = dht_collision; + R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + R["dht_get_time"] = dht_get_time; + R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + R["dht_fill_time"] = dht_fill_time; + R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + } + + free(workerlist); + free(timings); if (dht_enabled) - { - - if (dht_strategy == 0) - { - if(world_rank != 0) { - DHT_free(dht_object, NULL, NULL); - } + free(dht_perfs); + + cout << "CPP: Done! Results are stored as R objects into <" << out_dir + << "/timings.rds>" << endl; + /*exporting results and profiling data*/ + + std::string r_vis_code; + r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + R.parseEval(r_vis_code); + } else { /*This is executed by the workers*/ + if (!dht_file.empty()) { + int res = file_to_table((char *)dht_file.c_str()); + if (res != DHT_SUCCESS) { + if (res == DHT_WRONG_FILE) { + if (world_rank == 2) + cerr << "CPP: Worker: Wrong File" << endl; } else { - DHT_free(dht_object, NULL, NULL); + if (world_rank == 2) + cerr << "CPP: Worker: Error in loading current state of DHT from " + "file" + << endl; } + return EXIT_FAILURE; + } else { + if (world_rank == 2) + cout << "CPP: Worker: Successfully loaded state of DHT from file " + << dht_file << endl; + std::cout.flush(); + } } + worker_function(R); + free(mpi_buffer_results); + } - free(mpi_buffer); - MPI_Finalize(); - - if (world_rank==0) { - cout << "CPP: done, bye!" << endl; + cout << "CPP: finished, cleanup of process " << world_rank << endl; + + if (dht_enabled) { + + if (dht_strategy == 0) { + if (world_rank != 0) { + DHT_free(dht_object, NULL, NULL); + } + } else { + DHT_free(dht_object, NULL, NULL); } - - exit(0); + } + + free(mpi_buffer); + MPI_Finalize(); + + if (world_rank == 0) { + cout << "CPP: done, bye!" << endl; + } + + exit(0); } - diff --git a/src/util/RRuntime.cpp b/src/util/RRuntime.cpp new file mode 100644 index 000000000..0aefaf0f7 --- /dev/null +++ b/src/util/RRuntime.cpp @@ -0,0 +1,56 @@ +#include "RRuntime.h" +#include +#include +#include + +using namespace poet; + +/** + * Convert a R dataframe into a C continious memory area. + * + * @param varname Name of the R internal variable name. + */ +void RRuntime::to_C_domain(double *buffer) { + size_t rowCount = dfbuff.nrow(); + size_t colCount = dfbuff.ncol(); + + for (size_t i = 0; i < rowCount; i++) { + for (size_t j = 0; j < colCount; j++) { + /* Access column vector j and extract value of line i */ + Rcpp::DoubleVector col = dfbuff[j]; + buffer[i * colCount + j] = col[i]; + } + } +} + +/** + * Convert continious C memory area into R dataframe and puts it into R runtime. + * + * @param buffer Pointer to memory area which should be converted into R + * dataframe. + * @param skeleton Defines the raw data frame structure and muste be defined + * inside the R runtime beforehand. + * @param varname Name of the R internal variable name. + */ +void RRuntime::from_C_domain(double *buffer) { + size_t rowCount = dfbuff.nrow(); + size_t colCount = dfbuff.ncol(); + + for (size_t i = 0; i < rowCount; i++) { + for (size_t j = 0; j < colCount; j++) { + /* Access column vector j and extract value of line i */ + Rcpp::DoubleVector col = dfbuff[j]; + col[i] = buffer[i * colCount + j]; + } + } +} + +void RRuntime::setBufferDataFrame(std::string dfname) { + this->dfbuff = parseEval(dfname); +} + +Rcpp::DataFrame RRuntime::getBufferDataFrame() { return this->dfbuff; } + +size_t RRuntime::getBufferNCol() { return (this->dfbuff).ncol(); } + +size_t RRuntime::getBufferNRow() { return (this->dfbuff).nrow(); } diff --git a/src/util/RRuntime.h b/src/util/RRuntime.h new file mode 100644 index 000000000..c231cf9fd --- /dev/null +++ b/src/util/RRuntime.h @@ -0,0 +1,36 @@ +#ifndef RRUNTIME_H +#define RRUNTIME_H + +#include +#include +#include + +namespace poet { + +/** + * RRuntime is a wrapper class around a RInside (R) runtime and provides several + * simplified methods to use R commands inside POET. + * + * If an instance of RRuntime is created a R runtime will also be spawned. + */ +class RRuntime : public RInside { + +public: + /** + * Constructor of class RRuntime calling constructor of RInside. + */ + RRuntime(const int argc, const char *const argv[]) : RInside(argc, argv){}; + + void to_C_domain(double *buffer); + void from_C_domain(double *buffer); + + void setBufferDataFrame(std::string dfname); + Rcpp::DataFrame getBufferDataFrame(); + size_t getBufferNCol(); + size_t getBufferNRow(); + +private: + Rcpp::DataFrame dfbuff; +}; +} // namespace poet +#endif // RRUNTIME_H diff --git a/src/worker.cpp b/src/worker.cpp index 880b71f51..d6b21064e 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -1,277 +1,294 @@ #include "worker.h" #include "dht_wrapper.h" #include "global_buffer.h" -#include "r_utils.h" -#include +#include "util/RRuntime.h" #include +#include +#include -void worker_function(RInside& R) -{ - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - MPI_Status probe_status; - int count; +using namespace poet; +using namespace Rcpp; - int local_work_package_size; - int iteration; - double dt, current_sim_time; +void worker_function(RRuntime R) { + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + MPI_Status probe_status; + int count; - double idle_a, idle_b; - double cummul_idle = 0.f; + int local_work_package_size; + int iteration; + double dt, current_sim_time; - double dht_get_start=0, dht_get_end=0; - double dht_fill_start=0, dht_fill_end=0; - double phreeqc_time_start=0, phreeqc_time_end=0; - int phreeqc_count = 0; + double idle_a, idle_b; + double cummul_idle = 0.f; - //timing[0] -> phreeqc - //timing[1] -> dht_get - //timing[2] -> dht_fill - double timing[3]; - timing[0] = 0.0; - timing[1] = 0.0; - timing[2] = 0.0; + double dht_get_start = 0, dht_get_end = 0; + double dht_fill_start = 0, dht_fill_end = 0; + double phreeqc_time_start = 0, phreeqc_time_end = 0; + int phreeqc_count = 0; - //dht_perf[0] -> hits - //dht_perf[1] -> miss - //dht_perf[2] -> collisions - uint64_t dht_perf[3]; + // timing[0] -> phreeqc + // timing[1] -> dht_get + // timing[2] -> dht_fill + double timing[3]; + timing[0] = 0.0; + timing[1] = 0.0; + timing[2] = 0.0; - if (dht_enabled) - { - dht_flags.resize(work_package_size, true); //set size - dht_flags.assign(work_package_size, true); //assign all elements to true (default) - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; + // dht_perf[0] -> hits + // dht_perf[1] -> miss + // dht_perf[2] -> collisions + uint64_t dht_perf[3]; + if (dht_enabled) { + dht_flags.resize(work_package_size, true); // set size + dht_flags.assign(work_package_size, + true); // assign all elements to true (default) + dht_hits = 0; + dht_miss = 0; + dht_collision = 0; - // MDL: This code has now been moved to kin.cpp - // /*Load significance vector from R setup file (or set default)*/ - // bool signif_vector_exists = R.parseEval("exists('signif_vector')"); - // if (signif_vector_exists) - // { - // dht_significant_digits_vector = as>(R["signif_vector"]); - // } else - // { - // dht_significant_digits_vector.assign(dht_object->key_size / sizeof(double), dht_significant_digits); - // } + // MDL: This code has now been moved to kin.cpp + // /*Load significance vector from R setup file (or set default)*/ + // bool signif_vector_exists = R.parseEval("exists('signif_vector')"); + // if (signif_vector_exists) + // { + // dht_significant_digits_vector = + // as>(R["signif_vector"]); + // } else + // { + // dht_significant_digits_vector.assign(dht_object->key_size / + // sizeof(double), dht_significant_digits); + // } - // /*Load property type vector from R setup file (or set default)*/ - // bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); - // if (prop_type_vector_exists) - // { - // prop_type_vector = as>(R["prop_type"]); - // } else - // { - // prop_type_vector.assign(dht_object->key_size / sizeof(double), "normal"); - // } - } + // /*Load property type vector from R setup file (or set default)*/ + // bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); + // if (prop_type_vector_exists) + // { + // prop_type_vector = as>(R["prop_type"]); + // } else + // { + // prop_type_vector.assign(dht_object->key_size / sizeof(double), + // "normal"); + // } + } - //initialization of helper variables - iteration = 0; - dt = 0; - current_sim_time = 0; - local_work_package_size = 0; + // initialization of helper variables + iteration = 0; + dt = 0; + current_sim_time = 0; + local_work_package_size = 0; - /*worker loop*/ - while(1) - { - /*Wait for Message*/ - idle_a = MPI_Wtime(); - MPI_Probe(0, MPI_ANY_TAG, MPI_COMM_WORLD, &probe_status); - idle_b = MPI_Wtime(); + /*worker loop*/ + while (1) { + /*Wait for Message*/ + idle_a = MPI_Wtime(); + MPI_Probe(0, MPI_ANY_TAG, MPI_COMM_WORLD, &probe_status); + idle_b = MPI_Wtime(); - if (probe_status.MPI_TAG == TAG_WORK) - { /* do work */ - - cummul_idle += idle_b - idle_a; + if (probe_status.MPI_TAG == TAG_WORK) { /* do work */ - /* get number of doubles sent */ - MPI_Get_count(&probe_status, MPI_DOUBLE, &count); + cummul_idle += idle_b - idle_a; - /* receive */ - MPI_Recv(mpi_buffer, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + /* get number of doubles sent */ + MPI_Get_count(&probe_status, MPI_DOUBLE, &count); - //decrement count of work_package by BUFFER_OFFSET - count -= BUFFER_OFFSET; - //check for changes on all additional variables given by the 'header' of mpi_buffer - if (mpi_buffer[count] != local_work_package_size) { //work_package_size - local_work_package_size = mpi_buffer[count]; - R["work_package_size"] = local_work_package_size; - R.parseEvalQ("mysetup$work_package_size <- work_package_size"); - } - if (mpi_buffer[count+1] != iteration) { //current iteration of simulation - iteration = mpi_buffer[count+1]; - R["iter"] = iteration; - R.parseEvalQ("mysetup$iter <- iter"); - } - if (mpi_buffer[count+2] != dt) { //current timestep size - dt = mpi_buffer[count+2]; - R["dt"] = dt; - R.parseEvalQ("mysetup$dt <- dt"); - } - if (mpi_buffer[count+3] != current_sim_time) { //current simulation time ('age' of simulation) - current_sim_time = mpi_buffer[count+3]; - R["simulation_time"] = current_sim_time; - R.parseEvalQ("mysetup$simulation_time <- simulation_time"); - } - /* 4th double value is currently a placeholder */ - // if (mpi_buffer[count+4] != placeholder) { - // placeholder = mpi_buffer[count+4]; - // R["mysetup$placeholder"] = placeholder; - // } + /* receive */ + MPI_Recv(mpi_buffer, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); - /* get df with right structure to fill in work package */ - R.parseEvalQ("tmp2 <- head(mysetup$state_C, work_package_size)"); - // R.parseEval("print(rownames(tmp2)[1:5])"); - // R.parseEval("print(head(tmp2, 2))"); - // R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); + // decrement count of work_package by BUFFER_OFFSET + count -= BUFFER_OFFSET; + // check for changes on all additional variables given by the 'header' of + // mpi_buffer + if (mpi_buffer[count] != local_work_package_size) { // work_package_size + local_work_package_size = mpi_buffer[count]; + R["work_package_size"] = local_work_package_size; + R.parseEvalQ("mysetup$work_package_size <- work_package_size"); + } + if (mpi_buffer[count + 1] != + iteration) { // current iteration of simulation + iteration = mpi_buffer[count + 1]; + R["iter"] = iteration; + R.parseEvalQ("mysetup$iter <- iter"); + } + if (mpi_buffer[count + 2] != dt) { // current timestep size + dt = mpi_buffer[count + 2]; + R["dt"] = dt; + R.parseEvalQ("mysetup$dt <- dt"); + } + if (mpi_buffer[count + 3] != + current_sim_time) { // current simulation time ('age' of simulation) + current_sim_time = mpi_buffer[count + 3]; + R["simulation_time"] = current_sim_time; + R.parseEvalQ("mysetup$simulation_time <- simulation_time"); + } + /* 4th double value is currently a placeholder */ + // if (mpi_buffer[count+4] != placeholder) { + // placeholder = mpi_buffer[count+4]; + // R["mysetup$placeholder"] = placeholder; + // } - Rcpp::DataFrame buffer = R.parseEval("tmp2"); + /* get df with right structure to fill in work package */ + R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); + // R.parseEval("print(rownames(tmp2)[1:5])"); + // R.parseEval("print(head(tmp2, 2))"); + // R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); - if (dht_enabled) - { - // DEBUG - // cout << "RANK " << world_rank << " start checking DHT\n"; + //Rcpp::DataFrame buffer = R.parseEval("tmp2"); + R.setBufferDataFrame("skeleton"); - //resize helper vector dht_flags of work_package_size changes - if ((int) dht_flags.size() != local_work_package_size) { - dht_flags.resize(local_work_package_size, true); //set size - dht_flags.assign(local_work_package_size, true); //assign all elements to true (default) - } - - dht_get_start = MPI_Wtime(); - check_dht(R, local_work_package_size, dht_flags, mpi_buffer); - dht_get_end = MPI_Wtime(); + if (dht_enabled) { + // DEBUG + // cout << "RANK " << world_rank << " start checking DHT\n"; - //DEBUG - //cout << "RANK " << world_rank << " checking DHT complete \n"; - - R["dht_flags"] = as(wrap(dht_flags)); - //R.parseEvalQ("print(head(dht_flags))"); - } - - /* work */ - convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); - R["work_package_full"] = buffer; - //R["work_package"] = buffer; - - //DEBUG - //R.parseEvalQ("print(head(work_package_full))"); - //R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); - - if (dht_enabled) - { - R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); - } else { - R.parseEvalQ("work_package <- work_package_full"); - } - - //DEBUG - // R.parseEvalQ("print(head(work_package),2)"); - - // R.parseEvalQ("rownames(work_package) <- work_package$id"); - // R.parseEval("print(paste('id %in% colnames(work_package)', 'id' %in% colnames(work_package)"); - // R.parseEvalQ("id_store <- rownames(work_package)"); //"[, ncol(work_package)]"); - // R.parseEvalQ("work_package$id <- NULL"); - R.parseEvalQ("work_package <- as.matrix(work_package)"); - - unsigned int nrows = R.parseEval("nrow(work_package)"); - - if (nrows > 0) - { - /*Single Line error Workaround*/ - if (nrows <=1) - { - //duplicate line to enable correct simmulation - R.parseEvalQ("work_package <- work_package[rep(1:nrow(work_package), times = 2), ]"); - } - - phreeqc_count++; - - phreeqc_time_start = MPI_Wtime(); - // MDL - // R.parseEvalQ("print('Work_package:\n'); print(head(work_package , 2)); cat('RCpp: worker_function:', local_rank, ' \n')"); - R.parseEvalQ("result <- as.data.frame(slave_chemistry(setup=mysetup, data = work_package))"); - phreeqc_time_end = MPI_Wtime(); - // R.parseEvalQ("result$id <- id_store"); - } else - { - //cout << "Work-Package is empty, skipping phreeqc!" << endl; - } - - - if (dht_enabled) - { - R.parseEvalQ("result_full <- work_package_full"); - if (nrows > 0) - R.parseEvalQ("result_full[dht_flags,] <- result"); - } else { - R.parseEvalQ("result_full <- result"); - } - - Rcpp::DataFrame result = R.parseEval("result_full"); - convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); - - /* send results to master */ - MPI_Request send_req; - MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, &send_req); - - if (dht_enabled) - { - dht_fill_start = MPI_Wtime(); - fill_dht(R, local_work_package_size, dht_flags, mpi_buffer, mpi_buffer_results); - dht_fill_end = MPI_Wtime(); - - timing[1] += dht_get_end - dht_get_start; - timing[2] += dht_fill_end - dht_fill_start; - } - - timing[0] += phreeqc_time_end - phreeqc_time_start; - - MPI_Wait(&send_req,MPI_STATUS_IGNORE); - - } else if (probe_status.MPI_TAG == TAG_FINISH) - { /* recv and die */ - /* before death, submit profiling/timings to master*/ - - MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_FINISH, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - - //timings - MPI_Send(timing, 3, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); - - MPI_Send(&phreeqc_count, 1, MPI_INT, 0, TAG_TIMING, MPI_COMM_WORLD); - MPI_Send(&cummul_idle, 1, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); - - if(dht_enabled) - { - //dht_perf - dht_perf[0] = dht_hits; - dht_perf[1] = dht_miss; - dht_perf[2] = dht_collision; - MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, MPI_COMM_WORLD); - } - - break; - - } else if ((probe_status.MPI_TAG == TAG_DHT_STATS)) { - MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_DHT_STATS, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - print_statistics(); - MPI_Barrier(MPI_COMM_WORLD); - } else if ((probe_status.MPI_TAG == TAG_DHT_STORE)) { - char* outdir; - MPI_Get_count(&probe_status, MPI_CHAR, &count); - outdir = (char *) calloc(count + 1, sizeof(char)); - MPI_Recv(outdir, count, MPI_CHAR, 0, TAG_DHT_STORE, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - int res = table_to_file((char *) outdir); - if (res != DHT_SUCCESS) { - if (world_rank == 2) cerr << "CPP: Worker: Error in writing current state of DHT to file (TAG_DHT_STORE)" << endl; - } else { - if (world_rank == 2) cout << "CPP: Worker: Successfully written DHT to file " << outdir << endl; - } - free(outdir); - MPI_Barrier(MPI_COMM_WORLD); + // resize helper vector dht_flags of work_package_size changes + if ((int)dht_flags.size() != local_work_package_size) { + dht_flags.resize(local_work_package_size, true); // set size + dht_flags.assign(local_work_package_size, + true); // assign all elements to true (default) } + + dht_get_start = MPI_Wtime(); + check_dht(R, local_work_package_size, dht_flags, mpi_buffer); + dht_get_end = MPI_Wtime(); + + // DEBUG + // cout << "RANK " << world_rank << " checking DHT complete \n"; + + R["dht_flags"] = as(wrap(dht_flags)); + // R.parseEvalQ("print(head(dht_flags))"); + } + + /* work */ + R.from_C_domain(mpi_buffer); + //convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); + R["work_package_full"] = R.getBufferDataFrame(); + // R["work_package"] = buffer; + + // DEBUG + // R.parseEvalQ("print(head(work_package_full))"); + // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); + + if (dht_enabled) { + R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); + } else { + R.parseEvalQ("work_package <- work_package_full"); + } + + // DEBUG + // R.parseEvalQ("print(head(work_package),2)"); + + // R.parseEvalQ("rownames(work_package) <- work_package$id"); + // R.parseEval("print(paste('id %in% colnames(work_package)', 'id' %in% + // colnames(work_package)"); R.parseEvalQ("id_store <- + // rownames(work_package)"); //"[, ncol(work_package)]"); + // R.parseEvalQ("work_package$id <- NULL"); + R.parseEvalQ("work_package <- as.matrix(work_package)"); + + unsigned int nrows = R.parseEval("nrow(work_package)"); + + if (nrows > 0) { + /*Single Line error Workaround*/ + if (nrows <= 1) { + // duplicate line to enable correct simmulation + R.parseEvalQ("work_package <- work_package[rep(1:nrow(work_package), " + "times = 2), ]"); + } + + phreeqc_count++; + + phreeqc_time_start = MPI_Wtime(); + // MDL + // R.parseEvalQ("print('Work_package:\n'); print(head(work_package , + // 2)); cat('RCpp: worker_function:', local_rank, ' \n')"); + R.parseEvalQ("result <- as.data.frame(slave_chemistry(setup=mysetup, " + "data = work_package))"); + phreeqc_time_end = MPI_Wtime(); + // R.parseEvalQ("result$id <- id_store"); + } else { + // cout << "Work-Package is empty, skipping phreeqc!" << endl; + } + + if (dht_enabled) { + R.parseEvalQ("result_full <- work_package_full"); + if (nrows > 0) + R.parseEvalQ("result_full[dht_flags,] <- result"); + } else { + R.parseEvalQ("result_full <- result"); + } + + R.setBufferDataFrame("result_full"); + //Rcpp::DataFrame result = R.parseEval("result_full"); + //convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); + R.to_C_domain(mpi_buffer_results); + + /* send results to master */ + MPI_Request send_req; + MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, + MPI_COMM_WORLD, &send_req); + + if (dht_enabled) { + dht_fill_start = MPI_Wtime(); + fill_dht(R, local_work_package_size, dht_flags, mpi_buffer, + mpi_buffer_results); + dht_fill_end = MPI_Wtime(); + + timing[1] += dht_get_end - dht_get_start; + timing[2] += dht_fill_end - dht_fill_start; + } + + timing[0] += phreeqc_time_end - phreeqc_time_start; + + MPI_Wait(&send_req, MPI_STATUS_IGNORE); + + } else if (probe_status.MPI_TAG == TAG_FINISH) { /* recv and die */ + /* before death, submit profiling/timings to master*/ + + MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_FINISH, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + + // timings + MPI_Send(timing, 3, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); + + MPI_Send(&phreeqc_count, 1, MPI_INT, 0, TAG_TIMING, MPI_COMM_WORLD); + MPI_Send(&cummul_idle, 1, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); + + if (dht_enabled) { + // dht_perf + dht_perf[0] = dht_hits; + dht_perf[1] = dht_miss; + dht_perf[2] = dht_collision; + MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, + MPI_COMM_WORLD); + } + + break; + + } else if ((probe_status.MPI_TAG == TAG_DHT_STATS)) { + MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_DHT_STATS, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + print_statistics(); + MPI_Barrier(MPI_COMM_WORLD); + } else if ((probe_status.MPI_TAG == TAG_DHT_STORE)) { + char *outdir; + MPI_Get_count(&probe_status, MPI_CHAR, &count); + outdir = (char *)calloc(count + 1, sizeof(char)); + MPI_Recv(outdir, count, MPI_CHAR, 0, TAG_DHT_STORE, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + int res = table_to_file((char *)outdir); + if (res != DHT_SUCCESS) { + if (world_rank == 2) + cerr << "CPP: Worker: Error in writing current state of DHT to file " + "(TAG_DHT_STORE)" + << endl; + } else { + if (world_rank == 2) + cout << "CPP: Worker: Successfully written DHT to file " << outdir + << endl; + } + free(outdir); + MPI_Barrier(MPI_COMM_WORLD); } + } } diff --git a/src/worker.h b/src/worker.h index 221a1f7a3..72faa87e8 100644 --- a/src/worker.h +++ b/src/worker.h @@ -1,11 +1,10 @@ #pragma once -#include +#include "util/RRuntime.h" using namespace std; -using namespace Rcpp; - +using namespace poet; /*Functions*/ -void worker_function(RInside &R); +void worker_function(RRuntime R); /*Globals*/ @@ -14,4 +13,4 @@ void worker_function(RInside &R); #define TAG_TIMING 44 #define TAG_DHT_PERF 45 #define TAG_DHT_STATS 46 -#define TAG_DHT_STORE 47 \ No newline at end of file +#define TAG_DHT_STORE 47 From 3e0c931c4169f4b503373641e9562f4d744ad1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 16 Dec 2020 14:15:14 +0100 Subject: [PATCH 04/50] added Grid Class --- src/CMakeLists.txt | 4 +-- src/dht_wrapper.cpp | 37 +++++++++++---------------- src/dht_wrapper.h | 6 ++--- src/kin.cpp | 48 +++++++++++++++++------------------ src/model/Grid.cpp | 61 +++++++++++++++++++++++++++++++++++++++++++++ src/model/Grid.h | 32 ++++++++++++++++++++++++ src/worker.cpp | 45 +++++++++++++++++++-------------- src/worker.h | 3 ++- 8 files changed, 164 insertions(+), 72 deletions(-) create mode 100644 src/model/Grid.cpp create mode 100644 src/model/Grid.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35b93c905..4eb121d63 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(MPI REQUIRED) -set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS -Wall -O3") +set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") add_definitions(${GCC_CXX_FLAGS}) # prepare R environment (Rcpp + RInside) @@ -89,7 +89,7 @@ include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(POET_Libs OBJECT util/RRuntime.cpp dht_wrapper.cpp worker.cpp DHT.cpp) +add_library(POET_Libs STATIC util/RRuntime.cpp dht_wrapper.cpp worker.cpp DHT.cpp model/Grid.cpp) target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) diff --git a/src/dht_wrapper.cpp b/src/dht_wrapper.cpp index 5042cfb6e..832aea641 100644 --- a/src/dht_wrapper.cpp +++ b/src/dht_wrapper.cpp @@ -39,20 +39,20 @@ uint64_t get_md5(int key_size, void *key) { return retval; } -double Round_off(RRuntime R, double N, double n) { - double result; - R["roundsig"] = n; - R["roundin"] = N; - - result = R.parseEval("signif(roundin, digits=roundsig)"); - - return result; -} +//double Round_off(double N, double n) { +// double result; +// R["roundsig"] = n; +// R["roundin"] = N; +// +// result = R.parseEval("signif(roundin, digits=roundsig)"); +// +// return result; +//} /* * Stores fuzzed version of key in fuzzing_buffer */ -void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt) { +void fuzz_for_dht(int var_count, void *key, double dt) { unsigned int i = 0; //introduce fuzzing to allow more hits in DHT for (i = 0; i < (unsigned int)var_count; i++) { @@ -64,15 +64,12 @@ void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt) { else if (((double *)key)[i] == 0) fuzzing_buffer[i] = 0; else - //fuzzing_buffer[i] = Round_off(R, std::log10(((double *)key)[i]), dht_significant_digits_vector[i] - 1); fuzzing_buffer[i] = ROUND(-(std::log10(((double *)key)[i])), dht_significant_digits_vector[i]); } else { //without log10 - //fuzzing_buffer[i] = Round_off(R, ((double *)key)[i], dht_significant_digits_vector[i]); fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_significant_digits_vector[i]); } } else if (prop_type_vector[i] == "logact") { - //fuzzing_buffer[i] = Round_off(R, ((double *)key)[i], dht_significant_digits_vector[i]); fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_significant_digits_vector[i]); } else if (prop_type_vector[i] == "ignore") { fuzzing_buffer[i] = 0; @@ -84,19 +81,16 @@ void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt) { fuzzing_buffer[var_count] = dt; } -void check_dht(RRuntime R, int length, std::vector &out_result_index, double *work_package) { +void check_dht(int length, std::vector &out_result_index, double *work_package, double dt) { void *key; int res; int var_count = prop_type_vector.size(); - double dt; - - dt = R.parseEval("mysetup$dt"); for (int i = 0; i < length; i++) { key = (void *)&(work_package[i * var_count]); //fuzz data (round, logarithm etc.) - fuzz_for_dht(R, var_count, key, dt); + fuzz_for_dht(var_count, key, dt); //overwrite input with data from DHT, IF value is found in DHT res = DHT_read(dht_object, fuzzing_buffer, key); @@ -116,14 +110,11 @@ void check_dht(RRuntime R, int length, std::vector &out_result_index, doub } } -void fill_dht(RRuntime R, int length, std::vector &result_index, double *work_package, double *results) { +void fill_dht(int length, std::vector &result_index, double *work_package, double *results, double dt) { void *key; void *data; int res; int var_count = prop_type_vector.size(); - double dt; - - dt = R.parseEval("mysetup$dt"); for (int i = 0; i < length; i++) { key = (void *)&(work_package[i * var_count]); @@ -133,7 +124,7 @@ void fill_dht(RRuntime R, int length, std::vector &result_index, double *w //If true -> was simulated, needs to be inserted into dht //fuzz data (round, logarithm etc.) - fuzz_for_dht(R, var_count, key, dt); + fuzz_for_dht(var_count, key, dt); res = DHT_write(dht_object, fuzzing_buffer, data); diff --git a/src/dht_wrapper.h b/src/dht_wrapper.h index 36aed4585..2d16c06dc 100644 --- a/src/dht_wrapper.h +++ b/src/dht_wrapper.h @@ -10,9 +10,9 @@ using namespace poet; /*Functions*/ uint64_t get_md5(int key_size, void* key); -void fuzz_for_dht(RRuntime R, int var_count, void *key, double dt); -void check_dht(RRuntime R, int length, std::vector &out_result_index, double *work_package); -void fill_dht(RRuntime R, int length, std::vector &result_index, double *work_package, double *results); +void fuzz_for_dht(int var_count, void *key, double dt); +void check_dht(int length, std::vector &out_result_index, double *work_package, double dt); +void fill_dht(int length, std::vector &result_index, double *work_package, double *results, double dt); void print_statistics(); int table_to_file(char* filename); int file_to_table(char* filename); diff --git a/src/kin.cpp b/src/kin.cpp index 7786f50ef..9b2e15ada 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -11,8 +11,9 @@ #include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" -#include "worker.h" #include "util/RRuntime.h" +#include "worker.h" +#include "model/Grid.h" using namespace std; using namespace poet; @@ -266,8 +267,10 @@ int main(int argc, char *argv[]) { std::string init_chemistry_code = "mysetup <- init_chemistry(setup=mysetup)"; R.parseEval(init_chemistry_code); + Grid grid(R); + grid.init(); /* Retrieve state_C from R context for MPI buffer generation */ - Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); + //Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); /* Init Parallel helper functions */ R["n_procs"] = world_size - 1; /* worker count */ @@ -276,12 +279,12 @@ int main(int argc, char *argv[]) { // Removed additional field for ID in previous versions if (world_rank == 0) { mpi_buffer = - (double *)calloc(state_C.nrow() * (state_C.ncol()), sizeof(double)); + (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); } else { mpi_buffer = (double *)calloc( - (work_package_size * (state_C.ncol())) + BUFFER_OFFSET, sizeof(double)); + (work_package_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); mpi_buffer_results = - (double *)calloc(work_package_size * (state_C.ncol()), sizeof(double)); + (double *)calloc(work_package_size * (grid.getCols()), sizeof(double)); } if (world_rank == 0) { @@ -296,9 +299,9 @@ int main(int argc, char *argv[]) { if (dht_enabled) { // cout << "\nCreating DHT\n"; // determine size of dht entries - int dht_data_size = state_C.ncol() * sizeof(double); + int dht_data_size = grid.getCols() * sizeof(double); int dht_key_size = - state_C.ncol() * sizeof(double) + (dt_differ * sizeof(double)); + grid.getCols() * sizeof(double) + (dt_differ * sizeof(double)); // determine bucket count for preset memory usage // bucket size is key + value + 1 byte for status @@ -391,7 +394,7 @@ int main(int argc, char *argv[]) { // a temporary send buffer double *send_buffer; send_buffer = (double *)calloc( - (work_package_size * (state_C.ncol())) + BUFFER_OFFSET, sizeof(double)); + (work_package_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); // helper variables int iteration; @@ -469,19 +472,16 @@ int main(int argc, char *argv[]) { */ // R.parseEval("tmp <- // shuffle_field(RedModRphree::Act2pH(mysetup$state_T), ordered_ids)"); - R.parseEval("tmp <- shuffle_field(mysetup$state_T, ordered_ids)"); - R.setBufferDataFrame("tmp"); - R.to_C_domain(mpi_buffer); - //Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); - - //convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); + // Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); + + // convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); // cout << "CPP: shuffle_field() done" << endl; - + grid.shuffleAndExport(mpi_buffer); /* send and receive work; this is done by counting * the wp */ int pkg_to_send = n_wp; int pkg_to_recv = n_wp; - size_t colCount = R.getBufferNCol(); + size_t colCount = grid.getCols(); int free_workers = world_size - 1; double *work_pointer = mpi_buffer; sim_c_chemistry = MPI_Wtime(); @@ -522,7 +522,6 @@ int main(int argc, char *argv[]) { /* end visual progress */ if (pkg_to_send > 0) { - master_send_a = MPI_Wtime(); /*search for free workers and send work*/ for (int p = 0; p < world_size - 1; p++) { @@ -614,14 +613,15 @@ int main(int argc, char *argv[]) { sim_d_chemistry = MPI_Wtime(); cummul_workers += sim_d_chemistry - sim_c_chemistry; - //convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); - R.from_C_domain(mpi_buffer); + // convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); + //R.from_C_domain(mpi_buffer); - R["chemistry_data"] = R.getBufferDataFrame(); + //R["chemistry_data"] = R.getBufferDataFrame(); - /* unshuffle results */ - R.parseEval("result <- unshuffle_field(chemistry_data, ordered_ids)"); + ///* unshuffle results */ + //R.parseEval("result <- unshuffle_field(chemistry_data, ordered_ids)"); + grid.importAndUnshuffle(mpi_buffer); /* do master stuff */ sim_e_chemistry = MPI_Wtime(); R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); @@ -649,7 +649,7 @@ int main(int argc, char *argv[]) { MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_STATS, MPI_COMM_WORLD); } - // MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); if (dht_snaps == 2) { std::stringstream outfile; @@ -817,7 +817,7 @@ int main(int argc, char *argv[]) { std::cout.flush(); } } - worker_function(R); + worker_function(R, grid); free(mpi_buffer_results); } diff --git a/src/model/Grid.cpp b/src/model/Grid.cpp new file mode 100644 index 000000000..ae61ff3d0 --- /dev/null +++ b/src/model/Grid.cpp @@ -0,0 +1,61 @@ +#include "Grid.h" +#include "Rcpp.h" + +using namespace poet; +using namespace Rcpp; +/** + * At this moment init will only declare and define a variable inside the R + * runtime called grid_tmp since the whole Grid initialization and management is + * done by the R runtime. This may change in the future. + */ +void Grid::init() { + R.parseEval("GRID_TMP <- mysetup$state_C"); + this->ncol = R.parseEval("ncol(GRID_TMP)"); + this->nrow = R.parseEval("nrow(GRID_TMP)"); +} + +/** + * Returns the number of elements for each gridcell. + */ +unsigned int Grid::getCols() { return this->ncol; } + +/** + * Returns the number of gridcells. + */ +unsigned int Grid::getRows() { return this->nrow; } + +void Grid::shuffleAndExport(double *buffer) { + R.parseEval("tmp <- shuffle_field(mysetup$state_T, ordered_ids)"); + R.setBufferDataFrame("tmp"); + R.to_C_domain(buffer); +} + +void Grid::importAndUnshuffle(double *buffer) { + R.setBufferDataFrame("GRID_TMP"); + R.from_C_domain(buffer); + R["GRID_CHEM_DATA"] = R.getBufferDataFrame(); + R.parseEval("result <- unshuffle_field(GRID_CHEM_DATA, ordered_ids)"); +} + +void Grid::importWP(double *buffer, unsigned int p_size) { + R["GRID_WP_SKELETON"] = getSkeletonDataFrame(p_size); + R.setBufferDataFrame("GRID_WP_SKELETON"); + R.from_C_domain(buffer); + R["work_package_full"] = R.getBufferDataFrame(); +} +void Grid::exportWP(double *buffer) { + R.setBufferDataFrame("result_full"); + R.to_C_domain(buffer); +} +/** + * Create a data frame with n rows. + * + * @return Can be seen as a skeleton. The content of the data frame might be + * irrelevant. + */ +Rcpp::DataFrame Grid::getSkeletonDataFrame(unsigned int rows) { + R["GRID_ROWS"] = rows; + + Rcpp::DataFrame tmp = R.parseEval("head(GRID_TMP,GRID_ROWS)"); + return tmp; +} diff --git a/src/model/Grid.h b/src/model/Grid.h new file mode 100644 index 000000000..a2c066505 --- /dev/null +++ b/src/model/Grid.h @@ -0,0 +1,32 @@ +#ifndef GRID_H +#define GRID_H + +#include "../util/RRuntime.h" +#include + +namespace poet { +class Grid { + +public: + Grid(RRuntime &R) : R(R){}; + + void init(); + + unsigned int getCols(); + unsigned int getRows(); + + void shuffleAndExport(double *buffer); + void importAndUnshuffle(double *buffer); + + void importWP(double *buffer, unsigned int p_size); + void exportWP(double *buffer); + +private: + RRuntime R; + unsigned int ncol; + unsigned int nrow; + + Rcpp::DataFrame getSkeletonDataFrame(unsigned int rows); +}; +} // namespace poet +#endif // GRID_H diff --git a/src/worker.cpp b/src/worker.cpp index d6b21064e..d53fc5d5d 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -1,15 +1,16 @@ #include "worker.h" #include "dht_wrapper.h" #include "global_buffer.h" +#include "model/Grid.h" #include "util/RRuntime.h" +#include #include #include -#include using namespace poet; using namespace Rcpp; -void worker_function(RRuntime R) { +void worker_function(RRuntime &R, Grid &grid) { int world_rank; MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); MPI_Status probe_status; @@ -130,13 +131,14 @@ void worker_function(RRuntime R) { // } /* get df with right structure to fill in work package */ - R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); - // R.parseEval("print(rownames(tmp2)[1:5])"); - // R.parseEval("print(head(tmp2, 2))"); - // R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); + // R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); + // R["skeleton"] = grid.buildDataFrame(work_package_size); + //// R.parseEval("print(rownames(tmp2)[1:5])"); + //// R.parseEval("print(head(tmp2, 2))"); + //// R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); - //Rcpp::DataFrame buffer = R.parseEval("tmp2"); - R.setBufferDataFrame("skeleton"); + ////Rcpp::DataFrame buffer = R.parseEval("tmp2"); + // R.setBufferDataFrame("skeleton"); if (dht_enabled) { // DEBUG @@ -150,7 +152,7 @@ void worker_function(RRuntime R) { } dht_get_start = MPI_Wtime(); - check_dht(R, local_work_package_size, dht_flags, mpi_buffer); + check_dht(local_work_package_size, dht_flags, mpi_buffer, dt); dht_get_end = MPI_Wtime(); // DEBUG @@ -160,16 +162,19 @@ void worker_function(RRuntime R) { // R.parseEvalQ("print(head(dht_flags))"); } + /* work */ - R.from_C_domain(mpi_buffer); - //convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); - R["work_package_full"] = R.getBufferDataFrame(); + // R.from_C_domain(mpi_buffer); + ////convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); + // R["work_package_full"] = R.getBufferDataFrame(); // R["work_package"] = buffer; // DEBUG // R.parseEvalQ("print(head(work_package_full))"); // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); + grid.importWP(mpi_buffer, work_package_size); + if (dht_enabled) { R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); } else { @@ -218,11 +223,12 @@ void worker_function(RRuntime R) { R.parseEvalQ("result_full <- result"); } - R.setBufferDataFrame("result_full"); - //Rcpp::DataFrame result = R.parseEval("result_full"); - //convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); - R.to_C_domain(mpi_buffer_results); + // R.setBufferDataFrame("result_full"); + ////Rcpp::DataFrame result = R.parseEval("result_full"); + ////convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); + // R.to_C_domain(mpi_buffer_results); + grid.exportWP(mpi_buffer_results); /* send results to master */ MPI_Request send_req; MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, @@ -230,8 +236,8 @@ void worker_function(RRuntime R) { if (dht_enabled) { dht_fill_start = MPI_Wtime(); - fill_dht(R, local_work_package_size, dht_flags, mpi_buffer, - mpi_buffer_results); + fill_dht(local_work_package_size, dht_flags, mpi_buffer, + mpi_buffer_results, dt); dht_fill_end = MPI_Wtime(); timing[1] += dht_get_end - dht_get_start; @@ -242,6 +248,7 @@ void worker_function(RRuntime R) { MPI_Wait(&send_req, MPI_STATUS_IGNORE); + } else if (probe_status.MPI_TAG == TAG_FINISH) { /* recv and die */ /* before death, submit profiling/timings to master*/ @@ -262,7 +269,6 @@ void worker_function(RRuntime R) { MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, MPI_COMM_WORLD); } - break; } else if ((probe_status.MPI_TAG == TAG_DHT_STATS)) { @@ -270,6 +276,7 @@ void worker_function(RRuntime R) { MPI_STATUS_IGNORE); print_statistics(); MPI_Barrier(MPI_COMM_WORLD); + } else if ((probe_status.MPI_TAG == TAG_DHT_STORE)) { char *outdir; MPI_Get_count(&probe_status, MPI_CHAR, &count); diff --git a/src/worker.h b/src/worker.h index 72faa87e8..349333811 100644 --- a/src/worker.h +++ b/src/worker.h @@ -1,10 +1,11 @@ #pragma once #include "util/RRuntime.h" +#include "model/Grid.h" using namespace std; using namespace poet; /*Functions*/ -void worker_function(RRuntime R); +void worker_function(RRuntime &R, Grid &grid); /*Globals*/ From a6e3aa4fe337e3629682569dc2624245318a6b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Thu, 17 Dec 2020 18:16:13 +0100 Subject: [PATCH 05/50] Declare new type SimParams as struct --- src/dht_wrapper.cpp | 220 ++++++++++++++++++++++--------------------- src/dht_wrapper.h | 57 ++++++----- src/kin.cpp | 215 +++++++++++++++++++++--------------------- src/util/SimParams.h | 26 +++++ src/worker.cpp | 26 +++-- src/worker.h | 3 +- 6 files changed, 294 insertions(+), 253 deletions(-) create mode 100644 src/util/SimParams.h diff --git a/src/dht_wrapper.cpp b/src/dht_wrapper.cpp index 832aea641..f03707c45 100644 --- a/src/dht_wrapper.cpp +++ b/src/dht_wrapper.cpp @@ -4,15 +4,15 @@ using namespace poet; /*init globals*/ -bool dht_enabled; -int dht_snaps; -int dht_strategy; -int dht_significant_digits; -std::string dht_file; -std::vector dht_significant_digits_vector; -std::vector prop_type_vector; -bool dht_logarithm; -uint64_t dht_size_per_process; +//bool dht_enabled; +//int dht_snaps; +//int dht_strategy; +//int dht_significant_digits; +//std::string dht_file; +//std::vector dht_significant_digits_vector; +//std::vector prop_type_vector; +//bool dht_logarithm; +//uint64_t dht_size_per_process; uint64_t dht_hits, dht_miss, dht_collision; RInside *R_DHT; std::vector dht_flags; @@ -20,26 +20,26 @@ DHT *dht_object; double *fuzzing_buffer; -bool dt_differ; +//bool dt_differ; /*functions*/ uint64_t get_md5(int key_size, void *key) { - MD5_CTX ctx; - unsigned char sum[MD5_DIGEST_LENGTH]; - uint64_t retval, *v1, *v2; + MD5_CTX ctx; + unsigned char sum[MD5_DIGEST_LENGTH]; + uint64_t retval, *v1, *v2; - MD5_Init(&ctx); - MD5_Update(&ctx, key, key_size); - MD5_Final(sum, &ctx); + MD5_Init(&ctx); + MD5_Update(&ctx, key, key_size); + MD5_Final(sum, &ctx); - v1 = (uint64_t *)&sum[0]; - v2 = (uint64_t *)&sum[8]; - retval = *v1 ^ *v2; + v1 = (uint64_t *)&sum[0]; + v2 = (uint64_t *)&sum[8]; + retval = *v1 ^ *v2; - return retval; + return retval; } -//double Round_off(double N, double n) { +// double Round_off(double N, double n) { // double result; // R["roundsig"] = n; // R["roundin"] = N; @@ -50,119 +50,127 @@ uint64_t get_md5(int key_size, void *key) { //} /* -* Stores fuzzed version of key in fuzzing_buffer -*/ -void fuzz_for_dht(int var_count, void *key, double dt) { - unsigned int i = 0; - //introduce fuzzing to allow more hits in DHT - for (i = 0; i < (unsigned int)var_count; i++) { - if (prop_type_vector[i] == "act") { - //with log10 - if (dht_logarithm) { - if (((double *)key)[i] < 0) - cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value at key!" << endl; - else if (((double *)key)[i] == 0) - fuzzing_buffer[i] = 0; - else - fuzzing_buffer[i] = ROUND(-(std::log10(((double *)key)[i])), dht_significant_digits_vector[i]); - } else { - //without log10 - fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_significant_digits_vector[i]); - } - } else if (prop_type_vector[i] == "logact") { - fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_significant_digits_vector[i]); - } else if (prop_type_vector[i] == "ignore") { - fuzzing_buffer[i] = 0; - } else { - cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Probably wrong prop_type!" << endl; - } + * Stores fuzzed version of key in fuzzing_buffer + */ +void fuzz_for_dht(int var_count, void *key, double dt, t_simparams *params) { + unsigned int i = 0; + // introduce fuzzing to allow more hits in DHT + for (i = 0; i < (unsigned int)var_count; i++) { + if (params->dht_prop_type_vector[i] == "act") { + // with log10 + if (params->dht_log) { + if (((double *)key)[i] < 0) + cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value in " + "key!" + << endl; + else if (((double *)key)[i] == 0) + fuzzing_buffer[i] = 0; + else + fuzzing_buffer[i] = ROUND(-(std::log10(((double *)key)[i])), + params->dht_signif_vector[i]); + } else { + // without log10 + fuzzing_buffer[i] = + ROUND((((double *)key)[i]), params->dht_signif_vector[i]); + } + } else if (params->dht_prop_type_vector[i] == "logact") { + fuzzing_buffer[i] = + ROUND((((double *)key)[i]), params->dht_signif_vector[i]); + } else if (params->dht_prop_type_vector[i] == "ignore") { + fuzzing_buffer[i] = 0; + } else { + cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Probably wrong " + "prop_type!" + << endl; } - if (dt_differ) - fuzzing_buffer[var_count] = dt; + } + if (params->dt_differ) + fuzzing_buffer[var_count] = dt; } -void check_dht(int length, std::vector &out_result_index, double *work_package, double dt) { - void *key; - int res; - int var_count = prop_type_vector.size(); +void check_dht(int length, std::vector &out_result_index, + double *work_package, double dt, t_simparams *params) { + void *key; + int res; + int var_count = params->dht_prop_type_vector.size(); + for (int i = 0; i < length; i++) { + key = (void *)&(work_package[i * var_count]); - for (int i = 0; i < length; i++) { - key = (void *)&(work_package[i * var_count]); + // fuzz data (round, logarithm etc.) + fuzz_for_dht(var_count, key, dt, params); - //fuzz data (round, logarithm etc.) - fuzz_for_dht(var_count, key, dt); + // overwrite input with data from DHT, IF value is found in DHT + res = DHT_read(dht_object, fuzzing_buffer, key); - //overwrite input with data from DHT, IF value is found in DHT - res = DHT_read(dht_object, fuzzing_buffer, key); - - if (res == DHT_SUCCESS) { - //flag that this line is replaced by DHT-value, do not simulate!! - out_result_index[i] = false; - dht_hits++; - } else if (res == DHT_READ_ERROR) { - //this line is untouched, simulation is needed - out_result_index[i] = true; - dht_miss++; - } else { - //MPI ERROR ... WHAT TO DO NOW? - //RUNNING CIRCLES WHILE SCREAMING - } + if (res == DHT_SUCCESS) { + // flag that this line is replaced by DHT-value, do not simulate!! + out_result_index[i] = false; + dht_hits++; + } else if (res == DHT_READ_ERROR) { + // this line is untouched, simulation is needed + out_result_index[i] = true; + dht_miss++; + } else { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING } + } } -void fill_dht(int length, std::vector &result_index, double *work_package, double *results, double dt) { - void *key; - void *data; - int res; - int var_count = prop_type_vector.size(); +void fill_dht(int length, std::vector &result_index, double *work_package, + double *results, double dt, t_simparams *params) { + void *key; + void *data; + int res; + int var_count = params->dht_prop_type_vector.size(); - for (int i = 0; i < length; i++) { - key = (void *)&(work_package[i * var_count]); - data = (void *)&(results[i * var_count]); + for (int i = 0; i < length; i++) { + key = (void *)&(work_package[i * var_count]); + data = (void *)&(results[i * var_count]); - if (result_index[i]) { - //If true -> was simulated, needs to be inserted into dht + if (result_index[i]) { + // If true -> was simulated, needs to be inserted into dht - //fuzz data (round, logarithm etc.) - fuzz_for_dht(var_count, key, dt); + // fuzz data (round, logarithm etc.) + fuzz_for_dht(var_count, key, dt, params); - res = DHT_write(dht_object, fuzzing_buffer, data); + res = DHT_write(dht_object, fuzzing_buffer, data); - if (res != DHT_SUCCESS) { - if (res == DHT_WRITE_SUCCESS_WITH_COLLISION) { - dht_collision++; - } else { - //MPI ERROR ... WHAT TO DO NOW? - //RUNNING CIRCLES WHILE SCREAMING - } - } + if (res != DHT_SUCCESS) { + if (res == DHT_WRITE_SUCCESS_WITH_COLLISION) { + dht_collision++; + } else { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING } + } } + } } void print_statistics() { - int res; + int res; - res = DHT_print_statistics(dht_object); + res = DHT_print_statistics(dht_object); - if (res != DHT_SUCCESS) { - //MPI ERROR ... WHAT TO DO NOW? - //RUNNING CIRCLES WHILE SCREAMING - } + if (res != DHT_SUCCESS) { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING + } } int table_to_file(char *filename) { - int res = DHT_to_file(dht_object, filename); - return res; + int res = DHT_to_file(dht_object, filename); + return res; } int file_to_table(char *filename) { - int res = DHT_from_file(dht_object, filename); - if (res != DHT_SUCCESS) - return res; + int res = DHT_from_file(dht_object, filename); + if (res != DHT_SUCCESS) + return res; - DHT_print_statistics(dht_object); + DHT_print_statistics(dht_object); - return DHT_SUCCESS; + return DHT_SUCCESS; } diff --git a/src/dht_wrapper.h b/src/dht_wrapper.h index 2d16c06dc..f6e6cb7f5 100644 --- a/src/dht_wrapper.h +++ b/src/dht_wrapper.h @@ -1,50 +1,55 @@ #pragma once +#include "DHT.h" #include "util/RRuntime.h" +#include "util/SimParams.h" +#include #include #include -#include -#include "DHT.h" using namespace std; using namespace poet; /*Functions*/ -uint64_t get_md5(int key_size, void* key); -void fuzz_for_dht(int var_count, void *key, double dt); -void check_dht(int length, std::vector &out_result_index, double *work_package, double dt); -void fill_dht(int length, std::vector &result_index, double *work_package, double *results, double dt); +uint64_t get_md5(int key_size, void *key); +void fuzz_for_dht(int var_count, void *key, double dt, t_simparams *params); +void check_dht(int length, std::vector &out_result_index, + double *work_package, double dt, t_simparams *params); +void fill_dht(int length, std::vector &result_index, double *work_package, + double *results, double dt, t_simparams *params); void print_statistics(); -int table_to_file(char* filename); -int file_to_table(char* filename); +int table_to_file(char *filename); +int file_to_table(char *filename); /*globals*/ -extern bool dht_enabled; -extern int dht_snaps; -extern std::string dht_file; -extern bool dt_differ; +//extern bool dht_enabled; +//extern int dht_snaps; +//extern std::string dht_file; +//extern bool dt_differ; -//Default DHT Size per process in Byte (defaults to 1 GiB) +// Default DHT Size per process in Byte (defaults to 1 GiB) #define DHT_SIZE_PER_PROCESS 1073741824 -//sets default dht access and distribution strategy +// sets default dht access and distribution strategy #define DHT_STRATEGY 0 // 0 -> DHT is on workers, access from workers only -// 1 -> DHT is on workers + master, access from master only !NOT IMPLEMENTED YET! +// 1 -> DHT is on workers + master, access from master only !NOT IMPLEMENTED +// YET! -#define ROUND(value,signif) (((int) (pow(10.0, (double) signif) * value)) * pow(10.0, (double) -signif)) +#define ROUND(value, signif) \ + (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) -extern int dht_strategy; -extern int dht_significant_digits; -extern std::vector dht_significant_digits_vector; -extern std::vector prop_type_vector; -extern bool dht_logarithm; -extern uint64_t dht_size_per_process; +//extern int dht_strategy; +//extern int dht_significant_digits; +//extern std::vector dht_significant_digits_vector; +//extern std::vector prop_type_vector; +//extern bool dht_logarithm; +//extern uint64_t dht_size_per_process; -//global DHT object, can be NULL if not initialized, check strategy -extern DHT* dht_object; +// global DHT object, can be NULL if not initialized, check strategy +extern DHT *dht_object; -//DHT Performance counter +// DHT Performance counter extern uint64_t dht_hits, dht_miss, dht_collision; -extern double* fuzzing_buffer; +extern double *fuzzing_buffer; extern std::vector dht_flags; diff --git a/src/kin.cpp b/src/kin.cpp index 9b2e15ada..cdddb1ad0 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -11,9 +11,10 @@ #include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" -#include "util/RRuntime.h" -#include "worker.h" #include "model/Grid.h" +#include "util/RRuntime.h" +#include "util/SimParams.h" +#include "worker.h" using namespace std; using namespace poet; @@ -100,41 +101,40 @@ int main(int argc, char *argv[]) { double sim_e_chemistry, sim_f_chemistry; argh::parser cmdl(argv); - + int dht_significant_digits; // cout << "CPP: Start Init (MPI)" << endl; + t_simparams params; + MPI_Init(&argc, &argv); - int world_size; - MPI_Comm_size(MPI_COMM_WORLD, &world_size); + MPI_Comm_size(MPI_COMM_WORLD, &(params.world_size)); - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &(params.world_rank)); /*Create custom Communicator with all processes except 0 (the master) for DHT * storage*/ // only needed if strategy == 0, but done anyway - MPI_Group group_world; - MPI_Group dht_group; + MPI_Group dht_group, group_world; MPI_Comm dht_comm; int *process_ranks; // make a list of processes in the new communicator - process_ranks = (int *)malloc(world_size * sizeof(int)); - for (int I = 1; I < world_size; I++) + process_ranks = (int *)malloc(params.world_size * sizeof(int)); + for (int I = 1; I < params.world_size; I++) process_ranks[I - 1] = I; // get the group under MPI_COMM_WORLD MPI_Comm_group(MPI_COMM_WORLD, &group_world); // create the new group - MPI_Group_incl(group_world, world_size - 1, process_ranks, &dht_group); + MPI_Group_incl(group_world, params.world_size - 1, process_ranks, &dht_group); // create the new communicator MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); free(process_ranks); // cleanup // cout << "Done"; if (cmdl[{"help", "h"}]) { - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "Todo" << endl << "See README.md for further information." << endl; } @@ -144,7 +144,7 @@ int main(int argc, char *argv[]) { /*INIT is now done separately in an R file provided here as argument!*/ if (!cmdl(2)) { - if (world_rank == 0) { + if (params.world_rank == 0) { cerr << "ERROR. Kin needs 2 positional arguments: " << endl << "1) the R script defining your simulation and" << endl << "2) the directory prefix where to save results and profiling" @@ -156,7 +156,7 @@ int main(int argc, char *argv[]) { std::list optionsError = checkOptions(cmdl); if (!optionsError.empty()) { - if (world_rank == 0) { + if (params.world_rank == 0) { cerr << "Unrecognized option(s):\n" << endl; for (auto option : optionsError) { cerr << option << endl; @@ -168,61 +168,61 @@ int main(int argc, char *argv[]) { } /*Parse DHT arguments*/ - dht_enabled = cmdl["dht"]; + params.dht_enabled = cmdl["dht"]; // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - if (dht_enabled) { - cmdl("dht-strategy", 0) >> dht_strategy; + if (params.dht_enabled) { + cmdl("dht-strategy", 0) >> params.dht_strategy; // cout << "CPP: DHT strategy is " << dht_strategy << endl; cmdl("dht-signif", 5) >> dht_significant_digits; // cout << "CPP: DHT significant digits = " << dht_significant_digits << // endl; - dht_logarithm = cmdl["dht-log"]; + params.dht_log = cmdl["dht-log"]; // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" // : "OFF" ) << endl; - cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> dht_size_per_process; + cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> params.dht_size_per_process; // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << // endl; - cmdl("dht-snaps", 0) >> dht_snaps; + cmdl("dht-snaps", 0) >> params.dht_snaps; - cmdl("dht-file") >> dht_file; + cmdl("dht-file") >> params.dht_file; } /*Parse work package size*/ - cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> work_package_size; + cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> params.wp_size; /*Parse output options*/ store_result = !cmdl["ignore-result"]; - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "CPP: Complete results storage is " << (store_result ? "ON" : "OFF") << endl; - cout << "CPP: Work Package Size: " << work_package_size << endl; - cout << "CPP: DHT is " << (dht_enabled ? "ON" : "OFF") << '\n'; + cout << "CPP: Work Package Size: " << params.wp_size << endl; + cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; - if (dht_enabled) { - cout << "CPP: DHT strategy is " << dht_strategy << endl; + if (params.dht_enabled) { + cout << "CPP: DHT strategy is " << params.dht_strategy << endl; cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " "defined) = " << dht_significant_digits << endl; cout << "CPP: DHT logarithm before rounding: " - << (dht_logarithm ? "ON" : "OFF") << endl; - cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process - << endl; - cout << "CPP: DHT save snapshots is " << dht_snaps << endl; - cout << "CPP: DHT load file is " << dht_file << endl; + << (params.dht_log ? "ON" : "OFF") << endl; + cout << "CPP: DHT size per process (Byte) = " + << params.dht_size_per_process << endl; + cout << "CPP: DHT save snapshots is " << params.dht_snaps << endl; + cout << "CPP: DHT load file is " << params.dht_file << endl; } } - cout << "CPP: R Init (RInside) on process " << world_rank << endl; + cout << "CPP: R Init (RInside) on process " << params.world_rank << endl; RRuntime R(argc, argv); // if local_rank == 0 then master else worker - R["local_rank"] = world_rank; + R["local_rank"] = params.world_rank; /*Loading Dependencies*/ std::string r_load_dependencies = "suppressMessages(library(Rmufits));" @@ -236,11 +236,10 @@ int main(int argc, char *argv[]) { R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' R.parseEvalQ("source(filesim)"); // eval the init string, ignoring any returns - std::string out_dir; - if (world_rank == + if (params.world_rank == 0) { // only rank 0 initializes goes through the whole initialization - cmdl(2) >> out_dir; // <- second positional argument - R["fileout"] = wrap(out_dir); // assign a char* (string) to 'fileout' + cmdl(2) >> params.out_dir; // <- second positional argument + R["fileout"] = wrap(params.out_dir); // assign a char* (string) to 'fileout' // Note: R::sim_init() checks if the directory already exists, // if not it makes it @@ -252,15 +251,17 @@ int main(int argc, char *argv[]) { std::string master_init_code = "mysetup <- master_init(setup=setup)"; R.parseEval(master_init_code); - dt_differ = R.parseEval("mysetup$dt_differ"); - MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + params.dt_differ = + R.parseEval("mysetup$dt_differ"); // TODO: Set in DHTWrapper + MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } else { // workers will only read the setup DataFrame defined by input file R.parseEval("mysetup <- setup"); - MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } - if (world_rank == 0) { - cout << "CPP: R init done on process with rank " << world_rank << endl; + if (params.world_rank == 0) { + cout << "CPP: R init done on process with rank " << params.world_rank + << endl; } // initialize chemistry on all processes @@ -270,63 +271,64 @@ int main(int argc, char *argv[]) { Grid grid(R); grid.init(); /* Retrieve state_C from R context for MPI buffer generation */ - //Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); + // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); /* Init Parallel helper functions */ - R["n_procs"] = world_size - 1; /* worker count */ - R["work_package_size"] = work_package_size; + R["n_procs"] = params.world_size - 1; /* worker count */ + R["work_package_size"] = params.wp_size; // Removed additional field for ID in previous versions - if (world_rank == 0) { + if (params.world_rank == 0) { mpi_buffer = (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); } else { mpi_buffer = (double *)calloc( - (work_package_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); + (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); mpi_buffer_results = - (double *)calloc(work_package_size * (grid.getCols()), sizeof(double)); + (double *)calloc(params.wp_size * (grid.getCols()), sizeof(double)); } - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "CPP: parallel init completed (buffers allocated)!" << endl; } // MDL: pass to R the DHT stuff (basically, only for storing of // simulation parameters). These 2 variables are always defined: - R["dht_enabled"] = dht_enabled; - R["dht_log"] = dht_logarithm; + R["dht_enabled"] = params.dht_enabled; + R["dht_log"] = params.dht_log; - if (dht_enabled) { + if (params.dht_enabled) { // cout << "\nCreating DHT\n"; // determine size of dht entries int dht_data_size = grid.getCols() * sizeof(double); int dht_key_size = - grid.getCols() * sizeof(double) + (dt_differ * sizeof(double)); + grid.getCols() * sizeof(double) + (params.dt_differ * sizeof(double)); // determine bucket count for preset memory usage // bucket size is key + value + 1 byte for status int dht_buckets_per_process = - dht_size_per_process / (1 + dht_data_size + dht_key_size); + params.dht_size_per_process / (1 + dht_data_size + dht_key_size); // MDL : following code moved here from worker.cpp /*Load significance vector from R setup file (or set default)*/ bool signif_vector_exists = R.parseEval("exists('signif_vector')"); if (signif_vector_exists) { - dht_significant_digits_vector = as>(R["signif_vector"]); + params.dht_signif_vector = as>(R["signif_vector"]); } else { - dht_significant_digits_vector.assign( - dht_object->key_size / sizeof(double), dht_significant_digits); + params.dht_signif_vector.assign(dht_object->key_size / sizeof(double), + dht_significant_digits); } /*Load property type vector from R setup file (or set default)*/ bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); if (prop_type_vector_exists) { - prop_type_vector = as>(R["prop_type"]); + params.dht_prop_type_vector = as>(R["prop_type"]); } else { - prop_type_vector.assign(dht_object->key_size / sizeof(double), "act"); + params.dht_prop_type_vector.assign(dht_object->key_size / sizeof(double), + "act"); } - if (world_rank == 0) { + if (params.world_rank == 0) { // print only on master, values are equal on all workes cout << "CPP: dht_data_size: " << dht_data_size << "\n"; cout << "CPP: dht_key_size: " << dht_key_size << "\n"; @@ -345,12 +347,12 @@ int main(int argc, char *argv[]) { // MDL: pass to R the DHT stuff. These variables exist // only if dht_enabled is true - R["dht_final_signif"] = dht_significant_digits_vector; - R["dht_final_proptype"] = prop_type_vector; + R["dht_final_signif"] = params.dht_signif_vector; + R["dht_final_proptype"] = params.dht_prop_type_vector; } - if (dht_strategy == 0) { - if (world_rank != 0) { + if (params.dht_strategy == 0) { + if (params.world_rank != 0) { dht_object = DHT_create(dht_comm, dht_buckets_per_process, dht_data_size, dht_key_size, get_md5); @@ -362,20 +364,20 @@ int main(int argc, char *argv[]) { dht_data_size, dht_key_size, get_md5); } - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "CPP: DHT successfully created!" << endl; } } // MDL: store all parameters - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "CPP: Calling R Function to store calling parameters" << endl; R.parseEvalQ("StoreSetup(setup=mysetup)"); } MPI_Barrier(MPI_COMM_WORLD); - if (world_rank == 0) { /* This is executed by the master */ + if (params.world_rank == 0) { /* This is executed by the master */ Rcpp::NumericVector master_send; Rcpp::NumericVector master_recv; @@ -383,7 +385,7 @@ int main(int argc, char *argv[]) { sim_a_seq = MPI_Wtime(); worker_struct *workerlist = - (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); + (worker_struct *)calloc(params.world_size - 1, sizeof(worker_struct)); int need_to_receive; MPI_Status probe_status; double *timings; @@ -394,7 +396,7 @@ int main(int argc, char *argv[]) { // a temporary send buffer double *send_buffer; send_buffer = (double *)calloc( - (work_package_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); + (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); // helper variables int iteration; @@ -440,7 +442,7 @@ int main(int argc, char *argv[]) { /*Fallback for sequential execution*/ sim_b_chemistry = MPI_Wtime(); - if (world_size == 1) { + if (params.world_size == 1) { // MDL : the transformation of values into pH and pe // takes now place in master_advection() so the // following line is unneeded @@ -473,7 +475,7 @@ int main(int argc, char *argv[]) { // R.parseEval("tmp <- // shuffle_field(RedModRphree::Act2pH(mysetup$state_T), ordered_ids)"); // Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); - + // convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); // cout << "CPP: shuffle_field() done" << endl; grid.shuffleAndExport(mpi_buffer); @@ -482,7 +484,7 @@ int main(int argc, char *argv[]) { int pkg_to_send = n_wp; int pkg_to_recv = n_wp; size_t colCount = grid.getCols(); - int free_workers = world_size - 1; + int free_workers = params.world_size - 1; double *work_pointer = mpi_buffer; sim_c_chemistry = MPI_Wtime(); @@ -524,7 +526,7 @@ int main(int argc, char *argv[]) { if (pkg_to_send > 0) { master_send_a = MPI_Wtime(); /*search for free workers and send work*/ - for (int p = 0; p < world_size - 1; p++) { + for (int p = 0; p < params.world_size - 1; p++) { if (workerlist[p].has_work == 0 && pkg_to_send > 0) /* worker is free */ { @@ -614,12 +616,13 @@ int main(int argc, char *argv[]) { cummul_workers += sim_d_chemistry - sim_c_chemistry; // convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); - //R.from_C_domain(mpi_buffer); + // R.from_C_domain(mpi_buffer); - //R["chemistry_data"] = R.getBufferDataFrame(); + // R["chemistry_data"] = R.getBufferDataFrame(); ///* unshuffle results */ - //R.parseEval("result <- unshuffle_field(chemistry_data, ordered_ids)"); + // R.parseEval("result <- unshuffle_field(chemistry_data, + // ordered_ids)"); grid.importAndUnshuffle(mpi_buffer); /* do master stuff */ @@ -644,18 +647,18 @@ int main(int argc, char *argv[]) { << endl << endl; - if (dht_enabled) { - for (int i = 1; i < world_size; i++) { + if (params.dht_enabled) { + for (int i = 1; i < params.world_size; i++) { MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_STATS, MPI_COMM_WORLD); } MPI_Barrier(MPI_COMM_WORLD); - if (dht_snaps == 2) { + if (params.dht_snaps == 2) { std::stringstream outfile; - outfile << out_dir << "/iter_" << std::setfill('0') << std::setw(3) - << iter << ".dht"; - for (int i = 1; i < world_size; i++) { + outfile << params.out_dir << "/iter_" << std::setfill('0') + << std::setw(3) << iter << ".dht"; + for (int i = 1; i < params.world_size; i++) { MPI_Send(outfile.str().c_str(), outfile.str().size(), MPI_CHAR, i, TAG_DHT_STORE, MPI_COMM_WORLD); } @@ -675,11 +678,11 @@ int main(int argc, char *argv[]) { sim_end = MPI_Wtime(); - if (dht_enabled && dht_snaps > 0) { + if (params.dht_enabled && params.dht_snaps > 0) { cout << "CPP: Master: Instruct workers to write DHT to file ..." << endl; std::string outfile; - outfile = out_dir + ".dht"; - for (int i = 1; i < world_size; i++) { + outfile = params.out_dir + ".dht"; + for (int i = 1; i < params.world_size; i++) { MPI_Send(outfile.c_str(), outfile.size(), MPI_CHAR, i, TAG_DHT_STORE, MPI_COMM_WORLD); } @@ -697,7 +700,7 @@ int main(int argc, char *argv[]) { timings = (double *)calloc(3, sizeof(double)); - if (dht_enabled) { + if (params.dht_enabled) { dht_hits = 0; dht_miss = 0; dht_collision = 0; @@ -706,7 +709,7 @@ int main(int argc, char *argv[]) { double idle_worker_tmp; - for (int p = 0; p < world_size - 1; p++) { + for (int p = 0; p < params.world_size - 1; p++) { /* ATTENTION Worker p has rank p+1 */ /* Send termination message to worker */ MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); @@ -723,7 +726,7 @@ int main(int argc, char *argv[]) { MPI_COMM_WORLD, MPI_STATUS_IGNORE); idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - if (dht_enabled) { + if (params.dht_enabled) { dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); @@ -770,7 +773,7 @@ int main(int argc, char *argv[]) { R["phreeqc_count"] = phreeqc_counts; R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - if (dht_enabled) { + if (params.dht_enabled) { R["dht_hits"] = dht_hits; R.parseEvalQ("profiling$dht_hits <- dht_hits"); R["dht_miss"] = dht_miss; @@ -786,47 +789,47 @@ int main(int argc, char *argv[]) { free(workerlist); free(timings); - if (dht_enabled) + if (params.dht_enabled) free(dht_perfs); - cout << "CPP: Done! Results are stored as R objects into <" << out_dir - << "/timings.rds>" << endl; + cout << "CPP: Done! Results are stored as R objects into <" + << params.out_dir << "/timings.rds>" << endl; /*exporting results and profiling data*/ std::string r_vis_code; r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ - if (!dht_file.empty()) { - int res = file_to_table((char *)dht_file.c_str()); + if (!params.dht_file.empty()) { + int res = file_to_table((char *)params.dht_file.c_str()); if (res != DHT_SUCCESS) { if (res == DHT_WRONG_FILE) { - if (world_rank == 2) + if (params.world_rank == 2) cerr << "CPP: Worker: Wrong File" << endl; } else { - if (world_rank == 2) + if (params.world_rank == 2) cerr << "CPP: Worker: Error in loading current state of DHT from " "file" << endl; } return EXIT_FAILURE; } else { - if (world_rank == 2) + if (params.world_rank == 2) cout << "CPP: Worker: Successfully loaded state of DHT from file " - << dht_file << endl; + << params.dht_file << endl; std::cout.flush(); } } - worker_function(R, grid); + worker_function(R, grid, ¶ms); free(mpi_buffer_results); } - cout << "CPP: finished, cleanup of process " << world_rank << endl; + cout << "CPP: finished, cleanup of process " << params.world_rank << endl; - if (dht_enabled) { + if (params.dht_enabled) { - if (dht_strategy == 0) { - if (world_rank != 0) { + if (params.dht_strategy == 0) { + if (params.world_rank != 0) { DHT_free(dht_object, NULL, NULL); } } else { @@ -837,7 +840,7 @@ int main(int argc, char *argv[]) { free(mpi_buffer); MPI_Finalize(); - if (world_rank == 0) { + if (params.world_rank == 0) { cout << "CPP: done, bye!" << endl; } diff --git a/src/util/SimParams.h b/src/util/SimParams.h new file mode 100644 index 000000000..f54d864cd --- /dev/null +++ b/src/util/SimParams.h @@ -0,0 +1,26 @@ +#ifndef SIMPARAMS_H +#define SIMPARAMS_H + +#include +#include + +typedef struct { + int world_size; + int world_rank; + + bool dht_enabled; + bool dht_log; + bool dt_differ; + int dht_snaps; + int dht_strategy; + unsigned int dht_size_per_process; + std::vector dht_signif_vector; + std::vector dht_prop_type_vector; + std::string dht_file; + + unsigned int wp_size; + + std::string out_dir; +} t_simparams; + +#endif // SIMPARAMS_H diff --git a/src/worker.cpp b/src/worker.cpp index d53fc5d5d..0bc36dfb6 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -10,7 +10,7 @@ using namespace poet; using namespace Rcpp; -void worker_function(RRuntime &R, Grid &grid) { +void worker_function(RRuntime &R, Grid &grid, t_simparams *params) { int world_rank; MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); MPI_Status probe_status; @@ -41,9 +41,9 @@ void worker_function(RRuntime &R, Grid &grid) { // dht_perf[2] -> collisions uint64_t dht_perf[3]; - if (dht_enabled) { - dht_flags.resize(work_package_size, true); // set size - dht_flags.assign(work_package_size, + if (params->dht_enabled) { + dht_flags.resize(params->wp_size, true); // set size + dht_flags.assign(params->wp_size, true); // assign all elements to true (default) dht_hits = 0; dht_miss = 0; @@ -140,7 +140,7 @@ void worker_function(RRuntime &R, Grid &grid) { ////Rcpp::DataFrame buffer = R.parseEval("tmp2"); // R.setBufferDataFrame("skeleton"); - if (dht_enabled) { + if (params->dht_enabled) { // DEBUG // cout << "RANK " << world_rank << " start checking DHT\n"; @@ -152,7 +152,7 @@ void worker_function(RRuntime &R, Grid &grid) { } dht_get_start = MPI_Wtime(); - check_dht(local_work_package_size, dht_flags, mpi_buffer, dt); + check_dht(local_work_package_size, dht_flags, mpi_buffer, dt, params); dht_get_end = MPI_Wtime(); // DEBUG @@ -162,7 +162,6 @@ void worker_function(RRuntime &R, Grid &grid) { // R.parseEvalQ("print(head(dht_flags))"); } - /* work */ // R.from_C_domain(mpi_buffer); ////convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); @@ -173,9 +172,9 @@ void worker_function(RRuntime &R, Grid &grid) { // R.parseEvalQ("print(head(work_package_full))"); // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); - grid.importWP(mpi_buffer, work_package_size); + grid.importWP(mpi_buffer, params->wp_size); - if (dht_enabled) { + if (params->dht_enabled) { R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); } else { R.parseEvalQ("work_package <- work_package_full"); @@ -215,7 +214,7 @@ void worker_function(RRuntime &R, Grid &grid) { // cout << "Work-Package is empty, skipping phreeqc!" << endl; } - if (dht_enabled) { + if (params->dht_enabled) { R.parseEvalQ("result_full <- work_package_full"); if (nrows > 0) R.parseEvalQ("result_full[dht_flags,] <- result"); @@ -234,10 +233,10 @@ void worker_function(RRuntime &R, Grid &grid) { MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, &send_req); - if (dht_enabled) { + if (params->dht_enabled) { dht_fill_start = MPI_Wtime(); fill_dht(local_work_package_size, dht_flags, mpi_buffer, - mpi_buffer_results, dt); + mpi_buffer_results, dt, params); dht_fill_end = MPI_Wtime(); timing[1] += dht_get_end - dht_get_start; @@ -248,7 +247,6 @@ void worker_function(RRuntime &R, Grid &grid) { MPI_Wait(&send_req, MPI_STATUS_IGNORE); - } else if (probe_status.MPI_TAG == TAG_FINISH) { /* recv and die */ /* before death, submit profiling/timings to master*/ @@ -261,7 +259,7 @@ void worker_function(RRuntime &R, Grid &grid) { MPI_Send(&phreeqc_count, 1, MPI_INT, 0, TAG_TIMING, MPI_COMM_WORLD); MPI_Send(&cummul_idle, 1, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); - if (dht_enabled) { + if (params->dht_enabled) { // dht_perf dht_perf[0] = dht_hits; dht_perf[1] = dht_miss; diff --git a/src/worker.h b/src/worker.h index 349333811..435cfef7d 100644 --- a/src/worker.h +++ b/src/worker.h @@ -1,11 +1,12 @@ #pragma once +#include "util/SimParams.h" #include "util/RRuntime.h" #include "model/Grid.h" using namespace std; using namespace poet; /*Functions*/ -void worker_function(RRuntime &R, Grid &grid); +void worker_function(RRuntime &R, Grid &grid, t_simparams *params); /*Globals*/ From ea15c1e23d5b4cdb3b4d81f016e3c2dbd17762dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 21 Dec 2020 12:32:33 +0100 Subject: [PATCH 06/50] removed RRuntime and grid as a function parameter --- src/kin.cpp | 4 +++- src/util/SimParams.h | 4 ++++ src/worker.cpp | 4 +++- src/worker.h | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/kin.cpp b/src/kin.cpp index cdddb1ad0..8cdfa47f7 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -220,6 +220,7 @@ int main(int argc, char *argv[]) { cout << "CPP: R Init (RInside) on process " << params.world_rank << endl; RRuntime R(argc, argv); + params.R = &R; // if local_rank == 0 then master else worker R["local_rank"] = params.world_rank; @@ -269,6 +270,7 @@ int main(int argc, char *argv[]) { R.parseEval(init_chemistry_code); Grid grid(R); + params.grid = &grid; grid.init(); /* Retrieve state_C from R context for MPI buffer generation */ // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); @@ -820,7 +822,7 @@ int main(int argc, char *argv[]) { std::cout.flush(); } } - worker_function(R, grid, ¶ms); + worker_function(¶ms); free(mpi_buffer_results); } diff --git a/src/util/SimParams.h b/src/util/SimParams.h index f54d864cd..948dee481 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -3,6 +3,7 @@ #include #include +#include "RRuntime.h" typedef struct { int world_size; @@ -21,6 +22,9 @@ typedef struct { unsigned int wp_size; std::string out_dir; + + void* R; + void* grid; } t_simparams; #endif // SIMPARAMS_H diff --git a/src/worker.cpp b/src/worker.cpp index 0bc36dfb6..60643530d 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -10,7 +10,9 @@ using namespace poet; using namespace Rcpp; -void worker_function(RRuntime &R, Grid &grid, t_simparams *params) { +void worker_function(t_simparams *params) { + RRuntime R = *(static_cast(params->R)); + Grid grid = *(static_cast(params->grid)); int world_rank; MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); MPI_Status probe_status; diff --git a/src/worker.h b/src/worker.h index 435cfef7d..076ede6e2 100644 --- a/src/worker.h +++ b/src/worker.h @@ -6,7 +6,7 @@ using namespace std; using namespace poet; /*Functions*/ -void worker_function(RRuntime &R, Grid &grid, t_simparams *params); +void worker_function(t_simparams *params); /*Globals*/ From 4eb4702cff928517a59c7d8f44c5a67fb825ce18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 5 Jan 2021 14:39:56 +0100 Subject: [PATCH 07/50] updated gitignore --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index 801010414..a1bcba0ce 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,12 @@ vignettes/*.pdf # R package: bookdown caching files /*_files/ +### vscode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + # End of https://www.toptal.com/developers/gitignore/api/c,c++,r,cmake From 1d9c1d5710659106660181a5b9f4e19b7f2346a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 5 Jan 2021 18:07:27 +0100 Subject: [PATCH 08/50] Substitute ChemSim --- .gitignore | 8 +- src/CMakeLists.txt | 2 +- src/DHT/DHT.cpp | 423 +++++++++++++++++++++++++++++++++++ src/DHT/DHT.h | 112 ++++++++++ src/DHT/DHT_Wrapper.cpp | 165 ++++++++++++++ src/DHT/DHT_Wrapper.h | 56 +++++ src/kin.cpp | 472 +++++++++------------------------------ src/model/ChemMaster.cpp | 217 ++++++++++++++++++ src/model/ChemSim.cpp | 23 ++ src/model/ChemSim.h | 115 ++++++++++ src/model/ChemWorker.cpp | 325 +++++++++++++++++++++++++++ 11 files changed, 1552 insertions(+), 366 deletions(-) create mode 100644 src/DHT/DHT.cpp create mode 100644 src/DHT/DHT.h create mode 100644 src/DHT/DHT_Wrapper.cpp create mode 100644 src/DHT/DHT_Wrapper.h create mode 100644 src/model/ChemMaster.cpp create mode 100644 src/model/ChemSim.cpp create mode 100644 src/model/ChemSim.h create mode 100644 src/model/ChemWorker.cpp diff --git a/.gitignore b/.gitignore index a1bcba0ce..0889d14d1 100644 --- a/.gitignore +++ b/.gitignore @@ -137,12 +137,14 @@ vignettes/*.pdf # R package: bookdown caching files /*_files/ -### vscode ### +### VisualStudioCode ### .vscode/* -!.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json -!.vscode/extensions.json *.code-workspace +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide # End of https://www.toptal.com/developers/gitignore/api/c,c++,r,cmake diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4eb121d63..f0f865eef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,7 +89,7 @@ include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(POET_Libs STATIC util/RRuntime.cpp dht_wrapper.cpp worker.cpp DHT.cpp model/Grid.cpp) +add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp) target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) diff --git a/src/DHT/DHT.cpp b/src/DHT/DHT.cpp new file mode 100644 index 000000000..3a988cdad --- /dev/null +++ b/src/DHT/DHT.cpp @@ -0,0 +1,423 @@ +#include "DHT.h" +#include +#include +#include +#include + +/* + * determining destination rank and index by hash of key + * + * return values by reference + */ +static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int *dest_rank, unsigned int *index, unsigned int index_count) { + uint64_t tmp; + int char_hop = 9-index_count; + unsigned int i; + for (i = 0; i < index_count ; i++) { + tmp = 0; + memcpy(&tmp,(unsigned char *)&hash+i, char_hop); + index[i] = (unsigned int) (tmp % table_size); + } + *dest_rank = (unsigned int) (hash % comm_size); +} + +/** + * set write flag to 1 + */ +static void set_flag(char* flag_byte) { + *flag_byte = 0; + *flag_byte |= (1 << 0); +} + +/** + * return 1 if write flag is set + * else 0 + */ +static int read_flag(char flag_byte) { + if ((flag_byte & 0x01) == 0x01) { + return 1; + } else return 0; +} + +/* + * allocating memory for DHT object and buckets. + * creating MPI window for OSC + * filling DHT object with passed by parameters, window, 2 counters for R/W errors and 2 pointers with allocated memory for further use + * return DHT object + */ +DHT* DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, uint64_t(*hash_func) (int, void*)) { + DHT *object; + MPI_Win window; + void* mem_alloc; + int comm_size, tmp; + + tmp = (int) ceil(log2(size)); + if (tmp%8 != 0) tmp = tmp + (8-(tmp%8)); + + object = (DHT*) malloc(sizeof(DHT)); + if (object == NULL) return NULL; + + //every memory allocation has 1 additional byte for flags etc. + if (MPI_Alloc_mem(size * (1 + data_size + key_size), MPI_INFO_NULL, &mem_alloc) != 0) return NULL; + if (MPI_Comm_size(comm, &comm_size) != 0) return NULL; + + memset(mem_alloc, '\0', size * (1 + data_size + key_size)); + + if (MPI_Win_create(mem_alloc, size * (1 + data_size + key_size), (1 + data_size + key_size), MPI_INFO_NULL, comm, &window) != 0) return NULL; + + object->data_size = data_size; + object->key_size = key_size; + object->table_size = size; + object->window = window; + object->hash_func = hash_func; + object->comm_size = comm_size; + object->communicator = comm; + object->read_misses = 0; + object->collisions = 0; + object->recv_entry = malloc(1 + data_size + key_size); + object->send_entry = malloc(1 + data_size + key_size); + object->index_count = 9-(tmp/8); + object->index = (unsigned int*) malloc((9-(tmp/8))*sizeof(int)); + object->mem_alloc = mem_alloc; + + DHT_stats *stats; + + stats = (DHT_stats*) malloc(sizeof(DHT_stats)); + if (stats == NULL) return NULL; + + object->stats = stats; + object->stats->writes_local = (int*) calloc(comm_size, sizeof(int)); + object->stats->old_writes = 0; + object->stats->read_misses = 0; + object->stats->collisions = 0; + object->stats->w_access = 0; + object->stats->r_access = 0; + + return object; +} + +/* + * puts passed by data with key to DHT + * + * returning DHT_MPI_ERROR = -1 if MPI error occurred + * else DHT_SUCCESS = 0 if success + */ +int DHT_write(DHT *table, void* send_key, void* send_data) { + unsigned int dest_rank, i; + int result = DHT_SUCCESS; + + table->stats->w_access++; + + //determine destination rank and index by hash of key + determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); + + //concatenating key with data to write entry to DHT + set_flag((char *) table->send_entry); + memcpy((char *) table->send_entry + 1, (char *) send_key, table->key_size); + memcpy((char *) table->send_entry + table->key_size + 1, (char *) send_data, table->data_size); + + //locking window of target rank with exclusive lock + if (MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dest_rank, 0, table->window) != 0) + return DHT_MPI_ERROR; + for (i = 0; i < table->index_count; i++) + { + if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; + if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + + //increment collision counter if receiving key doesn't match sending key + //,entry has write flag + last index is reached + + if (read_flag(*(char *)table->recv_entry)) { + if (memcmp(send_key, (char *) table->recv_entry + 1, table->key_size) != 0) { + if (i == (table->index_count)-1) { + table->collisions += 1; + table->stats->collisions += 1; + result = DHT_WRITE_SUCCESS_WITH_COLLISION; + break; + } + } else break; + } else { + table->stats->writes_local[dest_rank]++; + break; + } + } + + //put data to DHT + if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; + //unlock window of target rank + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + + return result; +} + +/* + * gets data from the DHT by key + * + * return DHT_SUCCESS = 0 if success + * DHT_MPI_ERROR = -1 if MPI error occurred + * DHT_READ_ERROR = -2 if receiving key doesn't match sending key + */ +int DHT_read(DHT *table, void* send_key, void* destination) { + unsigned int dest_rank, i; + + table->stats->r_access++; + + //determine destination rank and index by hash of key + determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); + + //locking window of target rank with shared lock + if (MPI_Win_lock(MPI_LOCK_SHARED, dest_rank, 0, table->window) != 0) return DHT_MPI_ERROR; + //receive data + for (i = 0; i < table->index_count; i++) { + if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; + if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + + //increment read error counter if write flag isn't set or key doesn't match passed by key + last index reached + //else copy data to dereference of passed by destination pointer + + if ((read_flag(*(char *) table->recv_entry)) == 0) { + table->read_misses += 1; + table->stats->read_misses += 1; + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + return DHT_READ_ERROR; + } + + if (memcmp(((char*)table->recv_entry) + 1, send_key, table->key_size) != 0) { + if (i == (table->index_count)-1) { + table->read_misses += 1; + table->stats->read_misses += 1; + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + return DHT_READ_ERROR; + } + } else break; + } + + //unlock window of target rank + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + + memcpy((char *) destination, (char *) table->recv_entry + table->key_size + 1, table->data_size); + + return DHT_SUCCESS; +} + +int DHT_to_file(DHT* table, const char* filename) { + //open file + MPI_File file; + if (MPI_File_open(table->communicator, filename, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; + + int rank; + MPI_Comm_rank(table->communicator, &rank); + + //write header (key_size and data_size) + if (rank == 0) { + if (MPI_File_write(file, &table->key_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; + if (MPI_File_write(file, &table->data_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; + } + + if (MPI_File_seek_shared(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; + + char* ptr; + int bucket_size = table->key_size + table->data_size + 1; + + //iterate over local memory + for (unsigned int i = 0; i < table->table_size; i++) { + ptr = (char *) table->mem_alloc + (i * bucket_size); + //if bucket has been written to (checked by written_flag)... + if (read_flag(*ptr)) { + //write key and data to file + if (MPI_File_write_shared(file, ptr + 1, bucket_size - 1, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; + } + } + //close file + if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; + + return DHT_SUCCESS; +} + +int DHT_from_file(DHT* table, const char* filename) { + MPI_File file; + MPI_Offset f_size; + int e_size, m_size, cur_pos, rank, offset; + char* buffer; + void* key; + void* data; + + if (MPI_File_open(table->communicator, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; + + if (MPI_File_get_size(file, &f_size) != 0) return DHT_FILE_IO_ERROR; + + MPI_Comm_rank(table->communicator, &rank); + + e_size = table->key_size + table->data_size; + m_size = e_size > DHT_HEADER_SIZE ? e_size : DHT_HEADER_SIZE; + buffer = (char *) malloc(m_size); + + if (MPI_File_read(file, buffer, DHT_HEADER_SIZE, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; + + if (*(int *) buffer != table->key_size) return DHT_WRONG_FILE; + if (*(int *) (buffer + 4) != table->data_size) return DHT_WRONG_FILE; + + offset = e_size*table->comm_size; + + if (MPI_File_seek(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; + cur_pos = DHT_HEADER_SIZE + (rank * e_size); + + while(cur_pos < f_size) { + if (MPI_File_seek(file, cur_pos, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; + MPI_Offset tmp; + MPI_File_get_position(file, &tmp); + if (MPI_File_read(file, buffer, e_size, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; + key = buffer; + data = (buffer+table->key_size); + if (DHT_write(table, key, data) == DHT_MPI_ERROR) return DHT_MPI_ERROR; + + cur_pos += offset; + } + + free (buffer); + if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; + + return DHT_SUCCESS; +} + +/* + * frees up memory and accumulate counter + */ +int DHT_free(DHT* table, int* collision_counter, int* readerror_counter) { + int buf; + + if (collision_counter != NULL) { + buf = 0; + if (MPI_Reduce(&table->collisions, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + *collision_counter = buf; + } + if (readerror_counter != NULL) { + buf = 0; + if (MPI_Reduce(&table->read_misses, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + *readerror_counter = buf; + } + if (MPI_Win_free(&(table->window)) != 0) return DHT_MPI_ERROR; + if (MPI_Free_mem(table->mem_alloc) != 0) return DHT_MPI_ERROR; + free(table->recv_entry); + free(table->send_entry); + free(table->index); + + free(table->stats->writes_local); + free(table->stats); + + free(table); + + return DHT_SUCCESS; +} + +/* + * prints a table with statistics about current use of DHT + * for each participating process and summed up results containing: + * 1. occupied buckets (in respect to the memory of this process) + * 2. free buckets (in respect to the memory of this process) + * 3. calls of DHT_write (w_access) + * 4. calls of DHT_read (r_access) + * 5. read misses (see DHT_READ_ERROR) + * 6. collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) + * 3-6 will reset with every call of this function + * finally the amount of new written entries is printed out (in relation to last call of this funtion) + */ +int DHT_print_statistics(DHT *table) { + int *written_buckets; + int *read_misses, sum_read_misses; + int *collisions, sum_collisions; + int sum_w_access, sum_r_access, *w_access, *r_access; + int rank; + + MPI_Comm_rank(table->communicator, &rank); + + //disable possible warning of unitialized variable, which is not the case + //since we're using MPI_Gather to obtain all values only on rank 0 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" + + //obtaining all values from all processes in the communicator + if (rank == 0) read_misses = (int*) malloc(table->comm_size*sizeof(int)); + if (MPI_Gather(&table->stats->read_misses, 1, MPI_INT, read_misses, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->read_misses, &sum_read_misses, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + table->stats->read_misses = 0; + + if (rank == 0) collisions = (int*) malloc(table->comm_size*sizeof(int)); + if (MPI_Gather(&table->stats->collisions, 1, MPI_INT, collisions, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->collisions, &sum_collisions, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + table->stats->collisions = 0; + + if (rank == 0) w_access = (int*) malloc(table->comm_size*sizeof(int)); + if (MPI_Gather(&table->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + table->stats->w_access = 0; + + if (rank == 0) r_access = (int*) malloc(table->comm_size*sizeof(int)); + if (MPI_Gather(&table->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + table->stats->r_access = 0; + + if (rank == 0) written_buckets = (int*) calloc(table->comm_size, sizeof(int)); + if (MPI_Reduce(table->stats->writes_local, written_buckets, table->comm_size, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + + if (rank == 0) { //only process with rank 0 will print out results as a table + int sum_written_buckets = 0; + + for (int i=0; i < table->comm_size; i++) { + sum_written_buckets += written_buckets[i]; + } + + int members = 7; + int padsize = (members*12)+1; + char pad[padsize+1]; + + memset(pad, '-', padsize*sizeof(char)); + pad[padsize]= '\0'; + printf("\n"); + printf("%-35s||resets with every call of this function\n", " "); + printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s\n", + "rank", + "occupied", + "free", + "w_access", + "r_access", + "read misses", + "collisions"); + printf("%s\n", pad); + for (int i = 0; i < table->comm_size; i++) { + printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", + i, + written_buckets[i], + table->table_size-written_buckets[i], + w_access[i], + r_access[i], + read_misses[i], + collisions[i]); + } + printf("%s\n", pad); + printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", + "sum", + sum_written_buckets, + (table->table_size*table->comm_size)-sum_written_buckets, + sum_w_access, + sum_r_access, + sum_read_misses, + sum_collisions); + + printf("%s\n", pad); + printf("%s %d\n", + "new entries:", + sum_written_buckets - table->stats->old_writes); + + printf("\n"); + fflush(stdout); + + table->stats->old_writes = sum_written_buckets; + } + + //enable warning again + #pragma GCC diagnostic pop + + MPI_Barrier(table->communicator); + return DHT_SUCCESS; +} \ No newline at end of file diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h new file mode 100644 index 000000000..308281650 --- /dev/null +++ b/src/DHT/DHT.h @@ -0,0 +1,112 @@ +/* + * File: DHT.h + * Author: max luebke + * + * Created on 16. November 2017, 09:14 + */ + +#ifndef DHT_H +#define DHT_H + +#include +#include + +#define DHT_MPI_ERROR -1 +#define DHT_READ_ERROR -2 +#define DHT_SUCCESS 0 +#define DHT_WRITE_SUCCESS_WITH_COLLISION 1 + +#define DHT_WRONG_FILE 11 +#define DHT_FILE_IO_ERROR 12 +#define DHT_FILE_READ_ERROR 13 +#define DHT_FILE_WRITE_ERROR 14 + +#define DHT_HEADER_SIZE 8 + +typedef struct {; + int *writes_local, old_writes; + int read_misses, collisions; + int w_access, r_access; +} DHT_stats; + +typedef struct { + MPI_Win window; + int data_size; + int key_size; + unsigned int table_size; + MPI_Comm communicator; + int comm_size; + uint64_t(*hash_func) (int, void*); + void* recv_entry; + void* send_entry; + void* mem_alloc; + int read_misses; + int collisions; + unsigned int *index; + unsigned int index_count; + DHT_stats *stats; +} DHT; + + + +/* + * parameters: + * MPI_Comm comm - communicator of processes that are holding the DHT + * int size_per_process - number of buckets each process will create + * int data_size - size of data in bytes + * int key_size - size of key in bytes + * *hash_func - pointer to hashfunction + * + * return: + * NULL if error during initialization + * DHT* if success + */ +extern DHT* DHT_create(MPI_Comm comm, unsigned int size_per_process, int data_size, int key_size, uint64_t(*hash_func)(int, void*)); + +/* + * parameters: + * DHT *table - DHT_object created by DHT_create + * void* data - pointer to data + * void* - pointer to key + * + * return: + * error value (see above) + */ +extern int DHT_write(DHT *table, void* key, void* data); + +/* + * parameters: + * DHT *table - DHT_object created by DHT_create + * void* key - pointer to key + * void* destination - pointer which will hold the resulting data from DHT + * + * return: + * error value (see above) + */ +extern int DHT_read(DHT *table, void* key, void* destination); + +extern int DHT_to_file(DHT *table, const char* filename); + +extern int DHT_from_file(DHT *table, const char* filename); + +/* + * parameters: + * DHT *table - DHT_object created by DHT_create + * int* collision_counter - pointer which will hold the total count of collisions + * int* readerror_counter - pointer which will hold the total count of read errors + * + * return: + * error value (see above) + */ +extern int DHT_free(DHT *table, int* collision_counter, int* readerror_counter); + +/* + * parameters: + * DHT *table - DHT_object created by DHT_create + * + * return: + * error value (DHT_SUCCESS or DHT_MPI_ERROR) + */ +extern int DHT_print_statistics(DHT *table); + +#endif /* DHT_H */ \ No newline at end of file diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp new file mode 100644 index 000000000..4cdc77a88 --- /dev/null +++ b/src/DHT/DHT_Wrapper.cpp @@ -0,0 +1,165 @@ +#include "DHT_Wrapper.h" +#include "DHT.h" +#include +#include + +using namespace poet; +using namespace std; +uint64_t get_md5(int key_size, void *key) { + MD5_CTX ctx; + unsigned char sum[MD5_DIGEST_LENGTH]; + uint64_t retval, *v1, *v2; + + MD5_Init(&ctx); + MD5_Update(&ctx, key, key_size); + MD5_Final(sum, &ctx); + + v1 = (uint64_t *)&sum[0]; + v2 = (uint64_t *)&sum[8]; + retval = *v1 ^ *v2; + + return retval; +} + +DHT_Wrapper::DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, + int buckets_per_process, int data_size, int key_size) { + dht_object = + DHT_create(dht_comm, buckets_per_process, data_size, key_size, &get_md5); + fuzzing_buffer = (double *)malloc(key_size); + + this->dt_differ = params->dt_differ; + this->dht_log = params->dht_log; + this->dht_signif_vector = params->dht_signif_vector; + this->dht_prop_type_vector = params->dht_prop_type_vector; +} + +DHT_Wrapper::~DHT_Wrapper() { + DHT_free(dht_object, NULL, NULL); + free(fuzzing_buffer); +} + +void DHT_Wrapper::checkDHT(int length, std::vector &out_result_index, + double *work_package, double dt) { + void *key; + int res; + int var_count = dht_prop_type_vector.size(); + for (int i = 0; i < length; i++) { + key = (void *)&(work_package[i * var_count]); + + // fuzz data (round, logarithm etc.) + fuzzForDHT(var_count, key, dt); + + // overwrite input with data from DHT, IF value is found in DHT + res = DHT_read(dht_object, fuzzing_buffer, key); + + if (res == DHT_SUCCESS) { + // flag that this line is replaced by DHT-value, do not simulate!! + out_result_index[i] = false; + dht_hits++; + } else if (res == DHT_READ_ERROR) { + // this line is untouched, simulation is needed + out_result_index[i] = true; + dht_miss++; + } else { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING + } + } +} + +void DHT_Wrapper::fillDHT(int length, std::vector &result_index, + double *work_package, double *results, double dt) { + void *key; + void *data; + int res; + int var_count = dht_prop_type_vector.size(); + + for (int i = 0; i < length; i++) { + key = (void *)&(work_package[i * var_count]); + data = (void *)&(results[i * var_count]); + + if (result_index[i]) { + // If true -> was simulated, needs to be inserted into dht + + // fuzz data (round, logarithm etc.) + fuzzForDHT(var_count, key, dt); + + res = DHT_write(dht_object, fuzzing_buffer, data); + + if (res != DHT_SUCCESS) { + if (res == DHT_WRITE_SUCCESS_WITH_COLLISION) { + dht_evictions++; + } else { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING + } + } + } + } +} + +int DHT_Wrapper::tableToFile(const char *filename) { + int res = DHT_to_file(dht_object, filename); + return res; +} + +int DHT_Wrapper::fileToTable(const char *filename) { + int res = DHT_from_file(dht_object, filename); + if (res != DHT_SUCCESS) + return res; + + DHT_print_statistics(dht_object); + + return DHT_SUCCESS; +} + +void DHT_Wrapper::printStatistics() { + int res; + + res = DHT_print_statistics(dht_object); + + if (res != DHT_SUCCESS) { + // MPI ERROR ... WHAT TO DO NOW? + // RUNNING CIRCLES WHILE SCREAMING + } +} + +uint64_t DHT_Wrapper::getHits() { return this->dht_hits; } + +uint64_t DHT_Wrapper::getMisses() { return this->dht_miss; } + +uint64_t DHT_Wrapper::getEvictions() { return this->dht_evictions; } + +void DHT_Wrapper::fuzzForDHT(int var_count, void *key, double dt) { + unsigned int i = 0; + // introduce fuzzing to allow more hits in DHT + for (i = 0; i < (unsigned int)var_count; i++) { + if (dht_prop_type_vector[i] == "act") { + // with log10 + if (dht_log) { + if (((double *)key)[i] < 0) + cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value in " + "key!" + << endl; + else if (((double *)key)[i] == 0) + fuzzing_buffer[i] = 0; + else + fuzzing_buffer[i] = + ROUND(-(std::log10(((double *)key)[i])), dht_signif_vector[i]); + } else { + // without log10 + fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_signif_vector[i]); + } + } else if (dht_prop_type_vector[i] == "logact") { + fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_signif_vector[i]); + } else if (dht_prop_type_vector[i] == "ignore") { + fuzzing_buffer[i] = 0; + } else { + cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Probably wrong " + "prop_type!" + << endl; + } + } + if (dt_differ) + fuzzing_buffer[var_count] = dt; +} diff --git a/src/DHT/DHT_Wrapper.h b/src/DHT/DHT_Wrapper.h new file mode 100644 index 000000000..7c5218af2 --- /dev/null +++ b/src/DHT/DHT_Wrapper.h @@ -0,0 +1,56 @@ +#ifndef DHT_WRAPPER_H +#define DHT_WRAPPER_H + +#include + +#include +#include + +#include "../util/SimParams.h" +#include "DHT.h" + +#define ROUND(value, signif) \ + (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) + +uint64_t get_md5(int key_size, void *key); + +namespace poet { +class DHT_Wrapper { + public: + DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, int buckets_per_process, + int data_size, int key_size); + ~DHT_Wrapper(); + + void checkDHT(int length, std::vector &out_result_index, + double *work_package, double dt); + void fillDHT(int length, std::vector &result_index, + double *work_package, double *results, double dt); + + int tableToFile(const char *filename); + int fileToTable(const char *filename); + + void printStatistics(); + + uint64_t getHits(); + uint64_t getMisses(); + uint64_t getEvictions(); + + private: + void fuzzForDHT(int var_count, void *key, double dt); + + DHT *dht_object; + + uint64_t dht_hits = 0; + uint64_t dht_miss = 0; + uint64_t dht_evictions = 0; + + double *fuzzing_buffer; + + bool dt_differ; + bool dht_log; + std::vector dht_signif_vector; + std::vector dht_prop_type_vector; +}; +} // namespace poet + +#endif // DHT_WRAPPER_H diff --git a/src/kin.cpp b/src/kin.cpp index 8cdfa47f7..abfb6023d 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -1,20 +1,22 @@ +#include +#include // mpi header file + #include #include #include #include -#include - -#include // mpi header file - -#include "DHT.h" // MPI-DHT Implementation -#include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced -#include "dht_wrapper.h" -#include "global_buffer.h" +// #include "DHT.h" // MPI-DHT Implementation +#include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced +// #include "dht_wrapper.h" +// #include "global_buffer.h" +#include "model/ChemSim.h" #include "model/Grid.h" #include "util/RRuntime.h" #include "util/SimParams.h" -#include "worker.h" +// #include "worker.h" + +#define DHT_SIZE_PER_PROCESS 1073741824 using namespace std; using namespace poet; @@ -59,8 +61,7 @@ std::list checkOptions(argh::parser cmdl) { std::set plist = paramList(); for (auto &flag : cmdl.flags()) { - if (!(flist.find(flag) != flist.end())) - retList.push_back(flag); + if (!(flist.find(flag) != flist.end())) retList.push_back(flag); } for (auto ¶m : cmdl.params()) { @@ -85,7 +86,7 @@ int main(int argc, char *argv[]) { double cummul_workers = 0.f; double cummul_chemistry_master = 0.f; - double cummul_master_seq_pre_loop = 0.f; + double cummul_master_seq = 0.f; double cummul_master_seq_loop = 0.f; double master_idle = 0.f; @@ -121,8 +122,7 @@ int main(int argc, char *argv[]) { // make a list of processes in the new communicator process_ranks = (int *)malloc(params.world_size * sizeof(int)); - for (int I = 1; I < params.world_size; I++) - process_ranks[I - 1] = I; + for (int I = 1; I < params.world_size; I++) process_ranks[I - 1] = I; // get the group under MPI_COMM_WORLD MPI_Comm_group(MPI_COMM_WORLD, &group_world); @@ -130,7 +130,7 @@ int main(int argc, char *argv[]) { MPI_Group_incl(group_world, params.world_size - 1, process_ranks, &dht_group); // create the new communicator MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); - free(process_ranks); // cleanup + free(process_ranks); // cleanup // cout << "Done"; if (cmdl[{"help", "h"}]) { @@ -226,21 +226,24 @@ int main(int argc, char *argv[]) { R["local_rank"] = params.world_rank; /*Loading Dependencies*/ - std::string r_load_dependencies = "suppressMessages(library(Rmufits));" - "suppressMessages(library(RedModRphree));" - "source('kin_r_library.R');" - "source('parallel_r_library.R');"; + std::string r_load_dependencies = + "suppressMessages(library(Rmufits));" + "suppressMessages(library(RedModRphree));" + "source('kin_r_library.R');" + "source('parallel_r_library.R');"; R.parseEvalQ(r_load_dependencies); std::string filesim; - cmdl(1) >> filesim; // <- first positional argument - R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' - R.parseEvalQ("source(filesim)"); // eval the init string, ignoring any returns + cmdl(1) >> filesim; // <- first positional argument + R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' + R.parseEvalQ( + "source(filesim)"); // eval the init string, ignoring any returns if (params.world_rank == - 0) { // only rank 0 initializes goes through the whole initialization - cmdl(2) >> params.out_dir; // <- second positional argument - R["fileout"] = wrap(params.out_dir); // assign a char* (string) to 'fileout' + 0) { // only rank 0 initializes goes through the whole initialization + cmdl(2) >> params.out_dir; // <- second positional argument + R["fileout"] = + wrap(params.out_dir); // assign a char* (string) to 'fileout' // Note: R::sim_init() checks if the directory already exists, // if not it makes it @@ -253,9 +256,9 @@ int main(int argc, char *argv[]) { R.parseEval(master_init_code); params.dt_differ = - R.parseEval("mysetup$dt_differ"); // TODO: Set in DHTWrapper + R.parseEval("mysetup$dt_differ"); // TODO: Set in DHTWrapper MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); - } else { // workers will only read the setup DataFrame defined by input file + } else { // workers will only read the setup DataFrame defined by input file R.parseEval("mysetup <- setup"); MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } @@ -280,15 +283,15 @@ int main(int argc, char *argv[]) { R["work_package_size"] = params.wp_size; // Removed additional field for ID in previous versions - if (params.world_rank == 0) { - mpi_buffer = - (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); - } else { - mpi_buffer = (double *)calloc( - (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); - mpi_buffer_results = - (double *)calloc(params.wp_size * (grid.getCols()), sizeof(double)); - } + // if (params.world_rank == 0) { + // mpi_buffer = + // (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); + // } else { + // mpi_buffer = (double *)calloc( + // (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); + // mpi_buffer_results = + // (double *)calloc(params.wp_size * (grid.getCols()), sizeof(double)); + // } if (params.world_rank == 0) { cout << "CPP: parallel init completed (buffers allocated)!" << endl; @@ -302,14 +305,14 @@ int main(int argc, char *argv[]) { if (params.dht_enabled) { // cout << "\nCreating DHT\n"; // determine size of dht entries - int dht_data_size = grid.getCols() * sizeof(double); - int dht_key_size = - grid.getCols() * sizeof(double) + (params.dt_differ * sizeof(double)); + // int dht_data_size = grid.getCols() * sizeof(double); + // int dht_key_size = + // grid.getCols() * sizeof(double) + (params.dt_differ * sizeof(double)); - // determine bucket count for preset memory usage - // bucket size is key + value + 1 byte for status - int dht_buckets_per_process = - params.dht_size_per_process / (1 + dht_data_size + dht_key_size); + // // // determine bucket count for preset memory usage + // // // bucket size is key + value + 1 byte for status + // int dht_buckets_per_process = + // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); // MDL : following code moved here from worker.cpp /*Load significance vector from R setup file (or set default)*/ @@ -317,8 +320,8 @@ int main(int argc, char *argv[]) { if (signif_vector_exists) { params.dht_signif_vector = as>(R["signif_vector"]); } else { - params.dht_signif_vector.assign(dht_object->key_size / sizeof(double), - dht_significant_digits); + // params.dht_signif_vector.assign(dht_object->key_size / sizeof(double), + // dht_significant_digits); } /*Load property type vector from R setup file (or set default)*/ @@ -326,22 +329,24 @@ int main(int argc, char *argv[]) { if (prop_type_vector_exists) { params.dht_prop_type_vector = as>(R["prop_type"]); } else { - params.dht_prop_type_vector.assign(dht_object->key_size / sizeof(double), - "act"); + // params.dht_prop_type_vector.assign(dht_object->key_size / + // sizeof(double), + // "act"); } if (params.world_rank == 0) { - // print only on master, values are equal on all workes - cout << "CPP: dht_data_size: " << dht_data_size << "\n"; - cout << "CPP: dht_key_size: " << dht_key_size << "\n"; - cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process - << endl; + // // print only on master, values are equal on all workes + // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; + // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; + // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process + // << endl; // MDL: new output on signif_vector and prop_type if (signif_vector_exists) { cout << "CPP: using problem-specific rounding digits: " << endl; - R.parseEval("print(data.frame(prop=prop, type=prop_type, " - "digits=signif_vector))"); + R.parseEval( + "print(data.frame(prop=prop, type=prop_type, " + "digits=signif_vector))"); } else { cout << "CPP: using DHT default rounding digits = " << dht_significant_digits << endl; @@ -353,22 +358,22 @@ int main(int argc, char *argv[]) { R["dht_final_proptype"] = params.dht_prop_type_vector; } - if (params.dht_strategy == 0) { - if (params.world_rank != 0) { - dht_object = DHT_create(dht_comm, dht_buckets_per_process, - dht_data_size, dht_key_size, get_md5); + // if (params.dht_strategy == 0) { + // if (params.world_rank != 0) { + // dht_object = DHT_create(dht_comm, dht_buckets_per_process, + // dht_data_size, dht_key_size, get_md5); - // storing for access from worker and callback functions - fuzzing_buffer = (double *)malloc(dht_key_size); - } - } else { - dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, - dht_data_size, dht_key_size, get_md5); - } + // // storing for access from worker and callback functions + // fuzzing_buffer = (double *)malloc(dht_key_size); + // } + // } else { + // dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, + // dht_data_size, dht_key_size, get_md5); + // } - if (params.world_rank == 0) { - cout << "CPP: DHT successfully created!" << endl; - } + // if (params.world_rank == 0) { + // cout << "CPP: DHT successfully created!" << endl; + // } } // MDL: store all parameters @@ -380,52 +385,23 @@ int main(int argc, char *argv[]) { MPI_Barrier(MPI_COMM_WORLD); if (params.world_rank == 0) { /* This is executed by the master */ + ChemMaster master(¶ms, R, grid); Rcpp::NumericVector master_send; Rcpp::NumericVector master_recv; - sim_a_seq = MPI_Wtime(); - - worker_struct *workerlist = - (worker_struct *)calloc(params.world_size - 1, sizeof(worker_struct)); - int need_to_receive; - MPI_Status probe_status; double *timings; uint64_t *dht_perfs = NULL; - int local_work_package_size; - // a temporary send buffer - double *send_buffer; - send_buffer = (double *)calloc( - (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); - - // helper variables - int iteration; - double dt, current_sim_time; - - int n_wp = 1; // holds the actual number of wp which is - // computed later in R::distribute_work_packages() - std::vector wp_sizes_vector; // vector with the sizes of - // each package - sim_start = MPI_Wtime(); // Iteration Count is dynamic, retrieving value from R (is only needed by // master for the following loop) uint32_t maxiter = R.parseEval("mysetup$maxiter"); - sim_b_seq = MPI_Wtime(); - - cummul_master_seq_pre_loop += sim_b_seq - sim_a_seq; - /*SIMULATION LOOP*/ for (uint32_t iter = 1; iter < maxiter + 1; iter++) { - sim_a_seq = MPI_Wtime(); - - cummul_master_send = 0.f; - cummul_master_recv = 0.f; - cout << "CPP: Evaluating next time step" << endl; R.parseEvalQ("mysetup <- master_iteration_setup(mysetup)"); @@ -440,201 +416,19 @@ int main(int argc, char *argv[]) { R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); sim_a_transport = MPI_Wtime(); + if (iter == 1) master.prepareSimulation(); + cout << "CPP: Chemistry" << endl; /*Fallback for sequential execution*/ sim_b_chemistry = MPI_Wtime(); if (params.world_size == 1) { - // MDL : the transformation of values into pH and pe - // takes now place in master_advection() so the - // following line is unneeded - // R.parseEvalQ("mysetup$state_T <- - // RedModRphree::Act2pH(mysetup$state_T)"); - R.parseEvalQ( - "result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); - R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + master.runSeq(); } else { /*send work to workers*/ - - // NEW: only in the first iteration we call - // R::distribute_work_packages()!! - if (iter == 1) { - R.parseEvalQ( - "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_T), " - "package_size=work_package_size)"); - - // we only sort once the vector - R.parseEvalQ("ordered_ids <- order(wp_ids)"); - R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); - n_wp = (int)R.parseEval("length(wp_sizes_vector)"); - wp_sizes_vector = as>(R["wp_sizes_vector"]); - cout << "CPP: Total number of work packages: " << n_wp << endl; - R.parseEval("stat_wp_sizes(wp_sizes_vector)"); - } - - /* shuffle and extract data - MDL: we now apply :Act2pH directly in master_advection - */ - // R.parseEval("tmp <- - // shuffle_field(RedModRphree::Act2pH(mysetup$state_T), ordered_ids)"); - // Rcpp::DataFrame chemistry_data = R.parseEval("tmp"); - - // convert_R_Dataframe_2_C_buffer(mpi_buffer, chemistry_data); - // cout << "CPP: shuffle_field() done" << endl; - grid.shuffleAndExport(mpi_buffer); - /* send and receive work; this is done by counting - * the wp */ - int pkg_to_send = n_wp; - int pkg_to_recv = n_wp; - size_t colCount = grid.getCols(); - int free_workers = params.world_size - 1; - double *work_pointer = mpi_buffer; - sim_c_chemistry = MPI_Wtime(); - - /* visual progress */ - float progress = 0.0; - int barWidth = 70; - - // retrieve data from R runtime - iteration = (int)R.parseEval("mysetup$iter"); - dt = (double)R.parseEval("mysetup$requested_dt"); - current_sim_time = - (double)R.parseEval("mysetup$simulation_time-mysetup$requested_dt"); - - int count_pkgs = 0; - - sim_b_seq = MPI_Wtime(); - - sim_c_chemistry = MPI_Wtime(); - - while (pkg_to_recv > 0) // start dispatching work packages - { - /* visual progress */ - progress = (float)(count_pkgs + 1) / n_wp; - - cout << "["; - int pos = barWidth * progress; - for (int iprog = 0; iprog < barWidth; ++iprog) { - if (iprog < pos) - cout << "="; - else if (iprog == pos) - cout << ">"; - else - cout << " "; - } - std::cout << "] " << int(progress * 100.0) << " %\r"; - std::cout.flush(); - /* end visual progress */ - - if (pkg_to_send > 0) { - master_send_a = MPI_Wtime(); - /*search for free workers and send work*/ - for (int p = 0; p < params.world_size - 1; p++) { - if (workerlist[p].has_work == 0 && - pkg_to_send > 0) /* worker is free */ { - - // to enable different work_package_size, set local copy of - // work_package_size to either global work_package size or - // remaining 'to_send' packages to_send >= work_package_size ? - // local_work_package_size = work_package_size : - // local_work_package_size = to_send; - - local_work_package_size = (int)wp_sizes_vector[count_pkgs]; - count_pkgs++; - - // cout << "CPP: sending pkg n. " << count_pkgs << " with size " - // << local_work_package_size << endl; - - /*push pointer forward to next work package, after taking the - * current one*/ - workerlist[p].send_addr = work_pointer; - - int end_of_wp = local_work_package_size * colCount; - work_pointer = &(work_pointer[end_of_wp]); - - // fill send buffer starting with work_package ... - std::memcpy(send_buffer, workerlist[p].send_addr, - (end_of_wp) * sizeof(double)); - // followed by: work_package_size - send_buffer[end_of_wp] = (double)local_work_package_size; - // current iteration of simulation - send_buffer[end_of_wp + 1] = (double)iteration; - // size of timestep in seconds - send_buffer[end_of_wp + 2] = dt; - // current time of simulation (age) in seconds - send_buffer[end_of_wp + 3] = current_sim_time; - // placeholder for work_package_count - send_buffer[end_of_wp + 4] = 0.; - - /* ATTENTION Worker p has rank p+1 */ - MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, - p + 1, TAG_WORK, MPI_COMM_WORLD); - - workerlist[p].has_work = 1; - free_workers--; - pkg_to_send -= 1; - } - } - master_send_b = MPI_Wtime(); - cummul_master_send += master_send_b - master_send_a; - } - - /*check if there are results to receive and receive them*/ - need_to_receive = 1; - master_recv_a = MPI_Wtime(); - while (need_to_receive && pkg_to_recv > 0) { - - if (pkg_to_send > 0 && free_workers > 0) - MPI_Iprobe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, - &need_to_receive, &probe_status); - else { - idle_a = MPI_Wtime(); - MPI_Probe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, - &probe_status); - idle_b = MPI_Wtime(); - master_idle += idle_b - idle_a; - } - - if (need_to_receive) { - int p = probe_status.MPI_SOURCE; - int size; - MPI_Get_count(&probe_status, MPI_DOUBLE, &size); - MPI_Recv(workerlist[p - 1].send_addr, size, MPI_DOUBLE, p, - TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - workerlist[p - 1].has_work = 0; - pkg_to_recv -= 1; - free_workers++; - } - } - master_recv_b = MPI_Wtime(); - cummul_master_recv += master_recv_b - master_recv_a; - } - - sim_c_seq = MPI_Wtime(); - - // don't overwrite last progress - cout << endl; - - sim_d_chemistry = MPI_Wtime(); - cummul_workers += sim_d_chemistry - sim_c_chemistry; - - // convert_C_buffer_2_R_Dataframe(mpi_buffer, chemistry_data); - // R.from_C_domain(mpi_buffer); - - // R["chemistry_data"] = R.getBufferDataFrame(); - - ///* unshuffle results */ - // R.parseEval("result <- unshuffle_field(chemistry_data, - // ordered_ids)"); - - grid.importAndUnshuffle(mpi_buffer); - /* do master stuff */ - sim_e_chemistry = MPI_Wtime(); - R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); - sim_f_chemistry = MPI_Wtime(); - cummul_chemistry_master += sim_f_chemistry - sim_e_chemistry; + master.runIteration(); } sim_a_chemistry = MPI_Wtime(); - + double master_seq_a = MPI_Wtime(); // MDL master_iteration_end just writes on disk state_T and // state_C after every iteration if the cmdline option // --ignore-results is not given (and thus the R variable @@ -649,48 +443,22 @@ int main(int argc, char *argv[]) { << endl << endl; - if (params.dht_enabled) { - for (int i = 1; i < params.world_size; i++) { - MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_STATS, MPI_COMM_WORLD); - } + double master_seq_b = MPI_Wtime(); - MPI_Barrier(MPI_COMM_WORLD); + cummul_master_seq += master.getSeqTime() + (master_seq_b - master_seq_a); + master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); + master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); - if (params.dht_snaps == 2) { - std::stringstream outfile; - outfile << params.out_dir << "/iter_" << std::setfill('0') - << std::setw(3) << iter << ".dht"; - for (int i = 1; i < params.world_size; i++) { - MPI_Send(outfile.str().c_str(), outfile.str().size(), MPI_CHAR, i, - TAG_DHT_STORE, MPI_COMM_WORLD); - } - MPI_Barrier(MPI_COMM_WORLD); - } + for (int i = 1; i < params.world_size; i++) { + MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); } - sim_d_seq = MPI_Wtime(); + MPI_Barrier(MPI_COMM_WORLD); - cummul_master_seq_loop += - ((sim_b_seq - sim_a_seq) - (sim_a_transport - sim_b_transport)) + - (sim_d_seq - sim_c_seq); - master_send.push_back(cummul_master_send, "it_" + to_string(iter)); - master_recv.push_back(cummul_master_recv, "it_" + to_string(iter)); - - } // END SIMULATION LOOP + } // END SIMULATION LOOP sim_end = MPI_Wtime(); - - if (params.dht_enabled && params.dht_snaps > 0) { - cout << "CPP: Master: Instruct workers to write DHT to file ..." << endl; - std::string outfile; - outfile = params.out_dir + ".dht"; - for (int i = 1; i < params.world_size; i++) { - MPI_Send(outfile.c_str(), outfile.size(), MPI_CHAR, i, TAG_DHT_STORE, - MPI_COMM_WORLD); - } - MPI_Barrier(MPI_COMM_WORLD); - cout << "CPP: Master: ... done" << endl; - } + master.finishSimulation(); Rcpp::NumericVector phreeqc_time; Rcpp::NumericVector dht_get_time; @@ -702,6 +470,10 @@ int main(int argc, char *argv[]) { timings = (double *)calloc(3, sizeof(double)); + int dht_hits = 0; + int dht_miss = 0; + int dht_collision = 0; + if (params.dht_enabled) { dht_hits = 0; dht_miss = 0; @@ -748,23 +520,21 @@ int main(int argc, char *argv[]) { R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); R["simtime_chemistry"] = cummul_chemistry; R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - R["simtime_workers"] = cummul_workers; + R["simtime_workers"] = master.getWorkerTime(); R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - R["simtime_chemistry_master"] = cummul_chemistry_master; + R["simtime_chemistry_master"] = master.getChemMasterTime(); R.parseEvalQ( "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - R["seq_master_prep"] = cummul_master_seq_pre_loop; - R.parseEvalQ("profiling$seq_master_prep <- seq_master_prep"); - R["seq_master_loop"] = cummul_master_seq_loop; - R.parseEvalQ("profiling$seq_master_loop <- seq_master_loop"); + R["seq_master"] = cummul_master_seq; + R.parseEvalQ("profiling$seq_master <- seq_master"); // R["master_send"] = master_send; // R.parseEvalQ("profiling$master_send <- master_send"); // R["master_recv"] = master_recv; // R.parseEvalQ("profiling$master_recv <- master_recv"); - R["idle_master"] = master_idle; + R["idle_master"] = master.getIdleTime(); R.parseEvalQ("profiling$idle_master <- idle_master"); R["idle_worker"] = idle_worker; R.parseEvalQ("profiling$idle_worker <- idle_worker"); @@ -788,11 +558,9 @@ int main(int argc, char *argv[]) { R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); } - free(workerlist); free(timings); - if (params.dht_enabled) - free(dht_perfs); + if (params.dht_enabled) free(dht_perfs); cout << "CPP: Done! Results are stored as R objects into <" << params.out_dir << "/timings.rds>" << endl; @@ -802,42 +570,22 @@ int main(int argc, char *argv[]) { r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ - if (!params.dht_file.empty()) { - int res = file_to_table((char *)params.dht_file.c_str()); - if (res != DHT_SUCCESS) { - if (res == DHT_WRONG_FILE) { - if (params.world_rank == 2) - cerr << "CPP: Worker: Wrong File" << endl; - } else { - if (params.world_rank == 2) - cerr << "CPP: Worker: Error in loading current state of DHT from " - "file" - << endl; - } - return EXIT_FAILURE; - } else { - if (params.world_rank == 2) - cout << "CPP: Worker: Successfully loaded state of DHT from file " - << params.dht_file << endl; - std::cout.flush(); - } - } - worker_function(¶ms); - free(mpi_buffer_results); + ChemWorker worker(¶ms, R, grid); + worker.prepareSimulation(dht_comm); + worker.loop(); } cout << "CPP: finished, cleanup of process " << params.world_rank << endl; - if (params.dht_enabled) { - - if (params.dht_strategy == 0) { - if (params.world_rank != 0) { - DHT_free(dht_object, NULL, NULL); - } - } else { - DHT_free(dht_object, NULL, NULL); - } - } + // if (params.dht_enabled) { + // if (params.dht_strategy == 0) { + // if (params.world_rank != 0) { + // DHT_free(dht_object, NULL, NULL); + // } + // } else { + // DHT_free(dht_object, NULL, NULL); + // } + // } free(mpi_buffer); MPI_Finalize(); diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp new file mode 100644 index 000000000..fee86113a --- /dev/null +++ b/src/model/ChemMaster.cpp @@ -0,0 +1,217 @@ +#include +#include + +#include + +#include "ChemSim.h" + +using namespace poet; +using namespace std; +using namespace Rcpp; + +#define TAG_WORK 42 + +ChemMaster::ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_) + : ChemSim(params, R_, grid_) { + this->wp_size = params->wp_size; + this->out_dir = params->out_dir; +} + +void ChemMaster::runIteration() { + double seq_a, seq_b, seq_c, seq_d; + double worker_chemistry_a, worker_chemistry_b; + double sim_e_chemistry, sim_f_chemistry; + int pkg_to_send, pkg_to_recv; + int free_workers; + int i_pkgs; + + seq_a = MPI_Wtime(); + grid.shuffleAndExport(mpi_buffer); + // retrieve data from R runtime + iteration = (int)R.parseEval("mysetup$iter"); + dt = (double)R.parseEval("mysetup$requested_dt"); + current_sim_time = + (double)R.parseEval("mysetup$simulation_time-mysetup$requested_dt"); + + // setup local variables + pkg_to_send = wp_sizes_vector.size(); + pkg_to_recv = wp_sizes_vector.size(); + work_pointer = mpi_buffer; + free_workers = world_size - 1; + i_pkgs = 0; + + seq_b = MPI_Wtime(); + seq_t += seq_b - seq_a; + + worker_chemistry_a = MPI_Wtime(); + while (pkg_to_recv > 0) { + // TODO: Progressbar into IO instance. + printProgressbar((int)i_pkgs, (int)wp_sizes_vector.size()); + if (pkg_to_send > 0) { + sendPkgs(pkg_to_send, i_pkgs, free_workers); + } + recvPkgs(pkg_to_recv, pkg_to_send > 0, free_workers); + } + + // Just to complete the progressbar + cout << endl; + + worker_chemistry_b = MPI_Wtime(); + worker_t = worker_chemistry_b - worker_chemistry_a; + + seq_c = MPI_Wtime(); + grid.importAndUnshuffle(mpi_buffer); + /* do master stuff */ + sim_e_chemistry = MPI_Wtime(); + R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + sim_f_chemistry = MPI_Wtime(); + chem_master += sim_f_chemistry - sim_e_chemistry; + seq_d = MPI_Wtime(); + seq_t += seq_d - seq_c; +} + +void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, + int &free_workers) { + double master_send_a, master_send_b; + int local_work_package_size; + int end_of_wp; + + // start time measurement + master_send_a = MPI_Wtime(); + /*search for free workers and send work*/ + for (int p = 0; p < world_size - 1; p++) { + if (workerlist[p].has_work == 0 && pkg_to_send > 0) /* worker is free */ { + // to enable different work_package_size, set local copy of + // work_package_size to either global work_package size or + // remaining 'to_send' packages to_send >= work_package_size ? + // local_work_package_size = work_package_size : + // local_work_package_size = to_send; + + local_work_package_size = (int)wp_sizes_vector[count_pkgs]; + count_pkgs++; + + // cout << "CPP: sending pkg n. " << count_pkgs << " with size " + // << local_work_package_size << endl; + + /*push pointer forward to next work package, after taking the + * current one*/ + workerlist[p].send_addr = work_pointer; + + end_of_wp = local_work_package_size * grid.getCols(); + work_pointer = &(work_pointer[end_of_wp]); + + // fill send buffer starting with work_package ... + std::memcpy(send_buffer, workerlist[p].send_addr, + (end_of_wp) * sizeof(double)); + // followed by: work_package_size + send_buffer[end_of_wp] = (double)local_work_package_size; + // current iteration of simulation + send_buffer[end_of_wp + 1] = (double)iteration; + // size of timestep in seconds + send_buffer[end_of_wp + 2] = dt; + // current time of simulation (age) in seconds + send_buffer[end_of_wp + 3] = current_sim_time; + // placeholder for work_package_count + send_buffer[end_of_wp + 4] = 0.; + + /* ATTENTION Worker p has rank p+1 */ + MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, p + 1, + TAG_WORK, MPI_COMM_WORLD); + + workerlist[p].has_work = 1; + free_workers--; + pkg_to_send -= 1; + } + } + master_send_b = MPI_Wtime(); + send_t += master_send_b - master_send_a; +} + +void ChemMaster::recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers) { + int need_to_receive = 1; + double master_recv_a, master_recv_b; + double idle_a, idle_b; + int p, size; + + MPI_Status probe_status; + master_recv_a = MPI_Wtime(); + while (need_to_receive && pkg_to_recv > 0) { + if (to_send && free_workers > 0) + MPI_Iprobe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &need_to_receive, + &probe_status); + else { + idle_a = MPI_Wtime(); + MPI_Probe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &probe_status); + idle_b = MPI_Wtime(); + master_idle += idle_b - idle_a; + } + + if (need_to_receive) { + p = probe_status.MPI_SOURCE; + size; + MPI_Get_count(&probe_status, MPI_DOUBLE, &size); + MPI_Recv(workerlist[p - 1].send_addr, size, MPI_DOUBLE, p, TAG_WORK, + MPI_COMM_WORLD, MPI_STATUS_IGNORE); + workerlist[p - 1].has_work = 0; + pkg_to_recv -= 1; + free_workers++; + } + } + master_recv_b = MPI_Wtime(); + recv_t += master_recv_b - master_recv_a; +} + +void ChemMaster::printProgressbar(int count_pkgs, int n_wp, int barWidth) { + /* visual progress */ + double progress = (float)(count_pkgs + 1) / n_wp; + + cout << "["; + int pos = barWidth * progress; + for (int iprog = 0; iprog < barWidth; ++iprog) { + if (iprog < pos) + cout << "="; + else if (iprog == pos) + cout << ">"; + else + cout << " "; + } + std::cout << "] " << int(progress * 100.0) << " %\r"; + std::cout.flush(); + /* end visual progress */ +} + +void ChemMaster::prepareSimulation() { + workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); + send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, + sizeof(double)); + + R.parseEvalQ( + "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_T), " + "package_size=work_package_size)"); + + // we only sort once the vector + R.parseEvalQ("ordered_ids <- order(wp_ids)"); + R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); + R.parseEval("stat_wp_sizes(wp_sizes_vector)"); + wp_sizes_vector = as>(R["wp_sizes_vector"]); + + mpi_buffer = + (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); +} + +void ChemMaster::finishSimulation() { + free(mpi_buffer); + free(workerlist); +} + +double ChemMaster::getSendTime() { return this->send_t; } + +double ChemMaster::getRecvTime() { return this->recv_t; } + +double ChemMaster::getIdleTime() { return this->master_idle; } + +double ChemMaster::getWorkerTime() { return this->worker_t; } + +double ChemMaster::getChemMasterTime() { return this->chem_master; } + +double ChemMaster::getSeqTime() { return this->seq_t; } diff --git a/src/model/ChemSim.cpp b/src/model/ChemSim.cpp new file mode 100644 index 000000000..62b783481 --- /dev/null +++ b/src/model/ChemSim.cpp @@ -0,0 +1,23 @@ +#include "ChemSim.h" +#include "../util/RRuntime.h" +#include "Grid.h" +#include +#include +#include + +using namespace Rcpp; +using namespace poet; + +ChemSim::ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_) + : R(R_), grid(grid_) { + this->world_rank = params->world_rank; + this->world_size = params->world_size; + this->wp_size = params->wp_size; + this->out_dir = params->out_dir; +} + +void ChemSim::runSeq() { + R.parseEvalQ( + "result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); + R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); +} diff --git a/src/model/ChemSim.h b/src/model/ChemSim.h new file mode 100644 index 000000000..0d2c6a615 --- /dev/null +++ b/src/model/ChemSim.h @@ -0,0 +1,115 @@ +#ifndef CHEMSIM_H +#define CHEMSIM_H + +#include "../DHT/DHT_Wrapper.h" +#include "../util/RRuntime.h" +#include "../util/SimParams.h" +#include "Grid.h" + +#include +#include + +#define BUFFER_OFFSET 5 +#define TAG_WORK 42 +#define TAG_FINISH 43 +#define TAG_TIMING 44 +#define TAG_DHT_PERF 45 +#define TAG_DHT_STATS 46 +#define TAG_DHT_STORE 47 +#define TAG_DHT_ITER 48 + +namespace poet { +class ChemSim { +public: + ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_); + void runSeq(); + +protected: + double current_sim_time = 0; + int iteration = 0; + int dt = 0; + + int world_rank; + int world_size; + unsigned int wp_size; + + RRuntime &R; + Grid &grid; + + std::vector wp_sizes_vector; + std::string out_dir; + + double *send_buffer; + typedef struct { + char has_work; + double *send_addr; + } worker_struct; + worker_struct *workerlist; + + double *mpi_buffer; +}; + +class ChemMaster : public ChemSim { +public: + ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_); + + void prepareSimulation(); + void finishSimulation(); + + void runIteration(); + + double getSendTime(); + double getRecvTime(); + double getIdleTime(); + double getWorkerTime(); + double getChemMasterTime(); + double getSeqTime(); + +private: + void printProgressbar(int count_pkgs, int n_wp, int barWidth = 70); + void sendPkgs(int &pkg_to_send, int &count_pkgs, int &free_workers); + void recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers); + + unsigned int wp_size; + double *work_pointer; + + double send_t = 0.f; + double recv_t = 0.f; + double master_idle = 0.f; + double worker_t = 0.f; + double chem_master = 0.f; + double seq_t = 0.f; +}; + +class ChemWorker : public ChemSim { +public: + ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_); + + void prepareSimulation(MPI_Comm dht_comm); + void loop(); + +private: + void doWork(MPI_Status &probe_status); + void postIter(); + void finishWork(); + void writeFile(); + void readFile(); + + bool dht_enabled; + bool dt_differ; + int dht_snaps; + std::string dht_file; + unsigned int dht_size_per_process; + std::vector dht_flags; + t_simparams *params; + + double *mpi_buffer_results; + + DHT_Wrapper *dht; + + double timing[3]; + double idle_t = 0.f; + int phreeqc_count = 0; +}; +} // namespace poet +#endif // CHEMSIM_H diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp new file mode 100644 index 000000000..8649fbc11 --- /dev/null +++ b/src/model/ChemWorker.cpp @@ -0,0 +1,325 @@ +#include +#include + +#include +#include + +#include "ChemSim.h" + +using namespace poet; +using namespace std; +using namespace Rcpp; + +ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_) + : params(params_), ChemSim(params_, R_, grid_) { + this->dt_differ = params->dt_differ; + this->dht_enabled = params->dht_enabled; + this->dht_size_per_process = params->dht_size_per_process; + this->dht_file = params->dht_file; +} + +void ChemWorker::prepareSimulation(MPI_Comm dht_comm) { + mpi_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, + sizeof(double)); + mpi_buffer_results = + (double *)calloc(wp_size * (grid.getCols()), sizeof(double)); + + if (world_rank == 1) + cout << "Worker: DHT usage is " << (dht_enabled ? "ON" : "OFF") << endl; + + if (dht_enabled) { + int data_size = grid.getCols() * sizeof(double); + int key_size = + grid.getCols() * sizeof(double) + (dt_differ * sizeof(double)); + int dht_buckets_per_process = + dht_size_per_process / (1 + data_size + key_size); + + if (world_rank == 1) + cout << "CPP: Worker: data size: " << data_size << " bytes" << endl + << "CPP: Worker: key size: " << key_size << "bytes" << endl + << "CPP: Worker: buckets per process " << dht_buckets_per_process + << endl; + + dht = new DHT_Wrapper(params, dht_comm, dht_buckets_per_process, data_size, + key_size); + + if (world_rank == 1) cout << "CPP: Worker: DHT created!" << endl; + } + + if (!dht_file.empty()) readFile(); + + // set size + dht_flags.resize(params->wp_size, true); + // assign all elements to true (default) + dht_flags.assign(params->wp_size, true); + + timing[0] = 0.0; + timing[1] = 0.0; + timing[2] = 0.0; +} + +void ChemWorker::loop() { + MPI_Status probe_status; + while (1) { + double idle_a = MPI_Wtime(); + MPI_Probe(0, MPI_ANY_TAG, MPI_COMM_WORLD, &probe_status); + double idle_b = MPI_Wtime(); + + if (probe_status.MPI_TAG == TAG_WORK) { + idle_t += idle_b - idle_a; + doWork(probe_status); + } else if (probe_status.MPI_TAG == TAG_FINISH) { + finishWork(); + break; + } else if (probe_status.MPI_TAG == TAG_DHT_ITER) { + postIter(); + } + } +} + +void ChemWorker::doWork(MPI_Status &probe_status) { + int count; + int local_work_package_size = 0; + + double dht_get_start, dht_get_end; + double phreeqc_time_start, phreeqc_time_end; + double dht_fill_start, dht_fill_end; + + /* get number of doubles sent */ + MPI_Get_count(&probe_status, MPI_DOUBLE, &count); + + /* receive */ + MPI_Recv(mpi_buffer, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + + // decrement count of work_package by BUFFER_OFFSET + count -= BUFFER_OFFSET; + // check for changes on all additional variables given by the 'header' of + // mpi_buffer + + // work_package_size + if (mpi_buffer[count] != local_work_package_size) { // work_package_size + local_work_package_size = mpi_buffer[count]; + R["work_package_size"] = local_work_package_size; + R.parseEvalQ("mysetup$work_package_size <- work_package_size"); + } + + // current iteration of simulation + if (mpi_buffer[count + 1] != iteration) { + iteration = mpi_buffer[count + 1]; + R["iter"] = iteration; + R.parseEvalQ("mysetup$iter <- iter"); + } + + // current timestep size + if (mpi_buffer[count + 2] != dt) { + dt = mpi_buffer[count + 2]; + R["dt"] = dt; + R.parseEvalQ("mysetup$dt <- dt"); + } + + // current simulation time ('age' of simulation) + if (mpi_buffer[count + 3] != current_sim_time) { + current_sim_time = mpi_buffer[count + 3]; + R["simulation_time"] = current_sim_time; + R.parseEvalQ("mysetup$simulation_time <- simulation_time"); + } + /* 4th double value is currently a placeholder */ + // if (mpi_buffer[count+4] != placeholder) { + // placeholder = mpi_buffer[count+4]; + // R["mysetup$placeholder"] = placeholder; + // } + + /* get df with right structure to fill in work package */ + // R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); + // R["skeleton"] = grid.buildDataFrame(work_package_size); + //// R.parseEval("print(rownames(tmp2)[1:5])"); + //// R.parseEval("print(head(tmp2, 2))"); + //// R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); + + ////Rcpp::DataFrame buffer = R.parseEval("tmp2"); + // R.setBufferDataFrame("skeleton"); + + if (dht_enabled) { + // DEBUG + // cout << "RANK " << world_rank << " start checking DHT\n"; + + // resize helper vector dht_flags of work_package_size changes + if ((int)dht_flags.size() != local_work_package_size) { + dht_flags.resize(local_work_package_size, true); // set size + dht_flags.assign(local_work_package_size, + true); // assign all elements to true (default) + } + + dht_get_start = MPI_Wtime(); + dht->checkDHT(local_work_package_size, dht_flags, mpi_buffer, dt); + dht_get_end = MPI_Wtime(); + + // DEBUG + // cout << "RANK " << world_rank << " checking DHT complete \n"; + + R["dht_flags"] = as(wrap(dht_flags)); + // R.parseEvalQ("print(head(dht_flags))"); + } + + /* work */ + // R.from_C_domain(mpi_buffer); + ////convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); + // R["work_package_full"] = R.getBufferDataFrame(); + // R["work_package"] = buffer; + + // DEBUG + // R.parseEvalQ("print(head(work_package_full))"); + // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); + + grid.importWP(mpi_buffer, params->wp_size); + + if (params->dht_enabled) { + R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); + } else { + R.parseEvalQ("work_package <- work_package_full"); + } + + // DEBUG + // R.parseEvalQ("print(head(work_package),2)"); + + // R.parseEvalQ("rownames(work_package) <- work_package$id"); + // R.parseEval("print(paste('id %in% colnames(work_package)', 'id' %in% + // colnames(work_package)"); R.parseEvalQ("id_store <- + // rownames(work_package)"); //"[, ncol(work_package)]"); + // R.parseEvalQ("work_package$id <- NULL"); + R.parseEvalQ("work_package <- as.matrix(work_package)"); + + unsigned int nrows = R.parseEval("nrow(work_package)"); + + if (nrows > 0) { + /*Single Line error Workaround*/ + if (nrows <= 1) { + // duplicate line to enable correct simmulation + R.parseEvalQ( + "work_package <- work_package[rep(1:nrow(work_package), " + "times = 2), ]"); + } + + phreeqc_count++; + + phreeqc_time_start = MPI_Wtime(); + // MDL + // R.parseEvalQ("print('Work_package:\n'); print(head(work_package , + // 2)); cat('RCpp: worker_function:', local_rank, ' \n')"); + R.parseEvalQ( + "result <- as.data.frame(slave_chemistry(setup=mysetup, " + "data = work_package))"); + phreeqc_time_end = MPI_Wtime(); + // R.parseEvalQ("result$id <- id_store"); + } else { + // cout << "Work-Package is empty, skipping phreeqc!" << endl; + } + + if (dht_enabled) { + R.parseEvalQ("result_full <- work_package_full"); + if (nrows > 0) R.parseEvalQ("result_full[dht_flags,] <- result"); + } else { + R.parseEvalQ("result_full <- result"); + } + + // R.setBufferDataFrame("result_full"); + ////Rcpp::DataFrame result = R.parseEval("result_full"); + ////convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); + // R.to_C_domain(mpi_buffer_results); + + grid.exportWP(mpi_buffer_results); + /* send results to master */ + MPI_Request send_req; + MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, + &send_req); + + if (dht_enabled) { + dht_fill_start = MPI_Wtime(); + dht->fillDHT(local_work_package_size, dht_flags, mpi_buffer, + mpi_buffer_results, dt); + dht_fill_end = MPI_Wtime(); + + timing[1] += dht_get_end - dht_get_start; + timing[2] += dht_fill_end - dht_fill_start; + } + + timing[0] += phreeqc_time_end - phreeqc_time_start; + + MPI_Wait(&send_req, MPI_STATUS_IGNORE); +} + +void ChemWorker::postIter() { + MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_DHT_ITER, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + if (dht_enabled) { + dht->printStatistics(); + + if (dht_snaps == 2) { + writeFile(); + } + } + // synchronize all processes + MPI_Barrier(MPI_COMM_WORLD); +} + +void ChemWorker::writeFile() { + std::stringstream out; + out << out_dir << "/iter_" << setfill('0') << setw(3) << iteration << ".dht"; + int res = dht->tableToFile(out.str().c_str()); + if (res != DHT_SUCCESS && world_rank == 2) + cerr << "CPP: Worker: Errir in writing current state of DHT to file." + << endl; + else if (world_rank == 2) + cout << "CPP: Worker: Successfully written DHT to file " << out.str() + << endl; +} + +void ChemWorker::readFile() { + int res = dht->fileToTable((char *)dht_file.c_str()); + if (res != DHT_SUCCESS) { + if (res == DHT_WRONG_FILE) { + if (world_rank == 1) + cerr << "CPP: Worker: Wrong file layout! Continue with empty DHT ..." + << endl; + } else { + if (world_rank == 1) + cerr << "CPP: Worker: Error in loading current state of DHT from " + "file. Continue with empty DHT ..." + << endl; + } + } else { + if (world_rank == 2) + cout << "CPP: Worker: Successfully loaded state of DHT from file " + << dht_file << endl; + std::cout.flush(); + } +} + +void ChemWorker::finishWork() { + if (dht_enabled && dht_snaps > 0) writeFile(); + + double dht_perf[3]; + /* before death, submit profiling/timings to master*/ + MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_FINISH, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + + // timings + MPI_Send(timing, 3, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); + + MPI_Send(&phreeqc_count, 1, MPI_INT, 0, TAG_TIMING, MPI_COMM_WORLD); + MPI_Send(&idle_t, 1, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); + + if (dht_enabled) { + // dht_perf + dht_perf[0] = dht->getHits(); + dht_perf[1] = dht->getMisses(); + dht_perf[2] = dht->getEvictions(); + MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, + MPI_COMM_WORLD); + } + + free(mpi_buffer); + free(mpi_buffer_results); + delete dht; +} From 489a6b32729707ea73343f5a987787fad0714b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Fri, 8 Jan 2021 17:53:16 +0100 Subject: [PATCH 09/50] Substitute worker.c with ChemWorker.cpp --- src/kin.cpp | 2 +- src/model/ChemWorker.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/kin.cpp b/src/kin.cpp index abfb6023d..673b5c223 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -179,7 +179,7 @@ int main(int argc, char *argv[]) { // cout << "CPP: DHT significant digits = " << dht_significant_digits << // endl; - params.dht_log = cmdl["dht-log"]; + params.dht_log = !(cmdl["dht-nolog"]); // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" // : "OFF" ) << endl; diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index 8649fbc11..91f8937ec 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -25,7 +25,8 @@ void ChemWorker::prepareSimulation(MPI_Comm dht_comm) { (double *)calloc(wp_size * (grid.getCols()), sizeof(double)); if (world_rank == 1) - cout << "Worker: DHT usage is " << (dht_enabled ? "ON" : "OFF") << endl; + cout << "CPP: Worker: DHT usage is " << (dht_enabled ? "ON" : "OFF") + << endl; if (dht_enabled) { int data_size = grid.getCols() * sizeof(double); @@ -36,7 +37,7 @@ void ChemWorker::prepareSimulation(MPI_Comm dht_comm) { if (world_rank == 1) cout << "CPP: Worker: data size: " << data_size << " bytes" << endl - << "CPP: Worker: key size: " << key_size << "bytes" << endl + << "CPP: Worker: key size: " << key_size << " bytes" << endl << "CPP: Worker: buckets per process " << dht_buckets_per_process << endl; From 3f710bfea6a9b3cf40734f258e92e6da350f44e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Fri, 8 Jan 2021 17:54:31 +0100 Subject: [PATCH 10/50] Cleanup old source files --- src/DHT.cpp | 423 -------------------------------------------- src/DHT.h | 112 ------------ src/dht_wrapper.cpp | 176 ------------------ src/dht_wrapper.h | 55 ------ src/global_buffer.h | 9 - src/r_utils.cpp | 44 ----- src/r_utils.h | 6 - src/worker.cpp | 301 ------------------------------- src/worker.h | 18 -- 9 files changed, 1144 deletions(-) delete mode 100644 src/DHT.cpp delete mode 100644 src/DHT.h delete mode 100644 src/dht_wrapper.cpp delete mode 100644 src/dht_wrapper.h delete mode 100644 src/global_buffer.h delete mode 100644 src/r_utils.cpp delete mode 100644 src/r_utils.h delete mode 100644 src/worker.cpp delete mode 100644 src/worker.h diff --git a/src/DHT.cpp b/src/DHT.cpp deleted file mode 100644 index 546f7aa50..000000000 --- a/src/DHT.cpp +++ /dev/null @@ -1,423 +0,0 @@ -#include "DHT.h" -#include -#include -#include -#include - -/* - * determining destination rank and index by hash of key - * - * return values by reference - */ -static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int *dest_rank, unsigned int *index, unsigned int index_count) { - uint64_t tmp; - int char_hop = 9-index_count; - unsigned int i; - for (i = 0; i < index_count ; i++) { - tmp = 0; - memcpy(&tmp,(unsigned char *)&hash+i, char_hop); - index[i] = (unsigned int) (tmp % table_size); - } - *dest_rank = (unsigned int) (hash % comm_size); -} - -/** - * set write flag to 1 - */ -static void set_flag(char* flag_byte) { - *flag_byte = 0; - *flag_byte |= (1 << 0); -} - -/** - * return 1 if write flag is set - * else 0 - */ -static int read_flag(char flag_byte) { - if ((flag_byte & 0x01) == 0x01) { - return 1; - } else return 0; -} - -/* - * allocating memory for DHT object and buckets. - * creating MPI window for OSC - * filling DHT object with passed by parameters, window, 2 counters for R/W errors and 2 pointers with allocated memory for further use - * return DHT object - */ -DHT* DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, uint64_t(*hash_func) (int, void*)) { - DHT *object; - MPI_Win window; - void* mem_alloc; - int comm_size, tmp; - - tmp = (int) ceil(log2(size)); - if (tmp%8 != 0) tmp = tmp + (8-(tmp%8)); - - object = (DHT*) malloc(sizeof(DHT)); - if (object == NULL) return NULL; - - //every memory allocation has 1 additional byte for flags etc. - if (MPI_Alloc_mem(size * (1 + data_size + key_size), MPI_INFO_NULL, &mem_alloc) != 0) return NULL; - if (MPI_Comm_size(comm, &comm_size) != 0) return NULL; - - memset(mem_alloc, '\0', size * (1 + data_size + key_size)); - - if (MPI_Win_create(mem_alloc, size * (1 + data_size + key_size), (1 + data_size + key_size), MPI_INFO_NULL, comm, &window) != 0) return NULL; - - object->data_size = data_size; - object->key_size = key_size; - object->table_size = size; - object->window = window; - object->hash_func = hash_func; - object->comm_size = comm_size; - object->communicator = comm; - object->read_misses = 0; - object->collisions = 0; - object->recv_entry = malloc(1 + data_size + key_size); - object->send_entry = malloc(1 + data_size + key_size); - object->index_count = 9-(tmp/8); - object->index = (unsigned int*) malloc((9-(tmp/8))*sizeof(int)); - object->mem_alloc = mem_alloc; - - DHT_stats *stats; - - stats = (DHT_stats*) malloc(sizeof(DHT_stats)); - if (stats == NULL) return NULL; - - object->stats = stats; - object->stats->writes_local = (int*) calloc(comm_size, sizeof(int)); - object->stats->old_writes = 0; - object->stats->read_misses = 0; - object->stats->collisions = 0; - object->stats->w_access = 0; - object->stats->r_access = 0; - - return object; -} - -/* - * puts passed by data with key to DHT - * - * returning DHT_MPI_ERROR = -1 if MPI error occurred - * else DHT_SUCCESS = 0 if success - */ -int DHT_write(DHT *table, void* send_key, void* send_data) { - unsigned int dest_rank, i; - int result = DHT_SUCCESS; - - table->stats->w_access++; - - //determine destination rank and index by hash of key - determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); - - //concatenating key with data to write entry to DHT - set_flag((char *) table->send_entry); - memcpy((char *) table->send_entry + 1, (char *) send_key, table->key_size); - memcpy((char *) table->send_entry + table->key_size + 1, (char *) send_data, table->data_size); - - //locking window of target rank with exclusive lock - if (MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dest_rank, 0, table->window) != 0) - return DHT_MPI_ERROR; - for (i = 0; i < table->index_count; i++) - { - if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - - //increment collision counter if receiving key doesn't match sending key - //,entry has write flag + last index is reached - - if (read_flag(*(char *)table->recv_entry)) { - if (memcmp(send_key, (char *) table->recv_entry + 1, table->key_size) != 0) { - if (i == (table->index_count)-1) { - table->collisions += 1; - table->stats->collisions += 1; - result = DHT_WRITE_SUCCESS_WITH_COLLISION; - break; - } - } else break; - } else { - table->stats->writes_local[dest_rank]++; - break; - } - } - - //put data to DHT - if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - //unlock window of target rank - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - - return result; -} - -/* - * gets data from the DHT by key - * - * return DHT_SUCCESS = 0 if success - * DHT_MPI_ERROR = -1 if MPI error occurred - * DHT_READ_ERROR = -2 if receiving key doesn't match sending key - */ -int DHT_read(DHT *table, void* send_key, void* destination) { - unsigned int dest_rank, i; - - table->stats->r_access++; - - //determine destination rank and index by hash of key - determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); - - //locking window of target rank with shared lock - if (MPI_Win_lock(MPI_LOCK_SHARED, dest_rank, 0, table->window) != 0) return DHT_MPI_ERROR; - //receive data - for (i = 0; i < table->index_count; i++) { - if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - - //increment read error counter if write flag isn't set or key doesn't match passed by key + last index reached - //else copy data to dereference of passed by destination pointer - - if ((read_flag(*(char *) table->recv_entry)) == 0) { - table->read_misses += 1; - table->stats->read_misses += 1; - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; - } - - if (memcmp(((char*)table->recv_entry) + 1, send_key, table->key_size) != 0) { - if (i == (table->index_count)-1) { - table->read_misses += 1; - table->stats->read_misses += 1; - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; - } - } else break; - } - - //unlock window of target rank - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - - memcpy((char *) destination, (char *) table->recv_entry + table->key_size + 1, table->data_size); - - return DHT_SUCCESS; -} - -int DHT_to_file(DHT* table, char* filename) { - //open file - MPI_File file; - if (MPI_File_open(table->communicator, filename, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; - - int rank; - MPI_Comm_rank(table->communicator, &rank); - - //write header (key_size and data_size) - if (rank == 0) { - if (MPI_File_write(file, &table->key_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; - if (MPI_File_write(file, &table->data_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; - } - - if (MPI_File_seek_shared(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - - char* ptr; - int bucket_size = table->key_size + table->data_size + 1; - - //iterate over local memory - for (unsigned int i = 0; i < table->table_size; i++) { - ptr = (char *) table->mem_alloc + (i * bucket_size); - //if bucket has been written to (checked by written_flag)... - if (read_flag(*ptr)) { - //write key and data to file - if (MPI_File_write_shared(file, ptr + 1, bucket_size - 1, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; - } - } - //close file - if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; - - return DHT_SUCCESS; -} - -int DHT_from_file(DHT* table, char* filename) { - MPI_File file; - MPI_Offset f_size; - int e_size, m_size, cur_pos, rank, offset; - char* buffer; - void* key; - void* data; - - if (MPI_File_open(table->communicator, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; - - if (MPI_File_get_size(file, &f_size) != 0) return DHT_FILE_IO_ERROR; - - MPI_Comm_rank(table->communicator, &rank); - - e_size = table->key_size + table->data_size; - m_size = e_size > DHT_HEADER_SIZE ? e_size : DHT_HEADER_SIZE; - buffer = (char *) malloc(m_size); - - if (MPI_File_read(file, buffer, DHT_HEADER_SIZE, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; - - if (*(int *) buffer != table->key_size) return DHT_WRONG_FILE; - if (*(int *) (buffer + 4) != table->data_size) return DHT_WRONG_FILE; - - offset = e_size*table->comm_size; - - if (MPI_File_seek(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - cur_pos = DHT_HEADER_SIZE + (rank * e_size); - - while(cur_pos < f_size) { - if (MPI_File_seek(file, cur_pos, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - MPI_Offset tmp; - MPI_File_get_position(file, &tmp); - if (MPI_File_read(file, buffer, e_size, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; - key = buffer; - data = (buffer+table->key_size); - if (DHT_write(table, key, data) == DHT_MPI_ERROR) return DHT_MPI_ERROR; - - cur_pos += offset; - } - - free (buffer); - if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; - - return DHT_SUCCESS; -} - -/* - * frees up memory and accumulate counter - */ -int DHT_free(DHT* table, int* collision_counter, int* readerror_counter) { - int buf; - - if (collision_counter != NULL) { - buf = 0; - if (MPI_Reduce(&table->collisions, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - *collision_counter = buf; - } - if (readerror_counter != NULL) { - buf = 0; - if (MPI_Reduce(&table->read_misses, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - *readerror_counter = buf; - } - if (MPI_Win_free(&(table->window)) != 0) return DHT_MPI_ERROR; - if (MPI_Free_mem(table->mem_alloc) != 0) return DHT_MPI_ERROR; - free(table->recv_entry); - free(table->send_entry); - free(table->index); - - free(table->stats->writes_local); - free(table->stats); - - free(table); - - return DHT_SUCCESS; -} - -/* - * prints a table with statistics about current use of DHT - * for each participating process and summed up results containing: - * 1. occupied buckets (in respect to the memory of this process) - * 2. free buckets (in respect to the memory of this process) - * 3. calls of DHT_write (w_access) - * 4. calls of DHT_read (r_access) - * 5. read misses (see DHT_READ_ERROR) - * 6. collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) - * 3-6 will reset with every call of this function - * finally the amount of new written entries is printed out (in relation to last call of this funtion) - */ -int DHT_print_statistics(DHT *table) { - int *written_buckets; - int *read_misses, sum_read_misses; - int *collisions, sum_collisions; - int sum_w_access, sum_r_access, *w_access, *r_access; - int rank; - - MPI_Comm_rank(table->communicator, &rank); - - //disable possible warning of unitialized variable, which is not the case - //since we're using MPI_Gather to obtain all values only on rank 0 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" - - //obtaining all values from all processes in the communicator - if (rank == 0) read_misses = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->read_misses, 1, MPI_INT, read_misses, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->read_misses, &sum_read_misses, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->read_misses = 0; - - if (rank == 0) collisions = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->collisions, 1, MPI_INT, collisions, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->collisions, &sum_collisions, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->collisions = 0; - - if (rank == 0) w_access = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->w_access = 0; - - if (rank == 0) r_access = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->r_access = 0; - - if (rank == 0) written_buckets = (int*) calloc(table->comm_size, sizeof(int)); - if (MPI_Reduce(table->stats->writes_local, written_buckets, table->comm_size, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - - if (rank == 0) { //only process with rank 0 will print out results as a table - int sum_written_buckets = 0; - - for (int i=0; i < table->comm_size; i++) { - sum_written_buckets += written_buckets[i]; - } - - int members = 7; - int padsize = (members*12)+1; - char pad[padsize+1]; - - memset(pad, '-', padsize*sizeof(char)); - pad[padsize]= '\0'; - printf("\n"); - printf("%-35s||resets with every call of this function\n", " "); - printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s\n", - "rank", - "occupied", - "free", - "w_access", - "r_access", - "read misses", - "collisions"); - printf("%s\n", pad); - for (int i = 0; i < table->comm_size; i++) { - printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", - i, - written_buckets[i], - table->table_size-written_buckets[i], - w_access[i], - r_access[i], - read_misses[i], - collisions[i]); - } - printf("%s\n", pad); - printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", - "sum", - sum_written_buckets, - (table->table_size*table->comm_size)-sum_written_buckets, - sum_w_access, - sum_r_access, - sum_read_misses, - sum_collisions); - - printf("%s\n", pad); - printf("%s %d\n", - "new entries:", - sum_written_buckets - table->stats->old_writes); - - printf("\n"); - fflush(stdout); - - table->stats->old_writes = sum_written_buckets; - } - - //enable warning again - #pragma GCC diagnostic pop - - MPI_Barrier(table->communicator); - return DHT_SUCCESS; -} \ No newline at end of file diff --git a/src/DHT.h b/src/DHT.h deleted file mode 100644 index 1b118e690..000000000 --- a/src/DHT.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * File: DHT.h - * Author: max luebke - * - * Created on 16. November 2017, 09:14 - */ - -#ifndef DHT_H -#define DHT_H - -#include -#include - -#define DHT_MPI_ERROR -1 -#define DHT_READ_ERROR -2 -#define DHT_SUCCESS 0 -#define DHT_WRITE_SUCCESS_WITH_COLLISION 1 - -#define DHT_WRONG_FILE 11 -#define DHT_FILE_IO_ERROR 12 -#define DHT_FILE_READ_ERROR 13 -#define DHT_FILE_WRITE_ERROR 14 - -#define DHT_HEADER_SIZE 8 - -typedef struct {; - int *writes_local, old_writes; - int read_misses, collisions; - int w_access, r_access; -} DHT_stats; - -typedef struct { - MPI_Win window; - int data_size; - int key_size; - unsigned int table_size; - MPI_Comm communicator; - int comm_size; - uint64_t(*hash_func) (int, void*); - void* recv_entry; - void* send_entry; - void* mem_alloc; - int read_misses; - int collisions; - unsigned int *index; - unsigned int index_count; - DHT_stats *stats; -} DHT; - - - -/* - * parameters: - * MPI_Comm comm - communicator of processes that are holding the DHT - * int size_per_process - number of buckets each process will create - * int data_size - size of data in bytes - * int key_size - size of key in bytes - * *hash_func - pointer to hashfunction - * - * return: - * NULL if error during initialization - * DHT* if success - */ -extern DHT* DHT_create(MPI_Comm comm, unsigned int size_per_process, int data_size, int key_size, uint64_t(*hash_func)(int, void*)); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * void* data - pointer to data - * void* - pointer to key - * - * return: - * error value (see above) - */ -extern int DHT_write(DHT *table, void* key, void* data); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * void* key - pointer to key - * void* destination - pointer which will hold the resulting data from DHT - * - * return: - * error value (see above) - */ -extern int DHT_read(DHT *table, void* key, void* destination); - -extern int DHT_to_file(DHT *table, char* filename); - -extern int DHT_from_file(DHT *table, char* filename); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * int* collision_counter - pointer which will hold the total count of collisions - * int* readerror_counter - pointer which will hold the total count of read errors - * - * return: - * error value (see above) - */ -extern int DHT_free(DHT *table, int* collision_counter, int* readerror_counter); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * - * return: - * error value (DHT_SUCCESS or DHT_MPI_ERROR) - */ -extern int DHT_print_statistics(DHT *table); - -#endif /* DHT_H */ \ No newline at end of file diff --git a/src/dht_wrapper.cpp b/src/dht_wrapper.cpp deleted file mode 100644 index f03707c45..000000000 --- a/src/dht_wrapper.cpp +++ /dev/null @@ -1,176 +0,0 @@ -#include "dht_wrapper.h" -#include - -using namespace poet; - -/*init globals*/ -//bool dht_enabled; -//int dht_snaps; -//int dht_strategy; -//int dht_significant_digits; -//std::string dht_file; -//std::vector dht_significant_digits_vector; -//std::vector prop_type_vector; -//bool dht_logarithm; -//uint64_t dht_size_per_process; -uint64_t dht_hits, dht_miss, dht_collision; -RInside *R_DHT; -std::vector dht_flags; -DHT *dht_object; - -double *fuzzing_buffer; - -//bool dt_differ; -/*functions*/ - -uint64_t get_md5(int key_size, void *key) { - MD5_CTX ctx; - unsigned char sum[MD5_DIGEST_LENGTH]; - uint64_t retval, *v1, *v2; - - MD5_Init(&ctx); - MD5_Update(&ctx, key, key_size); - MD5_Final(sum, &ctx); - - v1 = (uint64_t *)&sum[0]; - v2 = (uint64_t *)&sum[8]; - retval = *v1 ^ *v2; - - return retval; -} - -// double Round_off(double N, double n) { -// double result; -// R["roundsig"] = n; -// R["roundin"] = N; -// -// result = R.parseEval("signif(roundin, digits=roundsig)"); -// -// return result; -//} - -/* - * Stores fuzzed version of key in fuzzing_buffer - */ -void fuzz_for_dht(int var_count, void *key, double dt, t_simparams *params) { - unsigned int i = 0; - // introduce fuzzing to allow more hits in DHT - for (i = 0; i < (unsigned int)var_count; i++) { - if (params->dht_prop_type_vector[i] == "act") { - // with log10 - if (params->dht_log) { - if (((double *)key)[i] < 0) - cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value in " - "key!" - << endl; - else if (((double *)key)[i] == 0) - fuzzing_buffer[i] = 0; - else - fuzzing_buffer[i] = ROUND(-(std::log10(((double *)key)[i])), - params->dht_signif_vector[i]); - } else { - // without log10 - fuzzing_buffer[i] = - ROUND((((double *)key)[i]), params->dht_signif_vector[i]); - } - } else if (params->dht_prop_type_vector[i] == "logact") { - fuzzing_buffer[i] = - ROUND((((double *)key)[i]), params->dht_signif_vector[i]); - } else if (params->dht_prop_type_vector[i] == "ignore") { - fuzzing_buffer[i] = 0; - } else { - cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Probably wrong " - "prop_type!" - << endl; - } - } - if (params->dt_differ) - fuzzing_buffer[var_count] = dt; -} - -void check_dht(int length, std::vector &out_result_index, - double *work_package, double dt, t_simparams *params) { - void *key; - int res; - int var_count = params->dht_prop_type_vector.size(); - for (int i = 0; i < length; i++) { - key = (void *)&(work_package[i * var_count]); - - // fuzz data (round, logarithm etc.) - fuzz_for_dht(var_count, key, dt, params); - - // overwrite input with data from DHT, IF value is found in DHT - res = DHT_read(dht_object, fuzzing_buffer, key); - - if (res == DHT_SUCCESS) { - // flag that this line is replaced by DHT-value, do not simulate!! - out_result_index[i] = false; - dht_hits++; - } else if (res == DHT_READ_ERROR) { - // this line is untouched, simulation is needed - out_result_index[i] = true; - dht_miss++; - } else { - // MPI ERROR ... WHAT TO DO NOW? - // RUNNING CIRCLES WHILE SCREAMING - } - } -} - -void fill_dht(int length, std::vector &result_index, double *work_package, - double *results, double dt, t_simparams *params) { - void *key; - void *data; - int res; - int var_count = params->dht_prop_type_vector.size(); - - for (int i = 0; i < length; i++) { - key = (void *)&(work_package[i * var_count]); - data = (void *)&(results[i * var_count]); - - if (result_index[i]) { - // If true -> was simulated, needs to be inserted into dht - - // fuzz data (round, logarithm etc.) - fuzz_for_dht(var_count, key, dt, params); - - res = DHT_write(dht_object, fuzzing_buffer, data); - - if (res != DHT_SUCCESS) { - if (res == DHT_WRITE_SUCCESS_WITH_COLLISION) { - dht_collision++; - } else { - // MPI ERROR ... WHAT TO DO NOW? - // RUNNING CIRCLES WHILE SCREAMING - } - } - } - } -} - -void print_statistics() { - int res; - - res = DHT_print_statistics(dht_object); - - if (res != DHT_SUCCESS) { - // MPI ERROR ... WHAT TO DO NOW? - // RUNNING CIRCLES WHILE SCREAMING - } -} - -int table_to_file(char *filename) { - int res = DHT_to_file(dht_object, filename); - return res; -} - -int file_to_table(char *filename) { - - int res = DHT_from_file(dht_object, filename); - if (res != DHT_SUCCESS) - return res; - - DHT_print_statistics(dht_object); - - return DHT_SUCCESS; -} diff --git a/src/dht_wrapper.h b/src/dht_wrapper.h deleted file mode 100644 index f6e6cb7f5..000000000 --- a/src/dht_wrapper.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once -#include "DHT.h" -#include "util/RRuntime.h" -#include "util/SimParams.h" -#include -#include -#include - -using namespace std; -using namespace poet; - -/*Functions*/ -uint64_t get_md5(int key_size, void *key); -void fuzz_for_dht(int var_count, void *key, double dt, t_simparams *params); -void check_dht(int length, std::vector &out_result_index, - double *work_package, double dt, t_simparams *params); -void fill_dht(int length, std::vector &result_index, double *work_package, - double *results, double dt, t_simparams *params); -void print_statistics(); -int table_to_file(char *filename); -int file_to_table(char *filename); - -/*globals*/ -//extern bool dht_enabled; -//extern int dht_snaps; -//extern std::string dht_file; -//extern bool dt_differ; - -// Default DHT Size per process in Byte (defaults to 1 GiB) -#define DHT_SIZE_PER_PROCESS 1073741824 - -// sets default dht access and distribution strategy -#define DHT_STRATEGY 0 -// 0 -> DHT is on workers, access from workers only -// 1 -> DHT is on workers + master, access from master only !NOT IMPLEMENTED -// YET! - -#define ROUND(value, signif) \ - (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) - -//extern int dht_strategy; -//extern int dht_significant_digits; -//extern std::vector dht_significant_digits_vector; -//extern std::vector prop_type_vector; -//extern bool dht_logarithm; -//extern uint64_t dht_size_per_process; - -// global DHT object, can be NULL if not initialized, check strategy -extern DHT *dht_object; - -// DHT Performance counter -extern uint64_t dht_hits, dht_miss, dht_collision; - -extern double *fuzzing_buffer; -extern std::vector dht_flags; diff --git a/src/global_buffer.h b/src/global_buffer.h deleted file mode 100644 index 69cc742f2..000000000 --- a/src/global_buffer.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#define BUFFER_OFFSET 5 - -/*Globals*/ -extern double* mpi_buffer; -extern double* mpi_buffer_results; - -extern uint32_t work_package_size; \ No newline at end of file diff --git a/src/r_utils.cpp b/src/r_utils.cpp deleted file mode 100644 index eafa33fa6..000000000 --- a/src/r_utils.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "r_utils.h" - -/* This function converts a pure double dataframe into a double array. - buffer <- double array, needs to be allocated before - df <- reference to input dataframe -*/ -void convert_R_Dataframe_2_C_buffer(double* buffer, Rcpp::DataFrame &df) -{ - size_t rowCount = df.nrow(); - size_t colCount = df.ncol(); - - for (size_t i = 0; i < rowCount; i++) - { - for (size_t j = 0; j < colCount; j++) - { - /* Access column vector j and extract value of line i */ - Rcpp::DoubleVector col = df[j]; - buffer[i * colCount + j] = col[i]; - } - } -} - -/* This function converts a double array into a double dataframe. - buffer <- input double array - df <- reference to output dataframe, needs to be of fitting size, structure will be taken from it -*/ -void convert_C_buffer_2_R_Dataframe(double* buffer, Rcpp::DataFrame &df) -{ - size_t rowCount = df.nrow(); - size_t colCount = df.ncol(); - - for (size_t i = 0; i < rowCount; i++) - { - for (size_t j = 0; j < colCount; j++) - { - /* Access column vector j and extract value of line i */ - Rcpp::DoubleVector col = df[j]; - col[i] = buffer[i * colCount + j]; - } - } -} - - - diff --git a/src/r_utils.h b/src/r_utils.h deleted file mode 100644 index 95000e97f..000000000 --- a/src/r_utils.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -/*Functions*/ -void convert_R_Dataframe_2_C_buffer(double* buffer, Rcpp::DataFrame &df); -void convert_C_buffer_2_R_Dataframe(double* buffer, Rcpp::DataFrame &df); \ No newline at end of file diff --git a/src/worker.cpp b/src/worker.cpp deleted file mode 100644 index 60643530d..000000000 --- a/src/worker.cpp +++ /dev/null @@ -1,301 +0,0 @@ -#include "worker.h" -#include "dht_wrapper.h" -#include "global_buffer.h" -#include "model/Grid.h" -#include "util/RRuntime.h" -#include -#include -#include - -using namespace poet; -using namespace Rcpp; - -void worker_function(t_simparams *params) { - RRuntime R = *(static_cast(params->R)); - Grid grid = *(static_cast(params->grid)); - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - MPI_Status probe_status; - int count; - - int local_work_package_size; - int iteration; - double dt, current_sim_time; - - double idle_a, idle_b; - double cummul_idle = 0.f; - - double dht_get_start = 0, dht_get_end = 0; - double dht_fill_start = 0, dht_fill_end = 0; - double phreeqc_time_start = 0, phreeqc_time_end = 0; - int phreeqc_count = 0; - - // timing[0] -> phreeqc - // timing[1] -> dht_get - // timing[2] -> dht_fill - double timing[3]; - timing[0] = 0.0; - timing[1] = 0.0; - timing[2] = 0.0; - - // dht_perf[0] -> hits - // dht_perf[1] -> miss - // dht_perf[2] -> collisions - uint64_t dht_perf[3]; - - if (params->dht_enabled) { - dht_flags.resize(params->wp_size, true); // set size - dht_flags.assign(params->wp_size, - true); // assign all elements to true (default) - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; - - // MDL: This code has now been moved to kin.cpp - // /*Load significance vector from R setup file (or set default)*/ - // bool signif_vector_exists = R.parseEval("exists('signif_vector')"); - // if (signif_vector_exists) - // { - // dht_significant_digits_vector = - // as>(R["signif_vector"]); - // } else - // { - // dht_significant_digits_vector.assign(dht_object->key_size / - // sizeof(double), dht_significant_digits); - // } - - // /*Load property type vector from R setup file (or set default)*/ - // bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); - // if (prop_type_vector_exists) - // { - // prop_type_vector = as>(R["prop_type"]); - // } else - // { - // prop_type_vector.assign(dht_object->key_size / sizeof(double), - // "normal"); - // } - } - - // initialization of helper variables - iteration = 0; - dt = 0; - current_sim_time = 0; - local_work_package_size = 0; - - /*worker loop*/ - while (1) { - /*Wait for Message*/ - idle_a = MPI_Wtime(); - MPI_Probe(0, MPI_ANY_TAG, MPI_COMM_WORLD, &probe_status); - idle_b = MPI_Wtime(); - - if (probe_status.MPI_TAG == TAG_WORK) { /* do work */ - - cummul_idle += idle_b - idle_a; - - /* get number of doubles sent */ - MPI_Get_count(&probe_status, MPI_DOUBLE, &count); - - /* receive */ - MPI_Recv(mpi_buffer, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - - // decrement count of work_package by BUFFER_OFFSET - count -= BUFFER_OFFSET; - // check for changes on all additional variables given by the 'header' of - // mpi_buffer - if (mpi_buffer[count] != local_work_package_size) { // work_package_size - local_work_package_size = mpi_buffer[count]; - R["work_package_size"] = local_work_package_size; - R.parseEvalQ("mysetup$work_package_size <- work_package_size"); - } - if (mpi_buffer[count + 1] != - iteration) { // current iteration of simulation - iteration = mpi_buffer[count + 1]; - R["iter"] = iteration; - R.parseEvalQ("mysetup$iter <- iter"); - } - if (mpi_buffer[count + 2] != dt) { // current timestep size - dt = mpi_buffer[count + 2]; - R["dt"] = dt; - R.parseEvalQ("mysetup$dt <- dt"); - } - if (mpi_buffer[count + 3] != - current_sim_time) { // current simulation time ('age' of simulation) - current_sim_time = mpi_buffer[count + 3]; - R["simulation_time"] = current_sim_time; - R.parseEvalQ("mysetup$simulation_time <- simulation_time"); - } - /* 4th double value is currently a placeholder */ - // if (mpi_buffer[count+4] != placeholder) { - // placeholder = mpi_buffer[count+4]; - // R["mysetup$placeholder"] = placeholder; - // } - - /* get df with right structure to fill in work package */ - // R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); - // R["skeleton"] = grid.buildDataFrame(work_package_size); - //// R.parseEval("print(rownames(tmp2)[1:5])"); - //// R.parseEval("print(head(tmp2, 2))"); - //// R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); - - ////Rcpp::DataFrame buffer = R.parseEval("tmp2"); - // R.setBufferDataFrame("skeleton"); - - if (params->dht_enabled) { - // DEBUG - // cout << "RANK " << world_rank << " start checking DHT\n"; - - // resize helper vector dht_flags of work_package_size changes - if ((int)dht_flags.size() != local_work_package_size) { - dht_flags.resize(local_work_package_size, true); // set size - dht_flags.assign(local_work_package_size, - true); // assign all elements to true (default) - } - - dht_get_start = MPI_Wtime(); - check_dht(local_work_package_size, dht_flags, mpi_buffer, dt, params); - dht_get_end = MPI_Wtime(); - - // DEBUG - // cout << "RANK " << world_rank << " checking DHT complete \n"; - - R["dht_flags"] = as(wrap(dht_flags)); - // R.parseEvalQ("print(head(dht_flags))"); - } - - /* work */ - // R.from_C_domain(mpi_buffer); - ////convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); - // R["work_package_full"] = R.getBufferDataFrame(); - // R["work_package"] = buffer; - - // DEBUG - // R.parseEvalQ("print(head(work_package_full))"); - // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); - - grid.importWP(mpi_buffer, params->wp_size); - - if (params->dht_enabled) { - R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); - } else { - R.parseEvalQ("work_package <- work_package_full"); - } - - // DEBUG - // R.parseEvalQ("print(head(work_package),2)"); - - // R.parseEvalQ("rownames(work_package) <- work_package$id"); - // R.parseEval("print(paste('id %in% colnames(work_package)', 'id' %in% - // colnames(work_package)"); R.parseEvalQ("id_store <- - // rownames(work_package)"); //"[, ncol(work_package)]"); - // R.parseEvalQ("work_package$id <- NULL"); - R.parseEvalQ("work_package <- as.matrix(work_package)"); - - unsigned int nrows = R.parseEval("nrow(work_package)"); - - if (nrows > 0) { - /*Single Line error Workaround*/ - if (nrows <= 1) { - // duplicate line to enable correct simmulation - R.parseEvalQ("work_package <- work_package[rep(1:nrow(work_package), " - "times = 2), ]"); - } - - phreeqc_count++; - - phreeqc_time_start = MPI_Wtime(); - // MDL - // R.parseEvalQ("print('Work_package:\n'); print(head(work_package , - // 2)); cat('RCpp: worker_function:', local_rank, ' \n')"); - R.parseEvalQ("result <- as.data.frame(slave_chemistry(setup=mysetup, " - "data = work_package))"); - phreeqc_time_end = MPI_Wtime(); - // R.parseEvalQ("result$id <- id_store"); - } else { - // cout << "Work-Package is empty, skipping phreeqc!" << endl; - } - - if (params->dht_enabled) { - R.parseEvalQ("result_full <- work_package_full"); - if (nrows > 0) - R.parseEvalQ("result_full[dht_flags,] <- result"); - } else { - R.parseEvalQ("result_full <- result"); - } - - // R.setBufferDataFrame("result_full"); - ////Rcpp::DataFrame result = R.parseEval("result_full"); - ////convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); - // R.to_C_domain(mpi_buffer_results); - - grid.exportWP(mpi_buffer_results); - /* send results to master */ - MPI_Request send_req; - MPI_Isend(mpi_buffer_results, count, MPI_DOUBLE, 0, TAG_WORK, - MPI_COMM_WORLD, &send_req); - - if (params->dht_enabled) { - dht_fill_start = MPI_Wtime(); - fill_dht(local_work_package_size, dht_flags, mpi_buffer, - mpi_buffer_results, dt, params); - dht_fill_end = MPI_Wtime(); - - timing[1] += dht_get_end - dht_get_start; - timing[2] += dht_fill_end - dht_fill_start; - } - - timing[0] += phreeqc_time_end - phreeqc_time_start; - - MPI_Wait(&send_req, MPI_STATUS_IGNORE); - - } else if (probe_status.MPI_TAG == TAG_FINISH) { /* recv and die */ - /* before death, submit profiling/timings to master*/ - - MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_FINISH, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - - // timings - MPI_Send(timing, 3, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); - - MPI_Send(&phreeqc_count, 1, MPI_INT, 0, TAG_TIMING, MPI_COMM_WORLD); - MPI_Send(&cummul_idle, 1, MPI_DOUBLE, 0, TAG_TIMING, MPI_COMM_WORLD); - - if (params->dht_enabled) { - // dht_perf - dht_perf[0] = dht_hits; - dht_perf[1] = dht_miss; - dht_perf[2] = dht_collision; - MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, - MPI_COMM_WORLD); - } - break; - - } else if ((probe_status.MPI_TAG == TAG_DHT_STATS)) { - MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_DHT_STATS, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - print_statistics(); - MPI_Barrier(MPI_COMM_WORLD); - - } else if ((probe_status.MPI_TAG == TAG_DHT_STORE)) { - char *outdir; - MPI_Get_count(&probe_status, MPI_CHAR, &count); - outdir = (char *)calloc(count + 1, sizeof(char)); - MPI_Recv(outdir, count, MPI_CHAR, 0, TAG_DHT_STORE, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - int res = table_to_file((char *)outdir); - if (res != DHT_SUCCESS) { - if (world_rank == 2) - cerr << "CPP: Worker: Error in writing current state of DHT to file " - "(TAG_DHT_STORE)" - << endl; - } else { - if (world_rank == 2) - cout << "CPP: Worker: Successfully written DHT to file " << outdir - << endl; - } - free(outdir); - MPI_Barrier(MPI_COMM_WORLD); - } - } -} diff --git a/src/worker.h b/src/worker.h deleted file mode 100644 index 076ede6e2..000000000 --- a/src/worker.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once -#include "util/SimParams.h" -#include "util/RRuntime.h" -#include "model/Grid.h" - -using namespace std; -using namespace poet; -/*Functions*/ -void worker_function(t_simparams *params); - - -/*Globals*/ -#define TAG_WORK 42 -#define TAG_FINISH 43 -#define TAG_TIMING 44 -#define TAG_DHT_PERF 45 -#define TAG_DHT_STATS 46 -#define TAG_DHT_STORE 47 From b461e05d5a1715fdc322252558edbfe6eebb473f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 12 Jan 2021 14:48:25 +0100 Subject: [PATCH 11/50] Introducing TransportSim and Parser --- src/CMakeLists.txt | 2 +- src/kin.cpp | 369 +++++++++++++++++++------------------ src/model/ChemMaster.cpp | 54 +++--- src/model/ChemSim.cpp | 18 +- src/model/ChemSim.h | 15 +- src/model/ChemWorker.cpp | 31 ++-- src/model/TransportSim.cpp | 19 ++ src/model/TransportSim.h | 22 +++ src/util/Parser.cpp | 144 +++++++++++++++ src/util/Parser.h | 42 +++++ src/util/SimParams.h | 3 + 11 files changed, 491 insertions(+), 228 deletions(-) create mode 100644 src/model/TransportSim.cpp create mode 100644 src/model/TransportSim.h create mode 100644 src/util/Parser.cpp create mode 100644 src/util/Parser.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f0f865eef..d1b71ab99 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,7 +89,7 @@ include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp) +add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp) target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) diff --git a/src/kin.cpp b/src/kin.cpp index 673b5c223..f67186f4a 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -12,70 +12,72 @@ // #include "global_buffer.h" #include "model/ChemSim.h" #include "model/Grid.h" +#include "util/Parser.h" #include "util/RRuntime.h" #include "util/SimParams.h" +#include "model/TransportSim.h" // #include "worker.h" -#define DHT_SIZE_PER_PROCESS 1073741824 +//#define DHT_SIZE_PER_PROCESS 1073741824 using namespace std; using namespace poet; using namespace Rcpp; -double *mpi_buffer; -double *mpi_buffer_results; +// double *mpi_buffer; +// double *mpi_buffer_results; -uint32_t work_package_size; -#define WORK_PACKAGE_SIZE_DEFAULT 5 +// uint32_t work_package_size; +// #define WORK_PACKAGE_SIZE_DEFAULT 5 -bool store_result; +// bool store_result; -std::set paramList() { - std::set options; - // global - options.insert("work-package-size"); - // only DHT - options.insert("dht-signif"); - options.insert("dht-strategy"); - options.insert("dht-size"); - options.insert("dht-snaps"); - options.insert("dht-file"); +// std::set paramList() { +// std::set options; +// // global +// options.insert("work-package-size"); +// // only DHT +// options.insert("dht-signif"); +// options.insert("dht-strategy"); +// options.insert("dht-size"); +// options.insert("dht-snaps"); +// options.insert("dht-file"); - return options; -} +// return options; +// } -std::set flagList() { - std::set options; - // global - options.insert("ignore-result"); - // only DHT - options.insert("dht"); - options.insert("dht-log"); +// std::set flagList() { +// std::set options; +// // global +// options.insert("ignore-result"); +// // only DHT +// options.insert("dht"); +// options.insert("dht-log"); - return options; -} +// return options; +// } -std::list checkOptions(argh::parser cmdl) { - std::list retList; - std::set flist = flagList(); - std::set plist = paramList(); +// std::list checkOptions(argh::parser cmdl) { +// std::list retList; +// std::set flist = flagList(); +// std::set plist = paramList(); - for (auto &flag : cmdl.flags()) { - if (!(flist.find(flag) != flist.end())) retList.push_back(flag); - } +// for (auto &flag : cmdl.flags()) { +// if (!(flist.find(flag) != flist.end())) retList.push_back(flag); +// } - for (auto ¶m : cmdl.params()) { - if (!(plist.find(param.first) != plist.end())) - retList.push_back(param.first); - } +// for (auto ¶m : cmdl.params()) { +// if (!(plist.find(param.first) != plist.end())) +// retList.push_back(param.first); +// } - return retList; -} +// return retList; +// } -typedef struct { - char has_work; - double *send_addr; -} worker_struct; +// typedef struct { +// char has_work; +// double *send_addr; +// } worker_struct; int main(int argc, char *argv[]) { double sim_start, sim_b_transport, sim_a_transport, sim_b_chemistry, @@ -83,35 +85,20 @@ int main(int argc, char *argv[]) { double cummul_transport = 0.f; double cummul_chemistry = 0.f; - double cummul_workers = 0.f; - double cummul_chemistry_master = 0.f; - double cummul_master_seq = 0.f; - double cummul_master_seq_loop = 0.f; - double master_idle = 0.f; - - double master_send_a, master_send_b; - double cummul_master_send = 0.f; - double master_recv_a, master_recv_b; - double cummul_master_recv = 0.f; - - double sim_a_seq, sim_b_seq, sim_c_seq, sim_d_seq; - double idle_a, idle_b; - - double sim_c_chemistry, sim_d_chemistry; - double sim_e_chemistry, sim_f_chemistry; argh::parser cmdl(argv); int dht_significant_digits; // cout << "CPP: Start Init (MPI)" << endl; t_simparams params; + int world_size, world_rank; MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &(params.world_size)); + MPI_Comm_size(MPI_COMM_WORLD, &world_size); - MPI_Comm_rank(MPI_COMM_WORLD, &(params.world_rank)); + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); /*Create custom Communicator with all processes except 0 (the master) for DHT * storage*/ @@ -121,109 +108,90 @@ int main(int argc, char *argv[]) { int *process_ranks; // make a list of processes in the new communicator - process_ranks = (int *)malloc(params.world_size * sizeof(int)); - for (int I = 1; I < params.world_size; I++) process_ranks[I - 1] = I; + process_ranks = (int *)malloc(world_size * sizeof(int)); + for (int I = 1; I < world_size; I++) process_ranks[I - 1] = I; // get the group under MPI_COMM_WORLD MPI_Comm_group(MPI_COMM_WORLD, &group_world); // create the new group - MPI_Group_incl(group_world, params.world_size - 1, process_ranks, &dht_group); + MPI_Group_incl(group_world, world_size - 1, process_ranks, &dht_group); // create the new communicator MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); free(process_ranks); // cleanup // cout << "Done"; - if (cmdl[{"help", "h"}]) { - if (params.world_rank == 0) { - cout << "Todo" << endl - << "See README.md for further information." << endl; - } - MPI_Finalize(); - return EXIT_SUCCESS; - } + // if (cmdl[{"help", "h"}]) { + // if (params.world_rank == 0) { + // cout << "Todo" << endl + // << "See README.md for further information." << endl; + // } + // MPI_Finalize(); + // return EXIT_SUCCESS; + // } - /*INIT is now done separately in an R file provided here as argument!*/ - if (!cmdl(2)) { - if (params.world_rank == 0) { - cerr << "ERROR. Kin needs 2 positional arguments: " << endl - << "1) the R script defining your simulation and" << endl - << "2) the directory prefix where to save results and profiling" - << endl; - } - MPI_Finalize(); - return EXIT_FAILURE; - } + // /*INIT is now done separately in an R file provided here as argument!*/ + // if (!cmdl(2)) { + // if (params.world_rank == 0) { + // cerr << "ERROR. Kin needs 2 positional arguments: " << endl + // << "1) the R script defining your simulation and" << endl + // << "2) the directory prefix where to save results and profiling" + // << endl; + // } + // MPI_Finalize(); + // return EXIT_FAILURE; + // } - std::list optionsError = checkOptions(cmdl); - if (!optionsError.empty()) { - if (params.world_rank == 0) { - cerr << "Unrecognized option(s):\n" << endl; - for (auto option : optionsError) { - cerr << option << endl; - } - cerr << "\nMake sure to use available options. Exiting!" << endl; - } - MPI_Finalize(); - return EXIT_FAILURE; - } + // std::list optionsError = checkOptions(cmdl); + // if (!optionsError.empty()) { + // if (params.world_rank == 0) { + // cerr << "Unrecognized option(s):\n" << endl; + // for (auto option : optionsError) { + // cerr << option << endl; + // } + // cerr << "\nMake sure to use available options. Exiting!" << endl; + // } + // MPI_Finalize(); + // return EXIT_FAILURE; + // } - /*Parse DHT arguments*/ - params.dht_enabled = cmdl["dht"]; - // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; + // /*Parse DHT arguments*/ + // params.dht_enabled = cmdl["dht"]; + // // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - if (params.dht_enabled) { - cmdl("dht-strategy", 0) >> params.dht_strategy; - // cout << "CPP: DHT strategy is " << dht_strategy << endl; + // if (params.dht_enabled) { + // cmdl("dht-strategy", 0) >> params.dht_strategy; + // // cout << "CPP: DHT strategy is " << dht_strategy << endl; - cmdl("dht-signif", 5) >> dht_significant_digits; - // cout << "CPP: DHT significant digits = " << dht_significant_digits << - // endl; + // cmdl("dht-signif", 5) >> dht_significant_digits; + // // cout << "CPP: DHT significant digits = " << dht_significant_digits << + // // endl; - params.dht_log = !(cmdl["dht-nolog"]); - // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" - // : "OFF" ) << endl; + // params.dht_log = !(cmdl["dht-nolog"]); + // // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? + // "ON" + // // : "OFF" ) << endl; - cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> params.dht_size_per_process; - // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << - // endl; + // cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> params.dht_size_per_process; + // // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process + // << + // // endl; - cmdl("dht-snaps", 0) >> params.dht_snaps; + // cmdl("dht-snaps", 0) >> params.dht_snaps; - cmdl("dht-file") >> params.dht_file; - } + // cmdl("dht-file") >> params.dht_file; + // } - /*Parse work package size*/ - cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> params.wp_size; + // /*Parse work package size*/ + // cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> params.wp_size; - /*Parse output options*/ - store_result = !cmdl["ignore-result"]; + // /*Parse output options*/ + // store_result = !cmdl["ignore-result"]; - if (params.world_rank == 0) { - cout << "CPP: Complete results storage is " << (store_result ? "ON" : "OFF") - << endl; - cout << "CPP: Work Package Size: " << params.wp_size << endl; - cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; - - if (params.dht_enabled) { - cout << "CPP: DHT strategy is " << params.dht_strategy << endl; - cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " - "defined) = " - << dht_significant_digits << endl; - cout << "CPP: DHT logarithm before rounding: " - << (params.dht_log ? "ON" : "OFF") << endl; - cout << "CPP: DHT size per process (Byte) = " - << params.dht_size_per_process << endl; - cout << "CPP: DHT save snapshots is " << params.dht_snaps << endl; - cout << "CPP: DHT load file is " << params.dht_file << endl; - } - } - - cout << "CPP: R Init (RInside) on process " << params.world_rank << endl; RRuntime R(argc, argv); - params.R = &R; + cout << "CPP: R Init (RInside) on process " << world_rank << endl; // if local_rank == 0 then master else worker - R["local_rank"] = params.world_rank; + // R["local_rank"] = params.world_rank; /*Loading Dependencies*/ std::string r_load_dependencies = @@ -233,37 +201,71 @@ int main(int argc, char *argv[]) { "source('parallel_r_library.R');"; R.parseEvalQ(r_load_dependencies); - std::string filesim; - cmdl(1) >> filesim; // <- first positional argument - R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' - R.parseEvalQ( - "source(filesim)"); // eval the init string, ignoring any returns + Parser parser(argv, world_rank, world_size); + int pret = parser.parseCmdl(); - if (params.world_rank == - 0) { // only rank 0 initializes goes through the whole initialization - cmdl(2) >> params.out_dir; // <- second positional argument - R["fileout"] = - wrap(params.out_dir); // assign a char* (string) to 'fileout' + if (pret == PARSER_ERROR) { + MPI_Finalize(); + return EXIT_FAILURE; + } else if (pret == PARSER_HELP) { + MPI_Finalize(); + return EXIT_SUCCESS; + } + + parser.parseR(R); + params = parser.getParams(); + + // if (params.world_rank == 0) { + // cout << "CPP: Complete results storage is " << (params.store_result ? "ON" : + // "OFF") + // << endl; + // cout << "CPP: Work Package Size: " << params.wp_size << endl; + // cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; + + // if (params.dht_enabled) { + // cout << "CPP: DHT strategy is " << params.dht_strategy << endl; + // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + // // "defined) = " + // // << dht_significant_digits << endl; + // cout << "CPP: DHT logarithm before rounding: " + // << (params.dht_log ? "ON" : "OFF") << endl; + // cout << "CPP: DHT size per process (Byte) = " + // << params.dht_size_per_process << endl; + // cout << "CPP: DHT save snapshots is " << params.dht_snaps << endl; + // cout << "CPP: DHT load file is " << params.dht_file << endl; + // } + // } + + // std::string filesim; + // cmdl(1) >> filesim; // <- first positional argument + // R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' + // R.parseEvalQ( + // "source(filesim)"); // eval the init string, ignoring any returns + + // only rank 0 initializes goes through the whole initialization + if (world_rank == 0) { + // cmdl(2) >> params.out_dir; // <- second positional argument + // R["fileout"] = + // wrap(params.out_dir); // assign a char* (string) to 'fileout' // Note: R::sim_init() checks if the directory already exists, // if not it makes it // pass the boolean "store_result" to the R process - R["store_result"] = store_result; + // R["store_result"] = store_result; // get timestep vector from grid_init function ... std::string master_init_code = "mysetup <- master_init(setup=setup)"; R.parseEval(master_init_code); - params.dt_differ = - R.parseEval("mysetup$dt_differ"); // TODO: Set in DHTWrapper + params.dt_differ = R.parseEval("mysetup$dt_differ"); MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } else { // workers will only read the setup DataFrame defined by input file R.parseEval("mysetup <- setup"); MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } - if (params.world_rank == 0) { + if (world_rank == 0) { cout << "CPP: R init done on process with rank " << params.world_rank << endl; } @@ -279,8 +281,8 @@ int main(int argc, char *argv[]) { // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); /* Init Parallel helper functions */ - R["n_procs"] = params.world_size - 1; /* worker count */ - R["work_package_size"] = params.wp_size; + // R["n_procs"] = params.world_size - 1; /* worker count */ + // R["work_package_size"] = params.wp_size; // Removed additional field for ID in previous versions // if (params.world_rank == 0) { @@ -293,21 +295,24 @@ int main(int argc, char *argv[]) { // (double *)calloc(params.wp_size * (grid.getCols()), sizeof(double)); // } - if (params.world_rank == 0) { - cout << "CPP: parallel init completed (buffers allocated)!" << endl; - } + // if (params.world_rank == 0) { + // cout << "CPP: parallel init completed (buffers allocated)!" << endl; + // } // MDL: pass to R the DHT stuff (basically, only for storing of // simulation parameters). These 2 variables are always defined: - R["dht_enabled"] = params.dht_enabled; - R["dht_log"] = params.dht_log; + // R["dht_enabled"] = params.dht_enabled; + // R["dht_log"] = params.dht_log; + + params.R = &R; if (params.dht_enabled) { // cout << "\nCreating DHT\n"; // determine size of dht entries // int dht_data_size = grid.getCols() * sizeof(double); // int dht_key_size = - // grid.getCols() * sizeof(double) + (params.dt_differ * sizeof(double)); + // grid.getCols() * sizeof(double) + (params.dt_differ * + // sizeof(double)); // // // determine bucket count for preset memory usage // // // bucket size is key + value + 1 byte for status @@ -320,8 +325,7 @@ int main(int argc, char *argv[]) { if (signif_vector_exists) { params.dht_signif_vector = as>(R["signif_vector"]); } else { - // params.dht_signif_vector.assign(dht_object->key_size / sizeof(double), - // dht_significant_digits); + params.dht_signif_vector.assign(grid.getCols(), dht_significant_digits); } /*Load property type vector from R setup file (or set default)*/ @@ -329,9 +333,7 @@ int main(int argc, char *argv[]) { if (prop_type_vector_exists) { params.dht_prop_type_vector = as>(R["prop_type"]); } else { - // params.dht_prop_type_vector.assign(dht_object->key_size / - // sizeof(double), - // "act"); + params.dht_prop_type_vector.assign(grid.getCols(), "act"); } if (params.world_rank == 0) { @@ -386,6 +388,7 @@ int main(int argc, char *argv[]) { if (params.world_rank == 0) { /* This is executed by the master */ ChemMaster master(¶ms, R, grid); + TransportSim trans(R); Rcpp::NumericVector master_send; Rcpp::NumericVector master_recv; @@ -412,22 +415,22 @@ int main(int argc, char *argv[]) { cout << "CPP: Calling Advection" << endl; - sim_b_transport = MPI_Wtime(); - R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); - sim_a_transport = MPI_Wtime(); + trans.runIteration(); + // sim_b_transport = MPI_Wtime(); + // R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); + // sim_a_transport = MPI_Wtime(); - if (iter == 1) master.prepareSimulation(); + // if (iter == 1) master.prepareSimulation(); cout << "CPP: Chemistry" << endl; /*Fallback for sequential execution*/ - sim_b_chemistry = MPI_Wtime(); if (params.world_size == 1) { master.runSeq(); } else { /*send work to workers*/ - master.runIteration(); + master.runPar(); } - sim_a_chemistry = MPI_Wtime(); + double master_seq_a = MPI_Wtime(); // MDL master_iteration_end just writes on disk state_T and // state_C after every iteration if the cmdline option @@ -435,8 +438,8 @@ int main(int argc, char *argv[]) { // store_result is TRUE) R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - cummul_transport += sim_a_transport - sim_b_transport; - cummul_chemistry += sim_a_chemistry - sim_b_chemistry; + cummul_transport += trans.getTransportTime(); + cummul_chemistry += master.getChemistryTime(); cout << endl << "CPP: End of *coupling* iteration " << iter << "/" << maxiter @@ -457,8 +460,9 @@ int main(int argc, char *argv[]) { } // END SIMULATION LOOP + cout << "CPP: finished simulation loop" << endl; + sim_end = MPI_Wtime(); - master.finishSimulation(); Rcpp::NumericVector phreeqc_time; Rcpp::NumericVector dht_get_time; @@ -483,6 +487,9 @@ int main(int argc, char *argv[]) { double idle_worker_tmp; + cout << "CPP: Advising worker to stop work and collect data from them" + << endl; + for (int p = 0; p < params.world_size - 1; p++) { /* ATTENTION Worker p has rank p+1 */ /* Send termination message to worker */ @@ -570,8 +577,8 @@ int main(int argc, char *argv[]) { r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ - ChemWorker worker(¶ms, R, grid); - worker.prepareSimulation(dht_comm); + ChemWorker worker(¶ms, R, grid, dht_comm); + // worker.prepareSimulation(dht_comm); worker.loop(); } @@ -587,7 +594,7 @@ int main(int argc, char *argv[]) { // } // } - free(mpi_buffer); + // free(mpi_buffer); MPI_Finalize(); if (params.world_rank == 0) { diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp index fee86113a..c4a96396b 100644 --- a/src/model/ChemMaster.cpp +++ b/src/model/ChemMaster.cpp @@ -15,9 +15,32 @@ ChemMaster::ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_) : ChemSim(params, R_, grid_) { this->wp_size = params->wp_size; this->out_dir = params->out_dir; + + workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); + send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, + sizeof(double)); + + R.parseEvalQ( + "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_C), " + "package_size=work_package_size)"); + + // we only sort once the vector + R.parseEvalQ("ordered_ids <- order(wp_ids)"); + R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); + R.parseEval("stat_wp_sizes(wp_sizes_vector)"); + wp_sizes_vector = as>(R["wp_sizes_vector"]); + + mpi_buffer = + (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); } -void ChemMaster::runIteration() { +ChemMaster::~ChemMaster() { + free(mpi_buffer); + free(workerlist); +} + +void ChemMaster::runPar() { + double chem_a, chem_b; double seq_a, seq_b, seq_c, seq_d; double worker_chemistry_a, worker_chemistry_b; double sim_e_chemistry, sim_f_chemistry; @@ -25,6 +48,8 @@ void ChemMaster::runIteration() { int free_workers; int i_pkgs; + chem_a = MPI_Wtime(); + seq_a = MPI_Wtime(); grid.shuffleAndExport(mpi_buffer); // retrieve data from R runtime @@ -68,6 +93,9 @@ void ChemMaster::runIteration() { chem_master += sim_f_chemistry - sim_e_chemistry; seq_d = MPI_Wtime(); seq_t += seq_d - seq_c; + + chem_b = MPI_Wtime(); + chem_t += chem_b - chem_a; } void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, @@ -180,30 +208,6 @@ void ChemMaster::printProgressbar(int count_pkgs, int n_wp, int barWidth) { /* end visual progress */ } -void ChemMaster::prepareSimulation() { - workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); - send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, - sizeof(double)); - - R.parseEvalQ( - "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_T), " - "package_size=work_package_size)"); - - // we only sort once the vector - R.parseEvalQ("ordered_ids <- order(wp_ids)"); - R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); - R.parseEval("stat_wp_sizes(wp_sizes_vector)"); - wp_sizes_vector = as>(R["wp_sizes_vector"]); - - mpi_buffer = - (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); -} - -void ChemMaster::finishSimulation() { - free(mpi_buffer); - free(workerlist); -} - double ChemMaster::getSendTime() { return this->send_t; } double ChemMaster::getRecvTime() { return this->recv_t; } diff --git a/src/model/ChemSim.cpp b/src/model/ChemSim.cpp index 62b783481..8a322ef4b 100644 --- a/src/model/ChemSim.cpp +++ b/src/model/ChemSim.cpp @@ -1,9 +1,12 @@ #include "ChemSim.h" + +#include +#include + +#include + #include "../util/RRuntime.h" #include "Grid.h" -#include -#include -#include using namespace Rcpp; using namespace poet; @@ -17,7 +20,16 @@ ChemSim::ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_) } void ChemSim::runSeq() { + double chem_a, chem_b; + + chem_a = MPI_Wtime(); + R.parseEvalQ( "result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + + chem_b = MPI_Wtime(); + chem_t += chem_b - chem_a; } + +double ChemSim::getChemistryTime() { return this->chem_t; } diff --git a/src/model/ChemSim.h b/src/model/ChemSim.h index 0d2c6a615..5b9b2f8ae 100644 --- a/src/model/ChemSim.h +++ b/src/model/ChemSim.h @@ -22,7 +22,9 @@ namespace poet { class ChemSim { public: ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_); + void runSeq(); + double getChemistryTime(); protected: double current_sim_time = 0; @@ -47,16 +49,17 @@ protected: worker_struct *workerlist; double *mpi_buffer; + + double chem_t = 0.f; }; class ChemMaster : public ChemSim { public: ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_); + ~ChemMaster(); - void prepareSimulation(); - void finishSimulation(); - - void runIteration(); + void runPar(); + void profile(); double getSendTime(); double getRecvTime(); @@ -83,9 +86,9 @@ private: class ChemWorker : public ChemSim { public: - ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_); + ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, MPI_Comm dht_comm); + ~ChemWorker(); - void prepareSimulation(MPI_Comm dht_comm); void loop(); private: diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index 91f8937ec..20746db3e 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -10,15 +10,15 @@ using namespace poet; using namespace std; using namespace Rcpp; -ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_) +ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, + MPI_Comm dht_comm) : params(params_), ChemSim(params_, R_, grid_) { this->dt_differ = params->dt_differ; this->dht_enabled = params->dht_enabled; this->dht_size_per_process = params->dht_size_per_process; this->dht_file = params->dht_file; -} + this->dht_snaps = params->dht_snaps; -void ChemWorker::prepareSimulation(MPI_Comm dht_comm) { mpi_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); mpi_buffer_results = @@ -59,6 +59,12 @@ void ChemWorker::prepareSimulation(MPI_Comm dht_comm) { timing[2] = 0.0; } +ChemWorker::~ChemWorker() { + free(mpi_buffer); + free(mpi_buffer_results); + delete dht; +} + void ChemWorker::loop() { MPI_Status probe_status; while (1) { @@ -69,11 +75,11 @@ void ChemWorker::loop() { if (probe_status.MPI_TAG == TAG_WORK) { idle_t += idle_b - idle_a; doWork(probe_status); + } else if (probe_status.MPI_TAG == TAG_DHT_ITER) { + postIter(); } else if (probe_status.MPI_TAG == TAG_FINISH) { finishWork(); break; - } else if (probe_status.MPI_TAG == TAG_DHT_ITER) { - postIter(); } } } @@ -265,11 +271,12 @@ void ChemWorker::postIter() { } void ChemWorker::writeFile() { + cout.flush(); std::stringstream out; out << out_dir << "/iter_" << setfill('0') << setw(3) << iteration << ".dht"; int res = dht->tableToFile(out.str().c_str()); if (res != DHT_SUCCESS && world_rank == 2) - cerr << "CPP: Worker: Errir in writing current state of DHT to file." + cerr << "CPP: Worker: Error in writing current state of DHT to file." << endl; else if (world_rank == 2) cout << "CPP: Worker: Successfully written DHT to file " << out.str() @@ -298,9 +305,6 @@ void ChemWorker::readFile() { } void ChemWorker::finishWork() { - if (dht_enabled && dht_snaps > 0) writeFile(); - - double dht_perf[3]; /* before death, submit profiling/timings to master*/ MPI_Recv(NULL, 0, MPI_DOUBLE, 0, TAG_FINISH, MPI_COMM_WORLD, MPI_STATUS_IGNORE); @@ -313,6 +317,7 @@ void ChemWorker::finishWork() { if (dht_enabled) { // dht_perf + double dht_perf[3]; dht_perf[0] = dht->getHits(); dht_perf[1] = dht->getMisses(); dht_perf[2] = dht->getEvictions(); @@ -320,7 +325,9 @@ void ChemWorker::finishWork() { MPI_COMM_WORLD); } - free(mpi_buffer); - free(mpi_buffer_results); - delete dht; + if (dht_enabled && dht_snaps > 0) writeFile(); + + // free(mpi_buffer); + // free(mpi_buffer_results); + // delete dht; } diff --git a/src/model/TransportSim.cpp b/src/model/TransportSim.cpp new file mode 100644 index 000000000..566bef69e --- /dev/null +++ b/src/model/TransportSim.cpp @@ -0,0 +1,19 @@ +#include "TransportSim.h" + +#include + +using namespace poet; + +TransportSim::TransportSim(RRuntime &R_) : R(_R) {} + +void TransportSim::runIteration() { + double sim_a_transport, sim_b_transport; + + sim_b_transport = MPI_Wtime(); + R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); + sim_a_transport = MPI_Wtime(); + + transport_t += sim_a_transport - sim_b_transport; +} + +double TransportSim::getTransportTime { return this->transport_t; } diff --git a/src/model/TransportSim.h b/src/model/TransportSim.h new file mode 100644 index 000000000..7ceb44daf --- /dev/null +++ b/src/model/TransportSim.h @@ -0,0 +1,22 @@ +#ifndef TRANSPORT_SIM_H +#define TRANSPORT_SIM_H + +#include "../util/RRuntime.h" + +namespace poet { +class TransportSim { + public: + TransportSim(RRuntime &R); + + void runIteration(); + + double getTransportTime(); + + private: + RRuntime &R; + + double transport_t = 0.f; +}; +} // namespace poet + +#endif // TRANSPORT_SIM_H \ No newline at end of file diff --git a/src/util/Parser.cpp b/src/util/Parser.cpp new file mode 100644 index 000000000..51b3c73a5 --- /dev/null +++ b/src/util/Parser.cpp @@ -0,0 +1,144 @@ +#include "Parser.h" + +#include + +#include + +using namespace poet; +using namespace std; +using namespace Rcpp; + +Parser::Parser(char *argv[], int world_rank_, int world_size_) + : cmdl(argv), world_rank(world_rank_), world_size(world_size_) { + this->simparams.world_rank = world_rank_; + this->simparams.world_size = world_size; +} + +int Parser::parseCmdl() { + // if user asked for help + if (cmdl[{"help", "h"}]) { + if (world_rank == 0) { + cout << "Todo" << endl + << "See README.md for further information." << endl; + } + return PARSER_HELP; + } + // if positional arguments are missing + else if (!cmdl(2)) { + if (world_rank == 0) { + cerr << "ERROR. Kin needs 2 positional arguments: " << endl + << "1) the R script defining your simulation and" << endl + << "2) the directory prefix where to save results and profiling" + << endl; + } + return PARSER_ERROR; + } + + std::list optionsError = checkOptions(cmdl); + if (!optionsError.empty()) { + if (world_rank == 0) { + cerr << "Unrecognized option(s):\n" << endl; + for (auto option : optionsError) { + cerr << option << endl; + } + cerr << "\nMake sure to use available options. Exiting!" << endl; + } + return PARSER_ERROR; + } + + /*Parse DHT arguments*/ + simparams.dht_enabled = cmdl["dht"]; + // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; + + if (simparams.dht_enabled) { + cmdl("dht-strategy", 0) >> simparams.dht_strategy; + // cout << "CPP: DHT strategy is " << dht_strategy << endl; + + cmdl("dht-signif", 5) >> dht_significant_digits; + // cout << "CPP: DHT significant digits = " << dht_significant_digits << + // endl; + + simparams.dht_log = !(cmdl["dht-nolog"]); + // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" + // : "OFF" ) << endl; + + cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> simparams.dht_size_per_process; + // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << + // endl; + + cmdl("dht-snaps", 0) >> simparams.dht_snaps; + + cmdl("dht-file") >> simparams.dht_file; + } + /*Parse work package size*/ + cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> simparams.wp_size; + + /*Parse output options*/ + simparams.store_result = !cmdl["ignore-result"]; + + if (world_rank == 0) { + cout << "CPP: Complete results storage is " + << (simparams.store_result ? "ON" : "OFF") << endl; + cout << "CPP: Work Package Size: " << simparams.wp_size << endl; + cout << "CPP: DHT is " << (simparams.dht_enabled ? "ON" : "OFF") << '\n'; + + if (simparams.dht_enabled) { + cout << "CPP: DHT strategy is " << simparams.dht_strategy << endl; + cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + "defined) = " + << dht_significant_digits << endl; + cout << "CPP: DHT logarithm before rounding: " + << (simparams.dht_log ? "ON" : "OFF") << endl; + cout << "CPP: DHT size per process (Byte) = " + << simparams.dht_size_per_process << endl; + cout << "CPP: DHT save snapshots is " << simparams.dht_snaps << endl; + cout << "CPP: DHT load file is " << simparams.dht_file << endl; + } + } + + cmdl(1) >> simparams.filesim; + cmdl(2) >> simparams.out_dir; + + return PARSER_OK; +} + +void Parser::parseR(RRuntime &R) { + // if local_rank == 0 then master else worker + R["local_rank"] = simparams.world_rank; + // assign a char* (string) to 'filesim' + R["filesim"] = wrap(simparams.filesim); + // assign a char* (string) to 'fileout' + R["fileout"] = wrap(simparams.out_dir); + // pass the boolean "store_result" to the R process + R["store_result"] = simparams.store_result; + // worker count + R["n_procs"] = simparams.world_size - 1; + // work package size + R["work_package_size"] = simparams.wp_size; + // dht enabled? + R["dht_enabled"] = simparams.dht_enabled; + // log before rounding? + R["dht_log"] = simparams.dht_log; + + // eval the init string, ignoring any returns + R.parseEvalQ("source(filesim)"); +} + +t_simparams Parser::getParams() { return this->simparams; } + +std::list Parser::checkOptions(argh::parser cmdl) { + std::list retList; + // std::set flist = flagList(); + // std::set plist = paramList(); + + for (auto &flag : cmdl.flags()) { + if (!(flaglist.find(flag) != flaglist.end())) retList.push_back(flag); + } + + for (auto ¶m : cmdl.params()) { + if (!(paramlist.find(param.first) != paramlist.end())) + retList.push_back(param.first); + } + + return retList; +} diff --git a/src/util/Parser.h b/src/util/Parser.h new file mode 100644 index 000000000..d79a7ad35 --- /dev/null +++ b/src/util/Parser.h @@ -0,0 +1,42 @@ +#ifndef PARSER_H +#define PARSER_H + +#include + +#include "../argh.h" +#include "RRuntime.h" +#include "SimParams.h" + +#define PARSER_OK 0 +#define PARSER_ERROR 1 +#define PARSER_HELP 2 + +#define DHT_SIZE_PER_PROCESS 1073741824 +#define WORK_PACKAGE_SIZE_DEFAULT 5 + +namespace poet { +class Parser { + public: + Parser(char *argv[], int world_rank, int world_size); + int parseCmdl(); + void parseR(RRuntime &R); + + t_simparams getParams(); + + private: + std::list checkOptions(argh::parser cmdl); + std::set flaglist{"ignore-result", "dht", "dht-nolog"}; + std::set paramlist{"work-package-size", "dht-signif", + "dht-strategy", "dht-size", + "dht-snaps", "dht-file"}; + argh::parser cmdl; + + t_simparams simparams; + + int world_rank; + int world_size; + + int dht_significant_digits; +}; +} // namespace poet +#endif // PARSER_H \ No newline at end of file diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 948dee481..8c982f17a 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -21,8 +21,11 @@ typedef struct { unsigned int wp_size; + std::string filesim; std::string out_dir; + bool store_result; + void* R; void* grid; } t_simparams; From 0705772204e12e929bd05341dcb159f26cc9aa37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 13 Jan 2021 19:04:28 +0100 Subject: [PATCH 12/50] fixed typo in constructor --- src/model/TransportSim.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/model/TransportSim.cpp b/src/model/TransportSim.cpp index 566bef69e..ea4fe3899 100644 --- a/src/model/TransportSim.cpp +++ b/src/model/TransportSim.cpp @@ -4,7 +4,7 @@ using namespace poet; -TransportSim::TransportSim(RRuntime &R_) : R(_R) {} +TransportSim::TransportSim(RRuntime &R_) : R(R_) {} void TransportSim::runIteration() { double sim_a_transport, sim_b_transport; @@ -16,4 +16,4 @@ void TransportSim::runIteration() { transport_t += sim_a_transport - sim_b_transport; } -double TransportSim::getTransportTime { return this->transport_t; } +double TransportSim::getTransportTime() { return this->transport_t; } From 4c86f9fffd35b2b822e7a2868086fb79b03a2f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 13 Jan 2021 21:02:57 +0100 Subject: [PATCH 13/50] documentation in DHT and compiler macro DHT_STATISTICS --- src/DHT/DHT.cpp | 761 ++++++++++++++++++++++------------------ src/DHT/DHT.h | 308 +++++++++++----- src/DHT/DHT_Wrapper.cpp | 18 +- src/DHT/DHT_Wrapper.h | 2 - 4 files changed, 663 insertions(+), 426 deletions(-) diff --git a/src/DHT/DHT.cpp b/src/DHT/DHT.cpp index 3a988cdad..2ffb3d9ad 100644 --- a/src/DHT/DHT.cpp +++ b/src/DHT/DHT.cpp @@ -1,423 +1,510 @@ #include "DHT.h" -#include -#include + #include #include +#include +#include -/* - * determining destination rank and index by hash of key +/** + * @brief Determine destination rank and index. * - * return values by reference + * This is done by looping over all possbile indices. First of all, set a + * temporary index to zero and copy count of bytes for each index into the + * memory area of the temporary index. After that the current index is + * calculated by the temporary index modulo the table size. The destination rank + * of the process is simply determined by hash modulo the communicator size. + * + * @param hash Calculated 64 bit hash. + * @param comm_size Communicator size. + * @param table_size Count of buckets per process. + * @param dest_rank Reference to the destination rank variable. + * @param index Pointer to the array index. + * @param index_count Count of possible indeces. */ -static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int *dest_rank, unsigned int *index, unsigned int index_count) { - uint64_t tmp; - int char_hop = 9-index_count; - unsigned int i; - for (i = 0; i < index_count ; i++) { - tmp = 0; - memcpy(&tmp,(unsigned char *)&hash+i, char_hop); - index[i] = (unsigned int) (tmp % table_size); - } - *dest_rank = (unsigned int) (hash % comm_size); +static void determine_dest(uint64_t hash, int comm_size, + unsigned int table_size, unsigned int *dest_rank, + unsigned int *index, unsigned int index_count) { + /** temporary index */ + uint64_t tmp_index; + /** how many bytes for one index? */ + int index_size = 9 - index_count; + for (unsigned int i = 0; i < index_count; i++) { + tmp_index = 0; + memcpy(&tmp_index, (unsigned char *)&hash + i, index_size); + index[i] = (unsigned int)(tmp_index % table_size); + } + *dest_rank = (unsigned int)(hash % comm_size); } /** - * set write flag to 1 + * @brief Set the occupied flag. + * + * This will set the first bit of a bucket to 1. + * + * @param flag_byte First byte of a bucket. */ -static void set_flag(char* flag_byte) { - *flag_byte = 0; - *flag_byte |= (1 << 0); -} +static void set_flag(char *flag_byte) { + *flag_byte = 0; + *flag_byte |= (1 << 0); +} /** - * return 1 if write flag is set - * else 0 + * @brief Get the occupied flag. + * + * This function determines whether the occupied flag of a bucket was set or + * not. + * + * @param flag_byte First byte of a bucket. + * @return int Returns 1 for true or 0 for false. */ static int read_flag(char flag_byte) { - if ((flag_byte & 0x01) == 0x01) { - return 1; - } else return 0; + if ((flag_byte & 0x01) == 0x01) { + return 1; + } else + return 0; } -/* - * allocating memory for DHT object and buckets. - * creating MPI window for OSC - * filling DHT object with passed by parameters, window, 2 counters for R/W errors and 2 pointers with allocated memory for further use - * return DHT object - */ -DHT* DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, uint64_t(*hash_func) (int, void*)) { - DHT *object; - MPI_Win window; - void* mem_alloc; - int comm_size, tmp; +DHT *DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, + uint64_t (*hash_func)(int, void *)) { + DHT *object; + MPI_Win window; + void *mem_alloc; + int comm_size, index_bytes; - tmp = (int) ceil(log2(size)); - if (tmp%8 != 0) tmp = tmp + (8-(tmp%8)); + // calculate how much bytes for the index are needed to address count of + // buckets per process + index_bytes = (int)ceil(log2(size)); + if (index_bytes % 8 != 0) index_bytes = index_bytes + (8 - (index_bytes % 8)); - object = (DHT*) malloc(sizeof(DHT)); - if (object == NULL) return NULL; + // allocate memory for dht-object + object = (DHT *)malloc(sizeof(DHT)); + if (object == NULL) return NULL; - //every memory allocation has 1 additional byte for flags etc. - if (MPI_Alloc_mem(size * (1 + data_size + key_size), MPI_INFO_NULL, &mem_alloc) != 0) return NULL; - if (MPI_Comm_size(comm, &comm_size) != 0) return NULL; + // every memory allocation has 1 additional byte for flags etc. + if (MPI_Alloc_mem(size * (1 + data_size + key_size), MPI_INFO_NULL, + &mem_alloc) != 0) + return NULL; + if (MPI_Comm_size(comm, &comm_size) != 0) return NULL; - memset(mem_alloc, '\0', size * (1 + data_size + key_size)); + // since MPI_Alloc_mem doesn't provide memory allocation with the memory set + // to zero, we're doing this here + memset(mem_alloc, '\0', size * (1 + data_size + key_size)); - if (MPI_Win_create(mem_alloc, size * (1 + data_size + key_size), (1 + data_size + key_size), MPI_INFO_NULL, comm, &window) != 0) return NULL; + // create windows on previously allocated memory + if (MPI_Win_create(mem_alloc, size * (1 + data_size + key_size), + (1 + data_size + key_size), MPI_INFO_NULL, comm, + &window) != 0) + return NULL; - object->data_size = data_size; - object->key_size = key_size; - object->table_size = size; - object->window = window; - object->hash_func = hash_func; - object->comm_size = comm_size; - object->communicator = comm; - object->read_misses = 0; - object->collisions = 0; - object->recv_entry = malloc(1 + data_size + key_size); - object->send_entry = malloc(1 + data_size + key_size); - object->index_count = 9-(tmp/8); - object->index = (unsigned int*) malloc((9-(tmp/8))*sizeof(int)); - object->mem_alloc = mem_alloc; + // fill dht-object + object->data_size = data_size; + object->key_size = key_size; + object->table_size = size; + object->window = window; + object->hash_func = hash_func; + object->comm_size = comm_size; + object->communicator = comm; + object->read_misses = 0; + object->evictions = 0; + object->recv_entry = malloc(1 + data_size + key_size); + object->send_entry = malloc(1 + data_size + key_size); + object->index_count = 9 - (index_bytes / 8); + object->index = (unsigned int *)malloc((9 - (index_bytes / 8)) * sizeof(int)); + object->mem_alloc = mem_alloc; - DHT_stats *stats; + // if set, initialize dht_stats +#ifdef DHT_STATISTICS + DHT_stats *stats; - stats = (DHT_stats*) malloc(sizeof(DHT_stats)); - if (stats == NULL) return NULL; + stats = (DHT_stats *)malloc(sizeof(DHT_stats)); + if (stats == NULL) return NULL; - object->stats = stats; - object->stats->writes_local = (int*) calloc(comm_size, sizeof(int)); - object->stats->old_writes = 0; - object->stats->read_misses = 0; - object->stats->collisions = 0; - object->stats->w_access = 0; - object->stats->r_access = 0; + object->stats = stats; + object->stats->writes_local = (int *)calloc(comm_size, sizeof(int)); + object->stats->old_writes = 0; + object->stats->read_misses = 0; + object->stats->evictions = 0; + object->stats->w_access = 0; + object->stats->r_access = 0; +#endif - return object; + return object; } -/* - * puts passed by data with key to DHT - * - * returning DHT_MPI_ERROR = -1 if MPI error occurred - * else DHT_SUCCESS = 0 if success - */ -int DHT_write(DHT *table, void* send_key, void* send_data) { - unsigned int dest_rank, i; - int result = DHT_SUCCESS; +int DHT_write(DHT *table, void *send_key, void *send_data) { + unsigned int dest_rank, i; + int result = DHT_SUCCESS; - table->stats->w_access++; +#ifdef DHT_STATISTICS + table->stats->w_access++; +#endif - //determine destination rank and index by hash of key - determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); + // determine destination rank and index by hash of key + determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, + table->table_size, &dest_rank, table->index, + table->index_count); - //concatenating key with data to write entry to DHT - set_flag((char *) table->send_entry); - memcpy((char *) table->send_entry + 1, (char *) send_key, table->key_size); - memcpy((char *) table->send_entry + table->key_size + 1, (char *) send_data, table->data_size); + // concatenating key with data to write entry to DHT + set_flag((char *)table->send_entry); + memcpy((char *)table->send_entry + 1, (char *)send_key, table->key_size); + memcpy((char *)table->send_entry + table->key_size + 1, (char *)send_data, + table->data_size); - //locking window of target rank with exclusive lock - if (MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dest_rank, 0, table->window) != 0) - return DHT_MPI_ERROR; - for (i = 0; i < table->index_count; i++) - { - if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + // locking window of target rank with exclusive lock + if (MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dest_rank, 0, table->window) != 0) + return DHT_MPI_ERROR; + for (i = 0; i < table->index_count; i++) { + if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, + MPI_BYTE, dest_rank, table->index[i], + 1 + table->data_size + table->key_size, MPI_BYTE, + table->window) != 0) + return DHT_MPI_ERROR; + if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - //increment collision counter if receiving key doesn't match sending key - //,entry has write flag + last index is reached - - if (read_flag(*(char *)table->recv_entry)) { - if (memcmp(send_key, (char *) table->recv_entry + 1, table->key_size) != 0) { - if (i == (table->index_count)-1) { - table->collisions += 1; - table->stats->collisions += 1; - result = DHT_WRITE_SUCCESS_WITH_COLLISION; - break; - } - } else break; - } else { - table->stats->writes_local[dest_rank]++; - break; + // increment eviction counter if receiving key doesn't match sending key + // entry has write flag and last index is reached. + if (read_flag(*(char *)table->recv_entry)) { + if (memcmp(send_key, (char *)table->recv_entry + 1, table->key_size) != + 0) { + if (i == (table->index_count) - 1) { + table->evictions += 1; +#ifdef DHT_STATISTICS + table->stats->evictions += 1; +#endif + result = DHT_WRITE_SUCCESS_WITH_COLLISION; + break; } + } else + break; + } else { +#ifdef DHT_STATISTICS + table->stats->writes_local[dest_rank]++; +#endif + break; } + } - //put data to DHT - if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - //unlock window of target rank - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + // put data to DHT (with last selected index by value i) + if (MPI_Put(table->send_entry, 1 + table->data_size + table->key_size, + MPI_BYTE, dest_rank, table->index[i], + 1 + table->data_size + table->key_size, MPI_BYTE, + table->window) != 0) + return DHT_MPI_ERROR; + // unlock window of target rank + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return result; + return result; } -/* - * gets data from the DHT by key - * - * return DHT_SUCCESS = 0 if success - * DHT_MPI_ERROR = -1 if MPI error occurred - * DHT_READ_ERROR = -2 if receiving key doesn't match sending key - */ -int DHT_read(DHT *table, void* send_key, void* destination) { - unsigned int dest_rank, i; +int DHT_read(DHT *table, void *send_key, void *destination) { + unsigned int dest_rank, i; - table->stats->r_access++; +#ifdef DHT_STATISTICS + table->stats->r_access++; +#endif - //determine destination rank and index by hash of key - determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, table->table_size, &dest_rank, table->index, table->index_count); + // determine destination rank and index by hash of key + determine_dest(table->hash_func(table->key_size, send_key), table->comm_size, + table->table_size, &dest_rank, table->index, + table->index_count); - //locking window of target rank with shared lock - if (MPI_Win_lock(MPI_LOCK_SHARED, dest_rank, 0, table->window) != 0) return DHT_MPI_ERROR; - //receive data - for (i = 0; i < table->index_count; i++) { - if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, MPI_BYTE, dest_rank, table->index[i], 1 + table->data_size + table->key_size, MPI_BYTE, table->window) != 0) return DHT_MPI_ERROR; - if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - - //increment read error counter if write flag isn't set or key doesn't match passed by key + last index reached - //else copy data to dereference of passed by destination pointer + // locking window of target rank with shared lock + if (MPI_Win_lock(MPI_LOCK_SHARED, dest_rank, 0, table->window) != 0) + return DHT_MPI_ERROR; + // receive data + for (i = 0; i < table->index_count; i++) { + if (MPI_Get(table->recv_entry, 1 + table->data_size + table->key_size, + MPI_BYTE, dest_rank, table->index[i], + 1 + table->data_size + table->key_size, MPI_BYTE, + table->window) != 0) + return DHT_MPI_ERROR; + if (MPI_Win_flush(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - if ((read_flag(*(char *) table->recv_entry)) == 0) { - table->read_misses += 1; - table->stats->read_misses += 1; - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; - } - - if (memcmp(((char*)table->recv_entry) + 1, send_key, table->key_size) != 0) { - if (i == (table->index_count)-1) { - table->read_misses += 1; - table->stats->read_misses += 1; - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; - } - } else break; + // increment read error counter if write flag isn't set ... + if ((read_flag(*(char *)table->recv_entry)) == 0) { + table->read_misses += 1; +#ifdef DHT_STATISTICS + table->stats->read_misses += 1; +#endif + // unlock window and return + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + return DHT_READ_ERROR; } - //unlock window of target rank - if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + // ... or key doesn't match passed by key and last index reached. + if (memcmp(((char *)table->recv_entry) + 1, send_key, table->key_size) != + 0) { + if (i == (table->index_count) - 1) { + table->read_misses += 1; +#ifdef DHT_STATISTICS + table->stats->read_misses += 1; +#endif + // unlock window an return + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; + return DHT_READ_ERROR; + } + } else + break; + } - memcpy((char *) destination, (char *) table->recv_entry + table->key_size + 1, table->data_size); + // unlock window of target rank + if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_SUCCESS; + // if matching key was found copy data into memory of passed pointer + memcpy((char *)destination, (char *)table->recv_entry + table->key_size + 1, + table->data_size); + + return DHT_SUCCESS; } -int DHT_to_file(DHT* table, const char* filename) { - //open file - MPI_File file; - if (MPI_File_open(table->communicator, filename, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; +int DHT_to_file(DHT *table, const char *filename) { + // open file + MPI_File file; + if (MPI_File_open(table->communicator, filename, + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, + &file) != 0) + return DHT_FILE_IO_ERROR; - int rank; - MPI_Comm_rank(table->communicator, &rank); + int rank; + MPI_Comm_rank(table->communicator, &rank); - //write header (key_size and data_size) - if (rank == 0) { - if (MPI_File_write(file, &table->key_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; - if (MPI_File_write(file, &table->data_size, 1, MPI_INT, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; + // write header (key_size and data_size) + if (rank == 0) { + if (MPI_File_write(file, &table->key_size, 1, MPI_INT, MPI_STATUS_IGNORE) != + 0) + return DHT_FILE_WRITE_ERROR; + if (MPI_File_write(file, &table->data_size, 1, MPI_INT, + MPI_STATUS_IGNORE) != 0) + return DHT_FILE_WRITE_ERROR; + } + + // seek file pointer behind header for all processes + if (MPI_File_seek_shared(file, DHT_FILEHEADER_SIZE, MPI_SEEK_SET) != 0) + return DHT_FILE_IO_ERROR; + + char *ptr; + int bucket_size = table->key_size + table->data_size + 1; + + // iterate over local memory + for (unsigned int i = 0; i < table->table_size; i++) { + ptr = (char *)table->mem_alloc + (i * bucket_size); + // if bucket has been written to (checked by written_flag)... + if (read_flag(*ptr)) { + // write key and data to file + if (MPI_File_write_shared(file, ptr + 1, bucket_size - 1, MPI_BYTE, + MPI_STATUS_IGNORE) != 0) + return DHT_FILE_WRITE_ERROR; } + } + // close file + if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; - if (MPI_File_seek_shared(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - - char* ptr; - int bucket_size = table->key_size + table->data_size + 1; - - //iterate over local memory - for (unsigned int i = 0; i < table->table_size; i++) { - ptr = (char *) table->mem_alloc + (i * bucket_size); - //if bucket has been written to (checked by written_flag)... - if (read_flag(*ptr)) { - //write key and data to file - if (MPI_File_write_shared(file, ptr + 1, bucket_size - 1, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_WRITE_ERROR; - } - } - //close file - if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; - - return DHT_SUCCESS; + return DHT_SUCCESS; } -int DHT_from_file(DHT* table, const char* filename) { - MPI_File file; - MPI_Offset f_size; - int e_size, m_size, cur_pos, rank, offset; - char* buffer; - void* key; - void* data; +int DHT_from_file(DHT *table, const char *filename) { + MPI_File file; + MPI_Offset f_size; + int bucket_size, buffer_size, cur_pos, rank, offset; + char *buffer; + void *key; + void *data; - if (MPI_File_open(table->communicator, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &file) != 0) return DHT_FILE_IO_ERROR; + // open file + if (MPI_File_open(table->communicator, filename, MPI_MODE_RDONLY, + MPI_INFO_NULL, &file) != 0) + return DHT_FILE_IO_ERROR; - if (MPI_File_get_size(file, &f_size) != 0) return DHT_FILE_IO_ERROR; + // get file size + if (MPI_File_get_size(file, &f_size) != 0) return DHT_FILE_IO_ERROR; - MPI_Comm_rank(table->communicator, &rank); + MPI_Comm_rank(table->communicator, &rank); - e_size = table->key_size + table->data_size; - m_size = e_size > DHT_HEADER_SIZE ? e_size : DHT_HEADER_SIZE; - buffer = (char *) malloc(m_size); + // calculate bucket size + bucket_size = table->key_size + table->data_size; + // buffer size is either bucket size or, if bucket size is smaller than the + // file header, the size of DHT_FILEHEADER_SIZE + buffer_size = + bucket_size > DHT_FILEHEADER_SIZE ? bucket_size : DHT_FILEHEADER_SIZE; + // allocate buffer + buffer = (char *)malloc(buffer_size); - if (MPI_File_read(file, buffer, DHT_HEADER_SIZE, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; + // read file header + if (MPI_File_read(file, buffer, DHT_FILEHEADER_SIZE, MPI_BYTE, + MPI_STATUS_IGNORE) != 0) + return DHT_FILE_READ_ERROR; - if (*(int *) buffer != table->key_size) return DHT_WRONG_FILE; - if (*(int *) (buffer + 4) != table->data_size) return DHT_WRONG_FILE; + // compare if written header data and key size matches current sizes + if (*(int *)buffer != table->key_size) return DHT_WRONG_FILE; + if (*(int *)(buffer + 4) != table->data_size) return DHT_WRONG_FILE; - offset = e_size*table->comm_size; + // set offset for each process + offset = bucket_size * table->comm_size; - if (MPI_File_seek(file, DHT_HEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - cur_pos = DHT_HEADER_SIZE + (rank * e_size); + // seek behind header of DHT file + if (MPI_File_seek(file, DHT_FILEHEADER_SIZE, MPI_SEEK_SET) != 0) + return DHT_FILE_IO_ERROR; + + // current position is rank * bucket_size + OFFSET + cur_pos = DHT_FILEHEADER_SIZE + (rank * bucket_size); - while(cur_pos < f_size) { - if (MPI_File_seek(file, cur_pos, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - MPI_Offset tmp; - MPI_File_get_position(file, &tmp); - if (MPI_File_read(file, buffer, e_size, MPI_BYTE, MPI_STATUS_IGNORE) != 0) return DHT_FILE_READ_ERROR; - key = buffer; - data = (buffer+table->key_size); - if (DHT_write(table, key, data) == DHT_MPI_ERROR) return DHT_MPI_ERROR; + // loop over file and write data to DHT with DHT_write + while (cur_pos < f_size) { + if (MPI_File_seek(file, cur_pos, MPI_SEEK_SET) != 0) + return DHT_FILE_IO_ERROR; + // TODO: really necessary? + MPI_Offset tmp; + MPI_File_get_position(file, &tmp); + if (MPI_File_read(file, buffer, bucket_size, MPI_BYTE, MPI_STATUS_IGNORE) != + 0) + return DHT_FILE_READ_ERROR; + // extract key and data and write to DHT + key = buffer; + data = (buffer + table->key_size); + if (DHT_write(table, key, data) == DHT_MPI_ERROR) return DHT_MPI_ERROR; - cur_pos += offset; - } + // increment current position + cur_pos += offset; + } - free (buffer); - if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; + free(buffer); + if (MPI_File_close(&file) != 0) return DHT_FILE_IO_ERROR; - return DHT_SUCCESS; + return DHT_SUCCESS; } -/* - * frees up memory and accumulate counter - */ -int DHT_free(DHT* table, int* collision_counter, int* readerror_counter) { - int buf; +int DHT_free(DHT *table, int *eviction_counter, int *readerror_counter) { + int buf; - if (collision_counter != NULL) { - buf = 0; - if (MPI_Reduce(&table->collisions, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - *collision_counter = buf; - } - if (readerror_counter != NULL) { - buf = 0; - if (MPI_Reduce(&table->read_misses, &buf, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - *readerror_counter = buf; - } - if (MPI_Win_free(&(table->window)) != 0) return DHT_MPI_ERROR; - if (MPI_Free_mem(table->mem_alloc) != 0) return DHT_MPI_ERROR; - free(table->recv_entry); - free(table->send_entry); - free(table->index); + if (eviction_counter != NULL) { + buf = 0; + if (MPI_Reduce(&table->evictions, &buf, 1, MPI_INT, MPI_SUM, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + *eviction_counter = buf; + } + if (readerror_counter != NULL) { + buf = 0; + if (MPI_Reduce(&table->read_misses, &buf, 1, MPI_INT, MPI_SUM, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + *readerror_counter = buf; + } + if (MPI_Win_free(&(table->window)) != 0) return DHT_MPI_ERROR; + if (MPI_Free_mem(table->mem_alloc) != 0) return DHT_MPI_ERROR; + free(table->recv_entry); + free(table->send_entry); + free(table->index); - free(table->stats->writes_local); - free(table->stats); +#ifdef DHT_STATISTICS + free(table->stats->writes_local); + free(table->stats); +#endif + free(table); - free(table); - - return DHT_SUCCESS; + return DHT_SUCCESS; } -/* - * prints a table with statistics about current use of DHT - * for each participating process and summed up results containing: - * 1. occupied buckets (in respect to the memory of this process) - * 2. free buckets (in respect to the memory of this process) - * 3. calls of DHT_write (w_access) - * 4. calls of DHT_read (r_access) - * 5. read misses (see DHT_READ_ERROR) - * 6. collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) - * 3-6 will reset with every call of this function - * finally the amount of new written entries is printed out (in relation to last call of this funtion) - */ +#ifdef DHT_STATISTICS int DHT_print_statistics(DHT *table) { - int *written_buckets; - int *read_misses, sum_read_misses; - int *collisions, sum_collisions; - int sum_w_access, sum_r_access, *w_access, *r_access; - int rank; + int *written_buckets; + int *read_misses, sum_read_misses; + int *evictions, sum_evictions; + int sum_w_access, sum_r_access, *w_access, *r_access; + int rank; + MPI_Comm_rank(table->communicator, &rank); - MPI_Comm_rank(table->communicator, &rank); +// disable possible warning of unitialized variable, which is not the case +// since we're using MPI_Gather to obtain all values only on rank 0 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" - //disable possible warning of unitialized variable, which is not the case - //since we're using MPI_Gather to obtain all values only on rank 0 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" + // obtaining all values from all processes in the communicator + if (rank == 0) read_misses = (int *)malloc(table->comm_size * sizeof(int)); + if (MPI_Gather(&table->stats->read_misses, 1, MPI_INT, read_misses, 1, + MPI_INT, 0, table->communicator) != 0) + return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->read_misses, &sum_read_misses, 1, MPI_INT, + MPI_SUM, 0, table->communicator) != 0) + return DHT_MPI_ERROR; + table->stats->read_misses = 0; - //obtaining all values from all processes in the communicator - if (rank == 0) read_misses = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->read_misses, 1, MPI_INT, read_misses, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->read_misses, &sum_read_misses, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->read_misses = 0; - - if (rank == 0) collisions = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->collisions, 1, MPI_INT, collisions, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->collisions, &sum_collisions, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->collisions = 0; + if (rank == 0) evictions = (int *)malloc(table->comm_size * sizeof(int)); + if (MPI_Gather(&table->stats->evictions, 1, MPI_INT, evictions, 1, MPI_INT, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->evictions, &sum_evictions, 1, MPI_INT, MPI_SUM, + 0, table->communicator) != 0) + return DHT_MPI_ERROR; + table->stats->evictions = 0; - if (rank == 0) w_access = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->w_access = 0; + if (rank == 0) w_access = (int *)malloc(table->comm_size * sizeof(int)); + if (MPI_Gather(&table->stats->w_access, 1, MPI_INT, w_access, 1, MPI_INT, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->w_access, &sum_w_access, 1, MPI_INT, MPI_SUM, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + table->stats->w_access = 0; - if (rank == 0) r_access = (int*) malloc(table->comm_size*sizeof(int)); - if (MPI_Gather(&table->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0, table->communicator) != 0) return DHT_MPI_ERROR; - if (MPI_Reduce(&table->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; - table->stats->r_access = 0; - - if (rank == 0) written_buckets = (int*) calloc(table->comm_size, sizeof(int)); - if (MPI_Reduce(table->stats->writes_local, written_buckets, table->comm_size, MPI_INT, MPI_SUM, 0, table->communicator) != 0) return DHT_MPI_ERROR; + if (rank == 0) r_access = (int *)malloc(table->comm_size * sizeof(int)); + if (MPI_Gather(&table->stats->r_access, 1, MPI_INT, r_access, 1, MPI_INT, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + if (MPI_Reduce(&table->stats->r_access, &sum_r_access, 1, MPI_INT, MPI_SUM, 0, + table->communicator) != 0) + return DHT_MPI_ERROR; + table->stats->r_access = 0; - if (rank == 0) { //only process with rank 0 will print out results as a table - int sum_written_buckets = 0; + if (rank == 0) written_buckets = (int *)calloc(table->comm_size, sizeof(int)); + if (MPI_Reduce(table->stats->writes_local, written_buckets, table->comm_size, + MPI_INT, MPI_SUM, 0, table->communicator) != 0) + return DHT_MPI_ERROR; - for (int i=0; i < table->comm_size; i++) { - sum_written_buckets += written_buckets[i]; - } + if (rank == 0) { // only process with rank 0 will print out results as a + // table + int sum_written_buckets = 0; - int members = 7; - int padsize = (members*12)+1; - char pad[padsize+1]; - - memset(pad, '-', padsize*sizeof(char)); - pad[padsize]= '\0'; - printf("\n"); - printf("%-35s||resets with every call of this function\n", " "); - printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s\n", - "rank", - "occupied", - "free", - "w_access", - "r_access", - "read misses", - "collisions"); - printf("%s\n", pad); - for (int i = 0; i < table->comm_size; i++) { - printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", - i, - written_buckets[i], - table->table_size-written_buckets[i], - w_access[i], - r_access[i], - read_misses[i], - collisions[i]); - } - printf("%s\n", pad); - printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", - "sum", - sum_written_buckets, - (table->table_size*table->comm_size)-sum_written_buckets, - sum_w_access, - sum_r_access, - sum_read_misses, - sum_collisions); - - printf("%s\n", pad); - printf("%s %d\n", - "new entries:", - sum_written_buckets - table->stats->old_writes); - - printf("\n"); - fflush(stdout); - - table->stats->old_writes = sum_written_buckets; + for (int i = 0; i < table->comm_size; i++) { + sum_written_buckets += written_buckets[i]; } - //enable warning again - #pragma GCC diagnostic pop + int members = 7; + int padsize = (members * 12) + 1; + char pad[padsize + 1]; - MPI_Barrier(table->communicator); - return DHT_SUCCESS; -} \ No newline at end of file + memset(pad, '-', padsize * sizeof(char)); + pad[padsize] = '\0'; + printf("\n"); + printf("%-35s||resets with every call of this function\n", " "); + printf("%-11s|%-11s|%-11s||%-11s|%-11s|%-11s|%-11s\n", "rank", "occupied", + "free", "w_access", "r_access", "read misses", "evictions"); + printf("%s\n", pad); + for (int i = 0; i < table->comm_size; i++) { + printf("%-11d|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", i, + written_buckets[i], table->table_size - written_buckets[i], + w_access[i], r_access[i], read_misses[i], evictions[i]); + } + printf("%s\n", pad); + printf("%-11s|%-11d|%-11d||%-11d|%-11d|%-11d|%-11d\n", "sum", + sum_written_buckets, + (table->table_size * table->comm_size) - sum_written_buckets, + sum_w_access, sum_r_access, sum_read_misses, sum_evictions); + + printf("%s\n", pad); + printf("%s %d\n", + "new entries:", sum_written_buckets - table->stats->old_writes); + + printf("\n"); + fflush(stdout); + + table->stats->old_writes = sum_written_buckets; + } + +// enable warning again +#pragma GCC diagnostic pop + + MPI_Barrier(table->communicator); + return DHT_SUCCESS; +} +#endif \ No newline at end of file diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h index 308281650..751224407 100644 --- a/src/DHT/DHT.h +++ b/src/DHT/DHT.h @@ -1,8 +1,14 @@ -/* - * File: DHT.h - * Author: max luebke +/** + * @file DHT.h + * @author Max Lübke (mluebke@uni-potsdam.de) + * @brief API to interact with the DHT + * @version 0.1 + * @date 16 Nov 2017 * - * Created on 16. November 2017, 09:14 + * This file implements the creation of a DHT by using the MPI + * one-sided-communication. There is also the possibility to write or read data + * from or to the DHT. In addition, the current state of the DHT can be written + * to a file and read in again later. */ #ifndef DHT_H @@ -11,102 +17,242 @@ #include #include +/** Returned if some error in MPI routine occurs. */ #define DHT_MPI_ERROR -1 +/** Returned by a call of DHT_read if no bucket with given key was found. */ #define DHT_READ_ERROR -2 +/** Returned by DHT_write if a bucket was evicted. */ +#define DHT_WRITE_SUCCESS_WITH_COLLISION -3 +/** Returned when no errors occured. */ #define DHT_SUCCESS 0 -#define DHT_WRITE_SUCCESS_WITH_COLLISION 1 -#define DHT_WRONG_FILE 11 -#define DHT_FILE_IO_ERROR 12 -#define DHT_FILE_READ_ERROR 13 -#define DHT_FILE_WRITE_ERROR 14 +/** Returned by DHT_from_file if the given file does not match expected file. */ +#define DHT_WRONG_FILE -11 +/** Returned by DHT file operations if MPI file operation fails. */ +#define DHT_FILE_IO_ERROR -12 +/** Returned by DHT file operations if error occured in MPI_Read operation. */ +#define DHT_FILE_READ_ERROR -13 +/** Returned by DHT file operations if error occured in MPI_Write operation. */ +#define DHT_FILE_WRITE_ERROR -14 -#define DHT_HEADER_SIZE 8 +/** Size of the file header in byte. */ +#define DHT_FILEHEADER_SIZE 8 -typedef struct {; - int *writes_local, old_writes; - int read_misses, collisions; - int w_access, r_access; +/** + * Internal struct to store statistics about read and write accesses and also + * read misses and evictions. + * All values will be resetted to zero after a call of + * DHT_print_statistics(). + * Internal use only! + * + * @todo There's maybe a better solution than DHT_print_statistics and this + * struct + */ +typedef struct { + /** Count of writes to specific process this process did. */ + int* writes_local; + /** Writes after last call of DHT_print_statistics. */ + int old_writes; + /** How many read misses occur? */ + int read_misses; + /** How many buckets where evicted? */ + int evictions; + /** How many calls of DHT_write() did this process? */ + int w_access; + /** How many calls of DHT_read() did this process? */ + int r_access; } DHT_stats; +/** + * Struct which serves as a handler or so called \a DHT-object. Will + * be created by DHT_create and must be passed as a parameter to every following + * function. Stores all relevant data. + * Do not touch outside DHT functions! + */ typedef struct { - MPI_Win window; - int data_size; - int key_size; - unsigned int table_size; - MPI_Comm communicator; - int comm_size; - uint64_t(*hash_func) (int, void*); - void* recv_entry; - void* send_entry; - void* mem_alloc; - int read_misses; - int collisions; - unsigned int *index; - unsigned int index_count; - DHT_stats *stats; + /** Created MPI Window, which serves as the DHT memory area of the process. */ + MPI_Win window; + /** Size of the data of a bucket entry in byte. */ + int data_size; + /** Size of the key of a bucket entry in byte. */ + int key_size; + /** Count of buckets for each process. */ + unsigned int table_size; + /** MPI communicator of all participating processes. */ + MPI_Comm communicator; + /** Size of the MPI communicator respectively all participating processes. */ + int comm_size; + /** Pointer to a hashfunction. */ + uint64_t (*hash_func)(int, void*); + /** Pre-allocated memory where a bucket can be received. */ + void* recv_entry; + /** Pre-allocated memory where a bucket to send can be stored. */ + void* send_entry; + /** Allocated memory on which the MPI window was created. */ + void* mem_alloc; + /** Count of read misses over all time. */ + int read_misses; + /** Count of evictions over all time. */ + int evictions; + /** Array of indeces where a bucket can be stored. */ + unsigned int* index; + /** Count of possible indeces. */ + unsigned int index_count; +#ifdef DHT_STATISTICS + /** Detailed statistics of the usage of the DHT. */ + DHT_stats* stats; +#endif } DHT; - - -/* - * parameters: - * MPI_Comm comm - communicator of processes that are holding the DHT - * int size_per_process - number of buckets each process will create - * int data_size - size of data in bytes - * int key_size - size of key in bytes - * *hash_func - pointer to hashfunction +/** + * @brief Create a DHT. * - * return: - * NULL if error during initialization - * DHT* if success - */ -extern DHT* DHT_create(MPI_Comm comm, unsigned int size_per_process, int data_size, int key_size, uint64_t(*hash_func)(int, void*)); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * void* data - pointer to data - * void* - pointer to key + * When calling this function, the required memory is allocated and a + * MPI_Window is created. This allows the execution of MPI_Get and + * MPI_Put operations for one-sided communication. Then the number of + * indexes is calculated and finally all relevant data is entered into the + * \a DHT-object which is returned. * - * return: - * error value (see above) + * @param comm MPI communicator which addresses all participating process of the + * DHT. + * @param size_per_process Number of buckets per process. + * @param data_size Size of data in byte. + * @param key_size Size of the key in byte. + * @param hash_func Pointer to a hash function. This function must take the size + * of the key and a pointer to the key as input parameters and return a 64 bit + * hash. + * @return DHT* The returned value is the \a DHT-object which serves as a handle + * for all DHT operations. If an error occured NULL is returned. */ -extern int DHT_write(DHT *table, void* key, void* data); +extern DHT* DHT_create(MPI_Comm comm, unsigned int size_per_process, + int data_size, int key_size, + uint64_t (*hash_func)(int, void*)); -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * void* key - pointer to key - * void* destination - pointer which will hold the resulting data from DHT +/** + * @brief Write data into DHT. * - * return: - * error value (see above) - */ -extern int DHT_read(DHT *table, void* key, void* destination); - -extern int DHT_to_file(DHT *table, const char* filename); - -extern int DHT_from_file(DHT *table, const char* filename); - -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * int* collision_counter - pointer which will hold the total count of collisions - * int* readerror_counter - pointer which will hold the total count of read errors + * When DHT_write is called, the address window is locked with a + * LOCK_EXCLUSIVE for write access. Now the first bucket is received + * using MPI_Get and it is checked if the bucket is empty or if the received key + * matches the passed key. If this is the case, the data of the bucket is + * overwritten with the new value. If not, the function continues with the next + * index until no more indexes are available. When the last index is reached and + * there are no more indexes available, the last examined bucket is replaced. + * After successful writing, the memory window is released and the function + * returns. * - * return: - * error value (see above) + * @param table Pointer to the \a DHT-object. + * @param key Pointer to the key. + * @param data Pointer to the data. + * @return int Returns either DHT_SUCCESS on success or correspondending error + * value on eviction or error. */ -extern int DHT_free(DHT *table, int* collision_counter, int* readerror_counter); +extern int DHT_write(DHT* table, void* key, void* data); -/* - * parameters: - * DHT *table - DHT_object created by DHT_create - * - * return: - * error value (DHT_SUCCESS or DHT_MPI_ERROR) +/** + * @brief Read data from DHT. + * + * At the beginning, the target process and all possible indices are determined. + * After that a SHARED lock on the address window for read access is done + * and the first entry is retrieved. Now the received key is compared + * with the key passed to the function. If they coincide the correct data + * was found. If not it continues with the next index. If the last + * possible bucket is reached and the keys still do not match the read + * error counter is incremented. After the window has been released + * again, the function returns with a corresponding return value (read + * error or error-free read). The data to be read out is also written to + * the memory area of the passed pointer. + * + * @param table Pointer to the \a DHT-object. + * @param key Pointer to the key. + * @param destination Pointer to memory area where retreived data should be + * stored. + * @return int Returns either DHT_SUCCESS on success or correspondending error + * value on read miss or error. */ -extern int DHT_print_statistics(DHT *table); +extern int DHT_read(DHT* table, void* key, void* destination); + +/** + * @brief Write current state of DHT to file. + * + * All contents are written as a memory dump, so that no conversion takes place. + * First, an attempt is made to open or create a file. If this is successful the + * file header consisting of data and key size is written. Then each process + * reads its memory area of the DHT and each bucket that was marked as written + * is added to the file using MPI file operations. + * + * @param table Pointer to the \a DHT-object. + * @param filename Name of the file to write to. + * @return int Returns DHT_SUCCESS on succes, DHT_FILE_IO_ERROR if file can't be + * opened/closed or DHT_WRITE_ERROR if file is not writable. + */ +extern int DHT_to_file(DHT* table, const char* filename); + +/** + * @brief Read state of DHT from file. + * + * One needs a previously written DHT file (by DHT_from_file). + * First of all, an attempt is made to open the specified file. If this is + * succeeded the file header is read and compared with the current values of the + * DHT. If the data and key sizes do not differ, one can continue. Each process + * reads one line of the file and writes it to the DHT with DHT_write. This + * happens until no more lines are left. The writing is done by the + * implementation of DHT_write. + * + * @param table Pointer to the \a DHT-object. + * @param filename Name of the file to read from. + * @return int Returns DHT_SUCCESS on succes, DHT_FILE_IO_ERROR if file can't be + * opened/closed, DHT_READ_ERROR if file is not readable or DHT_WRONG_FILE if + * file doesn't match expectation. This is possible if the data size or key size + * is different. + */ +extern int DHT_from_file(DHT* table, const char* filename); + +/** + * @brief Free ressources of DHT. + * + * Finally, to free all resources after using the DHT, the function + * DHT_free must be used. This will free the MPI\_Window, as well as the + * associated memory. Also all internal variables are released. Optionally, the + * count of evictions and read misses can also be obtained. + * + * @param table Pointer to the \a DHT-object. + * @param eviction_counter \a optional: Pointer to integer where the count of + * evictions should be stored. + * @param readerror_counter \a optional: Pointer to integer where the count of + * read errors should be stored. + * @return int Returns either DHT_SUCCESS on success or DHT_MPI_ERROR on + * internal MPI error. + */ +extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter); + +/** + * @brief Prints a table with statistics about current use of DHT. + * + * These statistics are from each participated process and also summed up over + * all processes. Detailed statistics are: + * -# occupied buckets (in respect to the memory of this process) + * -# free buckets (in respect to the memory of this process) + * -# calls of DHT_write (w_access) + * -# calls of DHT_read (r_access) + * -# read misses (see DHT_READ_ERROR) + * -# collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) + * 3-6 will reset with every call of this function finally the amount of new + * written entries is printed out (since the last call of this funtion). + * + * This is done by collective MPI operations with the root process with rank 0, + * which will also print a table with all informations to stdout. + * + * Also, as this function was implemented for a special case (POET project) one + * need to define DHT_STATISTICS to the compiler macros to use this + * function (eg. gcc -DDHT_STATISTICS ... ). + * @param table Pointer to the \a DHT-object. + * @return int Returns DHT_SUCCESS on success or DHT_MPI_ERROR on internal MPI + * error. + */ + +#ifdef DHT_STATISTICS +extern int DHT_print_statistics(DHT* table); +#endif #endif /* DHT_H */ \ No newline at end of file diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index 4cdc77a88..0e2b8edef 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -1,10 +1,14 @@ #include "DHT_Wrapper.h" -#include "DHT.h" + #include + #include +#include "DHT.h" + using namespace poet; using namespace std; + uint64_t get_md5(int key_size, void *key) { MD5_CTX ctx; unsigned char sum[MD5_DIGEST_LENGTH]; @@ -105,10 +109,11 @@ int DHT_Wrapper::tableToFile(const char *filename) { int DHT_Wrapper::fileToTable(const char *filename) { int res = DHT_from_file(dht_object, filename); - if (res != DHT_SUCCESS) - return res; + if (res != DHT_SUCCESS) return res; +#ifdef DHT_STATISTICS DHT_print_statistics(dht_object); +#endif return DHT_SUCCESS; } @@ -116,7 +121,9 @@ int DHT_Wrapper::fileToTable(const char *filename) { void DHT_Wrapper::printStatistics() { int res; +#ifdef DHT_STATISTICS res = DHT_print_statistics(dht_object); +#endif if (res != DHT_SUCCESS) { // MPI ERROR ... WHAT TO DO NOW? @@ -160,6 +167,5 @@ void DHT_Wrapper::fuzzForDHT(int var_count, void *key, double dt) { << endl; } } - if (dt_differ) - fuzzing_buffer[var_count] = dt; -} + if (dt_differ) fuzzing_buffer[var_count] = dt; +} \ No newline at end of file diff --git a/src/DHT/DHT_Wrapper.h b/src/DHT/DHT_Wrapper.h index 7c5218af2..cbabf33a6 100644 --- a/src/DHT/DHT_Wrapper.h +++ b/src/DHT/DHT_Wrapper.h @@ -12,8 +12,6 @@ #define ROUND(value, signif) \ (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) -uint64_t get_md5(int key_size, void *key); - namespace poet { class DHT_Wrapper { public: From 7cef1e367b1ff5acddc64276b6de06b62a36d3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 13 Jan 2021 21:04:48 +0100 Subject: [PATCH 14/50] fixed wrong datatype in finishWork() --- src/model/ChemWorker.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index 20746db3e..27d500177 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -317,11 +317,12 @@ void ChemWorker::finishWork() { if (dht_enabled) { // dht_perf - double dht_perf[3]; + int dht_perf[3]; dht_perf[0] = dht->getHits(); dht_perf[1] = dht->getMisses(); + cout << "Worker " << world_rank << " had " << dht_perf[1] << " misses" << endl; dht_perf[2] = dht->getEvictions(); - MPI_Send(dht_perf, 3, MPI_UNSIGNED_LONG_LONG, 0, TAG_DHT_PERF, + MPI_Send(dht_perf, 3, MPI_INT, 0, TAG_DHT_PERF, MPI_COMM_WORLD); } From 24ae1486ad3c5e7bf2d2de1c5b2bd734cfaa8123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 13 Jan 2021 21:06:09 +0100 Subject: [PATCH 15/50] introduce Profiler --- src/CMakeLists.txt | 2 +- src/kin.cpp | 193 +++++++++++++++++++++--------------------- src/util/Profiler.cpp | 126 +++++++++++++++++++++++++++ src/util/Profiler.h | 18 ++++ 4 files changed, 240 insertions(+), 99 deletions(-) create mode 100644 src/util/Profiler.cpp create mode 100644 src/util/Profiler.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d1b71ab99..3541e1a4d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,7 +89,7 @@ include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp) +add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp util/Profiler.cpp) target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) diff --git a/src/kin.cpp b/src/kin.cpp index f67186f4a..192986ff7 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -12,10 +12,11 @@ // #include "global_buffer.h" #include "model/ChemSim.h" #include "model/Grid.h" +#include "model/TransportSim.h" #include "util/Parser.h" #include "util/RRuntime.h" #include "util/SimParams.h" -#include "model/TransportSim.h" +#include "util/Profiler.h" // #include "worker.h" //#define DHT_SIZE_PER_PROCESS 1073741824 @@ -216,15 +217,16 @@ int main(int argc, char *argv[]) { params = parser.getParams(); // if (params.world_rank == 0) { - // cout << "CPP: Complete results storage is " << (params.store_result ? "ON" : - // "OFF") + // cout << "CPP: Complete results storage is " << (params.store_result ? + // "ON" : "OFF") // << endl; // cout << "CPP: Work Package Size: " << params.wp_size << endl; // cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; // if (params.dht_enabled) { // cout << "CPP: DHT strategy is " << params.dht_strategy << endl; - // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is + // " // // "defined) = " // // << dht_significant_digits << endl; // cout << "CPP: DHT logarithm before rounding: " @@ -430,27 +432,23 @@ int main(int argc, char *argv[]) { } else { /*send work to workers*/ master.runPar(); } - - double master_seq_a = MPI_Wtime(); + // MDL master_iteration_end just writes on disk state_T and // state_C after every iteration if the cmdline option // --ignore-results is not given (and thus the R variable // store_result is TRUE) R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - cummul_transport += trans.getTransportTime(); - cummul_chemistry += master.getChemistryTime(); + // cummul_transport += trans.getTransportTime(); + // cummul_chemistry += master.getChemistryTime(); cout << endl << "CPP: End of *coupling* iteration " << iter << "/" << maxiter << endl << endl; - double master_seq_b = MPI_Wtime(); - - cummul_master_seq += master.getSeqTime() + (master_seq_b - master_seq_a); - master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); - master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); + // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); + // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); for (int i = 1; i < params.world_size; i++) { MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); @@ -464,118 +462,117 @@ int main(int argc, char *argv[]) { sim_end = MPI_Wtime(); - Rcpp::NumericVector phreeqc_time; - Rcpp::NumericVector dht_get_time; - Rcpp::NumericVector dht_fill_time; - Rcpp::IntegerVector phreeqc_counts; - Rcpp::NumericVector idle_worker; + Profiler::startProfiling(params, master, trans, R, sim_end - sim_start); - int phreeqc_tmp; + // Rcpp::NumericVector phreeqc_time; + // Rcpp::NumericVector dht_get_time; + // Rcpp::NumericVector dht_fill_time; + // Rcpp::IntegerVector phreeqc_counts; + // Rcpp::NumericVector idle_worker; - timings = (double *)calloc(3, sizeof(double)); + // int phreeqc_tmp; - int dht_hits = 0; - int dht_miss = 0; - int dht_collision = 0; + // timings = (double *)calloc(3, sizeof(double)); - if (params.dht_enabled) { - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; - dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); - } + // int dht_hits = 0; + // int dht_miss = 0; + // int dht_collision = 0; - double idle_worker_tmp; + // if (params.dht_enabled) { + // dht_hits = 0; + // dht_miss = 0; + // dht_collision = 0; + // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); + // } - cout << "CPP: Advising worker to stop work and collect data from them" - << endl; + // double idle_worker_tmp; - for (int p = 0; p < params.world_size - 1; p++) { - /* ATTENTION Worker p has rank p+1 */ - /* Send termination message to worker */ - MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + // for (int p = 0; p < params.world_size - 1; p++) { + // /* ATTENTION Worker p has rank p+1 */ + // /* Send termination message to worker */ + // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - if (params.dht_enabled) { - dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + // if (params.dht_enabled) { + // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - dht_hits += dht_perfs[0]; - dht_miss += dht_perfs[1]; - dht_collision += dht_perfs[2]; - } - } + // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // dht_hits += dht_perfs[0]; + // dht_miss += dht_perfs[1]; + // dht_collision += dht_perfs[2]; + // } + // } - R.parseEvalQ("profiling <- list()"); + // R.parseEvalQ("profiling <- list()"); - R["simtime"] = sim_end - sim_start; - R.parseEvalQ("profiling$simtime <- simtime"); - R["simtime_transport"] = cummul_transport; - R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - R["simtime_chemistry"] = cummul_chemistry; - R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - R["simtime_workers"] = master.getWorkerTime(); - R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - R["simtime_chemistry_master"] = master.getChemMasterTime(); - R.parseEvalQ( - "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + // R["simtime"] = sim_end - sim_start; + // R.parseEvalQ("profiling$simtime <- simtime"); + // R["simtime_transport"] = cummul_transport; + // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + // R["simtime_chemistry"] = cummul_chemistry; + // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + // R["simtime_workers"] = master.getWorkerTime(); + // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + // R["simtime_chemistry_master"] = master.getChemMasterTime(); + // R.parseEvalQ( + // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - R["seq_master"] = cummul_master_seq; - R.parseEvalQ("profiling$seq_master <- seq_master"); + // R["seq_master"] = cummul_master_seq; + // R.parseEvalQ("profiling$seq_master <- seq_master"); - // R["master_send"] = master_send; - // R.parseEvalQ("profiling$master_send <- master_send"); - // R["master_recv"] = master_recv; - // R.parseEvalQ("profiling$master_recv <- master_recv"); + // // R["master_send"] = master_send; + // // R.parseEvalQ("profiling$master_send <- master_send"); + // // R["master_recv"] = master_recv; + // // R.parseEvalQ("profiling$master_recv <- master_recv"); - R["idle_master"] = master.getIdleTime(); - R.parseEvalQ("profiling$idle_master <- idle_master"); - R["idle_worker"] = idle_worker; - R.parseEvalQ("profiling$idle_worker <- idle_worker"); + // R["idle_master"] = master.getIdleTime(); + // R.parseEvalQ("profiling$idle_master <- idle_master"); + // R["idle_worker"] = idle_worker; + // R.parseEvalQ("profiling$idle_worker <- idle_worker"); - R["phreeqc_time"] = phreeqc_time; - R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + // R["phreeqc_time"] = phreeqc_time; + // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - R["phreeqc_count"] = phreeqc_counts; - R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + // R["phreeqc_count"] = phreeqc_counts; + // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - if (params.dht_enabled) { - R["dht_hits"] = dht_hits; - R.parseEvalQ("profiling$dht_hits <- dht_hits"); - R["dht_miss"] = dht_miss; - R.parseEvalQ("profiling$dht_miss <- dht_miss"); - R["dht_collision"] = dht_collision; - R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - R["dht_get_time"] = dht_get_time; - R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - R["dht_fill_time"] = dht_fill_time; - R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - } + // if (params.dht_enabled) { + // R["dht_hits"] = dht_hits; + // R.parseEvalQ("profiling$dht_hits <- dht_hits"); + // R["dht_miss"] = dht_miss; + // R.parseEvalQ("profiling$dht_miss <- dht_miss"); + // R["dht_collision"] = dht_collision; + // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + // R["dht_get_time"] = dht_get_time; + // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + // R["dht_fill_time"] = dht_fill_time; + // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + // } - free(timings); + // free(timings); - if (params.dht_enabled) free(dht_perfs); + // if (params.dht_enabled) free(dht_perfs); cout << "CPP: Done! Results are stored as R objects into <" << params.out_dir << "/timings.rds>" << endl; /*exporting results and profiling data*/ - std::string r_vis_code; - r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - R.parseEval(r_vis_code); + // std::string r_vis_code; + // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + // R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ ChemWorker worker(¶ms, R, grid, dht_comm); // worker.prepareSimulation(dht_comm); diff --git a/src/util/Profiler.cpp b/src/util/Profiler.cpp new file mode 100644 index 000000000..3ba3af249 --- /dev/null +++ b/src/util/Profiler.cpp @@ -0,0 +1,126 @@ +#include "Profiler.h" + +#include +#include + +#include +#include + +using namespace Rcpp; +using namespace std; + +int poet::Profiler::startProfiling(t_simparams ¶ms, ChemMaster &chem, + TransportSim &trans, RRuntime &R, + double simtime) { + double *timings; + int *dht_perfs; + + Rcpp::NumericVector phreeqc_time; + Rcpp::NumericVector dht_get_time; + Rcpp::NumericVector dht_fill_time; + Rcpp::IntegerVector phreeqc_counts; + Rcpp::NumericVector idle_worker; + + int phreeqc_tmp; + + timings = (double *)calloc(3, sizeof(double)); + + int dht_hits = 0; + int dht_miss = 0; + int dht_collision = 0; + + if (params.dht_enabled) { + dht_hits = 0; + dht_miss = 0; + dht_collision = 0; + dht_perfs = (int *)calloc(3, sizeof(int)); + } + + double idle_worker_tmp; + + for (int p = 0; p < params.world_size - 1; p++) { + /* ATTENTION Worker p has rank p+1 */ + /* Send termination message to worker */ + MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + + MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + if (params.dht_enabled) { + dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + MPI_Recv(dht_perfs, 3, MPI_INT, p + 1, TAG_DHT_PERF, + MPI_COMM_WORLD, MPI_STATUS_IGNORE); + dht_hits += dht_perfs[0]; + dht_miss += dht_perfs[1]; + cout << "profiler miss = " << dht_miss << endl; + dht_collision += dht_perfs[2]; + } + } + + R.parseEvalQ("profiling <- list()"); + + R["simtime"] = simtime; + R.parseEvalQ("profiling$simtime <- simtime"); + R["simtime_transport"] = trans.getTransportTime(); + R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + R["simtime_chemistry"] = chem.getChemistryTime(); + R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + R["simtime_workers"] = chem.getWorkerTime(); + R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + R["simtime_chemistry_master"] = chem.getChemMasterTime(); + R.parseEvalQ( + "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + + R["seq_master"] = chem.getSeqTime(); + R.parseEvalQ("profiling$seq_master <- seq_master"); + + // R["master_send"] = master_send; + // R.parseEvalQ("profiling$master_send <- master_send"); + // R["master_recv"] = master_recv; + // R.parseEvalQ("profiling$master_recv <- master_recv"); + + R["idle_master"] = chem.getIdleTime(); + R.parseEvalQ("profiling$idle_master <- idle_master"); + R["idle_worker"] = idle_worker; + R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + R["phreeqc_time"] = phreeqc_time; + R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + R["phreeqc_count"] = phreeqc_counts; + R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + if (params.dht_enabled) { + R["dht_hits"] = dht_hits; + R.parseEvalQ("profiling$dht_hits <- dht_hits"); + R["dht_miss"] = dht_miss; + R.parseEvalQ("profiling$dht_miss <- dht_miss"); + R["dht_collision"] = dht_collision; + R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + R["dht_get_time"] = dht_get_time; + R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + R["dht_fill_time"] = dht_fill_time; + R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + } + + free(timings); + + if (params.dht_enabled) free(dht_perfs); + + string r_vis_code; + r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + R.parseEval(r_vis_code); + + return 0; +} diff --git a/src/util/Profiler.h b/src/util/Profiler.h new file mode 100644 index 000000000..71b3611e7 --- /dev/null +++ b/src/util/Profiler.h @@ -0,0 +1,18 @@ +#ifndef PROFILER_H +#define PROFILER_H + +#include "../model/ChemSim.h" +#include "../model/TransportSim.h" +#include "RRuntime.h" +#include "SimParams.h" + +namespace poet { +class Profiler { + public: + static int startProfiling(t_simparams ¶ms, ChemMaster &chem, + TransportSim &trans, RRuntime &R, double simtime); +}; + +} // namespace poet + +#endif // PROFILER_H \ No newline at end of file From 5b44d94730b2ae698ec87d7d8d0a919112beaa2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Thu, 21 Jan 2021 14:07:35 +0100 Subject: [PATCH 16/50] Fix freeing of DHT_Wrapper in Destructor if DHT is disabled --- src/model/ChemWorker.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index 27d500177..1f62371a1 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -62,7 +61,7 @@ ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, ChemWorker::~ChemWorker() { free(mpi_buffer); free(mpi_buffer_results); - delete dht; + if (dht_enabled) delete dht; } void ChemWorker::loop() { @@ -320,10 +319,8 @@ void ChemWorker::finishWork() { int dht_perf[3]; dht_perf[0] = dht->getHits(); dht_perf[1] = dht->getMisses(); - cout << "Worker " << world_rank << " had " << dht_perf[1] << " misses" << endl; dht_perf[2] = dht->getEvictions(); - MPI_Send(dht_perf, 3, MPI_INT, 0, TAG_DHT_PERF, - MPI_COMM_WORLD); + MPI_Send(dht_perf, 3, MPI_INT, 0, TAG_DHT_PERF, MPI_COMM_WORLD); } if (dht_enabled && dht_snaps > 0) writeFile(); From 76a32c12c0615c2d0c5c607b9deb711f2c63fb94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 25 Jan 2021 17:40:32 +0100 Subject: [PATCH 17/50] Build libraries for each directory --- CMake/FindR.cmake | 25 +++ CMake/FindRInside.cmake | 23 +++ CMake/FindRcpp.cmake | 23 +++ CMakeLists.txt | 21 +++ src/CMakeLists.txt | 107 +----------- src/DHT/CMakeLists.txt | 7 + src/DHT/{DHT.cpp => DHT.c} | 38 +---- src/DHT/DHT.h | 47 ++++- src/DHT/DHT_Wrapper.cpp | 5 +- src/DHT/DHT_Wrapper.h | 9 +- src/kin.cpp | 339 +++++++++++++++++++------------------ src/model/CMakeLists.txt | 3 + src/model/ChemMaster.cpp | 108 +++++++++++- src/model/ChemSim.cpp | 11 +- src/model/ChemSim.h | 42 +++-- src/model/Grid.cpp | 1 - src/model/Grid.h | 17 +- src/model/TransportSim.cpp | 7 +- src/model/TransportSim.h | 5 +- src/util/CMakeLists.txt | 3 + src/util/Parser.h | 9 +- src/util/Profiler.cpp | 126 -------------- src/util/Profiler.h | 18 -- src/util/RRuntime.cpp | 2 + src/util/RRuntime.h | 10 +- src/util/SimParams.h | 7 +- src/{ => util}/argh.h | 0 27 files changed, 509 insertions(+), 504 deletions(-) create mode 100644 CMake/FindR.cmake create mode 100644 CMake/FindRInside.cmake create mode 100644 CMake/FindRcpp.cmake create mode 100644 CMakeLists.txt create mode 100644 src/DHT/CMakeLists.txt rename src/DHT/{DHT.cpp => DHT.c} (93%) create mode 100644 src/model/CMakeLists.txt create mode 100644 src/util/CMakeLists.txt delete mode 100644 src/util/Profiler.cpp delete mode 100644 src/util/Profiler.h rename src/{ => util}/argh.h (100%) diff --git a/CMake/FindR.cmake b/CMake/FindR.cmake new file mode 100644 index 000000000..f07d36bd3 --- /dev/null +++ b/CMake/FindR.cmake @@ -0,0 +1,25 @@ +# prepare R environment (Rcpp + RInside) +find_program(R_EXE "R") + +# search for R executable, R header file and library path +if(R_EXE) + execute_process(COMMAND ${R_EXE} RHOME + OUTPUT_VARIABLE R_ROOT_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + find_path(R_INCLUDE_DIR R.h + HINTS ${R_ROOT_DIR} + PATHS /usr/inlcude /usr/local/include /usr/share + PATH_SUFFIXES include/R R/include + ) + + find_library(R_LIBRARY R + HINTS ${R_ROOT_DIR}/lib + ) +else() + message(FATAL_ERROR "No R runtime found!") +endif() + +set(R_LIBRARIES ${R_LIBRARY}) +set(R_INCLUDE_DIRS ${R_INCLUDE_DIR}) \ No newline at end of file diff --git a/CMake/FindRInside.cmake b/CMake/FindRInside.cmake new file mode 100644 index 000000000..c7757700f --- /dev/null +++ b/CMake/FindRInside.cmake @@ -0,0 +1,23 @@ +# find RInside libraries and include path +execute_process(COMMAND echo "cat(find.package('RInside'))" + COMMAND ${R_EXE} --vanilla --slave + RESULT_VARIABLE RINSIDE_NOT_FOUND + ERROR_QUIET + OUTPUT_VARIABLE RINSIDE_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +if(RInside_NOT_FOUND) + message(FATAL_ERROR "RInside not found!") +endif() + +find_library(R_RInside_LIBRARY libRInside.so + HINTS ${RINSIDE_PATH}/lib) + +list(APPEND R_LIBRARIES ${R_RInside_LIBRARY}) + +find_path(R_RInside_INCLUDE_DIR RInside.h + HINTS ${RINSIDE_PATH} + PATH_SUFFIXES include) + +list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR}) \ No newline at end of file diff --git a/CMake/FindRcpp.cmake b/CMake/FindRcpp.cmake new file mode 100644 index 000000000..be88a3780 --- /dev/null +++ b/CMake/FindRcpp.cmake @@ -0,0 +1,23 @@ +# find Rcpp include directory +execute_process(COMMAND echo "cat(find.package('Rcpp'))" + COMMAND ${R_EXE} --vanilla --slave + RESULT_VARIABLE RCPP_NOT_FOUND + ERROR_QUIET + OUTPUT_VARIABLE RCPP_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +if(RCPP_NOT_FOUND) + message(FATAL_ERROR "Rcpp not found!") +endif() + +# find_library(R_Rcpp_LIBRARY Rcpp.so +# HINTS ${RCPP_PATH}/libs) + +# list(APPEND R_LIBRARIES ${R_Rcpp_LIBRARY}) + +find_path(R_Rcpp_INCLUDE_DIR Rcpp.h + HINTS ${RCPP_PATH} + PATH_SUFFIXES include) + +list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR}) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..082be49c3 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,21 @@ +# Version 3.9+ offers new MPI package variables +cmake_minimum_required(VERSION 3.9) + +project(POET VERSION 0.1 LANGUAGES CXX C) + +# specify the C++ standard +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +list(APPEND CMAKE_MODULE_PATH "${POET_SOURCE_DIR}/CMake") + +set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") +add_definitions(${GCC_CXX_FLAGS}) + +find_package(MPI REQUIRED) + +find_package(R REQUIRED) +find_package(Rcpp REQUIRED) +find_package(RInside REQUIRED) + +add_subdirectory(src) \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3541e1a4d..b27714365 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,103 +1,6 @@ -# Version 3.9+ offers new MPI package variables -cmake_minimum_required(VERSION 3.9) - -project(POET VERSION 0.1) - -# Not needed until now -# specify the C++ standard -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_CXX_STANDARD_REQUIRED True) - -find_package(MPI REQUIRED) - -set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") -add_definitions(${GCC_CXX_FLAGS}) - -# prepare R environment (Rcpp + RInside) -find_program(R_EXE "R") - -# search for R executable, R header file and library path -if(R_EXE) - execute_process(COMMAND ${R_EXE} RHOME - OUTPUT_VARIABLE R_ROOT_DIR - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - - find_path(R_INCLUDE_DIR R.h - HINTS ${R_ROOT_DIR} - PATHS /usr/inlcude /usr/local/include /usr/share - PATH_SUFFIXES include/R R/include - ) - - find_library(R_LIBRARY R - HINTS ${R_ROOT_DIR}/lib - ) -else() - message(FATAL_ERROR "No R runtime found!") -endif() - -set(R_LIBRARIES ${R_LIBRARY}) -set(R_INCLUDE_DIRS ${R_INCLUDE_DIR}) - - -# find Rcpp include directory -execute_process(COMMAND echo "cat(find.package('Rcpp'))" - COMMAND ${R_EXE} --vanilla --slave - RESULT_VARIABLE RCPP_NOT_FOUND - ERROR_QUIET - OUTPUT_VARIABLE RCPP_PATH - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -if(RCPP_NOT_FOUND) - message(FATAL_ERROR "Rcpp not found!") -endif() - -find_path(R_Rcpp_INCLUDE_DIR Rcpp.h - HINTS ${RCPP_PATH} - PATH_SUFFIXES include) - -list(APPEND R_INCLUDE_DIRS ${R_Rcpp_INCLUDE_DIR}) - - -# find RInside libraries and include path -execute_process(COMMAND echo "cat(find.package('RInside'))" - COMMAND ${R_EXE} --vanilla --slave - RESULT_VARIABLE RINSIDE_NOT_FOUND - ERROR_QUIET - OUTPUT_VARIABLE RINSIDE_PATH - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -if(RInside_NOT_FOUND) - message(FATAL_ERROR "RInside not found!") -endif() - -find_library(R_RInside_LIBRARY libRInside.so - HINTS ${RINSIDE_PATH}/lib) - -list(APPEND R_LIBRARIES ${R_RInside_LIBRARY}) - -find_path(R_RInside_INCLUDE_DIR RInside.h - HINTS ${RINSIDE_PATH} - PATH_SUFFIXES include) - -list(APPEND R_INCLUDE_DIRS ${R_RInside_INCLUDE_DIR}) - -#include found directories for the whole scope (will be changed with modularization) -include_directories(${MPI_CXX_INCLUDE_DIRS}) - -#define program libraries - -add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp util/Profiler.cpp) -target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) -target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) - -#add_library(DHT OBJECT DHT.cpp dht_wrapper.cpp) -#target_link_libraries(DHT crypto R_Wrapper) - -#add_library(Worker OBJECT worker.cpp) -#target_link_libraries(Worker ${R_LIBRARIES} MPI::MPI_CXX R_Wrapper) - add_executable(poet kin.cpp) -target_link_libraries(poet PUBLIC MPI::MPI_CXX POET_Libs) +target_link_libraries(poet PRIVATE POET_Model POET_Util) + +add_subdirectory(DHT) +add_subdirectory(model) +add_subdirectory(util) diff --git a/src/DHT/CMakeLists.txt b/src/DHT/CMakeLists.txt new file mode 100644 index 000000000..086ec4903 --- /dev/null +++ b/src/DHT/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(DHT SHARED DHT.c DHT.h) +target_include_directories(DHT PRIVATE ${MPI_C_INCLUDE_DIRS}) +target_link_libraries(DHT MPI::MPI_C) + +add_library(DHT_Wrapper STATIC DHT_Wrapper.cpp DHT_Wrapper.h) +target_include_directories(DHT_Wrapper PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(DHT_Wrapper DHT m crypto POET_Util) \ No newline at end of file diff --git a/src/DHT/DHT.cpp b/src/DHT/DHT.c similarity index 93% rename from src/DHT/DHT.cpp rename to src/DHT/DHT.c index 2ffb3d9ad..f31161b01 100644 --- a/src/DHT/DHT.cpp +++ b/src/DHT/DHT.c @@ -5,22 +5,6 @@ #include #include -/** - * @brief Determine destination rank and index. - * - * This is done by looping over all possbile indices. First of all, set a - * temporary index to zero and copy count of bytes for each index into the - * memory area of the temporary index. After that the current index is - * calculated by the temporary index modulo the table size. The destination rank - * of the process is simply determined by hash modulo the communicator size. - * - * @param hash Calculated 64 bit hash. - * @param comm_size Communicator size. - * @param table_size Count of buckets per process. - * @param dest_rank Reference to the destination rank variable. - * @param index Pointer to the array index. - * @param index_count Count of possible indeces. - */ static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int *dest_rank, unsigned int *index, unsigned int index_count) { @@ -36,27 +20,11 @@ static void determine_dest(uint64_t hash, int comm_size, *dest_rank = (unsigned int)(hash % comm_size); } -/** - * @brief Set the occupied flag. - * - * This will set the first bit of a bucket to 1. - * - * @param flag_byte First byte of a bucket. - */ static void set_flag(char *flag_byte) { *flag_byte = 0; *flag_byte |= (1 << 0); } -/** - * @brief Get the occupied flag. - * - * This function determines whether the occupied flag of a bucket was set or - * not. - * - * @param flag_byte First byte of a bucket. - * @return int Returns 1 for true or 0 for false. - */ static int read_flag(char flag_byte) { if ((flag_byte & 0x01) == 0x01) { return 1; @@ -228,7 +196,7 @@ int DHT_read(DHT *table, void *send_key, void *destination) { #endif // unlock window and return if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; + return DHT_READ_MISS; } // ... or key doesn't match passed by key and last index reached. @@ -241,7 +209,7 @@ int DHT_read(DHT *table, void *send_key, void *destination) { #endif // unlock window an return if (MPI_Win_unlock(dest_rank, table->window) != 0) return DHT_MPI_ERROR; - return DHT_READ_ERROR; + return DHT_READ_MISS; } } else break; @@ -344,7 +312,7 @@ int DHT_from_file(DHT *table, const char *filename) { // seek behind header of DHT file if (MPI_File_seek(file, DHT_FILEHEADER_SIZE, MPI_SEEK_SET) != 0) return DHT_FILE_IO_ERROR; - + // current position is rank * bucket_size + OFFSET cur_pos = DHT_FILEHEADER_SIZE + (rank * bucket_size); diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h index 751224407..e5f83ad27 100644 --- a/src/DHT/DHT.h +++ b/src/DHT/DHT.h @@ -20,7 +20,7 @@ /** Returned if some error in MPI routine occurs. */ #define DHT_MPI_ERROR -1 /** Returned by a call of DHT_read if no bucket with given key was found. */ -#define DHT_READ_ERROR -2 +#define DHT_READ_MISS -2 /** Returned by DHT_write if a bucket was evicted. */ #define DHT_WRITE_SUCCESS_WITH_COLLISION -3 /** Returned when no errors occured. */ @@ -202,7 +202,7 @@ extern int DHT_to_file(DHT* table, const char* filename); * @param table Pointer to the \a DHT-object. * @param filename Name of the file to read from. * @return int Returns DHT_SUCCESS on succes, DHT_FILE_IO_ERROR if file can't be - * opened/closed, DHT_READ_ERROR if file is not readable or DHT_WRONG_FILE if + * opened/closed, DHT_READ_MISS if file is not readable or DHT_WRONG_FILE if * file doesn't match expectation. This is possible if the data size or key size * is different. */ @@ -235,7 +235,7 @@ extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter); * -# free buckets (in respect to the memory of this process) * -# calls of DHT_write (w_access) * -# calls of DHT_read (r_access) - * -# read misses (see DHT_READ_ERROR) + * -# read misses (see DHT_READ_MISS) * -# collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) * 3-6 will reset with every call of this function finally the amount of new * written entries is printed out (since the last call of this funtion). @@ -250,9 +250,48 @@ extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter); * @return int Returns DHT_SUCCESS on success or DHT_MPI_ERROR on internal MPI * error. */ - #ifdef DHT_STATISTICS extern int DHT_print_statistics(DHT* table); #endif +/** + * @brief Determine destination rank and index. + * + * This is done by looping over all possbile indices. First of all, set a + * temporary index to zero and copy count of bytes for each index into the + * memory area of the temporary index. After that the current index is + * calculated by the temporary index modulo the table size. The destination rank + * of the process is simply determined by hash modulo the communicator size. + * + * @param hash Calculated 64 bit hash. + * @param comm_size Communicator size. + * @param table_size Count of buckets per process. + * @param dest_rank Reference to the destination rank variable. + * @param index Pointer to the array index. + * @param index_count Count of possible indeces. + */ +static void determine_dest(uint64_t hash, int comm_size, + unsigned int table_size, unsigned int* dest_rank, + unsigned int* index, unsigned int index_count); + +/** + * @brief Set the occupied flag. + * + * This will set the first bit of a bucket to 1. + * + * @param flag_byte First byte of a bucket. + */ +static void set_flag(char* flag_byte); + +/** + * @brief Get the occupied flag. + * + * This function determines whether the occupied flag of a bucket was set or + * not. + * + * @param flag_byte First byte of a bucket. + * @return int Returns 1 for true or 0 for false. + */ +static int read_flag(char flag_byte); + #endif /* DHT_H */ \ No newline at end of file diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index 0e2b8edef..376e6ac8b 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -1,11 +1,10 @@ #include "DHT_Wrapper.h" +#include #include #include -#include "DHT.h" - using namespace poet; using namespace std; @@ -60,7 +59,7 @@ void DHT_Wrapper::checkDHT(int length, std::vector &out_result_index, // flag that this line is replaced by DHT-value, do not simulate!! out_result_index[i] = false; dht_hits++; - } else if (res == DHT_READ_ERROR) { + } else if (res == DHT_READ_MISS) { // this line is untouched, simulation is needed out_result_index[i] = true; dht_miss++; diff --git a/src/DHT/DHT_Wrapper.h b/src/DHT/DHT_Wrapper.h index cbabf33a6..fa836cdf5 100644 --- a/src/DHT/DHT_Wrapper.h +++ b/src/DHT/DHT_Wrapper.h @@ -1,13 +1,16 @@ #ifndef DHT_WRAPPER_H #define DHT_WRAPPER_H -#include +#include #include #include -#include "../util/SimParams.h" -#include "DHT.h" +extern "C" { +#include +} + +#include #define ROUND(value, signif) \ (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) diff --git a/src/kin.cpp b/src/kin.cpp index 192986ff7..b518f5e00 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -1,5 +1,4 @@ #include -#include // mpi header file #include #include @@ -7,16 +6,14 @@ #include // #include "DHT.h" // MPI-DHT Implementation -#include "argh.h" // Argument handler https://github.com/adishavit/argh BSD-licenced -// #include "dht_wrapper.h" -// #include "global_buffer.h" -#include "model/ChemSim.h" -#include "model/Grid.h" -#include "model/TransportSim.h" -#include "util/Parser.h" -#include "util/RRuntime.h" -#include "util/SimParams.h" -#include "util/Profiler.h" +// #include "argh.h" // Argument handler https://github.com/adishavit/argh +// BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" +#include +#include +#include +#include +#include +#include // #include "worker.h" //#define DHT_SIZE_PER_PROCESS 1073741824 @@ -277,7 +274,7 @@ int main(int argc, char *argv[]) { R.parseEval(init_chemistry_code); Grid grid(R); - params.grid = &grid; + // params.grid = &grid; grid.init(); /* Retrieve state_C from R context for MPI buffer generation */ // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); @@ -306,7 +303,7 @@ int main(int argc, char *argv[]) { // R["dht_enabled"] = params.dht_enabled; // R["dht_log"] = params.dht_log; - params.R = &R; + // params.R = &R; if (params.dht_enabled) { // cout << "\nCreating DHT\n"; @@ -417,7 +414,7 @@ int main(int argc, char *argv[]) { cout << "CPP: Calling Advection" << endl; - trans.runIteration(); + trans.run(); // sim_b_transport = MPI_Wtime(); // R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); // sim_a_transport = MPI_Wtime(); @@ -428,175 +425,193 @@ int main(int argc, char *argv[]) { /*Fallback for sequential execution*/ if (params.world_size == 1) { - master.runSeq(); - } else { /*send work to workers*/ - master.runPar(); + master.ChemSim::run(); + } else { + master.run(); } - // MDL master_iteration_end just writes on disk state_T and - // state_C after every iteration if the cmdline option - // --ignore-results is not given (and thus the R variable - // store_result is TRUE) - R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); + // MDL master_iteration_end just writes on disk state_T and + // state_C after every iteration if the cmdline option + // --ignore-results is not given (and thus the R variable + // store_result is TRUE) + R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - // cummul_transport += trans.getTransportTime(); - // cummul_chemistry += master.getChemistryTime(); + // cummul_transport += trans.getTransportTime(); + // cummul_chemistry += master.getChemistryTime(); - cout << endl - << "CPP: End of *coupling* iteration " << iter << "/" << maxiter - << endl - << endl; + cout << endl + << "CPP: End of *coupling* iteration " << iter << "/" << maxiter + << endl + << endl; - // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); - // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); + // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); + // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); - for (int i = 1; i < params.world_size; i++) { - MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); - } + for (int i = 1; i < params.world_size; i++) { + MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); + } - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); - } // END SIMULATION LOOP + } // END SIMULATION LOOP - cout << "CPP: finished simulation loop" << endl; + cout << "CPP: finished simulation loop" << endl; - sim_end = MPI_Wtime(); + sim_end = MPI_Wtime(); - Profiler::startProfiling(params, master, trans, R, sim_end - sim_start); + cout << "CPP: start timing profiling" << endl; - // Rcpp::NumericVector phreeqc_time; - // Rcpp::NumericVector dht_get_time; - // Rcpp::NumericVector dht_fill_time; - // Rcpp::IntegerVector phreeqc_counts; - // Rcpp::NumericVector idle_worker; + R.parseEvalQ("profiling <- list()"); - // int phreeqc_tmp; + R["simtime"] = sim_end - sim_start; + R.parseEvalQ("profiling$simtime <- simtime"); + + trans.end(); - // timings = (double *)calloc(3, sizeof(double)); - - // int dht_hits = 0; - // int dht_miss = 0; - // int dht_collision = 0; - - // if (params.dht_enabled) { - // dht_hits = 0; - // dht_miss = 0; - // dht_collision = 0; - // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); - // } - - // double idle_worker_tmp; - - // for (int p = 0; p < params.world_size - 1; p++) { - // /* ATTENTION Worker p has rank p+1 */ - // /* Send termination message to worker */ - // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - - // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - - // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - - // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - - // if (params.dht_enabled) { - // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - - // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // dht_hits += dht_perfs[0]; - // dht_miss += dht_perfs[1]; - // dht_collision += dht_perfs[2]; - // } - // } - - // R.parseEvalQ("profiling <- list()"); - - // R["simtime"] = sim_end - sim_start; - // R.parseEvalQ("profiling$simtime <- simtime"); - // R["simtime_transport"] = cummul_transport; - // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - // R["simtime_chemistry"] = cummul_chemistry; - // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - // R["simtime_workers"] = master.getWorkerTime(); - // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - // R["simtime_chemistry_master"] = master.getChemMasterTime(); - // R.parseEvalQ( - // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - - // R["seq_master"] = cummul_master_seq; - // R.parseEvalQ("profiling$seq_master <- seq_master"); - - // // R["master_send"] = master_send; - // // R.parseEvalQ("profiling$master_send <- master_send"); - // // R["master_recv"] = master_recv; - // // R.parseEvalQ("profiling$master_recv <- master_recv"); - - // R["idle_master"] = master.getIdleTime(); - // R.parseEvalQ("profiling$idle_master <- idle_master"); - // R["idle_worker"] = idle_worker; - // R.parseEvalQ("profiling$idle_worker <- idle_worker"); - - // R["phreeqc_time"] = phreeqc_time; - // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - - // R["phreeqc_count"] = phreeqc_counts; - // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - - // if (params.dht_enabled) { - // R["dht_hits"] = dht_hits; - // R.parseEvalQ("profiling$dht_hits <- dht_hits"); - // R["dht_miss"] = dht_miss; - // R.parseEvalQ("profiling$dht_miss <- dht_miss"); - // R["dht_collision"] = dht_collision; - // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - // R["dht_get_time"] = dht_get_time; - // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - // R["dht_fill_time"] = dht_fill_time; - // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - // } - - // free(timings); - - // if (params.dht_enabled) free(dht_perfs); - - cout << "CPP: Done! Results are stored as R objects into <" - << params.out_dir << "/timings.rds>" << endl; - /*exporting results and profiling data*/ - - // std::string r_vis_code; - // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - // R.parseEval(r_vis_code); - } else { /*This is executed by the workers*/ - ChemWorker worker(¶ms, R, grid, dht_comm); - // worker.prepareSimulation(dht_comm); - worker.loop(); + if (params.world_size == 1) { + master.ChemSim::end(); + } else { + master.end(); } - cout << "CPP: finished, cleanup of process " << params.world_rank << endl; + // Rcpp::NumericVector phreeqc_time; + // Rcpp::NumericVector dht_get_time; + // Rcpp::NumericVector dht_fill_time; + // Rcpp::IntegerVector phreeqc_counts; + // Rcpp::NumericVector idle_worker; + + // int phreeqc_tmp; + + // timings = (double *)calloc(3, sizeof(double)); + + // int dht_hits = 0; + // int dht_miss = 0; + // int dht_collision = 0; // if (params.dht_enabled) { - // if (params.dht_strategy == 0) { - // if (params.world_rank != 0) { - // DHT_free(dht_object, NULL, NULL); - // } - // } else { - // DHT_free(dht_object, NULL, NULL); + // dht_hits = 0; + // dht_miss = 0; + // dht_collision = 0; + // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); + // } + + // double idle_worker_tmp; + + // for (int p = 0; p < params.world_size - 1; p++) { + // /* ATTENTION Worker p has rank p+1 */ + // /* Send termination message to worker */ + // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + + // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + // if (params.dht_enabled) { + // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // dht_hits += dht_perfs[0]; + // dht_miss += dht_perfs[1]; + // dht_collision += dht_perfs[2]; // } // } - // free(mpi_buffer); - MPI_Finalize(); + // R.parseEvalQ("profiling <- list()"); - if (params.world_rank == 0) { - cout << "CPP: done, bye!" << endl; - } + // R["simtime"] = sim_end - sim_start; + // R.parseEvalQ("profiling$simtime <- simtime"); + // R["simtime_transport"] = cummul_transport; + // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + // R["simtime_chemistry"] = cummul_chemistry; + // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + // R["simtime_workers"] = master.getWorkerTime(); + // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + // R["simtime_chemistry_master"] = master.getChemMasterTime(); + // R.parseEvalQ( + // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - exit(0); + // R["seq_master"] = cummul_master_seq; + // R.parseEvalQ("profiling$seq_master <- seq_master"); + + // // R["master_send"] = master_send; + // // R.parseEvalQ("profiling$master_send <- master_send"); + // // R["master_recv"] = master_recv; + // // R.parseEvalQ("profiling$master_recv <- master_recv"); + + // R["idle_master"] = master.getIdleTime(); + // R.parseEvalQ("profiling$idle_master <- idle_master"); + // R["idle_worker"] = idle_worker; + // R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + // R["phreeqc_time"] = phreeqc_time; + // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + // R["phreeqc_count"] = phreeqc_counts; + // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + // if (params.dht_enabled) { + // R["dht_hits"] = dht_hits; + // R.parseEvalQ("profiling$dht_hits <- dht_hits"); + // R["dht_miss"] = dht_miss; + // R.parseEvalQ("profiling$dht_miss <- dht_miss"); + // R["dht_collision"] = dht_collision; + // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + // R["dht_get_time"] = dht_get_time; + // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + // R["dht_fill_time"] = dht_fill_time; + // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + // } + + // free(timings); + + // if (params.dht_enabled) free(dht_perfs); + + string r_vis_code; + r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + R.parseEval(r_vis_code); + + cout << "CPP: Done! Results are stored as R objects into <" << params.out_dir + << "/timings.rds>" << endl; + /*exporting results and profiling data*/ + + // std::string r_vis_code; + // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + // R.parseEval(r_vis_code); +} +else { /*This is executed by the workers*/ + ChemWorker worker(¶ms, R, grid, dht_comm); + // worker.prepareSimulation(dht_comm); + worker.loop(); +} + +cout << "CPP: finished, cleanup of process " << params.world_rank << endl; + +// if (params.dht_enabled) { +// if (params.dht_strategy == 0) { +// if (params.world_rank != 0) { +// DHT_free(dht_object, NULL, NULL); +// } +// } else { +// DHT_free(dht_object, NULL, NULL); +// } +// } + +// free(mpi_buffer); +MPI_Finalize(); + +if (params.world_rank == 0) { + cout << "CPP: done, bye!" << endl; +} + +exit(0); } diff --git a/src/model/CMakeLists.txt b/src/model/CMakeLists.txt new file mode 100644 index 000000000..db68f3a5a --- /dev/null +++ b/src/model/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(POET_Model STATIC ChemSim.cpp ChemSim.h ChemMaster.cpp ChemWorker.cpp TransportSim.cpp Grid.cpp) +target_include_directories(POET_Model PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(POET_Model POET_Util DHT_Wrapper) \ No newline at end of file diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp index c4a96396b..f662572a0 100644 --- a/src/model/ChemMaster.cpp +++ b/src/model/ChemMaster.cpp @@ -1,5 +1,4 @@ #include -#include #include @@ -15,6 +14,7 @@ ChemMaster::ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_) : ChemSim(params, R_, grid_) { this->wp_size = params->wp_size; this->out_dir = params->out_dir; + this->dht_enabled = params->dht_enabled; workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, @@ -39,7 +39,7 @@ ChemMaster::~ChemMaster() { free(workerlist); } -void ChemMaster::runPar() { +void ChemMaster::run() { double chem_a, chem_b; double seq_a, seq_b, seq_c, seq_d; double worker_chemistry_a, worker_chemistry_b; @@ -208,6 +208,110 @@ void ChemMaster::printProgressbar(int count_pkgs, int n_wp, int barWidth) { /* end visual progress */ } +void ChemMaster::end() { + ChemSim::end(); + + double *timings; + int *dht_perfs; + + Rcpp::NumericVector phreeqc_time; + Rcpp::NumericVector dht_get_time; + Rcpp::NumericVector dht_fill_time; + Rcpp::IntegerVector phreeqc_counts; + Rcpp::NumericVector idle_worker; + + int phreeqc_tmp; + + timings = (double *)calloc(3, sizeof(double)); + + int dht_hits = 0; + int dht_miss = 0; + int dht_collision = 0; + + if (dht_enabled) { + dht_hits = 0; + dht_miss = 0; + dht_collision = 0; + dht_perfs = (int *)calloc(3, sizeof(int)); + } + + double idle_worker_tmp; + + for (int p = 0; p < world_size - 1; p++) { + /* ATTENTION Worker p has rank p+1 */ + /* Send termination message to worker */ + MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + + MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + if (dht_enabled) { + dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + MPI_Recv(dht_perfs, 3, MPI_INT, p + 1, TAG_DHT_PERF, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + dht_hits += dht_perfs[0]; + dht_miss += dht_perfs[1]; + cout << "profiler miss = " << dht_miss << endl; + dht_collision += dht_perfs[2]; + } + } + + R["simtime_chemistry"] = chem_t; + R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + R["simtime_workers"] = worker_t; + R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + R["simtime_chemistry_master"] = chem_master; + R.parseEvalQ( + "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + + R["seq_master"] = seq_t; + R.parseEvalQ("profiling$seq_master <- seq_master"); + + // R["master_send"] = master_send; + // R.parseEvalQ("profiling$master_send <- master_send"); + // R["master_recv"] = master_recv; + // R.parseEvalQ("profiling$master_recv <- master_recv"); + + R["idle_master"] = master_idle; + R.parseEvalQ("profiling$idle_master <- idle_master"); + R["idle_worker"] = idle_worker; + R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + R["phreeqc_time"] = phreeqc_time; + R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + R["phreeqc_count"] = phreeqc_counts; + R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + if (dht_enabled) { + R["dht_hits"] = dht_hits; + R.parseEvalQ("profiling$dht_hits <- dht_hits"); + R["dht_miss"] = dht_miss; + R.parseEvalQ("profiling$dht_miss <- dht_miss"); + R["dht_collision"] = dht_collision; + R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + R["dht_get_time"] = dht_get_time; + R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + R["dht_fill_time"] = dht_fill_time; + R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + } + + free(timings); + + if (dht_enabled) free(dht_perfs); +} + double ChemMaster::getSendTime() { return this->send_t; } double ChemMaster::getRecvTime() { return this->recv_t; } diff --git a/src/model/ChemSim.cpp b/src/model/ChemSim.cpp index 8a322ef4b..d6459512c 100644 --- a/src/model/ChemSim.cpp +++ b/src/model/ChemSim.cpp @@ -1,13 +1,9 @@ #include "ChemSim.h" #include -#include #include -#include "../util/RRuntime.h" -#include "Grid.h" - using namespace Rcpp; using namespace poet; @@ -19,7 +15,7 @@ ChemSim::ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_) this->out_dir = params->out_dir; } -void ChemSim::runSeq() { +void ChemSim::run() { double chem_a, chem_b; chem_a = MPI_Wtime(); @@ -32,4 +28,9 @@ void ChemSim::runSeq() { chem_t += chem_b - chem_a; } +void ChemSim::end() { + R["simtime_chemistry"] = chem_t; + R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); +} + double ChemSim::getChemistryTime() { return this->chem_t; } diff --git a/src/model/ChemSim.h b/src/model/ChemSim.h index 5b9b2f8ae..3765c3bc2 100644 --- a/src/model/ChemSim.h +++ b/src/model/ChemSim.h @@ -1,15 +1,17 @@ #ifndef CHEMSIM_H #define CHEMSIM_H -#include "../DHT/DHT_Wrapper.h" -#include "../util/RRuntime.h" -#include "../util/SimParams.h" -#include "Grid.h" - +#include +#include +#include #include + #include +#include "Grid.h" + #define BUFFER_OFFSET 5 + #define TAG_WORK 42 #define TAG_FINISH 43 #define TAG_TIMING 44 @@ -20,13 +22,15 @@ namespace poet { class ChemSim { -public: + public: ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_); - - void runSeq(); + + virtual void run(); + virtual void end(); + double getChemistryTime(); -protected: + protected: double current_sim_time = 0; int iteration = 0; int dt = 0; @@ -54,12 +58,12 @@ protected: }; class ChemMaster : public ChemSim { -public: + public: ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_); ~ChemMaster(); - void runPar(); - void profile(); + void run() override; + void end() override; double getSendTime(); double getRecvTime(); @@ -68,11 +72,12 @@ public: double getChemMasterTime(); double getSeqTime(); -private: + private: void printProgressbar(int count_pkgs, int n_wp, int barWidth = 70); void sendPkgs(int &pkg_to_send, int &count_pkgs, int &free_workers); void recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers); + bool dht_enabled; unsigned int wp_size; double *work_pointer; @@ -85,13 +90,14 @@ private: }; class ChemWorker : public ChemSim { -public: - ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, MPI_Comm dht_comm); + public: + ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, + MPI_Comm dht_comm); ~ChemWorker(); void loop(); -private: + private: void doWork(MPI_Status &probe_status); void postIter(); void finishWork(); @@ -114,5 +120,5 @@ private: double idle_t = 0.f; int phreeqc_count = 0; }; -} // namespace poet -#endif // CHEMSIM_H +} // namespace poet +#endif // CHEMSIM_H diff --git a/src/model/Grid.cpp b/src/model/Grid.cpp index ae61ff3d0..6a84798f0 100644 --- a/src/model/Grid.cpp +++ b/src/model/Grid.cpp @@ -1,5 +1,4 @@ #include "Grid.h" -#include "Rcpp.h" using namespace poet; using namespace Rcpp; diff --git a/src/model/Grid.h b/src/model/Grid.h index a2c066505..61267c670 100644 --- a/src/model/Grid.h +++ b/src/model/Grid.h @@ -1,32 +1,31 @@ #ifndef GRID_H #define GRID_H -#include "../util/RRuntime.h" +#include #include namespace poet { class Grid { - -public: + public: Grid(RRuntime &R) : R(R){}; - + void init(); - + unsigned int getCols(); unsigned int getRows(); - + void shuffleAndExport(double *buffer); void importAndUnshuffle(double *buffer); void importWP(double *buffer, unsigned int p_size); void exportWP(double *buffer); -private: + private: RRuntime R; unsigned int ncol; unsigned int nrow; Rcpp::DataFrame getSkeletonDataFrame(unsigned int rows); }; -} // namespace poet -#endif // GRID_H +} // namespace poet +#endif // GRID_H diff --git a/src/model/TransportSim.cpp b/src/model/TransportSim.cpp index ea4fe3899..739f04ffe 100644 --- a/src/model/TransportSim.cpp +++ b/src/model/TransportSim.cpp @@ -6,7 +6,7 @@ using namespace poet; TransportSim::TransportSim(RRuntime &R_) : R(R_) {} -void TransportSim::runIteration() { +void TransportSim::run() { double sim_a_transport, sim_b_transport; sim_b_transport = MPI_Wtime(); @@ -16,4 +16,9 @@ void TransportSim::runIteration() { transport_t += sim_a_transport - sim_b_transport; } +void TransportSim::end() { + R["simtime_transport"] = transport_t; + R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); +} + double TransportSim::getTransportTime() { return this->transport_t; } diff --git a/src/model/TransportSim.h b/src/model/TransportSim.h index 7ceb44daf..f644503b2 100644 --- a/src/model/TransportSim.h +++ b/src/model/TransportSim.h @@ -1,14 +1,15 @@ #ifndef TRANSPORT_SIM_H #define TRANSPORT_SIM_H -#include "../util/RRuntime.h" +#include namespace poet { class TransportSim { public: TransportSim(RRuntime &R); - void runIteration(); + void run(); + void end(); double getTransportTime(); diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt new file mode 100644 index 000000000..aa6140de3 --- /dev/null +++ b/src/util/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(POET_Util STATIC RRuntime.cpp Parser.cpp) +target_include_directories(POET_Util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${R_INCLUDE_DIRS}) +target_link_libraries(POET_Util ${R_LIBRARIES}) \ No newline at end of file diff --git a/src/util/Parser.h b/src/util/Parser.h index d79a7ad35..2c17b25e1 100644 --- a/src/util/Parser.h +++ b/src/util/Parser.h @@ -1,11 +1,12 @@ #ifndef PARSER_H #define PARSER_H +#include + #include -#include "../argh.h" -#include "RRuntime.h" #include "SimParams.h" +#include "argh.h" #define PARSER_OK 0 #define PARSER_ERROR 1 @@ -27,8 +28,8 @@ class Parser { std::list checkOptions(argh::parser cmdl); std::set flaglist{"ignore-result", "dht", "dht-nolog"}; std::set paramlist{"work-package-size", "dht-signif", - "dht-strategy", "dht-size", - "dht-snaps", "dht-file"}; + "dht-strategy", "dht-size", + "dht-snaps", "dht-file"}; argh::parser cmdl; t_simparams simparams; diff --git a/src/util/Profiler.cpp b/src/util/Profiler.cpp deleted file mode 100644 index 3ba3af249..000000000 --- a/src/util/Profiler.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "Profiler.h" - -#include -#include - -#include -#include - -using namespace Rcpp; -using namespace std; - -int poet::Profiler::startProfiling(t_simparams ¶ms, ChemMaster &chem, - TransportSim &trans, RRuntime &R, - double simtime) { - double *timings; - int *dht_perfs; - - Rcpp::NumericVector phreeqc_time; - Rcpp::NumericVector dht_get_time; - Rcpp::NumericVector dht_fill_time; - Rcpp::IntegerVector phreeqc_counts; - Rcpp::NumericVector idle_worker; - - int phreeqc_tmp; - - timings = (double *)calloc(3, sizeof(double)); - - int dht_hits = 0; - int dht_miss = 0; - int dht_collision = 0; - - if (params.dht_enabled) { - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; - dht_perfs = (int *)calloc(3, sizeof(int)); - } - - double idle_worker_tmp; - - for (int p = 0; p < params.world_size - 1; p++) { - /* ATTENTION Worker p has rank p+1 */ - /* Send termination message to worker */ - MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - - MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - - MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - - MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - - if (params.dht_enabled) { - dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - - MPI_Recv(dht_perfs, 3, MPI_INT, p + 1, TAG_DHT_PERF, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - dht_hits += dht_perfs[0]; - dht_miss += dht_perfs[1]; - cout << "profiler miss = " << dht_miss << endl; - dht_collision += dht_perfs[2]; - } - } - - R.parseEvalQ("profiling <- list()"); - - R["simtime"] = simtime; - R.parseEvalQ("profiling$simtime <- simtime"); - R["simtime_transport"] = trans.getTransportTime(); - R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - R["simtime_chemistry"] = chem.getChemistryTime(); - R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - R["simtime_workers"] = chem.getWorkerTime(); - R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - R["simtime_chemistry_master"] = chem.getChemMasterTime(); - R.parseEvalQ( - "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - - R["seq_master"] = chem.getSeqTime(); - R.parseEvalQ("profiling$seq_master <- seq_master"); - - // R["master_send"] = master_send; - // R.parseEvalQ("profiling$master_send <- master_send"); - // R["master_recv"] = master_recv; - // R.parseEvalQ("profiling$master_recv <- master_recv"); - - R["idle_master"] = chem.getIdleTime(); - R.parseEvalQ("profiling$idle_master <- idle_master"); - R["idle_worker"] = idle_worker; - R.parseEvalQ("profiling$idle_worker <- idle_worker"); - - R["phreeqc_time"] = phreeqc_time; - R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - - R["phreeqc_count"] = phreeqc_counts; - R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - - if (params.dht_enabled) { - R["dht_hits"] = dht_hits; - R.parseEvalQ("profiling$dht_hits <- dht_hits"); - R["dht_miss"] = dht_miss; - R.parseEvalQ("profiling$dht_miss <- dht_miss"); - R["dht_collision"] = dht_collision; - R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - R["dht_get_time"] = dht_get_time; - R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - R["dht_fill_time"] = dht_fill_time; - R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - } - - free(timings); - - if (params.dht_enabled) free(dht_perfs); - - string r_vis_code; - r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - R.parseEval(r_vis_code); - - return 0; -} diff --git a/src/util/Profiler.h b/src/util/Profiler.h deleted file mode 100644 index 71b3611e7..000000000 --- a/src/util/Profiler.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef PROFILER_H -#define PROFILER_H - -#include "../model/ChemSim.h" -#include "../model/TransportSim.h" -#include "RRuntime.h" -#include "SimParams.h" - -namespace poet { -class Profiler { - public: - static int startProfiling(t_simparams ¶ms, ChemMaster &chem, - TransportSim &trans, RRuntime &R, double simtime); -}; - -} // namespace poet - -#endif // PROFILER_H \ No newline at end of file diff --git a/src/util/RRuntime.cpp b/src/util/RRuntime.cpp index 0aefaf0f7..adacfd8a1 100644 --- a/src/util/RRuntime.cpp +++ b/src/util/RRuntime.cpp @@ -1,6 +1,8 @@ #include "RRuntime.h" + #include #include + #include using namespace poet; diff --git a/src/util/RRuntime.h b/src/util/RRuntime.h index c231cf9fd..34d2fa9db 100644 --- a/src/util/RRuntime.h +++ b/src/util/RRuntime.h @@ -3,6 +3,7 @@ #include #include + #include namespace poet { @@ -14,8 +15,7 @@ namespace poet { * If an instance of RRuntime is created a R runtime will also be spawned. */ class RRuntime : public RInside { - -public: + public: /** * Constructor of class RRuntime calling constructor of RInside. */ @@ -29,8 +29,8 @@ public: size_t getBufferNCol(); size_t getBufferNRow(); -private: + private: Rcpp::DataFrame dfbuff; }; -} // namespace poet -#endif // RRUNTIME_H +} // namespace poet +#endif // RRUNTIME_H diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 8c982f17a..b4ad7ac8d 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -3,7 +3,6 @@ #include #include -#include "RRuntime.h" typedef struct { int world_size; @@ -26,8 +25,8 @@ typedef struct { bool store_result; - void* R; - void* grid; + // void* R; + // void* grid; } t_simparams; -#endif // SIMPARAMS_H +#endif // SIMPARAMS_H diff --git a/src/argh.h b/src/util/argh.h similarity index 100% rename from src/argh.h rename to src/util/argh.h From 7d36c3f8b08170ef90d545cf49831eb643d7b692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 25 Jan 2021 17:51:32 +0100 Subject: [PATCH 18/50] substitute add_defintions by target_compile_defintions --- CMakeLists.txt | 4 ++-- src/DHT/CMakeLists.txt | 1 + src/model/CMakeLists.txt | 5 +++-- src/util/CMakeLists.txt | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 082be49c3..4362d9c9d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,8 +9,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) list(APPEND CMAKE_MODULE_PATH "${POET_SOURCE_DIR}/CMake") -set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") -add_definitions(${GCC_CXX_FLAGS}) +# set(GCC_CXX_FLAGS "-D STRICT_R_HEADERS") +# add_definitions(${GCC_CXX_FLAGS}) find_package(MPI REQUIRED) diff --git a/src/DHT/CMakeLists.txt b/src/DHT/CMakeLists.txt index 086ec4903..0a0340a7d 100644 --- a/src/DHT/CMakeLists.txt +++ b/src/DHT/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(DHT SHARED DHT.c DHT.h) target_include_directories(DHT PRIVATE ${MPI_C_INCLUDE_DIRS}) target_link_libraries(DHT MPI::MPI_C) +target_compile_definitions(DHT PUBLIC OMPI_SKIP_MPICXX) add_library(DHT_Wrapper STATIC DHT_Wrapper.cpp DHT_Wrapper.h) target_include_directories(DHT_Wrapper PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/model/CMakeLists.txt b/src/model/CMakeLists.txt index db68f3a5a..a2e770c25 100644 --- a/src/model/CMakeLists.txt +++ b/src/model/CMakeLists.txt @@ -1,3 +1,4 @@ add_library(POET_Model STATIC ChemSim.cpp ChemSim.h ChemMaster.cpp ChemWorker.cpp TransportSim.cpp Grid.cpp) -target_include_directories(POET_Model PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(POET_Model POET_Util DHT_Wrapper) \ No newline at end of file +target_include_directories(POET_Model PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${MPI_C_INCLUDE_DIRS}) +target_link_libraries(POET_Model POET_Util DHT_Wrapper MPI::MPI_C) +target_compile_definitions(POET_Model PUBLIC OMPI_SKIP_MPICXX) \ No newline at end of file diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index aa6140de3..ea98f6797 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,3 +1,4 @@ add_library(POET_Util STATIC RRuntime.cpp Parser.cpp) target_include_directories(POET_Util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${R_INCLUDE_DIRS}) -target_link_libraries(POET_Util ${R_LIBRARIES}) \ No newline at end of file +target_link_libraries(POET_Util ${R_LIBRARIES}) +target_compile_definitions(POET_Util PUBLIC STRICT_R_HEADERS) \ No newline at end of file From 019f5214419023e85f6625f09c824f16355f8531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Thu, 28 Jan 2021 14:41:36 +0100 Subject: [PATCH 19/50] fix index datatypes --- src/DHT/DHT.c | 20 +++++++++++--------- src/DHT/DHT.h | 8 ++++---- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/DHT/DHT.c b/src/DHT/DHT.c index f31161b01..a4f07f4b8 100644 --- a/src/DHT/DHT.c +++ b/src/DHT/DHT.c @@ -4,18 +4,20 @@ #include #include #include +#include +#include static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int *dest_rank, - unsigned int *index, unsigned int index_count) { + uint64_t *index, unsigned int index_count) { /** temporary index */ uint64_t tmp_index; - /** how many bytes for one index? */ - int index_size = 9 - index_count; - for (unsigned int i = 0; i < index_count; i++) { + /** how many bytes do we need for one index? */ + int index_size = sizeof(double) - (index_count - 1); + for (int i = 0; i < index_count; i++) { tmp_index = 0; - memcpy(&tmp_index, (unsigned char *)&hash + i, index_size); - index[i] = (unsigned int)(tmp_index % table_size); + memcpy(&tmp_index, (char *)&hash + i, index_size); + index[i] = (uint64_t)(tmp_index % table_size); } *dest_rank = (unsigned int)(hash % comm_size); } @@ -32,8 +34,8 @@ static int read_flag(char flag_byte) { return 0; } -DHT *DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, - uint64_t (*hash_func)(int, void *)) { +DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size, + unsigned int key_size, uint64_t (*hash_func)(int, void *)) { DHT *object; MPI_Win window; void *mem_alloc; @@ -77,7 +79,7 @@ DHT *DHT_create(MPI_Comm comm, unsigned int size, int data_size, int key_size, object->recv_entry = malloc(1 + data_size + key_size); object->send_entry = malloc(1 + data_size + key_size); object->index_count = 9 - (index_bytes / 8); - object->index = (unsigned int *)malloc((9 - (index_bytes / 8)) * sizeof(int)); + object->index = (uint64_t*)malloc((object->index_count) * sizeof(uint64_t)); object->mem_alloc = mem_alloc; // if set, initialize dht_stats diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h index e5f83ad27..88a4ab632 100644 --- a/src/DHT/DHT.h +++ b/src/DHT/DHT.h @@ -95,7 +95,7 @@ typedef struct { /** Count of evictions over all time. */ int evictions; /** Array of indeces where a bucket can be stored. */ - unsigned int* index; + uint64_t* index; /** Count of possible indeces. */ unsigned int index_count; #ifdef DHT_STATISTICS @@ -124,8 +124,8 @@ typedef struct { * @return DHT* The returned value is the \a DHT-object which serves as a handle * for all DHT operations. If an error occured NULL is returned. */ -extern DHT* DHT_create(MPI_Comm comm, unsigned int size_per_process, - int data_size, int key_size, +extern DHT* DHT_create(MPI_Comm comm, uint64_t size_per_process, + unsigned int data_size, unsigned int key_size, uint64_t (*hash_func)(int, void*)); /** @@ -272,7 +272,7 @@ extern int DHT_print_statistics(DHT* table); */ static void determine_dest(uint64_t hash, int comm_size, unsigned int table_size, unsigned int* dest_rank, - unsigned int* index, unsigned int index_count); + uint64_t* index, unsigned int index_count); /** * @brief Set the occupied flag. From 716d9963397d5feb78d3c3f074a09d2b82e89354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Thu, 28 Jan 2021 14:43:36 +0100 Subject: [PATCH 20/50] added DHT_statistics option + fix depencies --- src/CMakeLists.txt | 5 +++-- src/DHT/CMakeLists.txt | 19 ++++++++++++++----- src/model/CMakeLists.txt | 8 ++++---- src/util/CMakeLists.txt | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b27714365..165efb053 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(poet kin.cpp) -target_link_libraries(poet PRIVATE POET_Model POET_Util) +target_link_libraries(poet PRIVATE POET_Model POET_Util MPI::MPI_C) +target_compile_definitions(poet PRIVATE OMPI_SKIP_MPICXX) add_subdirectory(DHT) add_subdirectory(model) -add_subdirectory(util) +add_subdirectory(util) \ No newline at end of file diff --git a/src/DHT/CMakeLists.txt b/src/DHT/CMakeLists.txt index 0a0340a7d..6658bf7b6 100644 --- a/src/DHT/CMakeLists.txt +++ b/src/DHT/CMakeLists.txt @@ -1,8 +1,17 @@ -add_library(DHT SHARED DHT.c DHT.h) -target_include_directories(DHT PRIVATE ${MPI_C_INCLUDE_DIRS}) -target_link_libraries(DHT MPI::MPI_C) +add_library(DHT DHT.c DHT.h) +target_include_directories(DHT PUBLIC ${MPI_C_INCLUDE_DIRS}) +target_link_libraries(DHT PRIVATE MPI::MPI_C) target_compile_definitions(DHT PUBLIC OMPI_SKIP_MPICXX) -add_library(DHT_Wrapper STATIC DHT_Wrapper.cpp DHT_Wrapper.h) +set(DHT_Debug FALSE CACHE BOOL "Toggle output of stastic table for each iteration") + +if (DHT_Debug) + target_compile_definitions(DHT PUBLIC DHT_STATISTICS) +endif() + +find_library(MATH_LIBRARY m) +find_library(CRYPTO_LIBRARY crypto) + +add_library(DHT_Wrapper DHT_Wrapper.cpp DHT_Wrapper.h) target_include_directories(DHT_Wrapper PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(DHT_Wrapper DHT m crypto POET_Util) \ No newline at end of file +target_link_libraries(DHT_Wrapper PRIVATE DHT ${MATH_LIBRARY} ${CRYPTO_LIBRARY} POET_Util) \ No newline at end of file diff --git a/src/model/CMakeLists.txt b/src/model/CMakeLists.txt index a2e770c25..955da1094 100644 --- a/src/model/CMakeLists.txt +++ b/src/model/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(POET_Model STATIC ChemSim.cpp ChemSim.h ChemMaster.cpp ChemWorker.cpp TransportSim.cpp Grid.cpp) -target_include_directories(POET_Model PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${MPI_C_INCLUDE_DIRS}) -target_link_libraries(POET_Model POET_Util DHT_Wrapper MPI::MPI_C) -target_compile_definitions(POET_Model PUBLIC OMPI_SKIP_MPICXX) \ No newline at end of file +add_library(POET_Model ChemSim.cpp ChemSim.h ChemMaster.cpp ChemWorker.cpp TransportSim.cpp Grid.cpp) +target_include_directories(POET_Model PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${MPI_C_INCLUDE_DIRS}) +target_link_libraries(POET_Model PRIVATE POET_Util MPI::MPI_C PUBLIC DHT_Wrapper) +target_compile_definitions(POET_Model PRIVATE OMPI_SKIP_MPICXX) \ No newline at end of file diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index ea98f6797..9c1446c2c 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(POET_Util STATIC RRuntime.cpp Parser.cpp) +add_library(POET_Util RRuntime.cpp Parser.cpp) target_include_directories(POET_Util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${R_INCLUDE_DIRS}) target_link_libraries(POET_Util ${R_LIBRARIES}) target_compile_definitions(POET_Util PUBLIC STRICT_R_HEADERS) \ No newline at end of file From 9a57e09d7ed2d1e30613bca06cfdc113932dca7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Thu, 28 Jan 2021 15:16:35 +0100 Subject: [PATCH 21/50] use better communicator creation --- src/kin.cpp | 342 +++++++++++++++++++++++++--------------------------- 1 file changed, 166 insertions(+), 176 deletions(-) diff --git a/src/kin.cpp b/src/kin.cpp index b518f5e00..91ca04bfd 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -100,22 +100,13 @@ int main(int argc, char *argv[]) { /*Create custom Communicator with all processes except 0 (the master) for DHT * storage*/ - // only needed if strategy == 0, but done anyway - MPI_Group dht_group, group_world; MPI_Comm dht_comm; - int *process_ranks; - // make a list of processes in the new communicator - process_ranks = (int *)malloc(world_size * sizeof(int)); - for (int I = 1; I < world_size; I++) process_ranks[I - 1] = I; - - // get the group under MPI_COMM_WORLD - MPI_Comm_group(MPI_COMM_WORLD, &group_world); - // create the new group - MPI_Group_incl(group_world, world_size - 1, process_ranks, &dht_group); - // create the new communicator - MPI_Comm_create(MPI_COMM_WORLD, dht_group, &dht_comm); - free(process_ranks); // cleanup + if (world_rank == 0) { + MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, world_rank, &dht_comm); + } else { + MPI_Comm_split(MPI_COMM_WORLD, 1, world_rank, &dht_comm); + } // cout << "Done"; // if (cmdl[{"help", "h"}]) { @@ -430,188 +421,187 @@ int main(int argc, char *argv[]) { master.run(); } - // MDL master_iteration_end just writes on disk state_T and - // state_C after every iteration if the cmdline option - // --ignore-results is not given (and thus the R variable - // store_result is TRUE) - R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); + // MDL master_iteration_end just writes on disk state_T and + // state_C after every iteration if the cmdline option + // --ignore-results is not given (and thus the R variable + // store_result is TRUE) + R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - // cummul_transport += trans.getTransportTime(); - // cummul_chemistry += master.getChemistryTime(); + // cummul_transport += trans.getTransportTime(); + // cummul_chemistry += master.getChemistryTime(); - cout << endl - << "CPP: End of *coupling* iteration " << iter << "/" << maxiter - << endl - << endl; + cout << endl + << "CPP: End of *coupling* iteration " << iter << "/" << maxiter + << endl + << endl; - // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); - // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); + // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); + // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); - for (int i = 1; i < params.world_size; i++) { - MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); + for (int i = 1; i < params.world_size; i++) { + MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); + } + + MPI_Barrier(MPI_COMM_WORLD); + + } // END SIMULATION LOOP + + cout << "CPP: finished simulation loop" << endl; + + sim_end = MPI_Wtime(); + + cout << "CPP: start timing profiling" << endl; + + R.parseEvalQ("profiling <- list()"); + + R["simtime"] = sim_end - sim_start; + R.parseEvalQ("profiling$simtime <- simtime"); + + trans.end(); + + if (params.world_size == 1) { + master.ChemSim::end(); + } else { + master.end(); } - MPI_Barrier(MPI_COMM_WORLD); + // Rcpp::NumericVector phreeqc_time; + // Rcpp::NumericVector dht_get_time; + // Rcpp::NumericVector dht_fill_time; + // Rcpp::IntegerVector phreeqc_counts; + // Rcpp::NumericVector idle_worker; - } // END SIMULATION LOOP + // int phreeqc_tmp; - cout << "CPP: finished simulation loop" << endl; + // timings = (double *)calloc(3, sizeof(double)); - sim_end = MPI_Wtime(); + // int dht_hits = 0; + // int dht_miss = 0; + // int dht_collision = 0; - cout << "CPP: start timing profiling" << endl; + // if (params.dht_enabled) { + // dht_hits = 0; + // dht_miss = 0; + // dht_collision = 0; + // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); + // } - R.parseEvalQ("profiling <- list()"); + // double idle_worker_tmp; - R["simtime"] = sim_end - sim_start; - R.parseEvalQ("profiling$simtime <- simtime"); - - trans.end(); + // for (int p = 0; p < params.world_size - 1; p++) { + // /* ATTENTION Worker p has rank p+1 */ + // /* Send termination message to worker */ + // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - if (params.world_size == 1) { - master.ChemSim::end(); - } else { - master.end(); + // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + // if (params.dht_enabled) { + // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // dht_hits += dht_perfs[0]; + // dht_miss += dht_perfs[1]; + // dht_collision += dht_perfs[2]; + // } + // } + + // R.parseEvalQ("profiling <- list()"); + + // R["simtime"] = sim_end - sim_start; + // R.parseEvalQ("profiling$simtime <- simtime"); + // R["simtime_transport"] = cummul_transport; + // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + // R["simtime_chemistry"] = cummul_chemistry; + // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + // R["simtime_workers"] = master.getWorkerTime(); + // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + // R["simtime_chemistry_master"] = master.getChemMasterTime(); + // R.parseEvalQ( + // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + + // R["seq_master"] = cummul_master_seq; + // R.parseEvalQ("profiling$seq_master <- seq_master"); + + // // R["master_send"] = master_send; + // // R.parseEvalQ("profiling$master_send <- master_send"); + // // R["master_recv"] = master_recv; + // // R.parseEvalQ("profiling$master_recv <- master_recv"); + + // R["idle_master"] = master.getIdleTime(); + // R.parseEvalQ("profiling$idle_master <- idle_master"); + // R["idle_worker"] = idle_worker; + // R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + // R["phreeqc_time"] = phreeqc_time; + // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + // R["phreeqc_count"] = phreeqc_counts; + // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + // if (params.dht_enabled) { + // R["dht_hits"] = dht_hits; + // R.parseEvalQ("profiling$dht_hits <- dht_hits"); + // R["dht_miss"] = dht_miss; + // R.parseEvalQ("profiling$dht_miss <- dht_miss"); + // R["dht_collision"] = dht_collision; + // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + // R["dht_get_time"] = dht_get_time; + // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + // R["dht_fill_time"] = dht_fill_time; + // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + // } + + // free(timings); + + // if (params.dht_enabled) free(dht_perfs); + + string r_vis_code; + r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + R.parseEval(r_vis_code); + + cout << "CPP: Done! Results are stored as R objects into <" + << params.out_dir << "/timings.rds>" << endl; + /*exporting results and profiling data*/ + + // std::string r_vis_code; + // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + // R.parseEval(r_vis_code); + } else { /*This is executed by the workers*/ + ChemWorker worker(¶ms, R, grid, dht_comm); + // worker.prepareSimulation(dht_comm); + worker.loop(); } - // Rcpp::NumericVector phreeqc_time; - // Rcpp::NumericVector dht_get_time; - // Rcpp::NumericVector dht_fill_time; - // Rcpp::IntegerVector phreeqc_counts; - // Rcpp::NumericVector idle_worker; - - // int phreeqc_tmp; - - // timings = (double *)calloc(3, sizeof(double)); - - // int dht_hits = 0; - // int dht_miss = 0; - // int dht_collision = 0; + cout << "CPP: finished, cleanup of process " << params.world_rank << endl; // if (params.dht_enabled) { - // dht_hits = 0; - // dht_miss = 0; - // dht_collision = 0; - // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); - // } - - // double idle_worker_tmp; - - // for (int p = 0; p < params.world_size - 1; p++) { - // /* ATTENTION Worker p has rank p+1 */ - // /* Send termination message to worker */ - // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - - // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - - // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - - // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - - // if (params.dht_enabled) { - // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - - // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // dht_hits += dht_perfs[0]; - // dht_miss += dht_perfs[1]; - // dht_collision += dht_perfs[2]; + // if (params.dht_strategy == 0) { + // if (params.world_rank != 0) { + // DHT_free(dht_object, NULL, NULL); + // } + // } else { + // DHT_free(dht_object, NULL, NULL); // } // } - // R.parseEvalQ("profiling <- list()"); + // free(mpi_buffer); + MPI_Finalize(); - // R["simtime"] = sim_end - sim_start; - // R.parseEvalQ("profiling$simtime <- simtime"); - // R["simtime_transport"] = cummul_transport; - // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - // R["simtime_chemistry"] = cummul_chemistry; - // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - // R["simtime_workers"] = master.getWorkerTime(); - // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - // R["simtime_chemistry_master"] = master.getChemMasterTime(); - // R.parseEvalQ( - // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + if (params.world_rank == 0) { + cout << "CPP: done, bye!" << endl; + } - // R["seq_master"] = cummul_master_seq; - // R.parseEvalQ("profiling$seq_master <- seq_master"); - - // // R["master_send"] = master_send; - // // R.parseEvalQ("profiling$master_send <- master_send"); - // // R["master_recv"] = master_recv; - // // R.parseEvalQ("profiling$master_recv <- master_recv"); - - // R["idle_master"] = master.getIdleTime(); - // R.parseEvalQ("profiling$idle_master <- idle_master"); - // R["idle_worker"] = idle_worker; - // R.parseEvalQ("profiling$idle_worker <- idle_worker"); - - // R["phreeqc_time"] = phreeqc_time; - // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - - // R["phreeqc_count"] = phreeqc_counts; - // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - - // if (params.dht_enabled) { - // R["dht_hits"] = dht_hits; - // R.parseEvalQ("profiling$dht_hits <- dht_hits"); - // R["dht_miss"] = dht_miss; - // R.parseEvalQ("profiling$dht_miss <- dht_miss"); - // R["dht_collision"] = dht_collision; - // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - // R["dht_get_time"] = dht_get_time; - // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - // R["dht_fill_time"] = dht_fill_time; - // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - // } - - // free(timings); - - // if (params.dht_enabled) free(dht_perfs); - - string r_vis_code; - r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - R.parseEval(r_vis_code); - - cout << "CPP: Done! Results are stored as R objects into <" << params.out_dir - << "/timings.rds>" << endl; - /*exporting results and profiling data*/ - - // std::string r_vis_code; - // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - // R.parseEval(r_vis_code); -} -else { /*This is executed by the workers*/ - ChemWorker worker(¶ms, R, grid, dht_comm); - // worker.prepareSimulation(dht_comm); - worker.loop(); -} - -cout << "CPP: finished, cleanup of process " << params.world_rank << endl; - -// if (params.dht_enabled) { -// if (params.dht_strategy == 0) { -// if (params.world_rank != 0) { -// DHT_free(dht_object, NULL, NULL); -// } -// } else { -// DHT_free(dht_object, NULL, NULL); -// } -// } - -// free(mpi_buffer); -MPI_Finalize(); - -if (params.world_rank == 0) { - cout << "CPP: done, bye!" << endl; -} - -exit(0); + exit(0); } From 7ac6cef3b4ccc3d6cad00531fa54ff14441c60fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 1 Feb 2021 10:57:51 +0100 Subject: [PATCH 22/50] change handling of compiler macro --- src/DHT/DHT.c | 8 ++++---- src/DHT/DHT.h | 2 -- src/DHT/DHT_Wrapper.cpp | 9 ++++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/DHT/DHT.c b/src/DHT/DHT.c index a4f07f4b8..cb9ef3005 100644 --- a/src/DHT/DHT.c +++ b/src/DHT/DHT.c @@ -1,10 +1,10 @@ #include "DHT.h" +#include #include #include #include #include -#include #include static void determine_dest(uint64_t hash, int comm_size, @@ -79,7 +79,7 @@ DHT *DHT_create(MPI_Comm comm, uint64_t size, unsigned int data_size, object->recv_entry = malloc(1 + data_size + key_size); object->send_entry = malloc(1 + data_size + key_size); object->index_count = 9 - (index_bytes / 8); - object->index = (uint64_t*)malloc((object->index_count) * sizeof(uint64_t)); + object->index = (uint64_t *)malloc((object->index_count) * sizeof(uint64_t)); object->mem_alloc = mem_alloc; // if set, initialize dht_stats @@ -375,8 +375,8 @@ int DHT_free(DHT *table, int *eviction_counter, int *readerror_counter) { return DHT_SUCCESS; } -#ifdef DHT_STATISTICS int DHT_print_statistics(DHT *table) { +#ifdef DHT_STATISTICS int *written_buckets; int *read_misses, sum_read_misses; int *evictions, sum_evictions; @@ -476,5 +476,5 @@ int DHT_print_statistics(DHT *table) { MPI_Barrier(table->communicator); return DHT_SUCCESS; +#endif } -#endif \ No newline at end of file diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h index 88a4ab632..a17ac3c26 100644 --- a/src/DHT/DHT.h +++ b/src/DHT/DHT.h @@ -250,9 +250,7 @@ extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter); * @return int Returns DHT_SUCCESS on success or DHT_MPI_ERROR on internal MPI * error. */ -#ifdef DHT_STATISTICS extern int DHT_print_statistics(DHT* table); -#endif /** * @brief Determine destination rank and index. diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index 376e6ac8b..b112e53b5 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -8,7 +8,7 @@ using namespace poet; using namespace std; -uint64_t get_md5(int key_size, void *key) { +static uint64_t get_md5(int key_size, void *key) { MD5_CTX ctx; unsigned char sum[MD5_DIGEST_LENGTH]; uint64_t retval, *v1, *v2; @@ -120,9 +120,7 @@ int DHT_Wrapper::fileToTable(const char *filename) { void DHT_Wrapper::printStatistics() { int res; -#ifdef DHT_STATISTICS res = DHT_print_statistics(dht_object); -#endif if (res != DHT_SUCCESS) { // MPI ERROR ... WHAT TO DO NOW? @@ -143,11 +141,12 @@ void DHT_Wrapper::fuzzForDHT(int var_count, void *key, double dt) { if (dht_prop_type_vector[i] == "act") { // with log10 if (dht_log) { - if (((double *)key)[i] < 0) + if (((double *)key)[i] < 0) { cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value in " "key!" << endl; - else if (((double *)key)[i] == 0) + fuzzing_buffer[i] = 0; + } else if (((double *)key)[i] == 0) fuzzing_buffer[i] = 0; else fuzzing_buffer[i] = From 7f99ca8c59c1910d860f9ffbf6c01af5ca7783d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 1 Feb 2021 11:09:20 +0100 Subject: [PATCH 23/50] commenting DHT_Wrapper and change compiler macro --- src/DHT/DHT.c | 2 +- src/DHT/DHT.h | 4 +- src/DHT/DHT_Wrapper.cpp | 70 ++++++++++--- src/DHT/DHT_Wrapper.h | 222 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 17 deletions(-) diff --git a/src/DHT/DHT.c b/src/DHT/DHT.c index a4f07f4b8..165953540 100644 --- a/src/DHT/DHT.c +++ b/src/DHT/DHT.c @@ -141,7 +141,7 @@ int DHT_write(DHT *table, void *send_key, void *send_data) { #ifdef DHT_STATISTICS table->stats->evictions += 1; #endif - result = DHT_WRITE_SUCCESS_WITH_COLLISION; + result = DHT_WRITE_SUCCESS_WITH_EVICTION; break; } } else diff --git a/src/DHT/DHT.h b/src/DHT/DHT.h index 88a4ab632..4eff08179 100644 --- a/src/DHT/DHT.h +++ b/src/DHT/DHT.h @@ -22,7 +22,7 @@ /** Returned by a call of DHT_read if no bucket with given key was found. */ #define DHT_READ_MISS -2 /** Returned by DHT_write if a bucket was evicted. */ -#define DHT_WRITE_SUCCESS_WITH_COLLISION -3 +#define DHT_WRITE_SUCCESS_WITH_EVICTION -3 /** Returned when no errors occured. */ #define DHT_SUCCESS 0 @@ -236,7 +236,7 @@ extern int DHT_free(DHT* table, int* eviction_counter, int* readerror_counter); * -# calls of DHT_write (w_access) * -# calls of DHT_read (r_access) * -# read misses (see DHT_READ_MISS) - * -# collisions (see DHT_WRITE_SUCCESS_WITH_COLLISION) + * -# collisions (see DHT_WRITE_SUCCESS_WITH_EVICTION) * 3-6 will reset with every call of this function finally the amount of new * written entries is printed out (since the last call of this funtion). * diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index 376e6ac8b..a57c2ba8f 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -13,10 +13,12 @@ uint64_t get_md5(int key_size, void *key) { unsigned char sum[MD5_DIGEST_LENGTH]; uint64_t retval, *v1, *v2; + // calculate md5 using MD5 functions MD5_Init(&ctx); MD5_Update(&ctx, key, key_size); MD5_Final(sum, &ctx); + // divide hash in 2 64 bit parts and XOR them v1 = (uint64_t *)&sum[0]; v2 = (uint64_t *)&sum[8]; retval = *v1 ^ *v2; @@ -26,10 +28,14 @@ uint64_t get_md5(int key_size, void *key) { DHT_Wrapper::DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, int buckets_per_process, int data_size, int key_size) { + // initialize DHT object dht_object = DHT_create(dht_comm, buckets_per_process, data_size, key_size, &get_md5); + + // allocate memory for fuzzing buffer fuzzing_buffer = (double *)malloc(key_size); + // extract needed values from sim_param struct this->dt_differ = params->dt_differ; this->dht_log = params->dht_log; this->dht_signif_vector = params->dht_signif_vector; @@ -37,7 +43,9 @@ DHT_Wrapper::DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, } DHT_Wrapper::~DHT_Wrapper() { + // free DHT DHT_free(dht_object, NULL, NULL); + // free fuzzing buffer free(fuzzing_buffer); } @@ -45,8 +53,11 @@ void DHT_Wrapper::checkDHT(int length, std::vector &out_result_index, double *work_package, double dt) { void *key; int res; + // var count -> count of variables per grid cell int var_count = dht_prop_type_vector.size(); + // loop over every grid cell contained in work package for (int i = 0; i < length; i++) { + // point to current grid cell key = (void *)&(work_package[i * var_count]); // fuzz data (round, logarithm etc.) @@ -55,12 +66,16 @@ void DHT_Wrapper::checkDHT(int length, std::vector &out_result_index, // overwrite input with data from DHT, IF value is found in DHT res = DHT_read(dht_object, fuzzing_buffer, key); + // if DHT_SUCCESS value was found ... if (res == DHT_SUCCESS) { - // flag that this line is replaced by DHT-value, do not simulate!! + // ... and grid cell will be marked as 'not to be simulating' out_result_index[i] = false; dht_hits++; - } else if (res == DHT_READ_MISS) { - // this line is untouched, simulation is needed + + } + // ... otherwise ... + else if (res == DHT_READ_MISS) { + // grid cell needs to be simulated by PHREEQC out_result_index[i] = true; dht_miss++; } else { @@ -75,22 +90,26 @@ void DHT_Wrapper::fillDHT(int length, std::vector &result_index, void *key; void *data; int res; + // var count -> count of variables per grid cell int var_count = dht_prop_type_vector.size(); - + // loop over every grid cell contained in work package for (int i = 0; i < length; i++) { key = (void *)&(work_package[i * var_count]); data = (void *)&(results[i * var_count]); + // If true grid cell was simulated, needs to be inserted into dht if (result_index[i]) { - // If true -> was simulated, needs to be inserted into dht - // fuzz data (round, logarithm etc.) fuzzForDHT(var_count, key, dt); + // insert simulated data with fuzzed key into DHT res = DHT_write(dht_object, fuzzing_buffer, data); + // if data was successfully written ... if (res != DHT_SUCCESS) { - if (res == DHT_WRITE_SUCCESS_WITH_COLLISION) { + // ... also check if a previously written value was evicted + if (res == DHT_WRITE_SUCCESS_WITH_EVICTION) { + // and increment internal eviciton counter dht_evictions++; } else { // MPI ERROR ... WHAT TO DO NOW? @@ -139,32 +158,55 @@ uint64_t DHT_Wrapper::getEvictions() { return this->dht_evictions; } void DHT_Wrapper::fuzzForDHT(int var_count, void *key, double dt) { unsigned int i = 0; // introduce fuzzing to allow more hits in DHT + // loop over every variable of grid cell for (i = 0; i < (unsigned int)var_count; i++) { + // check if variable is defined as 'act' if (dht_prop_type_vector[i] == "act") { - // with log10 + // if log is enabled (default) if (dht_log) { - if (((double *)key)[i] < 0) + // if variable is smaller than 0, which would be a strange result, + // warn the user and set fuzzing_buffer to 0 at this index + if (((double *)key)[i] < 0) { cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Negative value in " "key!" << endl; + fuzzing_buffer[i] = 0; + } + // if variable is 0 set fuzzing buffer to 0 else if (((double *)key)[i] == 0) fuzzing_buffer[i] = 0; + // otherwise ... else + // round current variable value by applying log with base 10, negate + // (since the actual values will be between 0 and 1) and cut result + // after significant digit fuzzing_buffer[i] = ROUND(-(std::log10(((double *)key)[i])), dht_signif_vector[i]); - } else { - // without log10 + } + // if log is disabled + else { + // just round by cutting after signifanct digit fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_signif_vector[i]); } - } else if (dht_prop_type_vector[i] == "logact") { + } + // if variable is defined as 'logact' (log was already applied e.g. pH) + else if (dht_prop_type_vector[i] == "logact") { + // just round by cutting after signifanct digit fuzzing_buffer[i] = ROUND((((double *)key)[i]), dht_signif_vector[i]); - } else if (dht_prop_type_vector[i] == "ignore") { + } + // if defined ass 'ignore' ... + else if (dht_prop_type_vector[i] == "ignore") { + // ... just set fuzzing buffer to 0 fuzzing_buffer[i] = 0; - } else { + } + // and finally, if type is not defined, print error message + else { cerr << "dht_wrapper.cpp::fuzz_for_dht(): Warning! Probably wrong " "prop_type!" << endl; } } + // if timestep differs over iterations set current current time step at the + // end of fuzzing buffer if (dt_differ) fuzzing_buffer[var_count] = dt; } \ No newline at end of file diff --git a/src/DHT/DHT_Wrapper.h b/src/DHT/DHT_Wrapper.h index fa836cdf5..c1d0edd88 100644 --- a/src/DHT/DHT_Wrapper.h +++ b/src/DHT/DHT_Wrapper.h @@ -12,44 +12,266 @@ extern "C" { #include +/** + * @brief Cut double value after signif digit + * + * Macro to round a double value by cutting every digit after significant digit + * + */ #define ROUND(value, signif) \ (((int)(pow(10.0, (double)signif) * value)) * pow(10.0, (double)-signif)) namespace poet { +/** + * @brief Return user-defined md5sum + * + * This function will calculate a hashsum with the help of md5sum. Therefore the + * md5sum for a given key is calculated and divided into two 64-bit parts. These + * will be XORed and returned as the hash. + * + * @param key_size Size of key in bytes + * @param key Pointer to key + * @return uint64_t Hashsum as an unsigned 64-bit integer + */ +static uint64_t get_md5(int key_size, void *key); + +/** + * @brief C++-Wrapper around DHT implementation + * + * Provides an API to interact with the current DHT implentation. This class is + * POET specific and can't be used outside the POET application. + * + */ class DHT_Wrapper { public: + /** + * @brief Construct a new dht wrapper object + * + * The constructor will initialize the private dht_object of this class by + * calling DHT_create with all given parameters. Also the fuzzing buffer will + * be allocated and all needed parameters extracted from simparams struct. + * + * @param params Simulation parameter struct returned at initialization + * @param dht_comm Communicator which addresses all participating DHT + * processes + * @param buckets_per_process Count of buckets to allocate for each process + * @param data_size Size of data in bytes + * @param key_size Size of key in bytes + */ DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, int buckets_per_process, int data_size, int key_size); + /** + * @brief Destroy the dht wrapper object + * + * By destroying this object the DHT will also be freed. Since all statistics + * are stored inside this object, no statistics will be retrieved during the + * call of DHT_free. After freeing the DHT the fuzzing buffer will be also + * freed. + * + */ ~DHT_Wrapper(); + /** + * @brief Check if values of workpackage are stored in DHT + * + * Call DHT_read for all grid cells of the given workpackage and if a + * previously simulated grid cell was found mark this grid cell as 'not be + * simulated'. Therefore all values of a grid cell are fuzzed by fuzzForDHT + * and used as input key. The correspondending retrieved value might be stored + * directly into the memory area of the work_package and out_result_index is + * marked with false ('not to be simulated'). + * + * @param length Count of grid cells inside work package + * @param[out] out_result_index Indexing work packages which should be simulated + * @param[in,out] work_package Pointer to current work package + * @param dt Current timestep of simulation + */ void checkDHT(int length, std::vector &out_result_index, double *work_package, double dt); + + /** + * @brief Write simulated values into DHT + * + * Call DHT_write for all grid cells of the given workpackage which was + * simulated shortly before by the worker. Whether the grid cell was simulated + * is given by result_index. For every grid cell indicated with true inside + * result_index write the simulated value into the DHT. + * + * @param length Count of grid cells inside work package + * @param result_index Indexing work packages which was simulated + * @param work_package Pointer to current work package which was used as input + * of PHREEQC + * @param results Pointer to current work package which are the resulting + * outputs of the PHREEQC simulation + * @param dt Current timestep of simulation + */ void fillDHT(int length, std::vector &result_index, double *work_package, double *results, double dt); + /** + * @brief Dump current DHT state into file. + * + * This function will simply execute DHT_to_file with given file name (see + * DHT.h for more info). + * + * @param filename Name of the dump file + * @return int Returns 0 on success, otherwise an error value + */ int tableToFile(const char *filename); + + /** + * @brief Load dump file into DHT. + * + * This function will simply execute DHT_from_file with given file name (see + * DHT.h for more info). + * + * @param filename Name of the dump file + * @return int Returns 0 on success, otherwise an error value + */ int fileToTable(const char *filename); + /** + * @brief Print a detailed statistic of DHT usage. + * + * This function will simply execute DHT_print_statistics with given file name + * (see DHT.h for more info). + * + */ void printStatistics(); + /** + * @brief Get the Hits object + * + * @return uint64_t Count of hits + */ uint64_t getHits(); + + /** + * @brief Get the Misses object + * + * @return uint64_t Count of read misses + */ uint64_t getMisses(); + + /** + * @brief Get the Evictions object + * + * @return uint64_t Count of evictions + */ uint64_t getEvictions(); private: + /** + * @brief Transform given workpackage into DHT key + * + * A given workpackage will be transformed into a DHT key by rounding each + * value of a workpackage to a given significant digit. Three different types + * of variables 'act', 'logact' and 'ignore' are used. Those types are given + * via the dht_signif_vector. + * + * If a variable is defined as 'act', dht_log is true and non-negative, the + * logarithm with base 10 will be applied. After that the value is negated. In + * case the value is 0 the fuzzing_buffer is also set to 0 at this position. + * If the value is negative a correspondending warning will be printed to + * stderr and the fuzzing buffer will be set to 0 at this index. + * + * If a variable is defined as 'logact' the value will be cut after the + * significant digit. + * + * If a variable ist defined as 'ignore' the fuzzing_buffer will be set to 0 + * at the index of the variable. + * + * If dt_differ is true the current time step of the simulation will be set at + * the end of the fuzzing_buffer. + * + * @param var_count Count of variables for the current work package + * @param key Pointer to work package handled as the key + * @param dt Current time step of the simulation + */ void fuzzForDHT(int var_count, void *key, double dt); + /** + * @brief DHT handle + * + * Stores information about the DHT. Will be used as a handle for each DHT + * library call. + * + */ DHT *dht_object; + /** + * @brief Count of hits + * + * The counter will be incremented if a previously simulated workpackage can + * be retrieved with a given key. + * + */ uint64_t dht_hits = 0; + + /** + * @brief Count of read misses + * + * The counter will be incremented if a given key doesn't retrieve a value + * from the DHT. + * + */ uint64_t dht_miss = 0; + + /** + * @brief Count of evictions + * + * If a value in the DHT must be evicted because of lack of space/reaching the + * last index etc., this counter will be incremented. + * + */ uint64_t dht_evictions = 0; + /** + * @brief Rounded work package values + * + * Stores rounded work package values and serves as the DHT key pointer. + * + */ double *fuzzing_buffer; + /** + * @brief Indicates change in time step during simulation + * + * If set to true, the time step of simulation will differ between iterations, + * so the current time step must be stored inside the DHT key. Otherwise wrong + * values would be obtained. + * + * If set to false the time step doesn't need to be stored in the DHT key. + * + */ bool dt_differ; + + /** + * @brief Logarithm before rounding + * + * Indicates if the logarithm with base 10 will be applied to a variable + * before rounding. + * + * Defaults to true. + * + */ bool dht_log; + + /** + * @brief Significant digits for each variable + * + * Stores the rounding/significant digits for each variable of the work + * package. + * + */ std::vector dht_signif_vector; + + /** + * @brief Type of each variable + * + * Defines the type of each variable of the work package. + * + */ std::vector dht_prop_type_vector; }; } // namespace poet From a0120661070ac48382f815a5ba6df6581b58ea14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 1 Feb 2021 11:32:38 +0100 Subject: [PATCH 24/50] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28763df54..189cc2bf3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + @@ -23,7 +23,7 @@ DHT: --dht ... enable dht (default is off) ---dht-log ... enable logarithm application before rounding (default is off) +--dht-nolog ... enable logarithm application before rounding (default is off) --dht-signif=<1-n> ... set rounding to number of significant digits (default 5) (only used if no vector is given in setup file) From a748d85cfda208ca3880de1deff95b6a5a2ea7a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Mon, 1 Feb 2021 16:45:13 +0100 Subject: [PATCH 25/50] Documented RRuntime --- src/util/RRuntime.cpp | 14 ------- src/util/RRuntime.h | 85 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/util/RRuntime.cpp b/src/util/RRuntime.cpp index adacfd8a1..7ac939292 100644 --- a/src/util/RRuntime.cpp +++ b/src/util/RRuntime.cpp @@ -7,11 +7,6 @@ using namespace poet; -/** - * Convert a R dataframe into a C continious memory area. - * - * @param varname Name of the R internal variable name. - */ void RRuntime::to_C_domain(double *buffer) { size_t rowCount = dfbuff.nrow(); size_t colCount = dfbuff.ncol(); @@ -25,15 +20,6 @@ void RRuntime::to_C_domain(double *buffer) { } } -/** - * Convert continious C memory area into R dataframe and puts it into R runtime. - * - * @param buffer Pointer to memory area which should be converted into R - * dataframe. - * @param skeleton Defines the raw data frame structure and muste be defined - * inside the R runtime beforehand. - * @param varname Name of the R internal variable name. - */ void RRuntime::from_C_domain(double *buffer) { size_t rowCount = dfbuff.nrow(); size_t colCount = dfbuff.ncol(); diff --git a/src/util/RRuntime.h b/src/util/RRuntime.h index 34d2fa9db..89652f7dd 100644 --- a/src/util/RRuntime.h +++ b/src/util/RRuntime.h @@ -9,27 +9,108 @@ namespace poet { /** + * @brief Provides an interface to a R runtime. + * * RRuntime is a wrapper class around a RInside (R) runtime and provides several * simplified methods to use R commands inside POET. * - * If an instance of RRuntime is created a R runtime will also be spawned. */ class RRuntime : public RInside { public: /** - * Constructor of class RRuntime calling constructor of RInside. + * @brief Construct a new RRuntime object + * + * Since this is an inherited class of RInside the constructor of the super + * class is just called. + * + * @param argc Argument counter of the program + * @param argv Argument values of the program */ RRuntime(const int argc, const char *const argv[]) : RInside(argc, argv){}; + /** + * Convert a R dataframe into a C continious memory area. + * + * @param buffer Name of the R internal variable name. + */ + + /** + * @brief Convert a R dataframe into a C continious memory area. + * + * A buffer data frame must be set beforehand with setBufferDataFrame. Then + * each value will be set into the continious memory area. This is done row + * wise. + * + * @todo: Might be more performant if all columns would be loaded at once and + * not for each column seperatly + * + * @param buffer Pointer to pre-allocated memory + */ void to_C_domain(double *buffer); + + /** + * @brief Convert continious C memory area into R dataframe and puts it into R + * runtime. + * + * A buffer data frame must be set beforehand with setBufferDataFrame. Then + * each value will be set into buffered data frame of this object. This is + * done row wise. + * + * @todo: Might be more performant if all columns would be loaded at once and + * not for each column seperatly + * + * @param buffer Pointer to memory area which should be converted into R + * dataframe. + */ void from_C_domain(double *buffer); + /** + * @brief Set the Buffer Data Frame object + * + * Set the buffered data frame (will be mostly the grid) of this object. + * + * @param dfname Name of the data frame inside R runtime + */ void setBufferDataFrame(std::string dfname); + + /** + * @brief Get the Buffer Data Frame object + * + * Returning the current buffered data frame as a Rcpp data frame. + * + * @return Rcpp::DataFrame Current buffered data frame + */ Rcpp::DataFrame getBufferDataFrame(); + + /** + * @brief Get the Buffer N Col object + * + * Get the numbers of columns of the buffered data frame. + * + * @return size_t Count of columns of buffered data frame + */ size_t getBufferNCol(); + + /** + * @brief Get the Buffer N Row object + * + * Get the numbers of rows of the buffered data frame. + * + * @return size_t Count of rows of buffered data frame + */ size_t getBufferNRow(); private: + /** + * @brief Buffered data frame + * + * This is used to convert a R data frame into continious memory used by C/C++ + * runtime and vice versa. Must be set with setBufferDataFrame and can be + * manipulated with from_C_domain. + * + * @todo: Find a cleaner solution. Maybe abstraction via another class. + * + */ Rcpp::DataFrame dfbuff; }; } // namespace poet From 6fa92c68fad3ec0b2d0824ea735307109b5e4db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 12:17:35 +0100 Subject: [PATCH 26/50] Added dht_signif_digits to simparams --- src/kin.cpp | 8 ++++---- src/util/Parser.cpp | 4 ++-- src/util/Parser.h | 3 +-- src/util/SimParams.h | 1 + 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/kin.cpp b/src/kin.cpp index 91ca04bfd..fcd1e0fb7 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -85,8 +85,8 @@ int main(int argc, char *argv[]) { double cummul_chemistry = 0.f; double cummul_master_seq = 0.f; - argh::parser cmdl(argv); - int dht_significant_digits; + // argh::parser cmdl(argv); + // int dht_significant_digits; // cout << "CPP: Start Init (MPI)" << endl; t_simparams params; @@ -315,7 +315,7 @@ int main(int argc, char *argv[]) { if (signif_vector_exists) { params.dht_signif_vector = as>(R["signif_vector"]); } else { - params.dht_signif_vector.assign(grid.getCols(), dht_significant_digits); + params.dht_signif_vector.assign(grid.getCols(), params.dht_significant_digits); } /*Load property type vector from R setup file (or set default)*/ @@ -341,7 +341,7 @@ int main(int argc, char *argv[]) { "digits=signif_vector))"); } else { cout << "CPP: using DHT default rounding digits = " - << dht_significant_digits << endl; + << params.dht_significant_digits << endl; } // MDL: pass to R the DHT stuff. These variables exist diff --git a/src/util/Parser.cpp b/src/util/Parser.cpp index 51b3c73a5..b04692868 100644 --- a/src/util/Parser.cpp +++ b/src/util/Parser.cpp @@ -54,7 +54,7 @@ int Parser::parseCmdl() { cmdl("dht-strategy", 0) >> simparams.dht_strategy; // cout << "CPP: DHT strategy is " << dht_strategy << endl; - cmdl("dht-signif", 5) >> dht_significant_digits; + cmdl("dht-signif", 5) >> simparams.dht_significant_digits; // cout << "CPP: DHT significant digits = " << dht_significant_digits << // endl; @@ -86,7 +86,7 @@ int Parser::parseCmdl() { cout << "CPP: DHT strategy is " << simparams.dht_strategy << endl; cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " "defined) = " - << dht_significant_digits << endl; + << simparams.dht_significant_digits << endl; cout << "CPP: DHT logarithm before rounding: " << (simparams.dht_log ? "ON" : "OFF") << endl; cout << "CPP: DHT size per process (Byte) = " diff --git a/src/util/Parser.h b/src/util/Parser.h index 2c17b25e1..0b9440a0b 100644 --- a/src/util/Parser.h +++ b/src/util/Parser.h @@ -36,8 +36,7 @@ class Parser { int world_rank; int world_size; - - int dht_significant_digits; + }; } // namespace poet #endif // PARSER_H \ No newline at end of file diff --git a/src/util/SimParams.h b/src/util/SimParams.h index b4ad7ac8d..9c7a26168 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -14,6 +14,7 @@ typedef struct { int dht_snaps; int dht_strategy; unsigned int dht_size_per_process; + int dht_significant_digits; std::vector dht_signif_vector; std::vector dht_prop_type_vector; std::string dht_file; From 4fca0d2cd2691c81344252a2127957a6bc98ef53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 13:42:10 +0100 Subject: [PATCH 27/50] Substitute struct SimParams through class --- src/DHT/DHT_Wrapper.cpp | 13 ++- src/DHT/DHT_Wrapper.h | 2 +- src/kin.cpp | 154 +++++++++++++------------ src/model/ChemMaster.cpp | 13 ++- src/model/ChemSim.cpp | 11 +- src/model/ChemSim.h | 9 +- src/model/ChemWorker.cpp | 25 ++-- src/util/CMakeLists.txt | 2 +- src/util/Parser.cpp | 144 ----------------------- src/util/Parser.h | 42 ------- src/util/SimParams.cpp | 238 +++++++++++++++++++++++++++++++++++++++ src/util/SimParams.h | 67 ++++++++--- 12 files changed, 413 insertions(+), 307 deletions(-) delete mode 100644 src/util/Parser.cpp delete mode 100644 src/util/Parser.h create mode 100644 src/util/SimParams.cpp diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index b112e53b5..a4cc1cb66 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -24,16 +24,19 @@ static uint64_t get_md5(int key_size, void *key) { return retval; } -DHT_Wrapper::DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, +DHT_Wrapper::DHT_Wrapper(SimParams ¶ms, MPI_Comm dht_comm, int buckets_per_process, int data_size, int key_size) { dht_object = DHT_create(dht_comm, buckets_per_process, data_size, key_size, &get_md5); fuzzing_buffer = (double *)malloc(key_size); - this->dt_differ = params->dt_differ; - this->dht_log = params->dht_log; - this->dht_signif_vector = params->dht_signif_vector; - this->dht_prop_type_vector = params->dht_prop_type_vector; + t_simparams tmp = params.getNumParams(); + + this->dt_differ = tmp.dt_differ; + this->dht_log = tmp.dht_log; + + this->dht_signif_vector = params.getDHTSignifVector(); + this->dht_prop_type_vector = params.getDHTPropTypeVector(); } DHT_Wrapper::~DHT_Wrapper() { diff --git a/src/DHT/DHT_Wrapper.h b/src/DHT/DHT_Wrapper.h index fa836cdf5..affc8b695 100644 --- a/src/DHT/DHT_Wrapper.h +++ b/src/DHT/DHT_Wrapper.h @@ -18,7 +18,7 @@ extern "C" { namespace poet { class DHT_Wrapper { public: - DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, int buckets_per_process, + DHT_Wrapper(SimParams ¶ms, MPI_Comm dht_comm, int buckets_per_process, int data_size, int key_size); ~DHT_Wrapper(); diff --git a/src/kin.cpp b/src/kin.cpp index fcd1e0fb7..367b01f60 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -10,7 +10,6 @@ // BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" #include #include -#include #include #include #include @@ -89,7 +88,6 @@ int main(int argc, char *argv[]) { // int dht_significant_digits; // cout << "CPP: Start Init (MPI)" << endl; - t_simparams params; int world_size, world_rank; MPI_Init(&argc, &argv); @@ -190,8 +188,8 @@ int main(int argc, char *argv[]) { "source('parallel_r_library.R');"; R.parseEvalQ(r_load_dependencies); - Parser parser(argv, world_rank, world_size); - int pret = parser.parseCmdl(); + SimParams parser(world_rank, world_size); + int pret = parser.parseFromCmdl(argv, R); if (pret == PARSER_ERROR) { MPI_Finalize(); @@ -201,8 +199,8 @@ int main(int argc, char *argv[]) { return EXIT_SUCCESS; } - parser.parseR(R); - params = parser.getParams(); + // parser.parseR(R); + // params = parser.getParams(); // if (params.world_rank == 0) { // cout << "CPP: Complete results storage is " << (params.store_result ? @@ -233,6 +231,7 @@ int main(int argc, char *argv[]) { // "source(filesim)"); // eval the init string, ignoring any returns // only rank 0 initializes goes through the whole initialization + bool dt_differ; if (world_rank == 0) { // cmdl(2) >> params.out_dir; // <- second positional argument // R["fileout"] = @@ -248,13 +247,17 @@ int main(int argc, char *argv[]) { std::string master_init_code = "mysetup <- master_init(setup=setup)"; R.parseEval(master_init_code); - params.dt_differ = R.parseEval("mysetup$dt_differ"); - MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + dt_differ = R.parseEval("mysetup$dt_differ"); + MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } else { // workers will only read the setup DataFrame defined by input file R.parseEval("mysetup <- setup"); - MPI_Bcast(&(params.dt_differ), 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } + parser.setDtDiffer(dt_differ); + + t_simparams params = parser.getNumParams(); + if (world_rank == 0) { cout << "CPP: R init done on process with rank " << params.world_rank << endl; @@ -267,6 +270,8 @@ int main(int argc, char *argv[]) { Grid grid(R); // params.grid = &grid; grid.init(); + + parser.initVectorParams(R, grid.getCols()); /* Retrieve state_C from R context for MPI buffer generation */ // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); @@ -296,78 +301,79 @@ int main(int argc, char *argv[]) { // params.R = &R; - if (params.dht_enabled) { - // cout << "\nCreating DHT\n"; - // determine size of dht entries - // int dht_data_size = grid.getCols() * sizeof(double); - // int dht_key_size = - // grid.getCols() * sizeof(double) + (params.dt_differ * - // sizeof(double)); + // if (params.dht_enabled) { + // // cout << "\nCreating DHT\n"; + // // determine size of dht entries + // // int dht_data_size = grid.getCols() * sizeof(double); + // // int dht_key_size = + // // grid.getCols() * sizeof(double) + (params.dt_differ * + // // sizeof(double)); - // // // determine bucket count for preset memory usage - // // // bucket size is key + value + 1 byte for status - // int dht_buckets_per_process = - // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); + // // // // determine bucket count for preset memory usage + // // // // bucket size is key + value + 1 byte for status + // // int dht_buckets_per_process = + // // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); - // MDL : following code moved here from worker.cpp - /*Load significance vector from R setup file (or set default)*/ - bool signif_vector_exists = R.parseEval("exists('signif_vector')"); - if (signif_vector_exists) { - params.dht_signif_vector = as>(R["signif_vector"]); - } else { - params.dht_signif_vector.assign(grid.getCols(), params.dht_significant_digits); - } + // // MDL : following code moved here from worker.cpp + // /*Load significance vector from R setup file (or set default)*/ + // bool signif_vector_exists = R.parseEval("exists('signif_vector')"); + // if (signif_vector_exists) { + // params.dht_signif_vector = as>(R["signif_vector"]); + // } else { + // params.dht_signif_vector.assign(grid.getCols(), + // params.dht_significant_digits); + // } - /*Load property type vector from R setup file (or set default)*/ - bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); - if (prop_type_vector_exists) { - params.dht_prop_type_vector = as>(R["prop_type"]); - } else { - params.dht_prop_type_vector.assign(grid.getCols(), "act"); - } + // /*Load property type vector from R setup file (or set default)*/ + // bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); + // if (prop_type_vector_exists) { + // params.dht_prop_type_vector = as>(R["prop_type"]); + // } else { + // params.dht_prop_type_vector.assign(grid.getCols(), "act"); + // } - if (params.world_rank == 0) { - // // print only on master, values are equal on all workes - // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; - // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; - // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process - // << endl; + // if (params.world_rank == 0) { + // // // print only on master, values are equal on all workes + // // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; + // // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; + // // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process + // // << endl; - // MDL: new output on signif_vector and prop_type - if (signif_vector_exists) { - cout << "CPP: using problem-specific rounding digits: " << endl; - R.parseEval( - "print(data.frame(prop=prop, type=prop_type, " - "digits=signif_vector))"); - } else { - cout << "CPP: using DHT default rounding digits = " - << params.dht_significant_digits << endl; - } + // // MDL: new output on signif_vector and prop_type + // if (signif_vector_exists) { + // cout << "CPP: using problem-specific rounding digits: " << endl; + // R.parseEval( + // "print(data.frame(prop=prop, type=prop_type, " + // "digits=signif_vector))"); + // } else { + // cout << "CPP: using DHT default rounding digits = " + // << params.dht_significant_digits << endl; + // } - // MDL: pass to R the DHT stuff. These variables exist - // only if dht_enabled is true - R["dht_final_signif"] = params.dht_signif_vector; - R["dht_final_proptype"] = params.dht_prop_type_vector; - } + // // MDL: pass to R the DHT stuff. These variables exist + // // only if dht_enabled is true + // R["dht_final_signif"] = params.dht_signif_vector; + // R["dht_final_proptype"] = params.dht_prop_type_vector; + // } - // if (params.dht_strategy == 0) { - // if (params.world_rank != 0) { - // dht_object = DHT_create(dht_comm, dht_buckets_per_process, - // dht_data_size, dht_key_size, get_md5); + // // if (params.dht_strategy == 0) { + // // if (params.world_rank != 0) { + // // dht_object = DHT_create(dht_comm, dht_buckets_per_process, + // // dht_data_size, dht_key_size, get_md5); - // // storing for access from worker and callback functions - // fuzzing_buffer = (double *)malloc(dht_key_size); - // } - // } else { - // dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, - // dht_data_size, dht_key_size, get_md5); - // } + // // // storing for access from worker and callback functions + // // fuzzing_buffer = (double *)malloc(dht_key_size); + // // } + // // } else { + // // dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, + // // dht_data_size, dht_key_size, get_md5); + // // } - // if (params.world_rank == 0) { - // cout << "CPP: DHT successfully created!" << endl; - // } - } + // // if (params.world_rank == 0) { + // // cout << "CPP: DHT successfully created!" << endl; + // // } + // } // MDL: store all parameters if (params.world_rank == 0) { cout << "CPP: Calling R Function to store calling parameters" << endl; @@ -377,7 +383,7 @@ int main(int argc, char *argv[]) { MPI_Barrier(MPI_COMM_WORLD); if (params.world_rank == 0) { /* This is executed by the master */ - ChemMaster master(¶ms, R, grid); + ChemMaster master(parser, R, grid); TransportSim trans(R); Rcpp::NumericVector master_send; @@ -572,14 +578,14 @@ int main(int argc, char *argv[]) { R.parseEval(r_vis_code); cout << "CPP: Done! Results are stored as R objects into <" - << params.out_dir << "/timings.rds>" << endl; + << parser.getOutDir() << "/timings.rds>" << endl; /*exporting results and profiling data*/ // std::string r_vis_code; // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; // R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ - ChemWorker worker(¶ms, R, grid, dht_comm); + ChemWorker worker(parser, R, grid, dht_comm); // worker.prepareSimulation(dht_comm); worker.loop(); } diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp index f662572a0..64f1305c3 100644 --- a/src/model/ChemMaster.cpp +++ b/src/model/ChemMaster.cpp @@ -8,13 +8,14 @@ using namespace poet; using namespace std; using namespace Rcpp; -#define TAG_WORK 42 - -ChemMaster::ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_) +ChemMaster::ChemMaster(SimParams ¶ms, RRuntime &R_, Grid &grid_) : ChemSim(params, R_, grid_) { - this->wp_size = params->wp_size; - this->out_dir = params->out_dir; - this->dht_enabled = params->dht_enabled; + t_simparams tmp = params.getNumParams(); + + this->wp_size = tmp.wp_size; + this->dht_enabled = tmp.dht_enabled; + + this->out_dir = params.getOutDir(); workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, diff --git a/src/model/ChemSim.cpp b/src/model/ChemSim.cpp index d6459512c..7be1a7406 100644 --- a/src/model/ChemSim.cpp +++ b/src/model/ChemSim.cpp @@ -7,12 +7,13 @@ using namespace Rcpp; using namespace poet; -ChemSim::ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_) +ChemSim::ChemSim(SimParams ¶ms, RRuntime &R_, Grid &grid_) : R(R_), grid(grid_) { - this->world_rank = params->world_rank; - this->world_size = params->world_size; - this->wp_size = params->wp_size; - this->out_dir = params->out_dir; + t_simparams tmp = params.getNumParams(); + this->world_rank = tmp.world_rank; + this->world_size = tmp.world_size; + this->wp_size = tmp.wp_size; + this->out_dir = params.getOutDir(); } void ChemSim::run() { diff --git a/src/model/ChemSim.h b/src/model/ChemSim.h index 3765c3bc2..790f80982 100644 --- a/src/model/ChemSim.h +++ b/src/model/ChemSim.h @@ -3,8 +3,8 @@ #include #include -#include #include +#include #include @@ -23,7 +23,7 @@ namespace poet { class ChemSim { public: - ChemSim(t_simparams *params, RRuntime &R_, Grid &grid_); + ChemSim(SimParams ¶ms, RRuntime &R_, Grid &grid_); virtual void run(); virtual void end(); @@ -59,7 +59,7 @@ class ChemSim { class ChemMaster : public ChemSim { public: - ChemMaster(t_simparams *params, RRuntime &R_, Grid &grid_); + ChemMaster(SimParams ¶ms, RRuntime &R_, Grid &grid_); ~ChemMaster(); void run() override; @@ -91,7 +91,7 @@ class ChemMaster : public ChemSim { class ChemWorker : public ChemSim { public: - ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, + ChemWorker(SimParams ¶ms, RRuntime &R_, Grid &grid_, MPI_Comm dht_comm); ~ChemWorker(); @@ -110,7 +110,6 @@ class ChemWorker : public ChemSim { std::string dht_file; unsigned int dht_size_per_process; std::vector dht_flags; - t_simparams *params; double *mpi_buffer_results; diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index 1f62371a1..c81ea450f 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -9,14 +9,17 @@ using namespace poet; using namespace std; using namespace Rcpp; -ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, +ChemWorker::ChemWorker(SimParams ¶ms, RRuntime &R_, Grid &grid_, MPI_Comm dht_comm) - : params(params_), ChemSim(params_, R_, grid_) { - this->dt_differ = params->dt_differ; - this->dht_enabled = params->dht_enabled; - this->dht_size_per_process = params->dht_size_per_process; - this->dht_file = params->dht_file; - this->dht_snaps = params->dht_snaps; + : ChemSim(params, R_, grid_) { + t_simparams tmp = params.getNumParams(); + + this->dt_differ = tmp.dt_differ; + this->dht_enabled = tmp.dht_enabled; + this->dht_size_per_process = tmp.dht_size_per_process; + this->dht_snaps = tmp.dht_snaps; + + this->dht_file = params.getDHTFile(); mpi_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); @@ -49,9 +52,9 @@ ChemWorker::ChemWorker(t_simparams *params_, RRuntime &R_, Grid &grid_, if (!dht_file.empty()) readFile(); // set size - dht_flags.resize(params->wp_size, true); + dht_flags.resize(wp_size, true); // assign all elements to true (default) - dht_flags.assign(params->wp_size, true); + dht_flags.assign(wp_size, true); timing[0] = 0.0; timing[1] = 0.0; @@ -178,9 +181,9 @@ void ChemWorker::doWork(MPI_Status &probe_status) { // R.parseEvalQ("print(head(work_package_full))"); // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); - grid.importWP(mpi_buffer, params->wp_size); + grid.importWP(mpi_buffer, wp_size); - if (params->dht_enabled) { + if (dht_enabled) { R.parseEvalQ("work_package <- work_package_full[dht_flags,]"); } else { R.parseEvalQ("work_package <- work_package_full"); diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 9c1446c2c..fb6b282f5 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(POET_Util RRuntime.cpp Parser.cpp) +add_library(POET_Util RRuntime.cpp SimParams.cpp) target_include_directories(POET_Util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${R_INCLUDE_DIRS}) target_link_libraries(POET_Util ${R_LIBRARIES}) target_compile_definitions(POET_Util PUBLIC STRICT_R_HEADERS) \ No newline at end of file diff --git a/src/util/Parser.cpp b/src/util/Parser.cpp deleted file mode 100644 index b04692868..000000000 --- a/src/util/Parser.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "Parser.h" - -#include - -#include - -using namespace poet; -using namespace std; -using namespace Rcpp; - -Parser::Parser(char *argv[], int world_rank_, int world_size_) - : cmdl(argv), world_rank(world_rank_), world_size(world_size_) { - this->simparams.world_rank = world_rank_; - this->simparams.world_size = world_size; -} - -int Parser::parseCmdl() { - // if user asked for help - if (cmdl[{"help", "h"}]) { - if (world_rank == 0) { - cout << "Todo" << endl - << "See README.md for further information." << endl; - } - return PARSER_HELP; - } - // if positional arguments are missing - else if (!cmdl(2)) { - if (world_rank == 0) { - cerr << "ERROR. Kin needs 2 positional arguments: " << endl - << "1) the R script defining your simulation and" << endl - << "2) the directory prefix where to save results and profiling" - << endl; - } - return PARSER_ERROR; - } - - std::list optionsError = checkOptions(cmdl); - if (!optionsError.empty()) { - if (world_rank == 0) { - cerr << "Unrecognized option(s):\n" << endl; - for (auto option : optionsError) { - cerr << option << endl; - } - cerr << "\nMake sure to use available options. Exiting!" << endl; - } - return PARSER_ERROR; - } - - /*Parse DHT arguments*/ - simparams.dht_enabled = cmdl["dht"]; - // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - - if (simparams.dht_enabled) { - cmdl("dht-strategy", 0) >> simparams.dht_strategy; - // cout << "CPP: DHT strategy is " << dht_strategy << endl; - - cmdl("dht-signif", 5) >> simparams.dht_significant_digits; - // cout << "CPP: DHT significant digits = " << dht_significant_digits << - // endl; - - simparams.dht_log = !(cmdl["dht-nolog"]); - // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" - // : "OFF" ) << endl; - - cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> simparams.dht_size_per_process; - // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << - // endl; - - cmdl("dht-snaps", 0) >> simparams.dht_snaps; - - cmdl("dht-file") >> simparams.dht_file; - } - /*Parse work package size*/ - cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> simparams.wp_size; - - /*Parse output options*/ - simparams.store_result = !cmdl["ignore-result"]; - - if (world_rank == 0) { - cout << "CPP: Complete results storage is " - << (simparams.store_result ? "ON" : "OFF") << endl; - cout << "CPP: Work Package Size: " << simparams.wp_size << endl; - cout << "CPP: DHT is " << (simparams.dht_enabled ? "ON" : "OFF") << '\n'; - - if (simparams.dht_enabled) { - cout << "CPP: DHT strategy is " << simparams.dht_strategy << endl; - cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " - "defined) = " - << simparams.dht_significant_digits << endl; - cout << "CPP: DHT logarithm before rounding: " - << (simparams.dht_log ? "ON" : "OFF") << endl; - cout << "CPP: DHT size per process (Byte) = " - << simparams.dht_size_per_process << endl; - cout << "CPP: DHT save snapshots is " << simparams.dht_snaps << endl; - cout << "CPP: DHT load file is " << simparams.dht_file << endl; - } - } - - cmdl(1) >> simparams.filesim; - cmdl(2) >> simparams.out_dir; - - return PARSER_OK; -} - -void Parser::parseR(RRuntime &R) { - // if local_rank == 0 then master else worker - R["local_rank"] = simparams.world_rank; - // assign a char* (string) to 'filesim' - R["filesim"] = wrap(simparams.filesim); - // assign a char* (string) to 'fileout' - R["fileout"] = wrap(simparams.out_dir); - // pass the boolean "store_result" to the R process - R["store_result"] = simparams.store_result; - // worker count - R["n_procs"] = simparams.world_size - 1; - // work package size - R["work_package_size"] = simparams.wp_size; - // dht enabled? - R["dht_enabled"] = simparams.dht_enabled; - // log before rounding? - R["dht_log"] = simparams.dht_log; - - // eval the init string, ignoring any returns - R.parseEvalQ("source(filesim)"); -} - -t_simparams Parser::getParams() { return this->simparams; } - -std::list Parser::checkOptions(argh::parser cmdl) { - std::list retList; - // std::set flist = flagList(); - // std::set plist = paramList(); - - for (auto &flag : cmdl.flags()) { - if (!(flaglist.find(flag) != flaglist.end())) retList.push_back(flag); - } - - for (auto ¶m : cmdl.params()) { - if (!(paramlist.find(param.first) != paramlist.end())) - retList.push_back(param.first); - } - - return retList; -} diff --git a/src/util/Parser.h b/src/util/Parser.h deleted file mode 100644 index 0b9440a0b..000000000 --- a/src/util/Parser.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef PARSER_H -#define PARSER_H - -#include - -#include - -#include "SimParams.h" -#include "argh.h" - -#define PARSER_OK 0 -#define PARSER_ERROR 1 -#define PARSER_HELP 2 - -#define DHT_SIZE_PER_PROCESS 1073741824 -#define WORK_PACKAGE_SIZE_DEFAULT 5 - -namespace poet { -class Parser { - public: - Parser(char *argv[], int world_rank, int world_size); - int parseCmdl(); - void parseR(RRuntime &R); - - t_simparams getParams(); - - private: - std::list checkOptions(argh::parser cmdl); - std::set flaglist{"ignore-result", "dht", "dht-nolog"}; - std::set paramlist{"work-package-size", "dht-signif", - "dht-strategy", "dht-size", - "dht-snaps", "dht-file"}; - argh::parser cmdl; - - t_simparams simparams; - - int world_rank; - int world_size; - -}; -} // namespace poet -#endif // PARSER_H \ No newline at end of file diff --git a/src/util/SimParams.cpp b/src/util/SimParams.cpp new file mode 100644 index 000000000..d165bf887 --- /dev/null +++ b/src/util/SimParams.cpp @@ -0,0 +1,238 @@ +#include "SimParams.h" + +#include + +#include + +using namespace poet; +using namespace std; +using namespace Rcpp; + +SimParams::SimParams(int world_rank_, int world_size_) + : world_rank(world_rank_), world_size(world_size_) { + this->simparams.world_rank = world_rank_; + this->simparams.world_size = world_size; +} + +int SimParams::parseFromCmdl(char *argv[], RRuntime &R) { + // initialize argh object + argh::parser cmdl(argv); + + // if user asked for help + if (cmdl[{"help", "h"}]) { + if (world_rank == 0) { + cout << "Todo" << endl + << "See README.md for further information." << endl; + } + return PARSER_HELP; + } + // if positional arguments are missing + else if (!cmdl(2)) { + if (world_rank == 0) { + cerr << "ERROR. Kin needs 2 positional arguments: " << endl + << "1) the R script defining your simulation and" << endl + << "2) the directory prefix where to save results and profiling" + << endl; + } + return PARSER_ERROR; + } + + std::list optionsError = checkOptions(cmdl); + if (!optionsError.empty()) { + if (world_rank == 0) { + cerr << "Unrecognized option(s):\n" << endl; + for (auto option : optionsError) { + cerr << option << endl; + } + cerr << "\nMake sure to use available options. Exiting!" << endl; + } + return PARSER_ERROR; + } + + /*Parse DHT arguments*/ + simparams.dht_enabled = cmdl["dht"]; + // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; + + if (simparams.dht_enabled) { + cmdl("dht-strategy", 0) >> simparams.dht_strategy; + // cout << "CPP: DHT strategy is " << dht_strategy << endl; + + cmdl("dht-signif", 5) >> simparams.dht_significant_digits; + // cout << "CPP: DHT significant digits = " << dht_significant_digits << + // endl; + + simparams.dht_log = !(cmdl["dht-nolog"]); + // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? "ON" + // : "OFF" ) << endl; + + cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> simparams.dht_size_per_process; + // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process << + // endl; + + cmdl("dht-snaps", 0) >> simparams.dht_snaps; + + cmdl("dht-file") >> dht_file; + } + /*Parse work package size*/ + cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> simparams.wp_size; + + /*Parse output options*/ + simparams.store_result = !cmdl["ignore-result"]; + + if (world_rank == 0) { + cout << "CPP: Complete results storage is " + << (simparams.store_result ? "ON" : "OFF") << endl; + cout << "CPP: Work Package Size: " << simparams.wp_size << endl; + cout << "CPP: DHT is " << (simparams.dht_enabled ? "ON" : "OFF") << '\n'; + + if (simparams.dht_enabled) { + cout << "CPP: DHT strategy is " << simparams.dht_strategy << endl; + cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + "defined) = " + << simparams.dht_significant_digits << endl; + cout << "CPP: DHT logarithm before rounding: " + << (simparams.dht_log ? "ON" : "OFF") << endl; + cout << "CPP: DHT size per process (Byte) = " + << simparams.dht_size_per_process << endl; + cout << "CPP: DHT save snapshots is " << simparams.dht_snaps << endl; + cout << "CPP: DHT load file is " << dht_file << endl; + } + } + + cmdl(1) >> filesim; + cmdl(2) >> out_dir; + + /* distribute information to R runtime */ + // if local_rank == 0 then master else worker + R["local_rank"] = simparams.world_rank; + // assign a char* (string) to 'filesim' + R["filesim"] = wrap(filesim); + // assign a char* (string) to 'fileout' + R["fileout"] = wrap(out_dir); + // pass the boolean "store_result" to the R process + R["store_result"] = simparams.store_result; + // worker count + R["n_procs"] = simparams.world_size - 1; + // work package size + R["work_package_size"] = simparams.wp_size; + // dht enabled? + R["dht_enabled"] = simparams.dht_enabled; + // log before rounding? + R["dht_log"] = simparams.dht_log; + + // eval the init string, ignoring any returns + R.parseEvalQ("source(filesim)"); + + return PARSER_OK; +} + +void SimParams::initVectorParams(RRuntime &R, int col_count) { + if (simparams.dht_enabled) { + // cout << "\nCreating DHT\n"; + // determine size of dht entries + // int dht_data_size = grid.getCols() * sizeof(double); + // int dht_key_size = + // grid.getCols() * sizeof(double) + (params.dt_differ * + // sizeof(double)); + + // // // determine bucket count for preset memory usage + // // // bucket size is key + value + 1 byte for status + // int dht_buckets_per_process = + // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); + + // MDL : following code moved here from worker.cpp + /*Load significance vector from R setup file (or set default)*/ + bool signif_vector_exists = R.parseEval("exists('signif_vector')"); + if (signif_vector_exists) { + dht_signif_vector = as>(R["signif_vector"]); + } else { + dht_signif_vector.assign(col_count, simparams.dht_significant_digits); + } + + /*Load property type vector from R setup file (or set default)*/ + bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); + if (prop_type_vector_exists) { + dht_prop_type_vector = as>(R["prop_type"]); + } else { + dht_prop_type_vector.assign(col_count, "act"); + } + + if (simparams.world_rank == 0) { + // // print only on master, values are equal on all workes + // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; + // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; + // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process + // << endl; + + // MDL: new output on signif_vector and prop_type + if (signif_vector_exists) { + cout << "CPP: using problem-specific rounding digits: " << endl; + R.parseEval( + "print(data.frame(prop=prop, type=prop_type, " + "digits=signif_vector))"); + } else { + cout << "CPP: using DHT default rounding digits = " + << simparams.dht_significant_digits << endl; + } + + // MDL: pass to R the DHT stuff. These variables exist + // only if dht_enabled is true + R["dht_final_signif"] = dht_signif_vector; + R["dht_final_proptype"] = dht_prop_type_vector; + } + } +} + +// void SimParams::parseR(RRuntime &R) { +// // if local_rank == 0 then master else worker +// R["local_rank"] = simparams.world_rank; +// // assign a char* (string) to 'filesim' +// R["filesim"] = wrap(simparams.filesim); +// // assign a char* (string) to 'fileout' +// R["fileout"] = wrap(simparams.out_dir); +// // pass the boolean "store_result" to the R process +// R["store_result"] = simparams.store_result; +// // worker count +// R["n_procs"] = simparams.world_size - 1; +// // work package size +// R["work_package_size"] = simparams.wp_size; +// // dht enabled? +// R["dht_enabled"] = simparams.dht_enabled; +// // log before rounding? +// R["dht_log"] = simparams.dht_log; + +// // eval the init string, ignoring any returns +// R.parseEvalQ("source(filesim)"); +// } + +void SimParams::setDtDiffer(bool dt_differ) { simparams.dt_differ = dt_differ; } + +t_simparams SimParams::getNumParams() { return this->simparams; } + +std::vector SimParams::getDHTSignifVector() { + return this->dht_signif_vector; +} +std::vector SimParams::getDHTPropTypeVector() { + return this->dht_prop_type_vector; +} +std::string SimParams::getDHTFile() { return this->dht_file; } + +std::string SimParams::getFilesim() { return this->filesim; } +std::string SimParams::getOutDir() { return this->out_dir; } + +std::list SimParams::checkOptions(argh::parser cmdl) { + std::list retList; + // std::set flist = flagList(); + // std::set plist = paramList(); + + for (auto &flag : cmdl.flags()) { + if (!(flaglist.find(flag) != flaglist.end())) retList.push_back(flag); + } + + for (auto ¶m : cmdl.params()) { + if (!(paramlist.find(param.first) != paramlist.end())) + retList.push_back(param.first); + } + + return retList; +} diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 9c7a26168..17813f08a 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -1,8 +1,18 @@ -#ifndef SIMPARAMS_H -#define SIMPARAMS_H +#ifndef PARSER_H +#define PARSER_H + +#include "RRuntime.h" #include -#include + +#include "argh.h" + +#define PARSER_OK 0 +#define PARSER_ERROR 1 +#define PARSER_HELP 2 + +#define DHT_SIZE_PER_PROCESS 1073741824 +#define WORK_PACKAGE_SIZE_DEFAULT 5 typedef struct { int world_size; @@ -15,19 +25,50 @@ typedef struct { int dht_strategy; unsigned int dht_size_per_process; int dht_significant_digits; + + unsigned int wp_size; + + bool store_result; +} t_simparams; + +namespace poet { +class SimParams { + public: + SimParams(int world_rank, int world_size); + int parseFromCmdl(char *argv[], RRuntime &R); + // void parseR(RRuntime &R); + + void initVectorParams(RRuntime &R, int col_count); + + void setDtDiffer(bool dt_differ); + + t_simparams getNumParams(); + + std::vector getDHTSignifVector(); + std::vector getDHTPropTypeVector(); + std::string getDHTFile(); + + std::string getFilesim(); + std::string getOutDir(); + + private: + std::list checkOptions(argh::parser cmdl); + std::set flaglist{"ignore-result", "dht", "dht-nolog"}; + std::set paramlist{"work-package-size", "dht-signif", + "dht-strategy", "dht-size", + "dht-snaps", "dht-file"}; + + t_simparams simparams; + + int world_rank; + int world_size; + std::vector dht_signif_vector; std::vector dht_prop_type_vector; std::string dht_file; - unsigned int wp_size; - std::string filesim; std::string out_dir; - - bool store_result; - - // void* R; - // void* grid; -} t_simparams; - -#endif // SIMPARAMS_H +}; +} // namespace poet +#endif // PARSER_H \ No newline at end of file From ca5ce5d0dbb1bb45229747c3eaf4b23697dbbb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 13:43:57 +0100 Subject: [PATCH 28/50] Documented Parser --- src/DHT/DHT_Wrapper.cpp | 4 +- src/util/Parser.cpp | 16 +++--- src/util/Parser.h | 117 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 15 deletions(-) diff --git a/src/DHT/DHT_Wrapper.cpp b/src/DHT/DHT_Wrapper.cpp index a57c2ba8f..468d913d4 100644 --- a/src/DHT/DHT_Wrapper.cpp +++ b/src/DHT/DHT_Wrapper.cpp @@ -8,7 +8,7 @@ using namespace poet; using namespace std; -uint64_t get_md5(int key_size, void *key) { +uint64_t poet::get_md5(int key_size, void *key) { MD5_CTX ctx; unsigned char sum[MD5_DIGEST_LENGTH]; uint64_t retval, *v1, *v2; @@ -30,7 +30,7 @@ DHT_Wrapper::DHT_Wrapper(t_simparams *params, MPI_Comm dht_comm, int buckets_per_process, int data_size, int key_size) { // initialize DHT object dht_object = - DHT_create(dht_comm, buckets_per_process, data_size, key_size, &get_md5); + DHT_create(dht_comm, buckets_per_process, data_size, key_size, &poet::get_md5); // allocate memory for fuzzing buffer fuzzing_buffer = (double *)malloc(key_size); diff --git a/src/util/Parser.cpp b/src/util/Parser.cpp index 51b3c73a5..8fd3e0dce 100644 --- a/src/util/Parser.cpp +++ b/src/util/Parser.cpp @@ -34,11 +34,13 @@ int Parser::parseCmdl() { return PARSER_ERROR; } - std::list optionsError = checkOptions(cmdl); - if (!optionsError.empty()) { + // collect all parameters which are not known, print them to stderr and return + // with PARSER_ERROR + std::list unknownOptions = validateOptions(); + if (!unknownOptions.empty()) { if (world_rank == 0) { cerr << "Unrecognized option(s):\n" << endl; - for (auto option : optionsError) { + for (auto option : unknownOptions) { cerr << option << endl; } cerr << "\nMake sure to use available options. Exiting!" << endl; @@ -76,6 +78,7 @@ int Parser::parseCmdl() { /*Parse output options*/ simparams.store_result = !cmdl["ignore-result"]; + /* rank 0 will summarize all extracted values to stdout */ if (world_rank == 0) { cout << "CPP: Complete results storage is " << (simparams.store_result ? "ON" : "OFF") << endl; @@ -126,15 +129,16 @@ void Parser::parseR(RRuntime &R) { t_simparams Parser::getParams() { return this->simparams; } -std::list Parser::checkOptions(argh::parser cmdl) { +std::list Parser::validateOptions() { + /* store all unknown parameters here */ std::list retList; - // std::set flist = flagList(); - // std::set plist = paramList(); + /* loop over all flags and compare to given flaglist*/ for (auto &flag : cmdl.flags()) { if (!(flaglist.find(flag) != flaglist.end())) retList.push_back(flag); } + /* and loop also over params and compare to given paramlist */ for (auto ¶m : cmdl.params()) { if (!(paramlist.find(param.first) != paramlist.end())) retList.push_back(param.first); diff --git a/src/util/Parser.h b/src/util/Parser.h index 2c17b25e1..0dd8e7ce5 100644 --- a/src/util/Parser.h +++ b/src/util/Parser.h @@ -8,33 +8,136 @@ #include "SimParams.h" #include "argh.h" +/** Return value if no error occured */ #define PARSER_OK 0 -#define PARSER_ERROR 1 -#define PARSER_HELP 2 +/** Return value if error occured during parsing of program arguments */ +#define PARSER_ERROR -1 +/** Return value if user asked for help message with program parameter */ +#define PARSER_HELP -2 +/** Standard DHT Size (Defaults to 1 GiB) */ #define DHT_SIZE_PER_PROCESS 1073741824 +/** Standard work package size */ #define WORK_PACKAGE_SIZE_DEFAULT 5 namespace poet { +/** + * @brief Reads information from program arguments and R runtime + * + * Providing functions to initialize parameters of the simulation using command + * line parameters and parameters from the R runtime. This class will also parse + * arguments from the commandline and decides if argument is known or unknown. + * + */ class Parser { public: + /** + * @brief Construct a new Parser object + * + * With all given parameters a new instance of this class will be created. + * + * @param argv Argument value of the program + * @param world_rank Rank of process inside MPI_COMM_WORLD + * @param world_size Size of communicator MPI_COMM_WORLD + */ Parser(char *argv[], int world_rank, int world_size); + + /** + * @brief Parse program arguments + * + * This is done by the argh.h library. + * + * First, the function will check if there is a flag 'help' or 'h'. If this is + * the case a help message is printed and the function will return with + * PARSER_HELP. + * + * Second, if there are not 2 positional arguments an error will be printed to + * stderr and the function returns with PARSER_ERROR. + * + * Then all given program parameters and flags will be read and checked, if + * there are known by validateOptions. A list of all unknown options might be + * returned, printed out and the function will return with PARSER_ERROR. + * Oterhwise the function continuos. + * + * Now all program arguments will be stored inside t_simparams struct, printed + * out and the function returns with PARSER_OK. + * + * @return int Returns with 0 if no error occured, otherwise value less than 0 + * is returned. + */ int parseCmdl(); + + /** + * @brief Distribute all known parameters to R runtime. + * + * All stored parameters are distributed to the R runtime. + * + * @todo This function might be redundant and can be put into parseCmdl. + * + * @param R Instance of RRuntime + */ void parseR(RRuntime &R); + /** + * @brief Get the Params object + * + * Return all parsed simulation parameters. Should be called after parseCmdl. + * + * @return t_simparams Struct of t_simparams + */ t_simparams getParams(); private: - std::list checkOptions(argh::parser cmdl); - std::set flaglist{"ignore-result", "dht", "dht-nolog"}; - std::set paramlist{"work-package-size", "dht-signif", - "dht-strategy", "dht-size", - "dht-snaps", "dht-file"}; + /** + * @brief Validate program parameters and flags + * + * Therefore this function iterates over the list of flags and parameters and + * compare them to the class member flagList and paramList. If a program + * argument is not included it is put to a list. This list will be returned. + * + * @return std::list List with all unknown parameters. Might be + * empty. + */ + std::list validateOptions(); + + /** + * @brief Contains all valid program flags. + * + */ + const std::set flaglist{"ignore-result", "dht", "dht-nolog"}; + + /** + * @brief Contains all valid program parameters. + * + */ + const std::set paramlist{"work-package-size", "dht-signif", + "dht-strategy", "dht-size", + "dht-snaps", "dht-file"}; + + /** + * @brief Instance of argh class + * + * This class will be instantiate inside constructor of this class object. + * + */ argh::parser cmdl; + /** + * @brief Struct containing all simulation parameters + * + */ t_simparams simparams; + /** + * @brief Rank of process inside MPI_COMM_WORLD + * + */ int world_rank; + + /** + * @brief Size of MPI_COMM_WORLD + * + */ int world_size; int dht_significant_digits; From 98ee035c388e3cd5c1b3e34fc8aecfb343c14767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 14:14:22 +0100 Subject: [PATCH 29/50] Documenting SimParams --- src/util/SimParams.cpp | 41 ------------------ src/util/SimParams.h | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 41 deletions(-) diff --git a/src/util/SimParams.cpp b/src/util/SimParams.cpp index fdf95e685..d97734354 100644 --- a/src/util/SimParams.cpp +++ b/src/util/SimParams.cpp @@ -129,19 +129,6 @@ int SimParams::parseFromCmdl(char *argv[], RRuntime &R) { void SimParams::initVectorParams(RRuntime &R, int col_count) { if (simparams.dht_enabled) { - // cout << "\nCreating DHT\n"; - // determine size of dht entries - // int dht_data_size = grid.getCols() * sizeof(double); - // int dht_key_size = - // grid.getCols() * sizeof(double) + (params.dt_differ * - // sizeof(double)); - - // // // determine bucket count for preset memory usage - // // // bucket size is key + value + 1 byte for status - // int dht_buckets_per_process = - // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); - - // MDL : following code moved here from worker.cpp /*Load significance vector from R setup file (or set default)*/ bool signif_vector_exists = R.parseEval("exists('signif_vector')"); if (signif_vector_exists) { @@ -159,12 +146,6 @@ void SimParams::initVectorParams(RRuntime &R, int col_count) { } if (simparams.world_rank == 0) { - // // print only on master, values are equal on all workes - // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; - // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; - // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process - // << endl; - // MDL: new output on signif_vector and prop_type if (signif_vector_exists) { cout << "CPP: using problem-specific rounding digits: " << endl; @@ -184,28 +165,6 @@ void SimParams::initVectorParams(RRuntime &R, int col_count) { } } -// void SimParams::parseR(RRuntime &R) { -// // if local_rank == 0 then master else worker -// R["local_rank"] = simparams.world_rank; -// // assign a char* (string) to 'filesim' -// R["filesim"] = wrap(simparams.filesim); -// // assign a char* (string) to 'fileout' -// R["fileout"] = wrap(simparams.out_dir); -// // pass the boolean "store_result" to the R process -// R["store_result"] = simparams.store_result; -// // worker count -// R["n_procs"] = simparams.world_size - 1; -// // work package size -// R["work_package_size"] = simparams.wp_size; -// // dht enabled? -// R["dht_enabled"] = simparams.dht_enabled; -// // log before rounding? -// R["dht_log"] = simparams.dht_log; - -// // eval the init string, ignoring any returns -// R.parseEvalQ("source(filesim)"); -// } - void SimParams::setDtDiffer(bool dt_differ) { simparams.dt_differ = dt_differ; } t_simparams SimParams::getNumParams() { return this->simparams; } diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 9cb70f0eb..3ec3e027f 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -87,17 +87,88 @@ class SimParams { */ int parseFromCmdl(char *argv[], RRuntime &R); + /** + * @brief Init std::vector values + * + * This will initialize dht_signif_vector and dht_prop_type_vector internally + * depending on whether vectors are defined by R-Simulation file or not. + * 'init_chemistry' must be run beforehand. + * + * @param R R runtime + * @param col_count Count of variables per grid cell (typically the count of + * columns of each grid cell) + */ void initVectorParams(RRuntime &R, int col_count); + /** + * @brief Set if dt differs + * + * Set a boolean variable if the timestep differs between iterations of + * simulation. + * + * @param dt_differ Boolean value, if dt differs + */ void setDtDiffer(bool dt_differ); + /** + * @brief Get the numerical params struct + * + * Returns a struct which contains all numerical or boolean simulation + * parameters. + * + * @return t_simparams Parameter struct + */ t_simparams getNumParams(); + /** + * @brief Get the DHT_Signif_Vector + * + * Returns a vector indicating which significant values are used for each + * variable of a grid cell. + * + * @return std::vector Vector of integers containing information about + * significant digits + */ std::vector getDHTSignifVector(); + + /** + * @brief Get the DHT_Prop_Type_Vector + * + * Returns a vector indicating of which type a variable of a grid cell is. + * + * @return std::vector Vector if strings defining a type of a + * variable + */ std::vector getDHTPropTypeVector(); + + /** + * @brief Return name of DHT snapshot. + * + * Name of the DHT file which is used to initialize the DHT with a previously + * written snapshot. + * + * @return std::string Absolute paht to the DHT snapshot + */ std::string getDHTFile(); + /** + * @brief Get the filesim name + * + * Returns a string containing the absolute path to a R file defining the + * simulation. + * + * @return std::string Absolute path to R file + */ std::string getFilesim(); + + /** + * @brief Get the output directory + * + * Returns the name of an absolute path where all output files should be + * stored. + * + * @return std::string Absolute path to output path + */ std::string getOutDir(); private: @@ -135,11 +206,34 @@ class SimParams { */ t_simparams simparams; + /** + * @brief Defines significant digits for each variable of a grid cell + * + */ std::vector dht_signif_vector; + + /** + * @brief Defines the type of a variable + * + */ std::vector dht_prop_type_vector; + + /** + * @brief Absolute path to a DHT snapshot + * + */ std::string dht_file; + /** + * @brief Absolute path to R file containing simulation definitions + * + */ std::string filesim; + + /** + * @brief Absolute path to output dir + * + */ std::string out_dir; }; } // namespace poet From 049120ae96e9caa6a50200d428bea42b795ec3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 17:22:11 +0100 Subject: [PATCH 30/50] documented libraries --- src/model/ChemMaster.cpp | 84 +++++--- src/model/ChemSim.cpp | 3 + src/model/ChemSim.h | 437 ++++++++++++++++++++++++++++++++++++++- src/model/ChemWorker.cpp | 94 +++------ src/model/Grid.cpp | 23 +-- src/model/Grid.h | 97 ++++++++- src/model/TransportSim.h | 39 ++++ src/util/SimParams.h | 21 +- 8 files changed, 679 insertions(+), 119 deletions(-) diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp index 64f1305c3..9b4480db0 100644 --- a/src/model/ChemMaster.cpp +++ b/src/model/ChemMaster.cpp @@ -17,10 +17,14 @@ ChemMaster::ChemMaster(SimParams ¶ms, RRuntime &R_, Grid &grid_) this->out_dir = params.getOutDir(); + /* allocate memory */ workerlist = (worker_struct *)calloc(world_size - 1, sizeof(worker_struct)); send_buffer = (double *)calloc((wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); + mpi_buffer = + (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); + /* calculate distribution of work packages */ R.parseEvalQ( "wp_ids <- distribute_work_packages(len=nrow(mysetup$state_C), " "package_size=work_package_size)"); @@ -30,9 +34,6 @@ ChemMaster::ChemMaster(SimParams ¶ms, RRuntime &R_, Grid &grid_) R.parseEvalQ("wp_sizes_vector <- compute_wp_sizes(wp_ids)"); R.parseEval("stat_wp_sizes(wp_sizes_vector)"); wp_sizes_vector = as>(R["wp_sizes_vector"]); - - mpi_buffer = - (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); } ChemMaster::~ChemMaster() { @@ -41,6 +42,7 @@ ChemMaster::~ChemMaster() { } void ChemMaster::run() { + /* declare most of the needed variables here */ double chem_a, chem_b; double seq_a, seq_b, seq_c, seq_d; double worker_chemistry_a, worker_chemistry_b; @@ -49,83 +51,105 @@ void ChemMaster::run() { int free_workers; int i_pkgs; + /* start time measurement of whole chemistry simulation */ chem_a = MPI_Wtime(); + /* start time measurement of sequential part */ seq_a = MPI_Wtime(); + + /* shuffle grid */ grid.shuffleAndExport(mpi_buffer); - // retrieve data from R runtime + + /* retrieve needed data from R runtime */ iteration = (int)R.parseEval("mysetup$iter"); dt = (double)R.parseEval("mysetup$requested_dt"); current_sim_time = (double)R.parseEval("mysetup$simulation_time-mysetup$requested_dt"); - // setup local variables + /* setup local variables */ pkg_to_send = wp_sizes_vector.size(); pkg_to_recv = wp_sizes_vector.size(); work_pointer = mpi_buffer; free_workers = world_size - 1; i_pkgs = 0; + /* end time measurement of sequential part */ seq_b = MPI_Wtime(); seq_t += seq_b - seq_a; + /* start time measurement of chemistry time needed for send/recv loop */ worker_chemistry_a = MPI_Wtime(); + + /* start send/recv loop */ + // while there are still packages to recv while (pkg_to_recv > 0) { - // TODO: Progressbar into IO instance. + // print a progressbar to stdout printProgressbar((int)i_pkgs, (int)wp_sizes_vector.size()); + // while there are still packages to send if (pkg_to_send > 0) { + // send packages to all free workers ... sendPkgs(pkg_to_send, i_pkgs, free_workers); } + // ... and try to receive them from workers who has finished their work recvPkgs(pkg_to_recv, pkg_to_send > 0, free_workers); } // Just to complete the progressbar cout << endl; + /* stop time measurement of chemistry time needed for send/recv loop */ worker_chemistry_b = MPI_Wtime(); worker_t = worker_chemistry_b - worker_chemistry_a; + /* start time measurement of sequential part */ seq_c = MPI_Wtime(); + + /* unshuffle grid */ grid.importAndUnshuffle(mpi_buffer); + /* do master stuff */ + + /* start time measurement of master chemistry */ sim_e_chemistry = MPI_Wtime(); + R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + + /* end time measurement of master chemistry */ sim_f_chemistry = MPI_Wtime(); chem_master += sim_f_chemistry - sim_e_chemistry; + + /* end time measurement of sequential part */ seq_d = MPI_Wtime(); seq_t += seq_d - seq_c; + /* end time measurement of whole chemistry simulation */ chem_b = MPI_Wtime(); chem_t += chem_b - chem_a; } void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, int &free_workers) { + /* declare variables */ double master_send_a, master_send_b; int local_work_package_size; int end_of_wp; - // start time measurement + /* start time measurement */ master_send_a = MPI_Wtime(); - /*search for free workers and send work*/ + + /* search for free workers and send work */ for (int p = 0; p < world_size - 1; p++) { if (workerlist[p].has_work == 0 && pkg_to_send > 0) /* worker is free */ { - // to enable different work_package_size, set local copy of - // work_package_size to either global work_package size or - // remaining 'to_send' packages to_send >= work_package_size ? - // local_work_package_size = work_package_size : - // local_work_package_size = to_send; + /* to enable different work_package_size, set local copy of + * work_package_size to pre-calculated work package size vector */ local_work_package_size = (int)wp_sizes_vector[count_pkgs]; count_pkgs++; - // cout << "CPP: sending pkg n. " << count_pkgs << " with size " - // << local_work_package_size << endl; - - /*push pointer forward to next work package, after taking the - * current one*/ + /* note current processed work package in workerlist */ workerlist[p].send_addr = work_pointer; + /* push work pointer to next work package */ end_of_wp = local_work_package_size * grid.getCols(); work_pointer = &(work_pointer[end_of_wp]); @@ -147,6 +171,7 @@ void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, MPI_Send(send_buffer, end_of_wp + BUFFER_OFFSET, MPI_DOUBLE, p + 1, TAG_WORK, MPI_COMM_WORLD); + /* Mark that worker has work to do */ workerlist[p].has_work = 1; free_workers--; pkg_to_send -= 1; @@ -157,6 +182,7 @@ void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, } void ChemMaster::recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers) { + /* declare most of the variables here */ int need_to_receive = 1; double master_recv_a, master_recv_b; double idle_a, idle_b; @@ -164,20 +190,26 @@ void ChemMaster::recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers) { MPI_Status probe_status; master_recv_a = MPI_Wtime(); + /* start to loop as long there are packages to recv and the need to receive + */ while (need_to_receive && pkg_to_recv > 0) { + // only of there are still packages to send and free workers are available if (to_send && free_workers > 0) + // non blocking probing MPI_Iprobe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &need_to_receive, &probe_status); else { idle_a = MPI_Wtime(); + // blocking probing MPI_Probe(MPI_ANY_SOURCE, TAG_WORK, MPI_COMM_WORLD, &probe_status); idle_b = MPI_Wtime(); master_idle += idle_b - idle_a; } + /* if need_to_receive was set to true above, so there is a message to + * receive */ if (need_to_receive) { p = probe_status.MPI_SOURCE; - size; MPI_Get_count(&probe_status, MPI_DOUBLE, &size); MPI_Recv(workerlist[p - 1].send_addr, size, MPI_DOUBLE, p, TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); @@ -210,8 +242,10 @@ void ChemMaster::printProgressbar(int count_pkgs, int n_wp, int barWidth) { } void ChemMaster::end() { + /* call end() from base class */ ChemSim::end(); + /* now we get to the part of the master */ double *timings; int *dht_perfs; @@ -238,11 +272,13 @@ void ChemMaster::end() { double idle_worker_tmp; + /* loop over all workers * + * ATTENTION Worker p has rank p+1 */ for (int p = 0; p < world_size - 1; p++) { - /* ATTENTION Worker p has rank p+1 */ /* Send termination message to worker */ MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + /* ... and receive all timings and metrics from each worker */ MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, MPI_STATUS_IGNORE); phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); @@ -263,11 +299,11 @@ void ChemMaster::end() { MPI_STATUS_IGNORE); dht_hits += dht_perfs[0]; dht_miss += dht_perfs[1]; - cout << "profiler miss = " << dht_miss << endl; dht_collision += dht_perfs[2]; } } + /* distribute all data to the R runtime */ R["simtime_chemistry"] = chem_t; R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); R["simtime_workers"] = worker_t; @@ -279,11 +315,6 @@ void ChemMaster::end() { R["seq_master"] = seq_t; R.parseEvalQ("profiling$seq_master <- seq_master"); - // R["master_send"] = master_send; - // R.parseEvalQ("profiling$master_send <- master_send"); - // R["master_recv"] = master_recv; - // R.parseEvalQ("profiling$master_recv <- master_recv"); - R["idle_master"] = master_idle; R.parseEvalQ("profiling$idle_master <- idle_master"); R["idle_worker"] = idle_worker; @@ -308,6 +339,7 @@ void ChemMaster::end() { R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); } + /* do some cleanup */ free(timings); if (dht_enabled) free(dht_perfs); diff --git a/src/model/ChemSim.cpp b/src/model/ChemSim.cpp index 7be1a7406..0e6f4c735 100644 --- a/src/model/ChemSim.cpp +++ b/src/model/ChemSim.cpp @@ -19,13 +19,16 @@ ChemSim::ChemSim(SimParams ¶ms, RRuntime &R_, Grid &grid_) void ChemSim::run() { double chem_a, chem_b; + /* start time measuring */ chem_a = MPI_Wtime(); R.parseEvalQ( "result <- slave_chemistry(setup=mysetup, data=mysetup$state_T)"); R.parseEvalQ("mysetup <- master_chemistry(setup=mysetup, data=result)"); + /* end time measuring */ chem_b = MPI_Wtime(); + chem_t += chem_b - chem_a; } diff --git a/src/model/ChemSim.h b/src/model/ChemSim.h index 130de9950..6d5105ae8 100644 --- a/src/model/ChemSim.h +++ b/src/model/ChemSim.h @@ -10,112 +10,543 @@ #include "Grid.h" +/** Number of data elements that are kept free at each work package */ #define BUFFER_OFFSET 5 +/** Message tag indicating work */ #define TAG_WORK 42 +/** Message tag indicating to finish loop */ #define TAG_FINISH 43 +/** Message tag indicating timing profiling */ #define TAG_TIMING 44 +/** Message tag indicating collecting DHT performance */ #define TAG_DHT_PERF 45 -#define TAG_DHT_STATS 46 -#define TAG_DHT_STORE 47 -#define TAG_DHT_ITER 48 +/** Message tag indicating simulation reached the end of an itertation */ +#define TAG_DHT_ITER 47 namespace poet { +/** + * @brief Base class of the chemical simulation + * + * Providing member functions to run an iteration and to end a simulation. Also + * containing basic parameters for simulation. + * + */ class ChemSim { public: + /** + * @brief Construct a new ChemSim object + * + * Creating a new instance of class ChemSim will just extract simulation + * parameters from SimParams object. + * + * @param params SimParams object + * @param R_ R runtime + * @param grid_ Initialized grid + */ ChemSim(SimParams ¶ms, RRuntime &R_, Grid &grid_); + /** + * @brief Run iteration of simulation in sequential mode + * + * This will call the correspondending R function slave_chemistry, followed by + * the execution of master_chemistry. + * + * @todo change function name. Maybe 'slave' to 'seq'. + * + */ virtual void run(); + + /** + * @brief End simulation + * + * End the simulation by distribute the measured runtime of simulation to the + * R runtime. + * + */ virtual void end(); + /** + * @brief Get the Chemistry Time + * + * @return double Runtime of sequential chemistry simulation in seconds + */ double getChemistryTime(); protected: + /** + * @brief Current simulation time or 'age' of simulation + * + */ double current_sim_time = 0; + + /** + * @brief Current iteration + * + */ int iteration = 0; + + /** + * @brief Current simulation timestep + * + */ int dt = 0; + /** + * @brief Rank of process in MPI_COMM_WORLD + * + */ int world_rank; + + /** + * @brief Size of communicator MPI_COMM_WORLD + * + */ int world_size; + + /** + * @brief Number of grid cells in each work package + * + */ unsigned int wp_size; + /** + * @brief Instance of RRuntime object + * + */ RRuntime &R; + + /** + * @brief Initialized grid object + * + */ Grid &grid; + /** + * @brief Stores information about size of the current work package + * + */ std::vector wp_sizes_vector; + + /** + * @brief Absolute path to output path + * + */ std::string out_dir; + /** + * @brief Pointer to sending buffer + * + */ double *send_buffer; + + /** + * @brief Worker struct + * + * This struct contains information which worker as work and which work + * package he is working on. + * + */ typedef struct { char has_work; double *send_addr; } worker_struct; + + /** + * @brief Pointer to worker_struct + * + */ worker_struct *workerlist; + /** + * @brief Pointer to mpi_buffer + * + * Typically for the master this is a continous C memory area containing the + * grid. For worker the memory area will just have the size of a work package. + * + */ double *mpi_buffer; + /** + * @brief Total chemistry runtime + * + */ double chem_t = 0.f; }; +/** + * @brief Class providing execution of master chemistry + * + * Providing member functions to run an iteration and to end a simulation. Also + * a loop to send and recv pkgs from workers is implemented. + * + */ class ChemMaster : public ChemSim { public: + /** + * @brief Construct a new ChemMaster object + * + * The following steps are executed to create a new object of ChemMaster: + * -# all needed simulation parameters are extracted + * -# memory is allocated + * -# distribution of work packages is calculated + * + * @param params Simulation parameters as SimParams object + * @param R_ R runtime + * @param grid_ Grid object + */ ChemMaster(SimParams ¶ms, RRuntime &R_, Grid &grid_); + + /** + * @brief Destroy the ChemMaster object + * + * By freeing ChemMaster all buffers allocated in the Constructor are freed. + * + */ ~ChemMaster(); + /** + * @brief Run iteration of simulation in parallel mode + * + * To run the chemistry simulation parallel following steps are done: + * + * -# 'Shuffle' the grid by previously calculated distribution of work + * packages. Convert R grid to C memory area. + * -# Start the send/recv loop. + * Detailed description in sendPkgs respectively in recvPkgs. + * -# 'Unshuffle' + * the grid and convert C memory area to R grid. + * -# Run 'master_chemistry' + * + * The main tasks are instrumented with time measurements. + * + */ void run() override; + + /** + * @brief End chemistry simulation. + * + * Notify the worker to finish their 'work'-loop. This is done by sending + * every worker an empty message with the tag TAG_FINISH. Now the master will + * receive measured times and DHT metrics from all worker one after another. + * Finally he will write all data to the R runtime and return this function. + * + */ void end() override; + /** + * @brief Get the send time + * + * Time spent in send loop. + * + * @return double sent time in seconds + */ double getSendTime(); + + /** + * @brief Get the recv time + * + * Time spent in recv loop. + * + * @return double recv time in seconds + */ double getRecvTime(); + + /** + * @brief Get the idle time + * + * Time master was idling in MPI_Probe of recv loop. + * + * @return double idle time in seconds + */ double getIdleTime(); + + /** + * @brief Get the Worker time + * + * Time spent in whole send/recv loop. + * + * @return double worker time in seconds + */ double getWorkerTime(); + + /** + * @brief Get the ChemMaster time + * + * Time spent in 'master_chemistry' R function. + * + * @return double ChemMaster time in seconds + */ double getChemMasterTime(); + + /** + * @brief Get the sequential time + * + * Time master executed code which must be run sequential. + * + * @return double seqntial time in seconds. + */ double getSeqTime(); private: + /** + * @brief Print a progressbar + * + * Prints a progressbar to stdout according to count of processed work + * packages in this iteration. + * + * @param count_pkgs Last processed index of work package + * @param n_wp Number of work packages + * @param barWidth Width of the progressbar/Count of characters to display the + * bar + */ void printProgressbar(int count_pkgs, int n_wp, int barWidth = 70); + + /** + * @brief Start send loop + * + * Send a work package to every free worker, which are noted in a worker + * struct. After a work package was sent move pointer on work grid to the next + * work package. Use MPI_Send to transfer work package to worker. + * + * @param pkg_to_send Pointer to variable containing how much work packages + * are still to send + * @param count_pkgs Pointer to variable indexing the current work package + * @param free_workers Pointer to variable with the count of free workers + */ void sendPkgs(int &pkg_to_send, int &count_pkgs, int &free_workers); + + /** + * @brief Start recv loop + * + * Receive processed work packages by worker. This is done by first probing + * for a message. If a message is receivable, receive it and put result into + * respective memory area. Continue, but now with a non blocking MPI_Probe. If + * a message is receivable or if no work packages are left to send, receive + * it. Otherwise or if all remaining work packages are received exit loop. + * + * @param pkg_to_recv Pointer to variable counting the to receiving work + * packages + * @param to_send Bool indicating if there are still work packages to send + * @param free_workers Pointer to worker to variable holding the number of + * free workers + */ void recvPkgs(int &pkg_to_recv, bool to_send, int &free_workers); + /** + * @brief Indicating usage of DHT + * + */ bool dht_enabled; + + /** + * @brief Default number of grid cells in each work package + * + */ unsigned int wp_size; + + /** + * @brief Pointer to current to be processed work package + * + */ double *work_pointer; + /** + * @brief Time spent in send loop + * + */ double send_t = 0.f; + + /** + * @brief Time spent in recv loop + * + */ double recv_t = 0.f; + + /** + * @brief Time master is idling in MPI_Probe + * + */ double master_idle = 0.f; + + /** + * @brief Time spent in send/recv loop + * + */ double worker_t = 0.f; + + /** + * @brief Time spent in sequential chemistry part + * + */ double chem_master = 0.f; + + /** + * @brief Time spent in sequential instructions + * + */ double seq_t = 0.f; }; +/** + * @brief Class providing execution of worker chemistry + * + * Providing mainly a function to loop and wait for messages from the master. + * + */ class ChemWorker : public ChemSim { public: + /** + * @brief Construct a new ChemWorker object + * + * The following steps are executed to create a new object of ChemWorker: + * -# all needed simulation parameters are extracted + * -# memory is allocated + * -# Preparetion to create a DHT + * -# and finally create a new DHT_Wrapper + * + * @param params Simulation parameters as SimParams object + * @param R_ R runtime + * @param grid_ Grid object + * @param dht_comm Communicator addressing all processes marked as worker + */ ChemWorker(SimParams ¶ms, RRuntime &R_, Grid &grid_, MPI_Comm dht_comm); + + /** + * @brief Destroy the ChemWorker object + * + * Therefore all buffers are freed and the DHT_Wrapper object is destroyed. + * + */ ~ChemWorker(); + /** + * @brief Start the 'work' loop + * + * Loop in an endless loop. At the beginning probe for a message from the + * master process. If there is a receivable message evaluate the message tag. + * + */ void loop(); private: + /** + * @brief Evaluating message to receive as work package + * + * These steps are done: + * + * -# Receive message + * -# Evaluate message header containing information about work package size, + * current iteration and timestep, simulation age + * -# if DHT is enabled check DHT for previously simulated results + * -# run simulation of work package + * -# send results back to master + * -# if DHT is enabled write simulated grid points to DHT + * + * @param probe_status message status of produced by MPI_Probe in loop + */ void doWork(MPI_Status &probe_status); + + /** + * @brief Action to do after iteration + * + * If DHT is enabled print statistics and if dht_snaps is set to 2 write DHT + * snapshots. + * + */ void postIter(); + + /** + * @brief Message tag evaluates to TAG_FINISH + * + * Send all the collected timings and (possbile) DHT metrics to the master. + * + */ void finishWork(); + + /** + * @brief Write DHT snapshot + * + */ void writeFile(); + + /** + * @brief Read DHT snapshot + * + */ void readFile(); + /** + * @brief Indicates usage of DHT + * + */ bool dht_enabled; + + /** + * @brief Boolean if dt differs between iterations + * + */ bool dt_differ; + + /** + * @brief Value between 0 and 2, indicating when to write DHT snapshots + * + */ int dht_snaps; + + /** + * @brief Absolute path to DHT snapshot file + * + */ std::string dht_file; + + /** + * @brief Count of bytes each process should allocate for the DHT + * + */ unsigned int dht_size_per_process; + + /** + * @brief Indicates which grid cells were previously simulated and don't need + * to be simulated now + * + */ std::vector dht_flags; + /** + * @brief simulated results are stored here + * + */ double *mpi_buffer_results; + /** + * @brief Instance of DHT_Wrapper + * + */ DHT_Wrapper *dht; + /** + * @brief Array to store timings + * + * The values are stored in following order + * + * -# PHREEQC time + * -# DHT_get time + * -# DHT_fill time + * + */ double timing[3]; + + /** + * @brief Time worker is idling in MPI_Probe + * + */ double idle_t = 0.f; + + /** + * @brief Count of PHREEQC calls + * + */ int phreeqc_count = 0; }; } // namespace poet diff --git a/src/model/ChemWorker.cpp b/src/model/ChemWorker.cpp index c81ea450f..e9d21899b 100644 --- a/src/model/ChemWorker.cpp +++ b/src/model/ChemWorker.cpp @@ -47,15 +47,14 @@ ChemWorker::ChemWorker(SimParams ¶ms, RRuntime &R_, Grid &grid_, key_size); if (world_rank == 1) cout << "CPP: Worker: DHT created!" << endl; + + if (!dht_file.empty()) readFile(); + // set size + dht_flags.resize(wp_size, true); + // assign all elements to true (default) + dht_flags.assign(wp_size, true); } - if (!dht_file.empty()) readFile(); - - // set size - dht_flags.resize(wp_size, true); - // assign all elements to true (default) - dht_flags.assign(wp_size, true); - timing[0] = 0.0; timing[1] = 0.0; timing[2] = 0.0; @@ -74,12 +73,17 @@ void ChemWorker::loop() { MPI_Probe(0, MPI_ANY_TAG, MPI_COMM_WORLD, &probe_status); double idle_b = MPI_Wtime(); + /* there is a work package to receive */ if (probe_status.MPI_TAG == TAG_WORK) { idle_t += idle_b - idle_a; doWork(probe_status); - } else if (probe_status.MPI_TAG == TAG_DHT_ITER) { + } + /* end of iteration */ + else if (probe_status.MPI_TAG == TAG_DHT_ITER) { postIter(); - } else if (probe_status.MPI_TAG == TAG_FINISH) { + } + /* end of simulation */ + else if (probe_status.MPI_TAG == TAG_FINISH) { finishWork(); break; } @@ -94,17 +98,18 @@ void ChemWorker::doWork(MPI_Status &probe_status) { double phreeqc_time_start, phreeqc_time_end; double dht_fill_start, dht_fill_end; - /* get number of doubles sent */ + /* get number of doubles to be received */ MPI_Get_count(&probe_status, MPI_DOUBLE, &count); /* receive */ MPI_Recv(mpi_buffer, count, MPI_DOUBLE, 0, TAG_WORK, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // decrement count of work_package by BUFFER_OFFSET + /* decrement count of work_package by BUFFER_OFFSET */ count -= BUFFER_OFFSET; - // check for changes on all additional variables given by the 'header' of - // mpi_buffer + + /* check for changes on all additional variables given by the 'header' of + * mpi_buffer */ // work_package_size if (mpi_buffer[count] != local_work_package_size) { // work_package_size @@ -139,48 +144,24 @@ void ChemWorker::doWork(MPI_Status &probe_status) { // R["mysetup$placeholder"] = placeholder; // } - /* get df with right structure to fill in work package */ - // R.parseEvalQ("skeleton <- head(mysetup$state_C, work_package_size)"); - // R["skeleton"] = grid.buildDataFrame(work_package_size); - //// R.parseEval("print(rownames(tmp2)[1:5])"); - //// R.parseEval("print(head(tmp2, 2))"); - //// R.parseEvalQ("tmp2$id <- as.double(rownames(tmp2))"); - - ////Rcpp::DataFrame buffer = R.parseEval("tmp2"); - // R.setBufferDataFrame("skeleton"); - if (dht_enabled) { - // DEBUG - // cout << "RANK " << world_rank << " start checking DHT\n"; - - // resize helper vector dht_flags of work_package_size changes + /* resize helper vector dht_flags of work_package_size changes */ if ((int)dht_flags.size() != local_work_package_size) { dht_flags.resize(local_work_package_size, true); // set size dht_flags.assign(local_work_package_size, true); // assign all elements to true (default) } + /* check for values in DHT */ dht_get_start = MPI_Wtime(); dht->checkDHT(local_work_package_size, dht_flags, mpi_buffer, dt); dht_get_end = MPI_Wtime(); - // DEBUG - // cout << "RANK " << world_rank << " checking DHT complete \n"; - + /* distribute dht_flags to R Runtime */ R["dht_flags"] = as(wrap(dht_flags)); - // R.parseEvalQ("print(head(dht_flags))"); } - /* work */ - // R.from_C_domain(mpi_buffer); - ////convert_C_buffer_2_R_Dataframe(mpi_buffer, buffer); - // R["work_package_full"] = R.getBufferDataFrame(); - // R["work_package"] = buffer; - - // DEBUG - // R.parseEvalQ("print(head(work_package_full))"); - // R.parseEvalQ("print( c(length(dht_flags), nrow(work_package_full)) )"); - + /* Convert grid to R runtime */ grid.importWP(mpi_buffer, wp_size); if (dht_enabled) { @@ -189,14 +170,6 @@ void ChemWorker::doWork(MPI_Status &probe_status) { R.parseEvalQ("work_package <- work_package_full"); } - // DEBUG - // R.parseEvalQ("print(head(work_package),2)"); - - // R.parseEvalQ("rownames(work_package) <- work_package$id"); - // R.parseEval("print(paste('id %in% colnames(work_package)', 'id' %in% - // colnames(work_package)"); R.parseEvalQ("id_store <- - // rownames(work_package)"); //"[, ncol(work_package)]"); - // R.parseEvalQ("work_package$id <- NULL"); R.parseEvalQ("work_package <- as.matrix(work_package)"); unsigned int nrows = R.parseEval("nrow(work_package)"); @@ -209,22 +182,18 @@ void ChemWorker::doWork(MPI_Status &probe_status) { "work_package <- work_package[rep(1:nrow(work_package), " "times = 2), ]"); } - - phreeqc_count++; - + /* Run PHREEQC */ phreeqc_time_start = MPI_Wtime(); - // MDL - // R.parseEvalQ("print('Work_package:\n'); print(head(work_package , - // 2)); cat('RCpp: worker_function:', local_rank, ' \n')"); R.parseEvalQ( "result <- as.data.frame(slave_chemistry(setup=mysetup, " "data = work_package))"); phreeqc_time_end = MPI_Wtime(); - // R.parseEvalQ("result$id <- id_store"); } else { - // cout << "Work-Package is empty, skipping phreeqc!" << endl; + // undefined behaviour, isn't it? } + phreeqc_count++; + if (dht_enabled) { R.parseEvalQ("result_full <- work_package_full"); if (nrows > 0) R.parseEvalQ("result_full[dht_flags,] <- result"); @@ -232,11 +201,7 @@ void ChemWorker::doWork(MPI_Status &probe_status) { R.parseEvalQ("result_full <- result"); } - // R.setBufferDataFrame("result_full"); - ////Rcpp::DataFrame result = R.parseEval("result_full"); - ////convert_R_Dataframe_2_C_buffer(mpi_buffer_results, result); - // R.to_C_domain(mpi_buffer_results); - + /* convert grid to C domain */ grid.exportWP(mpi_buffer_results); /* send results to master */ MPI_Request send_req; @@ -244,6 +209,7 @@ void ChemWorker::doWork(MPI_Status &probe_status) { &send_req); if (dht_enabled) { + /* write results to DHT */ dht_fill_start = MPI_Wtime(); dht->fillDHT(local_work_package_size, dht_flags, mpi_buffer, mpi_buffer_results, dt); @@ -327,8 +293,4 @@ void ChemWorker::finishWork() { } if (dht_enabled && dht_snaps > 0) writeFile(); - - // free(mpi_buffer); - // free(mpi_buffer_results); - // delete dht; } diff --git a/src/model/Grid.cpp b/src/model/Grid.cpp index 6a84798f0..b08e551e0 100644 --- a/src/model/Grid.cpp +++ b/src/model/Grid.cpp @@ -2,25 +2,15 @@ using namespace poet; using namespace Rcpp; -/** - * At this moment init will only declare and define a variable inside the R - * runtime called grid_tmp since the whole Grid initialization and management is - * done by the R runtime. This may change in the future. - */ + void Grid::init() { R.parseEval("GRID_TMP <- mysetup$state_C"); this->ncol = R.parseEval("ncol(GRID_TMP)"); this->nrow = R.parseEval("nrow(GRID_TMP)"); } -/** - * Returns the number of elements for each gridcell. - */ unsigned int Grid::getCols() { return this->ncol; } -/** - * Returns the number of gridcells. - */ unsigned int Grid::getRows() { return this->nrow; } void Grid::shuffleAndExport(double *buffer) { @@ -36,8 +26,8 @@ void Grid::importAndUnshuffle(double *buffer) { R.parseEval("result <- unshuffle_field(GRID_CHEM_DATA, ordered_ids)"); } -void Grid::importWP(double *buffer, unsigned int p_size) { - R["GRID_WP_SKELETON"] = getSkeletonDataFrame(p_size); +void Grid::importWP(double *buffer, unsigned int wp_size) { + R["GRID_WP_SKELETON"] = getSkeletonDataFrame(wp_size); R.setBufferDataFrame("GRID_WP_SKELETON"); R.from_C_domain(buffer); R["work_package_full"] = R.getBufferDataFrame(); @@ -46,12 +36,7 @@ void Grid::exportWP(double *buffer) { R.setBufferDataFrame("result_full"); R.to_C_domain(buffer); } -/** - * Create a data frame with n rows. - * - * @return Can be seen as a skeleton. The content of the data frame might be - * irrelevant. - */ + Rcpp::DataFrame Grid::getSkeletonDataFrame(unsigned int rows) { R["GRID_ROWS"] = rows; diff --git a/src/model/Grid.h b/src/model/Grid.h index 61267c670..898f1232a 100644 --- a/src/model/Grid.h +++ b/src/model/Grid.h @@ -5,26 +5,121 @@ #include namespace poet { +/** + * @brief Class describing the grid + * + * Providing methods to shuffle and unshuffle grid (for the master) as also to + * import and export a work package (for worker). + * + * @todo find better abstraction + * + */ class Grid { public: + /** + * @brief Construct a new Grid object + * + * This will call the default constructor and initializes private RRuntime + * with given R runtime. + * + * @param R + */ Grid(RRuntime &R) : R(R){}; + /** + * @brief Init the grid + * + * At this moment init will only declare and define a variable inside the R + * runtime called grid_tmp since the whole Grid initialization and management + * is done by the R runtime. This may change in the future. + * + */ void init(); + /** + * @brief Returns the number of elements for each gridcell + * + * @return unsigned int Number of elements + */ unsigned int getCols(); + + /** + * @brief Returns the number of grid cells + * + * @return unsigned int Number of grid cells + */ unsigned int getRows(); + /** + * @brief Shuffle the grid and export it to C memory area + * + * This will call shuffle_field inside R runtime, set the resulting grid as + * buffered data frame in RRuntime object and write R grid to continous C + * memory area. + * + * @param[in,out] buffer Pointer to C memory area + */ void shuffleAndExport(double *buffer); + + /** + * @brief Unshuffle the grid and import it from C memory area into R runtime + * + * Write C memory area into temporary R grid variable and unshuffle it. + * + * @param buffer Pointer to C memory area + */ void importAndUnshuffle(double *buffer); - void importWP(double *buffer, unsigned int p_size); + /** + * @brief Import a C memory area as a work package into R runtime + * + * Get a skeleton from getSkeletonDataFrame inside R runtime and set this as + * buffer data frame of RRuntime object. Now convert C memory area to R data + * structure. + * + * @param buffer Pointer to C memory area + * @param wp_size Count of grid cells per work package + */ + void importWP(double *buffer, unsigned int wp_size); + + /** + * @brief Export a work package from R runtime into C memory area + * + * Set buffer data frame in RRuntime object to data frame holding the results + * and convert this to C memory area. + * + * @param buffer Pointer to C memory area + */ void exportWP(double *buffer); private: + /** + * @brief Instance of RRuntime + * + */ RRuntime R; + + /** + * @brief Number of columns of grid + * + */ unsigned int ncol; + + /** + * @brief Number of rows of grid + * + */ unsigned int nrow; + /** + * @brief Get a SkeletonDataFrame + * + * Return a skeleton with \e n rows of current grid + * + * @param rows number of rows to return skeleton + * @return Rcpp::DataFrame Can be seen as a skeleton. The content of the data + * frame might be irrelevant. + */ Rcpp::DataFrame getSkeletonDataFrame(unsigned int rows); }; } // namespace poet diff --git a/src/model/TransportSim.h b/src/model/TransportSim.h index f644503b2..c2cd8880a 100644 --- a/src/model/TransportSim.h +++ b/src/model/TransportSim.h @@ -4,18 +4,57 @@ #include namespace poet { +/** + * @brief Class describing transport simulation + * + * Offers simple methods to run an iteration and end the simulation. + * + */ class TransportSim { public: + /** + * @brief Construct a new TransportSim object + * + * The instance will only be initialized with given R object. + * + * @param R RRuntime object + */ TransportSim(RRuntime &R); + /** + * @brief Run simulation for one iteration + * + * This will simply call the R function 'master_advection' + * + */ void run(); + + /** + * @brief End simulation + * + * All measured timings are distributed to the R runtime + * + */ void end(); + /** + * @brief Get the transport time + * + * @return double time spent in transport + */ double getTransportTime(); private: + /** + * @brief Instance of RRuntime + * + */ RRuntime &R; + /** + * @brief time spent for transport + * + */ double transport_t = 0.f; }; } // namespace poet diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 3ec3e027f..1bda18dcf 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -18,24 +18,37 @@ /** Standard work package size */ #define WORK_PACKAGE_SIZE_DEFAULT 5 +namespace poet { + +/** + * @brief Defining all simulation parameters + * + */ typedef struct { + /** Count of processes in MPI_COMM_WORLD */ int world_size; + /** rank of proces in MPI_COMM_WORLD */ int world_rank; - + /** indicates if DHT should be used */ bool dht_enabled; + /** apply logarithm to key before rounding */ bool dht_log; + /** indicates if timestep dt differs between iterations */ bool dt_differ; + /** Indicates, when a DHT snapshot should be written */ int dht_snaps; + /** not implemented: How a DHT is distributed over processes */ int dht_strategy; + /** Size of DHt per process in byter */ unsigned int dht_size_per_process; + /** Default significant digit for rounding */ int dht_significant_digits; - + /** Default work package size */ unsigned int wp_size; - + /** indicates if resulting grid should be stored after every iteration */ bool store_result; } t_simparams; -namespace poet { /** * @brief Reads information from program arguments and R runtime * From 33d35eafdaf5e9b4eadebc39524da9943d6bc9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Tue, 2 Feb 2021 18:26:29 +0100 Subject: [PATCH 31/50] cleanup of kin.cpp --- src/kin.cpp | 499 ++++----------------------------------- src/model/ChemMaster.cpp | 5 + src/util/SimParams.h | 3 +- 3 files changed, 49 insertions(+), 458 deletions(-) diff --git a/src/kin.cpp b/src/kin.cpp index 367b01f60..4d10fcca5 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -1,81 +1,19 @@ +#include +#include +#include #include +#include +#include #include #include #include #include -// #include "DHT.h" // MPI-DHT Implementation -// #include "argh.h" // Argument handler https://github.com/adishavit/argh -// BSD-licenced #include "dht_wrapper.h" #include "global_buffer.h" -#include -#include -#include -#include -#include -// #include "worker.h" - -//#define DHT_SIZE_PER_PROCESS 1073741824 - using namespace std; using namespace poet; using namespace Rcpp; -// double *mpi_buffer; -// double *mpi_buffer_results; - -// uint32_t work_package_size; -// #define WORK_PACKAGE_SIZE_DEFAULT 5 - -// bool store_result; - -// std::set paramList() { -// std::set options; -// // global -// options.insert("work-package-size"); -// // only DHT -// options.insert("dht-signif"); -// options.insert("dht-strategy"); -// options.insert("dht-size"); -// options.insert("dht-snaps"); -// options.insert("dht-file"); - -// return options; -// } - -// std::set flagList() { -// std::set options; -// // global -// options.insert("ignore-result"); -// // only DHT -// options.insert("dht"); -// options.insert("dht-log"); - -// return options; -// } - -// std::list checkOptions(argh::parser cmdl) { -// std::list retList; -// std::set flist = flagList(); -// std::set plist = paramList(); - -// for (auto &flag : cmdl.flags()) { -// if (!(flist.find(flag) != flist.end())) retList.push_back(flag); -// } - -// for (auto ¶m : cmdl.params()) { -// if (!(plist.find(param.first) != plist.end())) -// retList.push_back(param.first); -// } - -// return retList; -// } - -// typedef struct { -// char has_work; -// double *send_addr; -// } worker_struct; - int main(int argc, char *argv[]) { double sim_start, sim_b_transport, sim_a_transport, sim_b_chemistry, sim_a_chemistry, sim_end; @@ -84,10 +22,6 @@ int main(int argc, char *argv[]) { double cummul_chemistry = 0.f; double cummul_master_seq = 0.f; - // argh::parser cmdl(argv); - // int dht_significant_digits; - // cout << "CPP: Start Init (MPI)" << endl; - int world_size, world_rank; MPI_Init(&argc, &argv); @@ -97,7 +31,7 @@ int main(int argc, char *argv[]) { MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); /*Create custom Communicator with all processes except 0 (the master) for DHT - * storage*/ + * storage */ MPI_Comm dht_comm; if (world_rank == 0) { @@ -105,80 +39,9 @@ int main(int argc, char *argv[]) { } else { MPI_Comm_split(MPI_COMM_WORLD, 1, world_rank, &dht_comm); } - // cout << "Done"; - - // if (cmdl[{"help", "h"}]) { - // if (params.world_rank == 0) { - // cout << "Todo" << endl - // << "See README.md for further information." << endl; - // } - // MPI_Finalize(); - // return EXIT_SUCCESS; - // } - - // /*INIT is now done separately in an R file provided here as argument!*/ - // if (!cmdl(2)) { - // if (params.world_rank == 0) { - // cerr << "ERROR. Kin needs 2 positional arguments: " << endl - // << "1) the R script defining your simulation and" << endl - // << "2) the directory prefix where to save results and profiling" - // << endl; - // } - // MPI_Finalize(); - // return EXIT_FAILURE; - // } - - // std::list optionsError = checkOptions(cmdl); - // if (!optionsError.empty()) { - // if (params.world_rank == 0) { - // cerr << "Unrecognized option(s):\n" << endl; - // for (auto option : optionsError) { - // cerr << option << endl; - // } - // cerr << "\nMake sure to use available options. Exiting!" << endl; - // } - // MPI_Finalize(); - // return EXIT_FAILURE; - // } - - // /*Parse DHT arguments*/ - // params.dht_enabled = cmdl["dht"]; - // // cout << "CPP: DHT is " << ( dht_enabled ? "ON" : "OFF" ) << '\n'; - - // if (params.dht_enabled) { - // cmdl("dht-strategy", 0) >> params.dht_strategy; - // // cout << "CPP: DHT strategy is " << dht_strategy << endl; - - // cmdl("dht-signif", 5) >> dht_significant_digits; - // // cout << "CPP: DHT significant digits = " << dht_significant_digits << - // // endl; - - // params.dht_log = !(cmdl["dht-nolog"]); - // // cout << "CPP: DHT logarithm before rounding: " << ( dht_logarithm ? - // "ON" - // // : "OFF" ) << endl; - - // cmdl("dht-size", DHT_SIZE_PER_PROCESS) >> params.dht_size_per_process; - // // cout << "CPP: DHT size per process (Byte) = " << dht_size_per_process - // << - // // endl; - - // cmdl("dht-snaps", 0) >> params.dht_snaps; - - // cmdl("dht-file") >> params.dht_file; - // } - - // /*Parse work package size*/ - // cmdl("work-package-size", WORK_PACKAGE_SIZE_DEFAULT) >> params.wp_size; - - // /*Parse output options*/ - // store_result = !cmdl["ignore-result"]; + /* initialize R runtime */ RRuntime R(argc, argv); - cout << "CPP: R Init (RInside) on process " << world_rank << endl; - - // if local_rank == 0 then master else worker - // R["local_rank"] = params.world_rank; /*Loading Dependencies*/ std::string r_load_dependencies = @@ -188,8 +51,8 @@ int main(int argc, char *argv[]) { "source('parallel_r_library.R');"; R.parseEvalQ(r_load_dependencies); - SimParams parser(world_rank, world_size); - int pret = parser.parseFromCmdl(argv, R); + SimParams params(world_rank, world_size); + int pret = params.parseFromCmdl(argv, R); if (pret == PARSER_ERROR) { MPI_Finalize(); @@ -199,231 +62,81 @@ int main(int argc, char *argv[]) { return EXIT_SUCCESS; } - // parser.parseR(R); - // params = parser.getParams(); + cout << "CPP: R Init (RInside) on process " << world_rank << endl; - // if (params.world_rank == 0) { - // cout << "CPP: Complete results storage is " << (params.store_result ? - // "ON" : "OFF") - // << endl; - // cout << "CPP: Work Package Size: " << params.wp_size << endl; - // cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; - - // if (params.dht_enabled) { - // cout << "CPP: DHT strategy is " << params.dht_strategy << endl; - // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is - // " - // // "defined) = " - // // << dht_significant_digits << endl; - // cout << "CPP: DHT logarithm before rounding: " - // << (params.dht_log ? "ON" : "OFF") << endl; - // cout << "CPP: DHT size per process (Byte) = " - // << params.dht_size_per_process << endl; - // cout << "CPP: DHT save snapshots is " << params.dht_snaps << endl; - // cout << "CPP: DHT load file is " << params.dht_file << endl; - // } - // } - - // std::string filesim; - // cmdl(1) >> filesim; // <- first positional argument - // R["filesim"] = wrap(filesim); // assign a char* (string) to 'filesim' - // R.parseEvalQ( - // "source(filesim)"); // eval the init string, ignoring any returns - - // only rank 0 initializes goes through the whole initialization bool dt_differ; if (world_rank == 0) { - // cmdl(2) >> params.out_dir; // <- second positional argument - // R["fileout"] = - // wrap(params.out_dir); // assign a char* (string) to 'fileout' - - // Note: R::sim_init() checks if the directory already exists, - // if not it makes it - - // pass the boolean "store_result" to the R process - // R["store_result"] = store_result; - // get timestep vector from grid_init function ... std::string master_init_code = "mysetup <- master_init(setup=setup)"; R.parseEval(master_init_code); dt_differ = R.parseEval("mysetup$dt_differ"); + // ... and broadcast it to every other rank unequal to 0 MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); - } else { // workers will only read the setup DataFrame defined by input file + } + /* workers will only read the setup DataFrame defined by input file */ + else { R.parseEval("mysetup <- setup"); MPI_Bcast(&dt_differ, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); } - parser.setDtDiffer(dt_differ); - - t_simparams params = parser.getNumParams(); - - if (world_rank == 0) { - cout << "CPP: R init done on process with rank " << params.world_rank - << endl; - } + params.setDtDiffer(dt_differ); // initialize chemistry on all processes std::string init_chemistry_code = "mysetup <- init_chemistry(setup=mysetup)"; R.parseEval(init_chemistry_code); Grid grid(R); - // params.grid = &grid; grid.init(); - parser.initVectorParams(R, grid.getCols()); - /* Retrieve state_C from R context for MPI buffer generation */ - // Rcpp::DataFrame state_C = R.parseEval("mysetup$state_C"); + params.initVectorParams(R, grid.getCols()); - /* Init Parallel helper functions */ - // R["n_procs"] = params.world_size - 1; /* worker count */ - // R["work_package_size"] = params.wp_size; - - // Removed additional field for ID in previous versions - // if (params.world_rank == 0) { - // mpi_buffer = - // (double *)calloc(grid.getRows() * grid.getCols(), sizeof(double)); - // } else { - // mpi_buffer = (double *)calloc( - // (params.wp_size * (grid.getCols())) + BUFFER_OFFSET, sizeof(double)); - // mpi_buffer_results = - // (double *)calloc(params.wp_size * (grid.getCols()), sizeof(double)); - // } - - // if (params.world_rank == 0) { - // cout << "CPP: parallel init completed (buffers allocated)!" << endl; - // } - - // MDL: pass to R the DHT stuff (basically, only for storing of - // simulation parameters). These 2 variables are always defined: - // R["dht_enabled"] = params.dht_enabled; - // R["dht_log"] = params.dht_log; - - // params.R = &R; - - // if (params.dht_enabled) { - // // cout << "\nCreating DHT\n"; - // // determine size of dht entries - // // int dht_data_size = grid.getCols() * sizeof(double); - // // int dht_key_size = - // // grid.getCols() * sizeof(double) + (params.dt_differ * - // // sizeof(double)); - - // // // // determine bucket count for preset memory usage - // // // // bucket size is key + value + 1 byte for status - // // int dht_buckets_per_process = - // // params.dht_size_per_process / (1 + dht_data_size + dht_key_size); - - // // MDL : following code moved here from worker.cpp - // /*Load significance vector from R setup file (or set default)*/ - // bool signif_vector_exists = R.parseEval("exists('signif_vector')"); - // if (signif_vector_exists) { - // params.dht_signif_vector = as>(R["signif_vector"]); - // } else { - // params.dht_signif_vector.assign(grid.getCols(), - // params.dht_significant_digits); - // } - - // /*Load property type vector from R setup file (or set default)*/ - // bool prop_type_vector_exists = R.parseEval("exists('prop_type')"); - // if (prop_type_vector_exists) { - // params.dht_prop_type_vector = as>(R["prop_type"]); - // } else { - // params.dht_prop_type_vector.assign(grid.getCols(), "act"); - // } - - // if (params.world_rank == 0) { - // // // print only on master, values are equal on all workes - // // cout << "CPP: dht_data_size: " << dht_data_size << "\n"; - // // cout << "CPP: dht_key_size: " << dht_key_size << "\n"; - // // cout << "CPP: dht_buckets_per_process: " << dht_buckets_per_process - // // << endl; - - // // MDL: new output on signif_vector and prop_type - // if (signif_vector_exists) { - // cout << "CPP: using problem-specific rounding digits: " << endl; - // R.parseEval( - // "print(data.frame(prop=prop, type=prop_type, " - // "digits=signif_vector))"); - // } else { - // cout << "CPP: using DHT default rounding digits = " - // << params.dht_significant_digits << endl; - // } - - // // MDL: pass to R the DHT stuff. These variables exist - // // only if dht_enabled is true - // R["dht_final_signif"] = params.dht_signif_vector; - // R["dht_final_proptype"] = params.dht_prop_type_vector; - // } - - // // if (params.dht_strategy == 0) { - // // if (params.world_rank != 0) { - // // dht_object = DHT_create(dht_comm, dht_buckets_per_process, - // // dht_data_size, dht_key_size, get_md5); - - // // // storing for access from worker and callback functions - // // fuzzing_buffer = (double *)malloc(dht_key_size); - // // } - // // } else { - // // dht_object = DHT_create(MPI_COMM_WORLD, dht_buckets_per_process, - // // dht_data_size, dht_key_size, get_md5); - // // } - - // // if (params.world_rank == 0) { - // // cout << "CPP: DHT successfully created!" << endl; - // // } - - // } // MDL: store all parameters - if (params.world_rank == 0) { + if (world_rank == 0) { cout << "CPP: Calling R Function to store calling parameters" << endl; R.parseEvalQ("StoreSetup(setup=mysetup)"); } + if (world_rank == 0) { + cout << "CPP: Init done on process with rank " << world_rank << endl; + } + MPI_Barrier(MPI_COMM_WORLD); - if (params.world_rank == 0) { /* This is executed by the master */ - ChemMaster master(parser, R, grid); + /* THIS IS EXECUTED BY THE MASTER */ + if (world_rank == 0) { + ChemMaster master(params, R, grid); TransportSim trans(R); - Rcpp::NumericVector master_send; - Rcpp::NumericVector master_recv; - - double *timings; - uint64_t *dht_perfs = NULL; - - // a temporary send buffer sim_start = MPI_Wtime(); - // Iteration Count is dynamic, retrieving value from R (is only needed by - // master for the following loop) + /* Iteration Count is dynamic, retrieving value from R (is only needed by + * master for the following loop) */ uint32_t maxiter = R.parseEval("mysetup$maxiter"); - /*SIMULATION LOOP*/ + /* SIMULATION LOOP */ for (uint32_t iter = 1; iter < maxiter + 1; iter++) { cout << "CPP: Evaluating next time step" << endl; R.parseEvalQ("mysetup <- master_iteration_setup(mysetup)"); - /*displaying iteration number, with C++ and R iterator*/ + /* displaying iteration number, with C++ and R iterator */ cout << "CPP: Going through iteration " << iter << endl; cout << "CPP: R's $iter: " << ((uint32_t)(R.parseEval("mysetup$iter"))) << ". Iteration" << endl; cout << "CPP: Calling Advection" << endl; + /* run transport */ trans.run(); - // sim_b_transport = MPI_Wtime(); - // R.parseEvalQ("mysetup <- master_advection(setup=mysetup)"); - // sim_a_transport = MPI_Wtime(); - - // if (iter == 1) master.prepareSimulation(); cout << "CPP: Chemistry" << endl; - /*Fallback for sequential execution*/ - if (params.world_size == 1) { + /* Fallback for sequential execution */ + if (world_size == 1) { master.ChemSim::run(); - } else { + } + /* otherwise run parallel */ + else { master.run(); } @@ -433,21 +146,11 @@ int main(int argc, char *argv[]) { // store_result is TRUE) R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - // cummul_transport += trans.getTransportTime(); - // cummul_chemistry += master.getChemistryTime(); - cout << endl << "CPP: End of *coupling* iteration " << iter << "/" << maxiter << endl << endl; - // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); - // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); - - for (int i = 1; i < params.world_size; i++) { - MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); - } - MPI_Barrier(MPI_COMM_WORLD); } // END SIMULATION LOOP @@ -465,147 +168,29 @@ int main(int argc, char *argv[]) { trans.end(); - if (params.world_size == 1) { + if (world_size == 1) { master.ChemSim::end(); } else { master.end(); } - // Rcpp::NumericVector phreeqc_time; - // Rcpp::NumericVector dht_get_time; - // Rcpp::NumericVector dht_fill_time; - // Rcpp::IntegerVector phreeqc_counts; - // Rcpp::NumericVector idle_worker; - - // int phreeqc_tmp; - - // timings = (double *)calloc(3, sizeof(double)); - - // int dht_hits = 0; - // int dht_miss = 0; - // int dht_collision = 0; - - // if (params.dht_enabled) { - // dht_hits = 0; - // dht_miss = 0; - // dht_collision = 0; - // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); - // } - - // double idle_worker_tmp; - - // for (int p = 0; p < params.world_size - 1; p++) { - // /* ATTENTION Worker p has rank p+1 */ - // /* Send termination message to worker */ - // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - - // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - - // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - // MPI_STATUS_IGNORE); - // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - - // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - - // if (params.dht_enabled) { - // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - - // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, - // MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // dht_hits += dht_perfs[0]; - // dht_miss += dht_perfs[1]; - // dht_collision += dht_perfs[2]; - // } - // } - - // R.parseEvalQ("profiling <- list()"); - - // R["simtime"] = sim_end - sim_start; - // R.parseEvalQ("profiling$simtime <- simtime"); - // R["simtime_transport"] = cummul_transport; - // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - // R["simtime_chemistry"] = cummul_chemistry; - // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - // R["simtime_workers"] = master.getWorkerTime(); - // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - // R["simtime_chemistry_master"] = master.getChemMasterTime(); - // R.parseEvalQ( - // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - - // R["seq_master"] = cummul_master_seq; - // R.parseEvalQ("profiling$seq_master <- seq_master"); - - // // R["master_send"] = master_send; - // // R.parseEvalQ("profiling$master_send <- master_send"); - // // R["master_recv"] = master_recv; - // // R.parseEvalQ("profiling$master_recv <- master_recv"); - - // R["idle_master"] = master.getIdleTime(); - // R.parseEvalQ("profiling$idle_master <- idle_master"); - // R["idle_worker"] = idle_worker; - // R.parseEvalQ("profiling$idle_worker <- idle_worker"); - - // R["phreeqc_time"] = phreeqc_time; - // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - - // R["phreeqc_count"] = phreeqc_counts; - // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - - // if (params.dht_enabled) { - // R["dht_hits"] = dht_hits; - // R.parseEvalQ("profiling$dht_hits <- dht_hits"); - // R["dht_miss"] = dht_miss; - // R.parseEvalQ("profiling$dht_miss <- dht_miss"); - // R["dht_collision"] = dht_collision; - // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - // R["dht_get_time"] = dht_get_time; - // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - // R["dht_fill_time"] = dht_fill_time; - // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - // } - - // free(timings); - - // if (params.dht_enabled) free(dht_perfs); - string r_vis_code; r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; R.parseEval(r_vis_code); cout << "CPP: Done! Results are stored as R objects into <" - << parser.getOutDir() << "/timings.rds>" << endl; - /*exporting results and profiling data*/ - - // std::string r_vis_code; - // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - // R.parseEval(r_vis_code); - } else { /*This is executed by the workers*/ - ChemWorker worker(parser, R, grid, dht_comm); - // worker.prepareSimulation(dht_comm); + << params.getOutDir() << "/timings.rds>" << endl; + } + /* THIS IS EXECUTED BY THE WORKERS */ + else { + ChemWorker worker(params, R, grid, dht_comm); worker.loop(); } - cout << "CPP: finished, cleanup of process " << params.world_rank << endl; - - // if (params.dht_enabled) { - // if (params.dht_strategy == 0) { - // if (params.world_rank != 0) { - // DHT_free(dht_object, NULL, NULL); - // } - // } else { - // DHT_free(dht_object, NULL, NULL); - // } - // } - - // free(mpi_buffer); + cout << "CPP: finished, cleanup of process " << world_rank << endl; MPI_Finalize(); - if (params.world_rank == 0) { + if (world_rank == 0) { cout << "CPP: done, bye!" << endl; } diff --git a/src/model/ChemMaster.cpp b/src/model/ChemMaster.cpp index 9b4480db0..4c6e1c54a 100644 --- a/src/model/ChemMaster.cpp +++ b/src/model/ChemMaster.cpp @@ -125,6 +125,11 @@ void ChemMaster::run() { /* end time measurement of whole chemistry simulation */ chem_b = MPI_Wtime(); chem_t += chem_b - chem_a; + + /* advise workers to end chemistry iteration */ + for (int i = 1; i < world_size; i++) { + MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); + } } void ChemMaster::sendPkgs(int &pkg_to_send, int &count_pkgs, diff --git a/src/util/SimParams.h b/src/util/SimParams.h index 1bda18dcf..494833bb2 100644 --- a/src/util/SimParams.h +++ b/src/util/SimParams.h @@ -4,7 +4,8 @@ #include #include "RRuntime.h" -#include "argh.h" +#include "argh.h" // Argument handler https://github.com/adishavit/argh +// BSD-licenced /** Return value if no error occured */ #define PARSER_OK 0 From 430b78a21d15d3fe4291b03e61d58bd92b2ed41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20L=C3=BCbke?= Date: Wed, 3 Feb 2021 10:28:08 +0100 Subject: [PATCH 32/50] add Doxyfile --- doc/Doxyfile | 2622 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2622 insertions(+) create mode 100644 doc/Doxyfile diff --git a/doc/Doxyfile b/doc/Doxyfile new file mode 100644 index 000000000..e549c4973 --- /dev/null +++ b/doc/Doxyfile @@ -0,0 +1,2622 @@ +# Doxyfile 1.9.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = POET + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = /tmp/doc + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. + +OUTPUT_TEXT_DIRECTION = None + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See https://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which efficively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# declarations. If set to NO, these declarations will be included in the +# documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# Possible values are: NO, YES and FAIL_ON_WARNINGS. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = /home/max/GFZ/POET/src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), +# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, +# *.ucf, *.qsf and *.ice. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = ../src/util/argh.h + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# entity all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see https://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: +# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the main .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. + +FORMULA_MACROFILE = + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# https://www.mathjax.org) which uses client side JavaScript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /