diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d1b71ab99..3541e1a4d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,7 +89,7 @@ include_directories(${MPI_CXX_INCLUDE_DIRS}) #define program libraries -add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp) +add_library(POET_Libs STATIC util/RRuntime.cpp model/Grid.cpp model/ChemSim.cpp model/ChemMaster.cpp model/ChemWorker.cpp DHT/DHT_Wrapper.cpp DHT/DHT.cpp util/Parser.cpp model/TransportSim.cpp util/Profiler.cpp) target_include_directories(POET_Libs PUBLIC ${R_INCLUDE_DIRS}) target_link_libraries(POET_Libs ${R_LIBRARIES} MPI::MPI_CXX crypto) diff --git a/src/kin.cpp b/src/kin.cpp index f67186f4a..192986ff7 100644 --- a/src/kin.cpp +++ b/src/kin.cpp @@ -12,10 +12,11 @@ // #include "global_buffer.h" #include "model/ChemSim.h" #include "model/Grid.h" +#include "model/TransportSim.h" #include "util/Parser.h" #include "util/RRuntime.h" #include "util/SimParams.h" -#include "model/TransportSim.h" +#include "util/Profiler.h" // #include "worker.h" //#define DHT_SIZE_PER_PROCESS 1073741824 @@ -216,15 +217,16 @@ int main(int argc, char *argv[]) { params = parser.getParams(); // if (params.world_rank == 0) { - // cout << "CPP: Complete results storage is " << (params.store_result ? "ON" : - // "OFF") + // cout << "CPP: Complete results storage is " << (params.store_result ? + // "ON" : "OFF") // << endl; // cout << "CPP: Work Package Size: " << params.wp_size << endl; // cout << "CPP: DHT is " << (params.dht_enabled ? "ON" : "OFF") << '\n'; // if (params.dht_enabled) { // cout << "CPP: DHT strategy is " << params.dht_strategy << endl; - // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is " + // // cout << "CPP: DHT key default digits (ignored if 'signif_vector' is + // " // // "defined) = " // // << dht_significant_digits << endl; // cout << "CPP: DHT logarithm before rounding: " @@ -430,27 +432,23 @@ int main(int argc, char *argv[]) { } else { /*send work to workers*/ master.runPar(); } - - double master_seq_a = MPI_Wtime(); + // MDL master_iteration_end just writes on disk state_T and // state_C after every iteration if the cmdline option // --ignore-results is not given (and thus the R variable // store_result is TRUE) R.parseEvalQ("mysetup <- master_iteration_end(setup=mysetup)"); - cummul_transport += trans.getTransportTime(); - cummul_chemistry += master.getChemistryTime(); + // cummul_transport += trans.getTransportTime(); + // cummul_chemistry += master.getChemistryTime(); cout << endl << "CPP: End of *coupling* iteration " << iter << "/" << maxiter << endl << endl; - double master_seq_b = MPI_Wtime(); - - cummul_master_seq += master.getSeqTime() + (master_seq_b - master_seq_a); - master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); - master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); + // master_send.push_back(master.getSendTime(), "it_" + to_string(iter)); + // master_recv.push_back(master.getRecvTime(), "it_" + to_string(iter)); for (int i = 1; i < params.world_size; i++) { MPI_Send(NULL, 0, MPI_DOUBLE, i, TAG_DHT_ITER, MPI_COMM_WORLD); @@ -464,118 +462,117 @@ int main(int argc, char *argv[]) { sim_end = MPI_Wtime(); - Rcpp::NumericVector phreeqc_time; - Rcpp::NumericVector dht_get_time; - Rcpp::NumericVector dht_fill_time; - Rcpp::IntegerVector phreeqc_counts; - Rcpp::NumericVector idle_worker; + Profiler::startProfiling(params, master, trans, R, sim_end - sim_start); - int phreeqc_tmp; + // Rcpp::NumericVector phreeqc_time; + // Rcpp::NumericVector dht_get_time; + // Rcpp::NumericVector dht_fill_time; + // Rcpp::IntegerVector phreeqc_counts; + // Rcpp::NumericVector idle_worker; - timings = (double *)calloc(3, sizeof(double)); + // int phreeqc_tmp; - int dht_hits = 0; - int dht_miss = 0; - int dht_collision = 0; + // timings = (double *)calloc(3, sizeof(double)); - if (params.dht_enabled) { - dht_hits = 0; - dht_miss = 0; - dht_collision = 0; - dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); - } + // int dht_hits = 0; + // int dht_miss = 0; + // int dht_collision = 0; - double idle_worker_tmp; + // if (params.dht_enabled) { + // dht_hits = 0; + // dht_miss = 0; + // dht_collision = 0; + // dht_perfs = (uint64_t *)calloc(3, sizeof(uint64_t)); + // } - cout << "CPP: Advising worker to stop work and collect data from them" - << endl; + // double idle_worker_tmp; - for (int p = 0; p < params.world_size - 1; p++) { - /* ATTENTION Worker p has rank p+1 */ - /* Send termination message to worker */ - MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + // for (int p = 0; p < params.world_size - 1; p++) { + // /* ATTENTION Worker p has rank p+1 */ + // /* Send termination message to worker */ + // MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); - MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + // MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); - MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, - MPI_STATUS_IGNORE); - phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + // MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + // MPI_STATUS_IGNORE); + // phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); - MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + // MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); - if (params.dht_enabled) { - dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); - dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + // if (params.dht_enabled) { + // dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + // dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); - MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, - MPI_COMM_WORLD, MPI_STATUS_IGNORE); - dht_hits += dht_perfs[0]; - dht_miss += dht_perfs[1]; - dht_collision += dht_perfs[2]; - } - } + // MPI_Recv(dht_perfs, 3, MPI_UNSIGNED_LONG_LONG, p + 1, TAG_DHT_PERF, + // MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // dht_hits += dht_perfs[0]; + // dht_miss += dht_perfs[1]; + // dht_collision += dht_perfs[2]; + // } + // } - R.parseEvalQ("profiling <- list()"); + // R.parseEvalQ("profiling <- list()"); - R["simtime"] = sim_end - sim_start; - R.parseEvalQ("profiling$simtime <- simtime"); - R["simtime_transport"] = cummul_transport; - R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); - R["simtime_chemistry"] = cummul_chemistry; - R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); - R["simtime_workers"] = master.getWorkerTime(); - R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); - R["simtime_chemistry_master"] = master.getChemMasterTime(); - R.parseEvalQ( - "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + // R["simtime"] = sim_end - sim_start; + // R.parseEvalQ("profiling$simtime <- simtime"); + // R["simtime_transport"] = cummul_transport; + // R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + // R["simtime_chemistry"] = cummul_chemistry; + // R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + // R["simtime_workers"] = master.getWorkerTime(); + // R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + // R["simtime_chemistry_master"] = master.getChemMasterTime(); + // R.parseEvalQ( + // "profiling$simtime_chemistry_master <- simtime_chemistry_master"); - R["seq_master"] = cummul_master_seq; - R.parseEvalQ("profiling$seq_master <- seq_master"); + // R["seq_master"] = cummul_master_seq; + // R.parseEvalQ("profiling$seq_master <- seq_master"); - // R["master_send"] = master_send; - // R.parseEvalQ("profiling$master_send <- master_send"); - // R["master_recv"] = master_recv; - // R.parseEvalQ("profiling$master_recv <- master_recv"); + // // R["master_send"] = master_send; + // // R.parseEvalQ("profiling$master_send <- master_send"); + // // R["master_recv"] = master_recv; + // // R.parseEvalQ("profiling$master_recv <- master_recv"); - R["idle_master"] = master.getIdleTime(); - R.parseEvalQ("profiling$idle_master <- idle_master"); - R["idle_worker"] = idle_worker; - R.parseEvalQ("profiling$idle_worker <- idle_worker"); + // R["idle_master"] = master.getIdleTime(); + // R.parseEvalQ("profiling$idle_master <- idle_master"); + // R["idle_worker"] = idle_worker; + // R.parseEvalQ("profiling$idle_worker <- idle_worker"); - R["phreeqc_time"] = phreeqc_time; - R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + // R["phreeqc_time"] = phreeqc_time; + // R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); - R["phreeqc_count"] = phreeqc_counts; - R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + // R["phreeqc_count"] = phreeqc_counts; + // R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); - if (params.dht_enabled) { - R["dht_hits"] = dht_hits; - R.parseEvalQ("profiling$dht_hits <- dht_hits"); - R["dht_miss"] = dht_miss; - R.parseEvalQ("profiling$dht_miss <- dht_miss"); - R["dht_collision"] = dht_collision; - R.parseEvalQ("profiling$dht_collisions <- dht_collision"); - R["dht_get_time"] = dht_get_time; - R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); - R["dht_fill_time"] = dht_fill_time; - R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); - } + // if (params.dht_enabled) { + // R["dht_hits"] = dht_hits; + // R.parseEvalQ("profiling$dht_hits <- dht_hits"); + // R["dht_miss"] = dht_miss; + // R.parseEvalQ("profiling$dht_miss <- dht_miss"); + // R["dht_collision"] = dht_collision; + // R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + // R["dht_get_time"] = dht_get_time; + // R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + // R["dht_fill_time"] = dht_fill_time; + // R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + // } - free(timings); + // free(timings); - if (params.dht_enabled) free(dht_perfs); + // if (params.dht_enabled) free(dht_perfs); cout << "CPP: Done! Results are stored as R objects into <" << params.out_dir << "/timings.rds>" << endl; /*exporting results and profiling data*/ - std::string r_vis_code; - r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; - R.parseEval(r_vis_code); + // std::string r_vis_code; + // r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + // R.parseEval(r_vis_code); } else { /*This is executed by the workers*/ ChemWorker worker(¶ms, R, grid, dht_comm); // worker.prepareSimulation(dht_comm); diff --git a/src/util/Profiler.cpp b/src/util/Profiler.cpp new file mode 100644 index 000000000..3ba3af249 --- /dev/null +++ b/src/util/Profiler.cpp @@ -0,0 +1,126 @@ +#include "Profiler.h" + +#include +#include + +#include +#include + +using namespace Rcpp; +using namespace std; + +int poet::Profiler::startProfiling(t_simparams ¶ms, ChemMaster &chem, + TransportSim &trans, RRuntime &R, + double simtime) { + double *timings; + int *dht_perfs; + + Rcpp::NumericVector phreeqc_time; + Rcpp::NumericVector dht_get_time; + Rcpp::NumericVector dht_fill_time; + Rcpp::IntegerVector phreeqc_counts; + Rcpp::NumericVector idle_worker; + + int phreeqc_tmp; + + timings = (double *)calloc(3, sizeof(double)); + + int dht_hits = 0; + int dht_miss = 0; + int dht_collision = 0; + + if (params.dht_enabled) { + dht_hits = 0; + dht_miss = 0; + dht_collision = 0; + dht_perfs = (int *)calloc(3, sizeof(int)); + } + + double idle_worker_tmp; + + for (int p = 0; p < params.world_size - 1; p++) { + /* ATTENTION Worker p has rank p+1 */ + /* Send termination message to worker */ + MPI_Send(NULL, 0, MPI_DOUBLE, p + 1, TAG_FINISH, MPI_COMM_WORLD); + + MPI_Recv(timings, 3, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_time.push_back(timings[0], "w" + to_string(p + 1)); + + MPI_Recv(&phreeqc_tmp, 1, MPI_INT, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + phreeqc_counts.push_back(phreeqc_tmp, "w" + to_string(p + 1)); + + MPI_Recv(&idle_worker_tmp, 1, MPI_DOUBLE, p + 1, TAG_TIMING, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + idle_worker.push_back(idle_worker_tmp, "w" + to_string(p + 1)); + + if (params.dht_enabled) { + dht_get_time.push_back(timings[1], "w" + to_string(p + 1)); + dht_fill_time.push_back(timings[2], "w" + to_string(p + 1)); + + MPI_Recv(dht_perfs, 3, MPI_INT, p + 1, TAG_DHT_PERF, + MPI_COMM_WORLD, MPI_STATUS_IGNORE); + dht_hits += dht_perfs[0]; + dht_miss += dht_perfs[1]; + cout << "profiler miss = " << dht_miss << endl; + dht_collision += dht_perfs[2]; + } + } + + R.parseEvalQ("profiling <- list()"); + + R["simtime"] = simtime; + R.parseEvalQ("profiling$simtime <- simtime"); + R["simtime_transport"] = trans.getTransportTime(); + R.parseEvalQ("profiling$simtime_transport <- simtime_transport"); + R["simtime_chemistry"] = chem.getChemistryTime(); + R.parseEvalQ("profiling$simtime_chemistry <- simtime_chemistry"); + R["simtime_workers"] = chem.getWorkerTime(); + R.parseEvalQ("profiling$simtime_workers <- simtime_workers"); + R["simtime_chemistry_master"] = chem.getChemMasterTime(); + R.parseEvalQ( + "profiling$simtime_chemistry_master <- simtime_chemistry_master"); + + R["seq_master"] = chem.getSeqTime(); + R.parseEvalQ("profiling$seq_master <- seq_master"); + + // R["master_send"] = master_send; + // R.parseEvalQ("profiling$master_send <- master_send"); + // R["master_recv"] = master_recv; + // R.parseEvalQ("profiling$master_recv <- master_recv"); + + R["idle_master"] = chem.getIdleTime(); + R.parseEvalQ("profiling$idle_master <- idle_master"); + R["idle_worker"] = idle_worker; + R.parseEvalQ("profiling$idle_worker <- idle_worker"); + + R["phreeqc_time"] = phreeqc_time; + R.parseEvalQ("profiling$phreeqc <- phreeqc_time"); + + R["phreeqc_count"] = phreeqc_counts; + R.parseEvalQ("profiling$phreeqc_count <- phreeqc_count"); + + if (params.dht_enabled) { + R["dht_hits"] = dht_hits; + R.parseEvalQ("profiling$dht_hits <- dht_hits"); + R["dht_miss"] = dht_miss; + R.parseEvalQ("profiling$dht_miss <- dht_miss"); + R["dht_collision"] = dht_collision; + R.parseEvalQ("profiling$dht_collisions <- dht_collision"); + R["dht_get_time"] = dht_get_time; + R.parseEvalQ("profiling$dht_get_time <- dht_get_time"); + R["dht_fill_time"] = dht_fill_time; + R.parseEvalQ("profiling$dht_fill_time <- dht_fill_time"); + } + + free(timings); + + if (params.dht_enabled) free(dht_perfs); + + string r_vis_code; + r_vis_code = "saveRDS(profiling, file=paste0(fileout,'/timings.rds'));"; + R.parseEval(r_vis_code); + + return 0; +} diff --git a/src/util/Profiler.h b/src/util/Profiler.h new file mode 100644 index 000000000..71b3611e7 --- /dev/null +++ b/src/util/Profiler.h @@ -0,0 +1,18 @@ +#ifndef PROFILER_H +#define PROFILER_H + +#include "../model/ChemSim.h" +#include "../model/TransportSim.h" +#include "RRuntime.h" +#include "SimParams.h" + +namespace poet { +class Profiler { + public: + static int startProfiling(t_simparams ¶ms, ChemMaster &chem, + TransportSim &trans, RRuntime &R, double simtime); +}; + +} // namespace poet + +#endif // PROFILER_H \ No newline at end of file