Commit 5a10fd72 authored by Sonja Happ's avatar Sonja Happ

use a dedicated MPI communicator in FBS sweeps

parent 9a0e46d0
......@@ -210,6 +210,10 @@ protected:
//!< shared memory window saving the latency matrix
double *latency_matrix; //!< Pointer to array that contains latency matrix
/* MPI communicator for FBS*/
MPI_Comm filled_ranks; //<! communication for all ranks that contain agents (omitting ranks that do not contain agents)
Villas_interface *interface; //!< Interface to manage villas node type
villas_node_config * villas_config_agents;
villas_node_config * villas_config_model;
......
......@@ -557,6 +557,14 @@ void Model::create_agents() {
IO->log_info("Highest filled rank: " + std::to_string(highest_filled_rank));
// create MPI communicator for FBS method:
int filled = 0;
if (rank <= highest_filled_rank){
filled = 1;
}
MPI_Comm_split(MPI_COMM_WORLD, filled, rank, &filled_ranks);
MPI_Barrier(MPI_COMM_WORLD);
//Set hop matrix in DF agent if required
......
......@@ -3,9 +3,13 @@
/*! \brief Perform the forward-backward-sweep algorithm for the current tick
* */
void Model::forward_backward_sweep() {
IO->log_info("####### now calculating electrical interactions #######");
if(agents_scheduling.empty()){
return;
}
IO->start_time_measurement(IO->tm_fb_sweep);
IO->log_info("####### now calculating electrical interactions #######");
repast::ScheduleRunner &runner = repast::RepastProcess::instance()->getScheduleRunner();
IO->reset_number_of_fbs_sweeps();
......@@ -26,7 +30,7 @@ void Model::forward_backward_sweep() {
// it issues an asynchronous broadcast
IO->start_time_measurement(IO->tm_bcast_FBS);
MPI_Irecv(&forward_backward_sweep_finished, 1, MPI_C_BOOL, agent_rank_relation[SLACK_ID-1],
FBS_COMPLETION_TAG, MPI_COMM_WORLD, &convergence_req[rank]);
FBS_COMPLETION_TAG, filled_ranks, &convergence_req[rank]);
IO->stop_time_measurement(IO->tm_bcast_FBS);
}
......@@ -65,9 +69,8 @@ void Model::forward_backward_sweep() {
// inform all other processes about convergence
for (int i = 0; i <world_size; i++) {
if (i != rank) {
MPI_Isend(&forward_backward_sweep_finished, 1, MPI_C_BOOL, i, FBS_COMPLETION_TAG,
MPI_COMM_WORLD,
&convergence_req[i]);
MPI_Isend(&forward_backward_sweep_finished, 1, MPI_C_BOOL, i,
FBS_COMPLETION_TAG, filled_ranks, &convergence_req[i]);
}
}
......@@ -86,7 +89,7 @@ void Model::forward_backward_sweep() {
// This barrier is required since all processes should wait for
// the convergence check of the process holding the slack before entering another loop iteration
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(filled_ranks);
IO->increment_number_of_fbs_sweeps();
//synchronize changes: not needed because in last forward sweep iteration latest changes have been synchronized
......@@ -141,7 +144,7 @@ void Model::do_backward_sweep() {
IO->start_time_measurement(IO->tm_bcast_backward_sweep);
IO->log_info("Backward sweep: issuing Irecv");
MPI_Irecv(&backward_sweep_finished, 1, MPI_C_BOOL, agent_rank_relation[SLACK_ID-1],
BACKWARD_COMPLETION_TAG, MPI_COMM_WORLD, &backward_req[rank]);
BACKWARD_COMPLETION_TAG, filled_ranks, &backward_req[rank]);
IO->stop_time_measurement(IO->tm_bcast_backward_sweep);
}
......@@ -179,8 +182,8 @@ void Model::do_backward_sweep() {
// inform all other processes about convergence
for(int i=0; i<=highest_filled_rank; i++){
if (i != rank) {
MPI_Isend(&backward_sweep_finished, 1, MPI_C_BOOL, i, BACKWARD_COMPLETION_TAG, MPI_COMM_WORLD,
&backward_req[i]);
MPI_Isend(&backward_sweep_finished, 1, MPI_C_BOOL, i,
BACKWARD_COMPLETION_TAG, filled_ranks, &backward_req[i]);
}
}
for(int i=0; i<=highest_filled_rank; i++){
......@@ -193,7 +196,7 @@ void Model::do_backward_sweep() {
} // check finished
// this barrier is required to make all processes know the
// result of the finished check before entering a new loop iteration
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(filled_ranks);
IO->log_info("Backward sweep finished: " + std::to_string(backward_sweep_finished));
IO->stop_time_measurement(IO->tm_bcast_backward_sweep);
......@@ -244,7 +247,7 @@ void Model::do_forward_sweep() {
IO->start_time_measurement(IO->tm_bcast_forward_sweep);
// if rank does not contain slack it starts an async. broadcast here
MPI_Irecv(&forward_sweep_finished, 1, MPI_C_BOOL, agent_rank_relation[SLACK_ID-1],
FORWARD_COMPLETION_TAG, MPI_COMM_WORLD, &forward_req[rank]);
FORWARD_COMPLETION_TAG, filled_ranks, &forward_req[rank]);
IO->stop_time_measurement(IO->tm_bcast_forward_sweep);
}
......@@ -297,14 +300,14 @@ void Model::do_forward_sweep() {
// inform all other processes about convergence
for(int i=0; i<=highest_filled_rank; i++){
if (i != rank) {
MPI_Isend(&forward_sweep_finished, 1, MPI_C_BOOL, i, FORWARD_COMPLETION_TAG, MPI_COMM_WORLD,
&forward_req[i]);
MPI_Isend(&forward_sweep_finished, 1, MPI_C_BOOL, i,
FORWARD_COMPLETION_TAG, filled_ranks, &forward_req[i]);
}
}
for(int i=0; i<=highest_filled_rank; i++){
if (i != rank) {
IO->log_info("Waiting for Irecv of rank " + std::to_string(i));
IO->log_info("Waiting for Irecv of rank " + std::to_string(i));
MPI_Wait(&forward_req[i], &status);
}
}
......@@ -314,7 +317,7 @@ void Model::do_forward_sweep() {
// This barrier is required to make sure that all processes
// know about the finished state before entering the next loop iteration
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(filled_ranks);
IO->stop_time_measurement(IO->tm_bcast_forward_sweep);
// in forward sweep info about next and previous electrical nodes are required
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment