/******************************************************************************
One-sided Communication example:
In this example all processes expose a memory region large enough to
contain one integer for each rank in MPI::COMM_WORLD. The MPI-2 standard
defines three synchronization mechanisms:
- Lock/Unlock: One rank requests either a shared or an exclusive
lock on the window exposed by some target rank.
- Fence: The first call to MPI::Win::Fence will open an epoch
where all ranks can access all other ranks. The epoch
is closed with another call to MPI::Win::Fence.
- Active Target: In this synchronization mode, the targets of remote
memory accesses explicitly grant permission to the
origins through the call MPI::Win::Post. An epoch is
closed thought a call to MPI::Win::Wait. Likewise,
the origins explicitly request permission through a
call to MPI::Win::Start and close the epoch through
MPI::Win::Complete. Note that every rank granted
access *must* call MPI::Win::Start and eventually
MPI::Win::Complete before an epoch is closed.
Recall that "origin" refers to the process that performs the remote memory
access call, while the "target" refers to the process in which memory is
accessed.
In this example we show to use each of the synchronization mechanisms.
More specifically:
--------------------------------
- Every process exposes a window large enough to store one integer per
process in MPI::COMM_WORLD.
- Lock/Unlock: Every rank x accesses rank x+1's memory, and fills the
exposed region with integers of value x.
- Fence: Every process writes one integer (value = rank) to all
other processes.
- ActiveTarget: Every rank x accesses rank x-1's memory, and fills the
exposed region with integers of value x.
- Get: Every process read the exposed memory region of every
process and displays the result.
- The window is freed and MPI::Finalize is called.
--------------------------------
Copyright 2003 (c) Critical Software SA
. http://www.criticalsoftware.com
. http://www.criticalsoftware.com/hpc
. csWMPI II@criticalsoftware.com
*****************************************************************************/
#include "OneSided_Comm.h"
#include <stdio.h>
#include <windows.h>
#include <mpi.h>
/*
We use a global variable for the MPI::Win used in all the synchronization
examples. Note: If you mix different types of synchronization:
YOU WILL EXPLICITLY HAVE TO SYNCHRONIZE ALL
PROCESSES IN THE GROUP OF THE WINDOW
- either by MPI::Comm::Barrier (as done in this example) or by some other
method.
*/
MPI::Win g_mwWin;
/******************************************************************************
Displays a list of integers
*****************************************************************************/
void DisplayInts(int* pn_ints, int n_count) {
int nCounter;
for (nCounter = 0; nCounter < n_count; nCounter++) {
printf("(%d)", pn_ints[nCounter]);
}
printf("\n");
}
/******************************************************************************
In this function we use the MPI::Win::Lock and MPI::Win::Unlock to open and
close epochs.
*****************************************************************************/
void LockUnlock(int n_my_rank, int n_comm_size) {
int nCounter;
int nTargetRank;
int nTargetDisp;
/* Compute the target rank. If my rank is x then my target will be x+1 */
/* (modulo the communicator size). */
nTargetRank = (1 + n_my_rank) % n_comm_size;
/* Lock the window on the target. Use an exclusive lock, since we are */
/* going to do Puts */
g_mwWin.Lock (MPI::LOCK_EXCLUSIVE, nTargetRank, 0);
/* Fill the memory of the target with integers with a value equal to my */
/* rank. We put only one integer at the time (a much more efficient */
/* approach would have been to put n_comm_size integers using a single */
/* Put operation, however the purpose here is just to show how */
/* one-sided communication works, and not how to write efficient */
/* applications). */
for (nCounter = 0; nCounter < n_comm_size; nCounter++) {
/* Compute the target displacement. Since the Win was created using */
/* a displacement of 1 [see main()] we have to compute the */
/* displacements in bytes. Alternatively, we could have created the */
/* window using a displacement unit of sizeof(int), and simply used */
/* nCounter as the displacement. */
nTargetDisp = nCounter * sizeof(int);
/* Perform the Put. Write one integer: */
g_mwWin.Put (&n_my_rank,
1,
MPI::INT,
nTargetRank,
nTargetDisp,
1,
MPI::INT);
}
/* Unlock the window, since we are done accessing it for now: */
g_mwWin.Unlock (nTargetRank);
/* Since we are going to use other types of synchronization methods */
/* later we synchronize all processes: */
MPI::COMM_WORLD.Barrier ();
}
/******************************************************************************
In this function we use MPI::Win::Fence to open and close epochs. In this
function we write one integer to all other processes.
*****************************************************************************/
void Fence(int n_my_rank, int n_comm_size) {
int nCounter;
int nTargetDisp;
nTargetDisp = n_my_rank * sizeof(int);
/* Open an epoch: */
g_mwWin.Fence (0);
for (nCounter = 0; nCounter < n_comm_size; nCounter++) {
/* Put an integer in the memory exposed by rank nCounter */
g_mwWin.Put (&n_my_rank,
1,
MPI::INT,
nCounter,
nTargetDisp,
1,
MPI::INT);
}
/* Close the epoch: */
g_mwWin.Fence (0);
/* Since we are going to use other types of synchronization methods */
/* later we synchronize all processes: */
MPI::COMM_WORLD.Barrier ();
}
/******************************************************************************
In this function we use MPI::Win::Post, MPI::Win::Wait, MPI::Win::Start, and
MPI::Win::Complete to open and close epochs.
If we have rank x, then we would like to access rank x-1 and to let rank
x+1 access our exposed memory.
*****************************************************************************/
void ActiveTarget(int n_my_rank, int n_comm_size) {
int nCounter;
int nPostRank;
int nStartRank;
int nTargetDisp;
MPI::Group mgCommWorld;
MPI::Group mgPostGroup;
MPI::Group mgStartGroup;
/* Compute the rank of the process, which will access our memory, and */
/* the rank exposing the memory that we are going to access: */
nPostRank = (n_my_rank + 1) % n_comm_size;
nStartRank = (n_comm_size + (n_my_rank - 1)) % n_comm_size;
/* Get the group of MPI::COMM_WORLD and create groups for the Post and */
/* Start ranks: */
mgCommWorld = MPI::COMM_WORLD.Get_group();
mgPostGroup = mgCommWorld.Incl(1, &nPostRank);
mgStartGroup = mgCommWorld.Incl(1, &nStartRank);
/* Open the epochs: */
g_mwWin.Post(mgPostGroup, 0);
g_mwWin.Start(mgStartGroup, 0);
for (nCounter = 0; nCounter < n_comm_size; nCounter++) {
nTargetDisp = nCounter * sizeof(int);
g_mwWin.Put (&n_my_rank,
1,
MPI::INT,
nStartRank,
nTargetDisp,
1,
MPI::INT);
}
/* Close the epochs: */
g_mwWin.Complete();
g_mwWin.Wait();
/* Free the groups: */
mgCommWorld.Free();
mgPostGroup.Free();
mgStartGroup.Free();
/* Since we are going to use other types of synchronization methods */
/* later we synchronize all processes: */
MPI::COMM_WORLD.Barrier();
}
/******************************************************************************
This function reads (using Get) the content of all exposed memory regions
and displays it:
*****************************************************************************/
void Get(int n_comm_size) {
int* pnTempBuffer;
int nCounter;
/* Allocate a buffer large enough to hold the content of one process´ */
/* exposed region */
pnTempBuffer = (int *) malloc(n_comm_size * sizeof(int));
/* Read the content of exposed regions one by one. We use shared locks */
/* since we only do reads. */
for (nCounter = 0; nCounter < n_comm_size; nCounter++) {
/* Open an epoch: */
g_mwWin.Lock(MPI::LOCK_SHARED, nCounter, 0);
g_mwWin.Get(pnTempBuffer,
n_comm_size,
MPI::INT,
nCounter,
0,
n_comm_size,
MPI::INT);
/* Close the epoch. We have to do before accessing pnTempBuffer to */
/* make sure that the Get operation has completed. */
g_mwWin.Unlock(nCounter);
/* Display the result: */
printf("Got the following from rank %d: ", nCounter);
DisplayInts(pnTempBuffer, n_comm_size);
fflush(stdout);
}
free(pnTempBuffer);
}
/******************************************************************************
Constructor
*****************************************************************************/
COneSided_Comm::COneSided_Comm(int argc, char * argv[])
{
/* Initialize csWMPI II II: */
MPI::Init(argc, argv);
/* Determine what the world looks like and our own position in it: */
m_nCommSize = MPI::COMM_WORLD.Get_size();
m_nCommRank = MPI::COMM_WORLD.Get_rank();
printf("I am rank %d of %d in MPI_COMM_WORLD\n", m_nCommRank, m_nCommSize);
fflush(stdout);
}
/******************************************************************************
Destructor
*****************************************************************************/
COneSided_Comm::~COneSided_Comm()
{
}
/******************************************************************************
main
*****************************************************************************/
int main(int argc, char *argv[])
{
int nWindowSize;
int* pchSharedMemory = NULL;
COneSided_Comm* pcOneSidedComm = new COneSided_Comm(argc, argv);
/* Compute the size of the memory region to expose and allocate memory: */
nWindowSize = pcOneSidedComm->m_nCommSize * sizeof(int);
pchSharedMemory = (int*) malloc(nWindowSize);
/* Create the window (same size for all processes. Note that we use a */
/* displacement of 1. This means that displacements in Puts and Gets */
/* have to be calculated as byte offsets. */
g_mwWin = MPI::Win::Create(pchSharedMemory,
nWindowSize,
1,
MPI::INFO_NULL,
MPI::COMM_WORLD);
/* Use the lock/unlock synchronization: */
LockUnlock(pcOneSidedComm->m_nCommRank, pcOneSidedComm->m_nCommSize);
printf("After LockUnlock - My memory region contains: ");
DisplayInts(pchSharedMemory, pcOneSidedComm->m_nCommSize);
fflush(stdout);
/* Use the fence synchronization: */
Fence(pcOneSidedComm->m_nCommRank, pcOneSidedComm->m_nCommSize);
printf("After Fence - My memory region contains: ");
DisplayInts(pchSharedMemory, pcOneSidedComm->m_nCommSize);
fflush(stdout);
/* Use the active target (post,wait,start,complete) synchronization: */
ActiveTarget(pcOneSidedComm->m_nCommRank, pcOneSidedComm->m_nCommSize);
printf("After PostWaitStartComplete - My memory region contains: ");
DisplayInts(pchSharedMemory, pcOneSidedComm->m_nCommSize);
fflush(stdout);
/* Use get to read the content of the exposed memory regions on all */
/* processes: */
printf("\nGetting content of all ranks exposed memory regions: \n");
Get(pcOneSidedComm->m_nCommSize);
fflush(stdout);
/* Free the window and the allocated memory: */
g_mwWin.Free();
free(pchSharedMemory);
/* Finalize csWMPI II II: */
MPI::Finalize();
/* Pause rank 0 so that the output can be verified: */
if (pcOneSidedComm->m_nCommRank == 0) {
printf("\nPress ENTER to exit...\n");
fflush (stdout);
getchar();
}
delete pcOneSidedComm;
return 0;
}
|
examples/cpp/onesided_communication_cpp