/******************************************************************************
    Parallel I/O example:

    This example uses some of the file access routines. A file is created and 
    different file write mehtods are used, one using explicit offsets, one 
    using the shared file pointer, and one using individual file pointers for 
    each process. After each write has been performed the file content is read 
    and dumped to stdout. 
    
    More specifically:
   --------------------------------
      - A file is created and opened for both reading and writing.
      - The file view is set to MPI::INT for all processes.
      - Set the file's size so that it has room for one MPI::INT per process.
      - Write to the file using explicit offsets.
      - Write to the file using the shared file pointer.
      - Write to the file using individual file pointers.
      - Close the file and exit.
   --------------------------------

    Copyright 2003 (c) Critical Software SA
    . http://www.criticalsoftware.com
    . http://www.criticalsoftware.com/hpc
    . csWMPI II@criticalsoftware.com

 *****************************************************************************/
#include "Parallel_IO.h"
#include <stdio.h>
#include <mpi.h>

/* The filename to use for the shared file                                   */
#define FILENAME   "sharedfile.mpi"   
/* Rank of the process to act as file server                                 */
#define SERVERRANK "0"                

//-----------------------------------------------------------------------------

/******************************************************************************
    This function dumps the content of the file using the process' individual 
    file pointer.
 *****************************************************************************/
void DumpFileContent(MPI::File mf_file, int n_comm_size, int n_comm_rank) {
    int nCounter;
    int nFileData;

    mf_file.Seek(0, MPI::SEEK_SET);
    printf("  - File content: ");
    for (nCounter = 0; nCounter < n_comm_size; nCounter++) {
        mf_file.Read(&nFileData, 1, MPI::INT);
        printf(" (%d)", nFileData);
    }
    printf("\n");
    fflush(stdout);
}

/******************************************************************************
   Constructor
 *****************************************************************************/
CParallel_IO::CParallel_IO(int argc, char * argv[])
{
    /* Initialize csWMPI II II:                                                   */
    MPI::Init(argc, argv);

    /* Determine what the world looks like and our own position in it:       */
    m_nCommSize = MPI::COMM_WORLD.Get_size();
    m_nCommRank = MPI::COMM_WORLD.Get_rank();
    printf("I am rank %d of %d in MPI_COMM_WORLD\n", m_nCommRank, m_nCommSize);
    fflush(stdout);
}

/******************************************************************************
   Destructor
 *****************************************************************************/
CParallel_IO::~CParallel_IO()
{
}

/******************************************************************************
    main
 *****************************************************************************/
int main(int argc, char *argv[])
{
    CParallel_IO* pcParallelIO = new CParallel_IO(argc, argv);

    MPI::File mfFile;
    MPI::Info miInfo;

    /* Create an instance of MPI::Info and set the server_rank key, 
    /* specifying which rank should act as the file server, to SERVER_RANK:  */
    miInfo = MPI::Info::Create();
    miInfo.Set("server_rank", SERVERRANK);

    /* Try creating a file (and open it in read/write mode)                  */
    printf ("Opening file: %s (server_rank=%s)..", FILENAME, SERVERRANK);

    mfFile = MPI::File::Open(MPI::COMM_WORLD,
                             FILENAME,
                             MPI::MODE_RDWR | MPI::MODE_CREATE,
                             miInfo);
    printf ("OK!\n\n");
    fflush(stdout);

    /* Free the info object:                                                 */
    miInfo.Free();

    /* Set the file view (all use the displacement 0, elementary datatype    */
    /* and filetype MPI::INT). The data representation is set to "native".   */
    /* Hence, we have a file of MPI::INTs viewed identically by all          */
    /* processes:                                                            */
    mfFile.Set_view(0, MPI::INT, MPI::INT, "native", MPI::INFO_NULL);

    /* Make room in the file for one MPI::INT per process                    */
    mfFile.Set_size(pcParallelIO->m_nCommSize);

    /* Use explicit offsets (offset equal to rank) to write one MPI::INT to  */
    /* the file. The result should be a file containing integers ascending   */
    /* from 0 to the size of MPI::COMM_WORLD - 1.                            */
    printf ("Write to file using explicit offsets..........");
    mfFile.Write_at(pcParallelIO->m_nCommRank,
                    &pcParallelIO->m_nCommRank,
                    1,
                    MPI::INT);
    printf("OK!\n");
    fflush(stdout);
    MPI::COMM_WORLD.Barrier();
    DumpFileContent(mfFile,
                    pcParallelIO->m_nCommSize,
                    pcParallelIO->m_nCommRank);
    MPI::COMM_WORLD.Barrier();

    /* Use a shared file pointer to write to the file. Since there is no     */
    /* synchronization between the processes, the result is                  */
    /* non-deterministic, however the file should contain one MPI::INT for   */
    /* every process in MPI::COMM_WORLD.                                     */
    printf ("Write to file using the shared file pointer...");
    mfFile.Write_shared(&pcParallelIO->m_nCommRank, 1, MPI::INT);
    printf("OK!\n");
    fflush(stdout);
    MPI::COMM_WORLD.Barrier();
    DumpFileContent (mfFile,
                     pcParallelIO->m_nCommSize,
                     pcParallelIO->m_nCommRank);
    MPI::COMM_WORLD.Barrier();

    /* Use individual file pointers to write to the file. First a seek is    */
    /* performed to from the _end_ of the file. Hence the should be a file   */
    /* containing integers descending from size of MPI::COMM_WORLD - 1 to 0. */
    printf ("Write to file using individual file pointers..");
    mfFile.Seek(-(pcParallelIO->m_nCommRank+1), MPI::SEEK_END);
    mfFile.Write(&pcParallelIO->m_nCommRank, 1, MPI::INT);
    printf("OK!\n");
    fflush(stdout);
    MPI::COMM_WORLD.Barrier ();
    DumpFileContent (mfFile,
                     pcParallelIO->m_nCommSize,
                     pcParallelIO->m_nCommRank);
    MPI::COMM_WORLD.Barrier ();

    /* Close the file and finalize MPI:                                      */
    mfFile.Close();
    MPI::Finalize();

    /* Pause rank 0 so that the output can be verified:                      */
    if (pcParallelIO->m_nCommRank == 0) {
        printf("\nPress ENTER to exit...\n");
        fflush (stdout);
        getchar();
    }

    delete pcParallelIO;
    return 0;
}