---

ContainerComm.h

Go to the documentation of this file.

#ifndef CONTAINERCOMM_H
#define CONTAINERCOMM_H

#include "SundanceDefs.h"
#include "TSFArray.h"
#include "MPIComm.h"
#include "MPITraits.h"

namespace Sundance
{
  /** \ingroup MPI
   * MPI communication of templated containers
   */

  template <class T> class ContainerComm
    {
    public:
      /** broadcast a single object */
      static void bcast(T& x, int src, const MPIComm& comm);

      /** send a single object */
      static void send(const T& x, int tag, int dest,
                       const MPIComm& comm);

      /** recv a single object */
      static void recv(T& x, int tag, int src,
                       const MPIComm& comm);

      /** bcast an array of objects */
      static void bcast(TSFArray<T>& x, int src, const MPIComm& comm);

      /** send an array of objects */
      static void send(const TSFArray<T>& x, int tag, int dest,
                       const MPIComm& comm);

      /** recv an array of objects */
      static void recv(TSFArray<T>& x, int tag, int src,
                       const MPIComm& comm);

      /** bcast an array of arrays  */
      static void bcast(TSFArray<TSFArray<T> >& x,
                        int src, const MPIComm& comm);

      /** AllGather: each process sends a single object to all other procs */
      static void allGather(const T& outgoing,
                            TSFArray<T>& incoming,
                            const MPIComm& comm);

      /** All-to-all scatter/gather, each proc sends an object to every proc */
      static void allToAll(const TSFArray<T>& outgoing,
                           TSFArray<TSFArray<T> >& incoming,
                           const MPIComm& comm);

      /** All-to-all scatter/gather, each proc sends an array to every proc */
      static void allToAll(const TSFArray<TSFArray<T> >& outgoing,
                           TSFArray<TSFArray<T> >& incoming,
                           const MPIComm& comm);

      /** sum local values from all processors with rank < myRank */
      static void accumulate(const T& localValue, TSFArray<T>& sums,
                             const MPIComm& comm);

    private:
      /** build a 1D array and an offset list from a 2D array */
      static void getBigTSFArray(const TSFArray<TSFArray<T> >& x,
                                 TSFArray<T>& bigTSFArray,
                                 TSFArray<int>& offsets);

      /** reassemble a 2D array from a 1D array and an offset table */
      static void getSmallTSFArrays(const TSFArray<T>& bigTSFArray,
                                    const TSFArray<int>& offsets,
                                    TSFArray<TSFArray<T> >& x);


    };

  /** \ingroup MPI
   * Specialiaztion of ContainerComm<T> to string
   */
  template <> class ContainerComm<string>
    {
    public:
      static void bcast(string& x, int src, const MPIComm& comm);

      static void send(const string& x, int tag, int dest,
                       const MPIComm& comm);

      static void recv(string& x, int tag, int src,
                       const MPIComm& comm);

      /** bcast an array of objects */
      static void bcast(TSFArray<string>& x, int src, const MPIComm& comm);

      /** send an array of objects */
      static void send(const TSFArray<string>& x, int tag, int dest,
                       const MPIComm& comm);

      /** recv an array of objects */
      static void recv(TSFArray<string>& x, int tag, int src,
                       const MPIComm& comm);

      /** bcast an array of arrays  */
      static void bcast(TSFArray<TSFArray<string> >& x,
                        int src, const MPIComm& comm);

      /** AllGather: each process sends a single object to all other procs */
      static void allGather(const string& outgoing,
                            TSFArray<string>& incoming,
                            const MPIComm& comm);

    private:
      /** get a single big array of characters from an array of strings */
      static void getBigTSFArray(const TSFArray<string>& x,
                                 TSFArray<char>& bigTSFArray,
                                 TSFArray<int>& offsets);

      /** recover an array of strings from a single big array and
       * and offset table */
      static void getStrings(const TSFArray<char>& bigTSFArray,
                             const TSFArray<int>& offsets,
                             TSFArray<string>& x);
    };


  /* --------- generic functions for primitives ------------------- */

  template <class T> inline void ContainerComm<T>::bcast(T& x, int src,
                                                         const MPIComm& comm)
    {
      comm.bcast((void*)&x, 1, MPITraits<T>::type(), src);
    }

  template <class T> inline void ContainerComm<T>::send(const T& x, int tag, int dest,
                                                        const MPIComm& comm)
    {
      comm.send((void*)&x, 1, MPITraits<T>::type(), tag, dest);
    }

  template <class T> inline void ContainerComm<T>::recv(T& x, int tag, int src,
                                                        const MPIComm& comm)
    {
      comm.recv((void*)&x, 1, MPITraits<T>::type(), tag, src);
    }

  /* ----------- generic functions for arrays of primitives ----------- */

  template <class T>
    inline void ContainerComm<T>::bcast(TSFArray<T>& x, int src, const MPIComm& comm)
    {
      try
        {
          int len = x.length();
          ContainerComm<int>::bcast(len, src, comm);

          if (comm.getRank() != src)
            {
              x.resize(len);
            }
          if (len==0) return;

          /* then broadcast the contents */
          comm.bcast((void*) &(x[0]), (int) len,
                     MPITraits<T>::type(), src);
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in TSFArrayComm::bcast(TSFArray<T>)");
        }
    }


  template <class T>
    inline void ContainerComm<T>::send(const TSFArray<T>& x, int tag,
                                       int dest, const MPIComm& comm)
    {
      try
        {
          /* first send the length */
          int len = x.length();
          ContainerComm<int>::send(len, tag, dest, comm);

          /* now send the data */
          comm.send((void*) &(x[0]), len,
                    MPITraits<T>::type(), tag, dest);
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in TSFArrayComm::send(int)");
        }
    }

  template <class T>
    inline void ContainerComm<T>::recv(TSFArray<T>& x, int tag, int src, const MPIComm& comm)
    {
      try
        {
          /* first recv the length */
          int len;
          ContainerComm<int>::recv(len, tag, src, comm);

          /* set the size of the array to the recvd length */
          x.resize(len);

          /* recv the data */
          comm.recv((void*)&(x[0]), len, MPITraits<T>::type(), tag, src);
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in TSFArrayComm::recv(int)");
        }
    }

  /* ---------- generic function for arrays of arrays ----------- */

  template <class T>
    inline void ContainerComm<T>::bcast(TSFArray<TSFArray<T> >& x, int src, const MPIComm& comm)
    {
      try
        {
          TSFArray<T> bigTSFArray;
          TSFArray<int> offsets;

          if (src==comm.getRank())
            {
              getBigTSFArray(x, bigTSFArray, offsets);
            }

          bcast(bigTSFArray, src, comm);
          ContainerComm<int>::bcast(offsets, src, comm);

          if (src != comm.getRank())
            {
              getSmallTSFArrays(bigTSFArray, offsets, x);
            }
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in TSFArrayComm::bcast(TSFArray<T>)");
        }
    }

  /* ---------- generic gather and scatter ------------------------ */

  template <class T> inline
    void ContainerComm<T>::allToAll(const TSFArray<T>& outgoing,
                                    TSFArray<TSFArray<T> >& incoming,
                                    const MPIComm& comm)
    {
      try
        {
          int numProcs = comm.getNProc();

          // catch degenerate case
          if (numProcs==1)
            {
              incoming.resize(1);
              incoming[0] = outgoing;
              return;
            }

          T* sendBuf = new T[numProcs * outgoing.length()];
          if (sendBuf==0)
            TSFError::raise("Comm::allToAll failed to allocate sendBuf");
          T* recvBuf = new T[numProcs * outgoing.length()];
          if (recvBuf==0)
            TSFError::raise("Comm::allToAll failed to allocate recvBuf");

          int i;
          for (i=0; i<numProcs; i++)
            {
              for (int j=0; j<outgoing.length(); j++)
                {
                  sendBuf[i*outgoing.length() + j] = outgoing[j];
                }
            }

          comm.allToAll(sendBuf, outgoing.length(), MPITraits<T>::type(),
                        recvBuf, outgoing.length(), MPITraits<T>::type());

          incoming.resize(numProcs);

          for (i=0; i<numProcs; i++)
            {
              incoming[i].resize(outgoing.length());
              for (int j=0; j<outgoing.length(); j++)
                {
                  incoming[i][j] = recvBuf[i*outgoing.length() + j];
                }
            }

          delete [] sendBuf;
          delete [] recvBuf;
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in Comm::allToAll(const TSFArray<int>& outgoing, ...)");
        }
    }

  template <class T> inline
    void ContainerComm<T>::allToAll(const TSFArray<TSFArray<T> >& outgoing,
                                    TSFArray<TSFArray<T> >& incoming, const MPIComm& comm)
    {
      try
        {
          int numProcs = comm.getNProc();

          // catch degenerate case
          if (numProcs==1)
            {
              incoming = outgoing;
              return;
            }

          int* sendMesgLength = new int[numProcs];
          if (sendMesgLength==0)
            TSFError::raise("failed to allocate sendMesgLength");
          int* recvMesgLength = new int[numProcs];
          if (recvMesgLength==0)
            TSFError::raise("failed to allocate recvMesgLength");

          int p = 0;
          for (p=0; p<numProcs; p++)
            {
              sendMesgLength[p] = outgoing[p].length();
            }

          comm.allToAll(sendMesgLength, 1, MPIComm::INT,
                        recvMesgLength, 1, MPIComm::INT);


          int totalSendLength = 0;
          int totalRecvLength = 0;
          for (p=0; p<numProcs; p++)
            {
              totalSendLength += sendMesgLength[p];
              totalRecvLength += recvMesgLength[p];
            }

          T* sendBuf = new T[totalSendLength];
          if (sendBuf==0)
            TSFError::raise("failed to allocate sendBuf");
          T* recvBuf = new T[totalRecvLength];
          if (recvBuf==0)
            TSFError::raise("failed to allocate recvBuf");

          int* sendDisp = new int[numProcs];
          if (sendDisp==0)
            TSFError::raise("failed to allocate sendDisp");
          int* recvDisp = new int[numProcs];
          if (recvDisp==0)
            TSFError::raise("failed to allocate recvDisp");

          int count = 0;
          sendDisp[0] = 0;
          recvDisp[0] = 0;

          for (p=0; p<numProcs; p++)
            {
              for (int i=0; i<outgoing[p].length(); i++)
                {
                  sendBuf[count] = outgoing[p][i];
                  count++;
                }
              if (p>0)
                {
                  sendDisp[p] = sendDisp[p-1] + sendMesgLength[p-1];
                  recvDisp[p] = recvDisp[p-1] + recvMesgLength[p-1];
                }
            }

          comm.allToAllv(sendBuf, sendMesgLength,
                         sendDisp, MPITraits<T>::type(),
                         recvBuf, recvMesgLength,
                         recvDisp, MPITraits<T>::type());

          incoming.resize(numProcs);
          for (p=0; p<numProcs; p++)
            {
              incoming[p].resize(recvMesgLength[p]);
              for (int i=0; i<recvMesgLength[p]; i++)
                {
                  incoming[p][i] = recvBuf[recvDisp[p] + i];
                }
            }

          delete [] sendBuf;
          delete [] sendMesgLength;
          delete [] sendDisp;
          delete [] recvBuf;
          delete [] recvMesgLength;
          delete [] recvDisp;
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in TSFArrayComm::allToAll(const TSFArray<TSFArray<T> >& outgoing...)");
        }
    }

  template <class T> inline
    void ContainerComm<T>::allGather(const T& outgoing, TSFArray<T>& incoming,
                                     const MPIComm& comm)
    {
      int nProc = comm.getNProc();
      incoming.resize(nProc);

      if (nProc==1)
        {
          incoming[0] = outgoing;
        }
      else
        {
          comm.allGather((void*) &outgoing, 1, MPITraits<T>::type(),
                         (void*) &(incoming[0]), 1, MPITraits<T>::type());
        }
    }

  template <class T> inline
    void ContainerComm<T>::accumulate(const T& localValue, TSFArray<T>& sums,
                                      const MPIComm& comm)
    {
      TSFArray<T> contributions;
      allGather(localValue, contributions, comm);
      sums.resize(comm.getNProc());
      sums[0] = 0;

      for (int i=0; i<comm.getNProc()-1; i++)
        {
          sums[i+1] = sums[i] + contributions[i];
        }
    }




  template <class T> inline
    void ContainerComm<T>::getBigTSFArray(const TSFArray<TSFArray<T> >& x, TSFArray<T>& bigTSFArray,
                                          TSFArray<int>& offsets)
    {
      offsets.resize(x.length()+1);
      int totalLength = 0;

      for (int i=0; i<x.length(); i++)
        {
          offsets[i] = totalLength;
          totalLength += x[i].length();
        }
      offsets[x.length()] = totalLength;

      bigTSFArray.resize(totalLength);

      for (int i=0; i<x.length(); i++)
        {
          for (int j=0; j<x[i].length(); j++)
            {
              bigTSFArray[offsets[i]+j] = x[i][j];
            }
        }
    }

  template <class T> inline
    void ContainerComm<T>::getSmallTSFArrays(const TSFArray<T>& bigTSFArray,
                                             const TSFArray<int>& offsets,
                                             TSFArray<TSFArray<T> >& x)
    {
      x.resize(offsets.length()-1);
      for (int i=0; i<x.length(); i++)
        {
          x[i].resize(offsets[i+1]-offsets[i]);
          for (int j=0; j<x[i].length(); j++)
            {
              x[i][j] = bigTSFArray[offsets[i] + j];
            }
        }
    }


  /* --------------- string specializations --------------------- */

  inline void ContainerComm<string>::bcast(string& x,
                                           int src, const MPIComm& comm)
    {
      int len = x.length();
      ContainerComm<int>::bcast(len, src, comm);

      x.resize(len);
      comm.bcast((void*)&(x[0]), len, MPITraits<char>::type(), src);
    }

  inline void ContainerComm<string>::send(const string& x, int tag, int dest,
                                          const MPIComm& comm)
    {
      int len = x.length();
      ContainerComm<int>::send(len, tag, dest, comm);

      void* start = (void*) x.c_str();
      comm.send(start, len, MPITraits<char>::type(), tag, dest);
    }

  inline void ContainerComm<string>::recv(string& x, int tag, int src,
                                          const MPIComm& comm)
    {
      int len;
      ContainerComm<int>::recv(len, tag, src, comm);

      x.resize(len);

      void* start = (void*) x.c_str();
      comm.recv(start, len, MPITraits<char>::type(), tag, src);
    }

  inline void ContainerComm<string>::bcast(TSFArray<string>& x, int src,
                                           const MPIComm& comm)
    {
      try
        {
          /* begin by packing all the data into a big char array. This will
           * take a little time, but will be cheaper than multiple MPI calls */
          TSFArray<char> bigTSFArray;
          TSFArray<int> offsets;
          if (comm.getRank()==src)
            {
              getBigTSFArray(x, bigTSFArray, offsets);
            }

          /* now broadcast the big array and the offsets */
          ContainerComm<char>::bcast(bigTSFArray, src, comm);
          ContainerComm<int>::bcast(offsets, src, comm);

          /* finally, reassemble the array of strings */
          if (comm.getRank() != src)
            {
              getStrings(bigTSFArray, offsets, x);
            }

        }
      catch(exception& e)
        {
          TSFError::trace(e, "in bcast(TSFArray<string>)");
        }
    }

  inline void ContainerComm<string>::bcast(TSFArray<TSFArray<string> >& x,
                                           int src, const MPIComm& comm)
    {
      try
        {
          int len = x.length();
          ContainerComm<int>::bcast(len, src, comm);

          x.resize(len);
          for (int i=0; i<len; i++)
            {
              ContainerComm<string>::bcast(x[i], src, comm);
            }
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in ContainerComm<string>::bcast(TSFArray<TSFArray<string>>)");
        }
    }


  inline void ContainerComm<string>::send(const TSFArray<string>& x, int tag,
                                          int dest, const MPIComm& comm)
    {
      try
        {
          TSFArray<char> bigTSFArray;
          TSFArray<int> offsets;

          getBigTSFArray(x, bigTSFArray, offsets);

          ContainerComm<int>::send(offsets, tag, dest, comm);
          ContainerComm<char>::send(bigTSFArray, tag, dest, comm);
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in send(TSFArray<string>)");
        }
    }

  inline void ContainerComm<string>::recv(TSFArray<string>& x,
                                          int tag, int src, const MPIComm& comm)
    {
      try
        {
          TSFArray<char> bigTSFArray;
          TSFArray<int> offsets;

          ContainerComm<int>::recv(offsets, tag, src, comm);
          ContainerComm<char>::recv(bigTSFArray, tag, src, comm);

          getStrings(bigTSFArray, offsets, x);
        }
      catch(exception& e)
        {
          TSFError::trace(e, "in recv(TSFArray<string>)");
        }
    }

  inline void ContainerComm<string>::allGather(const string& outgoing,
                                               TSFArray<string>& incoming,
                                               const MPIComm& comm)
    {
      int nProc = comm.getNProc();

      int sendCount = outgoing.length();

      incoming.resize(nProc);

      int* recvCounts = new int[nProc];
      int* recvDisplacements = new int[nProc];

      /* share lengths with all procs */
      comm.allGather((void*) &sendCount, 1, MPIComm::INT,
                     (void*) recvCounts, 1, MPIComm::INT);


      int recvSize = 0;
      recvDisplacements[0] = 0;
      for (int i=0; i<nProc; i++)
        {
          recvSize += recvCounts[i];
          if (i < nProc-1)
            {
              recvDisplacements[i+1] = recvDisplacements[i]+recvCounts[i];
            }
        }

      char* recvBuf = new char[recvSize];

      comm.allGatherv((void*) outgoing.c_str(), sendCount, MPIComm::CHAR,
                      recvBuf, recvCounts, recvDisplacements, MPIComm::CHAR);

      for (int j=0; j<nProc; j++)
        {
          char* start = recvBuf + recvDisplacements[j];
          char* tmp = new char[recvCounts[j]+1];
          memcpy(tmp, start, recvCounts[j]);
          tmp[recvCounts[j]] = '\0';
          incoming[j] = string(tmp);
          delete [] tmp;
        }

      delete [] recvCounts;
      delete [] recvDisplacements;
      delete [] recvBuf;
    }


  inline void ContainerComm<string>::getBigTSFArray(const TSFArray<string>& x,
                                                    TSFArray<char>& bigTSFArray,
                                                    TSFArray<int>& offsets)
    {
      offsets.resize(x.length()+1);
      int totalLength = 0;

      for (int i=0; i<x.length(); i++)
        {
          offsets[i] = totalLength;
          totalLength += x[i].length();
        }
      offsets[x.length()] = totalLength;

      bigTSFArray.resize(totalLength);

      for (int i=0; i<x.length(); i++)
        {
          for (unsigned int j=0; j<x[i].length(); j++)
            {
              bigTSFArray[offsets[i]+j] = x[i][j];
            }
        }
    }

  inline void ContainerComm<string>::getStrings(const TSFArray<char>& bigTSFArray,
                                                const TSFArray<int>& offsets,
                                                TSFArray<string>& x)
    {
      x.resize(offsets.length()-1);
      for (int i=0; i<x.length(); i++)
        {
          x[i].resize(offsets[i+1]-offsets[i]);
          for (unsigned int j=0; j<x[i].length(); j++)
            {
              x[i][j] = bigTSFArray[offsets[i] + j];
            }
        }
    }
}


#endif

---

Contact:

Kevin Long (krlong@ca.sandia.gov)


Documentation generated by