00001
00002
00003
00004
00005 #ifndef PLAYA_MPICONTAINERCOMM_H
00006 #define PLAYA_MPICONTAINERCOMM_H
00007
00008
00009
00010
00011
00012 #include "PlayaMPIComm.hpp"
00013 #include "PlayaMPITraits.hpp"
00014
00015 namespace Playa
00016 {
00017 using Teuchos::Array;
00018
00019
00020
00021
00022
00023
00024 template <class T> class MPIContainerComm
00025 {
00026 public:
00027
00028
00029 static void bcast(T& x, int src, const MPIComm& comm);
00030
00031
00032 static void bcast(Array<T>& x, int src, const MPIComm& comm);
00033
00034
00035 static void bcast(Array<Array<T> >& x,
00036 int src, const MPIComm& comm);
00037
00038
00039 static void allGather(const T& outgoing,
00040 Array<T>& incoming,
00041 const MPIComm& comm);
00042
00043
00044 static void allToAll(const Array<T>& outgoing,
00045 Array<Array<T> >& incoming,
00046 const MPIComm& comm);
00047
00048
00049 static void allToAll(const Array<Array<T> >& outgoing,
00050 Array<Array<T> >& incoming,
00051 const MPIComm& comm);
00052
00053
00054 static void gatherv(const Array<T>& outgoing,
00055 Array<Array<T> >& incoming,
00056 int rootRank,
00057 const MPIComm& comm);
00058
00059
00060 static void accumulate(const T& localValue, Array<T>& sums, T& total,
00061 const MPIComm& comm);
00062
00063 private:
00064
00065 static void getBigArray(const Array<Array<T> >& x,
00066 Array<T>& bigArray,
00067 Array<int>& offsets);
00068
00069
00070 static void getSmallArrays(const Array<T>& bigArray,
00071 const Array<int>& offsets,
00072 Array<Array<T> >& x);
00073
00074
00075 };
00076
00077
00078 #ifndef DOXYGEN_SHOULD_SKIP_THIS
00079
00080
00081
00082 template <> class MPIContainerComm<std::string>
00083 {
00084 public:
00085 static void bcast(std::string& x, int src, const MPIComm& comm);
00086
00087
00088 static void bcast(Array<std::string>& x, int src, const MPIComm& comm);
00089
00090
00091 static void bcast(Array<Array<std::string> >& x,
00092 int src, const MPIComm& comm);
00093
00094
00095 static void allGather(const std::string& outgoing,
00096 Array<std::string>& incoming,
00097 const MPIComm& comm);
00098
00099
00100 static void gatherv(const Array<std::string>& outgoing,
00101 Array<Array<std::string> >& incoming,
00102 int rootRank,
00103 const MPIComm& comm);
00104
00105
00106
00107
00108
00109
00110
00111
00112 static void pack(const Array<std::string>& x,
00113 Array<char>& packed);
00114
00115
00116
00117 static void unpack(const Array<char>& packed,
00118 Array<std::string>& x);
00119 private:
00120
00121 static void getBigArray(const Array<std::string>& x,
00122 Array<char>& bigArray,
00123 Array<int>& offsets);
00124
00125
00126
00127 static void getStrings(const Array<char>& bigArray,
00128 const Array<int>& offsets,
00129 Array<std::string>& x);
00130 };
00131
00132 #endif // DOXYGEN_SHOULD_SKIP_THIS
00133
00134
00135
00136 template <class T> inline void MPIContainerComm<T>::bcast(T& x, int src,
00137 const MPIComm& comm)
00138 {
00139 comm.bcast((void*)&x, 1, MPITraits<T>::type(), src);
00140 }
00141
00142
00143
00144
00145 template <class T>
00146 inline void MPIContainerComm<T>::bcast(Array<T>& x, int src, const MPIComm& comm)
00147 {
00148 int len = x.length();
00149 MPIContainerComm<int>::bcast(len, src, comm);
00150
00151 if (comm.getRank() != src)
00152 {
00153 x.resize(len);
00154 }
00155 if (len==0) return;
00156
00157
00158 comm.bcast((void*) &(x[0]), (int) len,
00159 MPITraits<T>::type(), src);
00160 }
00161
00162
00163
00164
00165
00166 template <class T>
00167 inline void MPIContainerComm<T>::bcast(Array<Array<T> >& x, int src, const MPIComm& comm)
00168 {
00169 Array<T> bigArray;
00170 Array<int> offsets;
00171
00172 if (src==comm.getRank())
00173 {
00174 getBigArray(x, bigArray, offsets);
00175 }
00176
00177 bcast(bigArray, src, comm);
00178 MPIContainerComm<int>::bcast(offsets, src, comm);
00179
00180 if (src != comm.getRank())
00181 {
00182 getSmallArrays(bigArray, offsets, x);
00183 }
00184 }
00185
00186
00187
00188 template <class T> inline
00189 void MPIContainerComm<T>::allToAll(const Array<T>& outgoing,
00190 Array<Array<T> >& incoming,
00191 const MPIComm& comm)
00192 {
00193 int numProcs = comm.getNProc();
00194
00195
00196 if (numProcs==1)
00197 {
00198 incoming.resize(1);
00199 incoming[0] = outgoing;
00200 return;
00201 }
00202
00203 Array<T> sb(numProcs * outgoing.length());
00204 Array<T> rb(numProcs * outgoing.length());
00205
00206 T* sendBuf = new T[numProcs * outgoing.length()];
00207 TEUCHOS_TEST_FOR_EXCEPTION(sendBuf==0,
00208 std::runtime_error, "Comm::allToAll failed to allocate sendBuf");
00209
00210 T* recvBuf = new T[numProcs * outgoing.length()];
00211 TEUCHOS_TEST_FOR_EXCEPTION(recvBuf==0,
00212 std::runtime_error, "Comm::allToAll failed to allocate recvBuf");
00213
00214 int i;
00215 for (i=0; i<numProcs; i++)
00216 {
00217 for (int j=0; j<outgoing.length(); j++)
00218 {
00219 sendBuf[i*outgoing.length() + j] = outgoing[j];
00220 }
00221 }
00222
00223
00224
00225 comm.allToAll(sendBuf, outgoing.length(), MPITraits<T>::type(),
00226 recvBuf, outgoing.length(), MPITraits<T>::type());
00227
00228 incoming.resize(numProcs);
00229
00230 for (i=0; i<numProcs; i++)
00231 {
00232 incoming[i].resize(outgoing.length());
00233 for (int j=0; j<outgoing.length(); j++)
00234 {
00235 incoming[i][j] = recvBuf[i*outgoing.length() + j];
00236 }
00237 }
00238
00239 delete [] sendBuf;
00240 delete [] recvBuf;
00241 }
00242
00243 template <class T> inline
00244 void MPIContainerComm<T>::allToAll(const Array<Array<T> >& outgoing,
00245 Array<Array<T> >& incoming, const MPIComm& comm)
00246 {
00247 int numProcs = comm.getNProc();
00248
00249
00250 if (numProcs==1)
00251 {
00252 incoming = outgoing;
00253 return;
00254 }
00255
00256 int* sendMesgLength = new int[numProcs];
00257 TEUCHOS_TEST_FOR_EXCEPTION(sendMesgLength==0,
00258 std::runtime_error, "failed to allocate sendMesgLength");
00259 int* recvMesgLength = new int[numProcs];
00260 TEUCHOS_TEST_FOR_EXCEPTION(recvMesgLength==0,
00261 std::runtime_error, "failed to allocate recvMesgLength");
00262
00263 int p = 0;
00264 for (p=0; p<numProcs; p++)
00265 {
00266 sendMesgLength[p] = outgoing[p].length();
00267 }
00268
00269 comm.allToAll(sendMesgLength, 1, MPIDataType::intType(),
00270 recvMesgLength, 1, MPIDataType::intType());
00271
00272
00273 int totalSendLength = 0;
00274 int totalRecvLength = 0;
00275 for (p=0; p<numProcs; p++)
00276 {
00277 totalSendLength += sendMesgLength[p];
00278 totalRecvLength += recvMesgLength[p];
00279 }
00280
00281 T* sendBuf = new T[totalSendLength];
00282 TEUCHOS_TEST_FOR_EXCEPTION(sendBuf==0,
00283 std::runtime_error, "failed to allocate sendBuf");
00284 T* recvBuf = new T[totalRecvLength];
00285 TEUCHOS_TEST_FOR_EXCEPTION(recvBuf==0,
00286 std::runtime_error, "failed to allocate recvBuf");
00287
00288 int* sendDisp = new int[numProcs];
00289 TEUCHOS_TEST_FOR_EXCEPTION(sendDisp==0,
00290 std::runtime_error, "failed to allocate sendDisp");
00291 int* recvDisp = new int[numProcs];
00292 TEUCHOS_TEST_FOR_EXCEPTION(recvDisp==0,
00293 std::runtime_error, "failed to allocate recvDisp");
00294
00295 int count = 0;
00296 sendDisp[0] = 0;
00297 recvDisp[0] = 0;
00298
00299 for (p=0; p<numProcs; p++)
00300 {
00301 for (int i=0; i<outgoing[p].length(); i++)
00302 {
00303 sendBuf[count] = outgoing[p][i];
00304 count++;
00305 }
00306 if (p>0)
00307 {
00308 sendDisp[p] = sendDisp[p-1] + sendMesgLength[p-1];
00309 recvDisp[p] = recvDisp[p-1] + recvMesgLength[p-1];
00310 }
00311 }
00312
00313 comm.allToAllv(sendBuf, sendMesgLength,
00314 sendDisp, MPITraits<T>::type(),
00315 recvBuf, recvMesgLength,
00316 recvDisp, MPITraits<T>::type());
00317
00318 incoming.resize(numProcs);
00319 for (p=0; p<numProcs; p++)
00320 {
00321 incoming[p].resize(recvMesgLength[p]);
00322 for (int i=0; i<recvMesgLength[p]; i++)
00323 {
00324 incoming[p][i] = recvBuf[recvDisp[p] + i];
00325 }
00326 }
00327
00328 delete [] sendBuf;
00329 delete [] sendMesgLength;
00330 delete [] sendDisp;
00331 delete [] recvBuf;
00332 delete [] recvMesgLength;
00333 delete [] recvDisp;
00334 }
00335
00336 template <class T> inline
00337 void MPIContainerComm<T>::allGather(const T& outgoing, Array<T>& incoming,
00338 const MPIComm& comm)
00339 {
00340 int nProc = comm.getNProc();
00341 incoming.resize(nProc);
00342
00343 if (nProc==1)
00344 {
00345 incoming[0] = outgoing;
00346 }
00347 else
00348 {
00349 comm.allGather((void*) &outgoing, 1, MPITraits<T>::type(),
00350 (void*) &(incoming[0]), 1, MPITraits<T>::type());
00351 }
00352 }
00353
00354 template <class T> inline
00355 void MPIContainerComm<T>::accumulate(const T& localValue, Array<T>& sums,
00356 T& total,
00357 const MPIComm& comm)
00358 {
00359 Array<T> contributions;
00360 allGather(localValue, contributions, comm);
00361 sums.resize(comm.getNProc());
00362 sums[0] = 0;
00363 total = contributions[0];
00364
00365 for (int i=0; i<comm.getNProc()-1; i++)
00366 {
00367 total += contributions[i+1];
00368 sums[i+1] = sums[i] + contributions[i];
00369 }
00370 }
00371
00372
00373
00374
00375 template <class T> inline
00376 void MPIContainerComm<T>::getBigArray(const Array<Array<T> >& x, Array<T>& bigArray,
00377 Array<int>& offsets)
00378 {
00379 offsets.resize(x.length()+1);
00380 int totalLength = 0;
00381
00382 for (int i=0; i<x.length(); i++)
00383 {
00384 offsets[i] = totalLength;
00385 totalLength += x[i].length();
00386 }
00387 offsets[x.length()] = totalLength;
00388
00389 bigArray.resize(totalLength);
00390
00391 for (int i=0; i<x.length(); i++)
00392 {
00393 for (int j=0; j<x[i].length(); j++)
00394 {
00395 bigArray[offsets[i]+j] = x[i][j];
00396 }
00397 }
00398 }
00399
00400 template <class T> inline
00401 void MPIContainerComm<T>::getSmallArrays(const Array<T>& bigArray,
00402 const Array<int>& offsets,
00403 Array<Array<T> >& x)
00404 {
00405 x.resize(offsets.length()-1);
00406 for (int i=0; i<x.length(); i++)
00407 {
00408 x[i].resize(offsets[i+1]-offsets[i]);
00409 for (int j=0; j<x[i].length(); j++)
00410 {
00411 x[i][j] = bigArray[offsets[i] + j];
00412 }
00413 }
00414 }
00415
00416
00417 #ifndef DOXYGEN_SHOULD_SKIP_THIS
00418
00419
00420
00421 inline void MPIContainerComm<std::string>::bcast(std::string& x,
00422 int src, const MPIComm& comm)
00423 {
00424 int len = x.length();
00425 MPIContainerComm<int>::bcast(len, src, comm);
00426
00427 x.resize(len);
00428 comm.bcast((void*)&(x[0]), len, MPITraits<char>::type(), src);
00429 }
00430
00431
00432 inline void MPIContainerComm<std::string>::bcast(Array<std::string>& x, int src,
00433 const MPIComm& comm)
00434 {
00435
00436
00437 Array<char> bigArray;
00438 Array<int> offsets;
00439 if (comm.getRank()==src)
00440 {
00441 getBigArray(x, bigArray, offsets);
00442 }
00443
00444
00445 MPIContainerComm<char>::bcast(bigArray, src, comm);
00446 MPIContainerComm<int>::bcast(offsets, src, comm);
00447
00448
00449 if (comm.getRank() != src)
00450 {
00451 getStrings(bigArray, offsets, x);
00452 }
00453 }
00454
00455 inline void MPIContainerComm<std::string>::bcast(Array<Array<std::string> >& x,
00456 int src, const MPIComm& comm)
00457 {
00458 int len = x.length();
00459 MPIContainerComm<int>::bcast(len, src, comm);
00460
00461 x.resize(len);
00462 for (int i=0; i<len; i++)
00463 {
00464 MPIContainerComm<std::string>::bcast(x[i], src, comm);
00465 }
00466 }
00467
00468
00469 inline void MPIContainerComm<std::string>::allGather(const std::string& outgoing,
00470 Array<std::string>& incoming,
00471 const MPIComm& comm)
00472 {
00473 int nProc = comm.getNProc();
00474
00475 int sendCount = outgoing.length();
00476
00477 incoming.resize(nProc);
00478
00479 int* recvCounts = new int[nProc];
00480 int* recvDisplacements = new int[nProc];
00481
00482
00483 comm.allGather((void*) &sendCount, 1, MPIDataType::intType(),
00484 (void*) recvCounts, 1, MPIDataType::intType());
00485
00486
00487 int recvSize = 0;
00488 recvDisplacements[0] = 0;
00489 for (int i=0; i<nProc; i++)
00490 {
00491 recvSize += recvCounts[i];
00492 if (i < nProc-1)
00493 {
00494 recvDisplacements[i+1] = recvDisplacements[i]+recvCounts[i];
00495 }
00496 }
00497
00498 char* recvBuf = new char[recvSize];
00499
00500 comm.allGatherv((void*) outgoing.c_str(), sendCount, MPIDataType::charType(),
00501 recvBuf, recvCounts, recvDisplacements, MPIDataType::charType());
00502
00503 for (int j=0; j<nProc; j++)
00504 {
00505 char* start = recvBuf + recvDisplacements[j];
00506 char* tmp = new char[recvCounts[j]+1];
00507 std::memcpy(tmp, start, recvCounts[j]);
00508 tmp[recvCounts[j]] = '\0';
00509 incoming[j] = std::string(tmp);
00510 delete [] tmp;
00511 }
00512
00513 delete [] recvCounts;
00514 delete [] recvDisplacements;
00515 delete [] recvBuf;
00516 }
00517
00518 inline void MPIContainerComm<std::string>::gatherv(const Array<std::string>& outgoing,
00519 Array<Array<std::string> >& incoming,
00520 int root,
00521 const MPIComm& comm)
00522 {
00523 int nProc = comm.getNProc();
00524
00525 Array<char> packedLocalArray;
00526 pack(outgoing, packedLocalArray);
00527
00528 int sendCount = packedLocalArray.size();
00529
00530
00531 Array<int> recvCounts(nProc);
00532 Array<int> recvDisplacements(nProc);
00533
00534 comm.gather((void*) &sendCount, 1, MPIDataType::intType(),
00535 (void*) &(recvCounts[0]), 1, MPIDataType::intType(), root);
00536
00537
00538 int recvSize = 0;
00539 if (root == comm.getRank())
00540 {
00541 recvDisplacements[0] = 0;
00542 for (int i=0; i<nProc; i++)
00543 {
00544 recvSize += recvCounts[i];
00545 if (i < nProc-1)
00546 {
00547 recvDisplacements[i+1] = recvDisplacements[i]+recvCounts[i];
00548 }
00549 }
00550 }
00551
00552
00553 Array<char> recvBuf(std::max(1,recvSize));
00554
00555
00556 void* sendBuf = (void*) &(packedLocalArray[0]);
00557 void* inBuf = (void*) &(recvBuf[0]);
00558 int* inCounts = &(recvCounts[0]);
00559 int* inDisps = &(recvDisplacements[0]);
00560
00561
00562 comm.gatherv( sendBuf, sendCount, MPIDataType::charType(),
00563 inBuf, inCounts, inDisps,
00564 MPIDataType::charType(), root);
00565
00566
00567 if (comm.getRank()==root)
00568 {
00569 incoming.resize(nProc);
00570 for (int j=0; j<nProc; j++)
00571 {
00572 char* start = &(recvBuf[0]) + recvDisplacements[j];
00573 Array<char> tmp(recvCounts[j]+1);
00574 std::memcpy(&(tmp[0]), start, recvCounts[j]);
00575 tmp[recvCounts[j]] = '\0';
00576 unpack(tmp, incoming[j]);
00577 }
00578 }
00579
00580
00581 }
00582
00583
00584 inline void MPIContainerComm<std::string>::getBigArray(const Array<std::string>& x,
00585 Array<char>& bigArray,
00586 Array<int>& offsets)
00587 {
00588 offsets.resize(x.length()+1);
00589 int totalLength = 0;
00590
00591 for (int i=0; i<x.length(); i++)
00592 {
00593 offsets[i] = totalLength;
00594 totalLength += x[i].length();
00595 }
00596 offsets[x.length()] = totalLength;
00597
00598 bigArray.resize(totalLength);
00599
00600 for (int i=0; i<x.length(); i++)
00601 {
00602 for (unsigned int j=0; j<x[i].length(); j++)
00603 {
00604 bigArray[offsets[i]+j] = x[i][j];
00605 }
00606 }
00607 }
00608
00609 inline void MPIContainerComm<std::string>::pack(const Array<std::string>& x,
00610 Array<char>& bigArray)
00611 {
00612 Array<int> offsets(x.size()+1);
00613 int headerSize = (x.size()+2) * sizeof(int);
00614
00615 int totalLength = headerSize;
00616
00617 for (int i=0; i<x.length(); i++)
00618 {
00619 offsets[i] = totalLength;
00620 totalLength += x[i].length();
00621 }
00622 offsets[x.length()] = totalLength;
00623
00624
00625
00626
00627
00628 bigArray.resize(totalLength);
00629
00630 int* header = reinterpret_cast<int*>( &(bigArray[0]) );
00631 header[0] = x.size();
00632 for (Array<std::string>::size_type i=0; i<=x.size(); i++)
00633 {
00634 header[i+1] = offsets[i];
00635 }
00636
00637 for (int i=0; i<x.length(); i++)
00638 {
00639 for (unsigned int j=0; j<x[i].length(); j++)
00640 {
00641 bigArray[offsets[i]+j] = x[i][j];
00642 }
00643 }
00644 }
00645
00646 inline void MPIContainerComm<std::string>::unpack(const Array<char>& packed,
00647 Array<std::string>& x)
00648 {
00649 const int* header = reinterpret_cast<const int*>( &(packed[0]) );
00650
00651 x.resize(header[0]);
00652 Array<int> offsets(x.size()+1);
00653 for (Array<std::string>::size_type i=0; i<=x.size(); i++) offsets[i] = header[i+1];
00654
00655 for (Array<std::string>::size_type i=0; i<x.size(); i++)
00656 {
00657 x[i].resize(offsets[i+1]-offsets[i]);
00658 for (std::string::size_type j=0; j<x[i].length(); j++)
00659 {
00660 x[i][j] = packed[offsets[i] + j];
00661 }
00662 }
00663 }
00664
00665 inline void MPIContainerComm<std::string>::getStrings(const Array<char>& bigArray,
00666 const Array<int>& offsets,
00667 Array<std::string>& x)
00668 {
00669 x.resize(offsets.length()-1);
00670 for (int i=0; i<x.length(); i++)
00671 {
00672 x[i].resize(offsets[i+1]-offsets[i]);
00673 for (unsigned int j=0; j<x[i].length(); j++)
00674 {
00675 x[i][j] = bigArray[offsets[i] + j];
00676 }
00677 }
00678 }
00679 #endif // DOXYGEN_SHOULD_SKIP_THIS
00680
00681 }
00682
00683
00684 #endif
00685
00686