3 #ifndef DUNE_COMMUNICATOR
4 #define DUNE_COMMUNICATOR
12 #include <type_traits>
146 typedef typename V::value_type IndexedType;
152 typedef SizeOne IndexedTypeFlag;
162 static const void* getAddress(
const V& v,
int index);
169 static int getSize(
const V&,
int index);
172 template<
class K,
int n>
class FieldVector;
174 template<
class B,
class A>
class VariableBlockVector;
176 template<
class K,
class A,
int n>
177 struct CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >
179 typedef VariableBlockVector<FieldVector<K, n>, A> Type;
181 typedef typename Type::B IndexedType;
183 typedef VariableSize IndexedTypeFlag;
185 static const void* getAddress(
const Type& v,
int i);
187 static int getSize(
const Type& v,
int i);
193 class CommunicationError :
public IOError
200 struct CopyGatherScatter
202 typedef typename CommPolicy<T>::IndexedType IndexedType;
204 static const IndexedType& gather(
const T& vec, std::size_t i);
206 static void scatter(T& vec,
const IndexedType& v, std::size_t i);
222 class DatatypeCommunicator :
public InterfaceBuilder
229 typedef T ParallelIndexSet;
234 typedef Dune::RemoteIndices<ParallelIndexSet> RemoteIndices;
239 typedef typename RemoteIndices::GlobalIndex GlobalIndex;
244 typedef typename RemoteIndices::Attribute Attribute;
249 typedef typename RemoteIndices::LocalIndex LocalIndex;
254 DatatypeCommunicator();
259 ~DatatypeCommunicator();
287 template<
class T1,
class T2,
class V>
288 void build(
const RemoteIndices& remoteIndices,
const T1& sourceFlags, V& sendData,
const T2& destFlags, V& receiveData);
315 const RemoteIndices* remoteIndices_;
317 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >
323 MessageTypeMap messageTypes;
330 MPI_Request* requests_[2];
340 template<
class V,
bool FORWARD>
341 void createRequests(V& sendData, V& receiveData);
346 template<
class T1,
class T2,
class V,
bool send>
347 void createDataTypes(
const T1& source,
const T2& destination, V& data);
352 void sendRecv(MPI_Request* req);
357 struct IndexedTypeInformation
367 displ =
new MPI_Aint[i];
401 struct MPIDatatypeInformation
407 MPIDatatypeInformation(
const V& data) : data_(data)
415 void reserve(
int proc,
int size)
417 information_[proc].build(size);
425 void add(
int proc,
int local)
427 IndexedTypeInformation& info=information_[proc];
428 assert((info.elements)<info.size);
429 MPI_Get_address(
const_cast<void*
>(CommPolicy<V>::getAddress(data_, local)),
430 info.displ+info.elements);
431 info.length[info.elements]=CommPolicy<V>::getSize(data_, local);
439 std::map<int,IndexedTypeInformation> information_;
458 class BufferedCommunicator
465 BufferedCommunicator();
473 template<
class Data,
class Interface>
474 typename std::enable_if<std::is_same<SizeOne,typename CommPolicy<Data>::IndexedTypeFlag>::value,
void>::type
475 build(
const Interface& interface);
484 template<
class Data,
class Interface>
485 void build(
const Data& source,
const Data& target,
const Interface& interface);
515 template<
class GatherScatter,
class Data>
516 void forward(
const Data& source, Data& dest);
546 template<
class GatherScatter,
class Data>
547 void backward(Data& source,
const Data& dest);
574 template<
class GatherScatter,
class Data>
575 void forward(Data& data);
602 template<
class GatherScatter,
class Data>
603 void backward(Data& data);
613 ~BufferedCommunicator();
620 typedef std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
627 template<
class Data,
typename IndexedTypeFlag>
628 struct MessageSizeCalculator
636 struct MessageSizeCalculator<Data,SizeOne>
644 inline int operator()(
const InterfaceInformation& info)
const;
653 inline int operator()(
const Data& data,
const InterfaceInformation& info)
const;
661 struct MessageSizeCalculator<Data,VariableSize>
671 inline int operator()(
const Data& data,
const InterfaceInformation& info)
const;
677 template<
class Data,
class GatherScatter,
bool send,
typename IndexedTypeFlag>
678 struct MessageGatherer
685 template<
class Data,
class GatherScatter,
bool send>
686 struct MessageGatherer<Data,GatherScatter,send,SizeOne>
689 typedef typename CommPolicy<Data>::IndexedType Type;
695 typedef GatherScatter Gatherer;
713 inline void operator()(
const InterfaceMap& interface,
const Data& data, Type* buffer,
size_t bufferSize)
const;
720 template<
class Data,
class GatherScatter,
bool send>
721 struct MessageGatherer<Data,GatherScatter,send,VariableSize>
724 typedef typename CommPolicy<Data>::IndexedType Type;
730 typedef GatherScatter Gatherer;
748 inline void operator()(
const InterfaceMap& interface,
const Data& data, Type* buffer,
size_t bufferSize)
const;
754 template<
class Data,
class GatherScatter,
bool send,
typename IndexedTypeFlag>
755 struct MessageScatterer
762 template<
class Data,
class GatherScatter,
bool send>
763 struct MessageScatterer<Data,GatherScatter,send,SizeOne>
766 typedef typename CommPolicy<Data>::IndexedType Type;
772 typedef GatherScatter Scatterer;
790 inline void operator()(
const InterfaceMap& interface, Data& data, Type* buffer,
const int& proc)
const;
796 template<
class Data,
class GatherScatter,
bool send>
797 struct MessageScatterer<Data,GatherScatter,send,VariableSize>
800 typedef typename CommPolicy<Data>::IndexedType Type;
806 typedef GatherScatter Scatterer;
824 inline void operator()(
const InterfaceMap& interface, Data& data, Type* buffer,
const int& proc)
const;
830 struct MessageInformation
834 : start_(0), size_(0)
844 MessageInformation(
size_t start,
size_t size)
845 : start_(start), size_(size)
863 typedef std::map<int,std::pair<MessageInformation,MessageInformation> >
868 InformationMap messageInformation_;
876 size_t bufferSize_[2];
888 std::map<int,std::pair<InterfaceInformation,InterfaceInformation> > interfaces_;
890 MPI_Comm communicator_;
895 template<
class GatherScatter,
bool FORWARD,
class Data>
896 void sendRecv(
const Data& source, Data& target);
903 inline const void* CommPolicy<V>::getAddress(
const V& v,
int index)
909 inline int CommPolicy<V>::getSize(
const V& v,
int index)
916 template<
class K,
class A,
int n>
917 inline const void* CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getAddress(
const Type& v,
int index)
919 return &(v[index][0]);
922 template<
class K,
class A,
int n>
923 inline int CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getSize(
const Type& v,
int index)
925 return v[index].getsize();
929 inline const typename CopyGatherScatter<T>::IndexedType& CopyGatherScatter<T>::gather(
const T & vec, std::size_t i)
935 inline void CopyGatherScatter<T>::scatter(T& vec,
const IndexedType& v, std::size_t i)
941 DatatypeCommunicator<T>::DatatypeCommunicator()
942 : remoteIndices_(0), created_(false)
951 DatatypeCommunicator<T>::~DatatypeCommunicator()
957 template<
class T1,
class T2,
class V>
958 inline void DatatypeCommunicator<T>::build(
const RemoteIndices& remoteIndices,
959 const T1& source, V& sendData,
960 const T2& destination, V& receiveData)
962 remoteIndices_ = &remoteIndices;
964 createDataTypes<T1,T2,V,false>(source,destination, receiveData);
965 createDataTypes<T1,T2,V,true>(source,destination, sendData);
966 createRequests<V,true>(sendData, receiveData);
967 createRequests<V,false>(receiveData, sendData);
972 void DatatypeCommunicator<T>::free()
975 delete[] requests_[0];
976 delete[] requests_[1];
977 typedef MessageTypeMap::iterator iterator;
978 typedef MessageTypeMap::const_iterator const_iterator;
980 const const_iterator end=messageTypes.end();
982 for(iterator process = messageTypes.begin(); process != end; ++process) {
983 MPI_Datatype *type = &(process->second.first);
985 MPI_Finalized(&finalized);
986 if(*type!=MPI_DATATYPE_NULL && !finalized)
988 type = &(process->second.second);
989 if(*type!=MPI_DATATYPE_NULL && !finalized)
992 messageTypes.clear();
999 template<
class T1,
class T2,
class V,
bool send>
1000 void DatatypeCommunicator<T>::createDataTypes(
const T1& sourceFlags,
const T2& destFlags, V& data)
1003 MPIDatatypeInformation<V> dataInfo(data);
1004 this->
template buildInterface<RemoteIndices,T1,T2,MPIDatatypeInformation<V>,send>(*remoteIndices_,sourceFlags, destFlags, dataInfo);
1006 typedef typename RemoteIndices::RemoteIndexMap::const_iterator const_iterator;
1007 const const_iterator end=this->remoteIndices_->end();
1010 for(const_iterator process=this->remoteIndices_->begin(); process != end; ++process) {
1011 IndexedTypeInformation& info=dataInfo.information_[process->first];
1014 MPI_Get_address(
const_cast<void *
>(CommPolicy<V>::getAddress(data, 0)), &base);
1016 for(
int i=0; i< info.elements; i++) {
1017 info.displ[i]-=base;
1021 MPI_Datatype* type = &( send ? messageTypes[process->first].first : messageTypes[process->first].second);
1022 MPI_Type_create_hindexed(info.elements, info.length, info.displ,
1023 MPITraits<
typename CommPolicy<V>::IndexedType>::getType(), type);
1024 MPI_Type_commit(type);
1030 template<
typename T>
1031 template<
class V,
bool createForward>
1032 void DatatypeCommunicator<T>::createRequests(V& sendData, V& receiveData)
1034 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >::const_iterator MapIterator;
1036 static int index = createForward ? 1 : 0;
1037 int noMessages = messageTypes.size();
1039 requests_[index] =
new MPI_Request[2*noMessages];
1040 const MapIterator end = messageTypes.end();
1042 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1045 for(MapIterator process = messageTypes.begin(); process != end;
1046 ++process, ++request) {
1047 MPI_Datatype type = createForward ? process->second.second : process->second.first;
1048 void* address =
const_cast<void*
>(CommPolicy<V>::getAddress(receiveData,0));
1049 MPI_Recv_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1054 for(MapIterator process = messageTypes.begin(); process != end;
1055 ++process, ++request) {
1056 MPI_Datatype type = createForward ? process->second.first : process->second.second;
1057 void* address =
const_cast<void*
>(CommPolicy<V>::getAddress(sendData, 0));
1058 MPI_Ssend_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1062 template<
typename T>
1063 void DatatypeCommunicator<T>::forward()
1065 sendRecv(requests_[1]);
1068 template<
typename T>
1069 void DatatypeCommunicator<T>::backward()
1071 sendRecv(requests_[0]);
1074 template<
typename T>
1075 void DatatypeCommunicator<T>::sendRecv(MPI_Request* requests)
1077 int noMessages = messageTypes.size();
1079 MPI_Startall(noMessages, requests);
1081 MPI_Startall(noMessages, requests+noMessages);
1084 MPI_Status* status=
new MPI_Status[2*noMessages];
1085 for(
int i=0; i<2*noMessages; i++)
1086 status[i].MPI_ERROR=MPI_SUCCESS;
1088 int send = MPI_Waitall(noMessages, requests+noMessages, status+noMessages);
1089 int receive = MPI_Waitall(noMessages, requests, status);
1092 int success=1, globalSuccess=0;
1093 if(send==MPI_ERR_IN_STATUS) {
1095 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1096 std::cerr<<rank<<
": Error in sending :"<<std::endl;
1098 for(
int i=noMessages; i< 2*noMessages; i++)
1099 if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1102 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1103 std::cerr<<
" source="<<status[i].MPI_SOURCE<<
" message: ";
1104 for(
int j = 0; j < messageLength; j++)
1105 std::cout << message[j];
1107 std::cerr<<std::endl;
1111 if(receive==MPI_ERR_IN_STATUS) {
1113 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1114 std::cerr<<rank<<
": Error in receiving!"<<std::endl;
1116 for(
int i=0; i< noMessages; i++)
1117 if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1120 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1121 std::cerr<<
" source="<<status[i].MPI_SOURCE<<
" message: ";
1122 for(
int j = 0; j < messageLength; j++)
1123 std::cerr << message[j];
1125 std::cerr<<std::endl;
1129 MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, this->remoteIndices_->communicator());
1134 DUNE_THROW(CommunicationError,
"A communication error occurred!");
1138 inline BufferedCommunicator::BufferedCommunicator()
1146 template<
class Data,
class Interface>
1147 typename std::enable_if<std::is_same<SizeOne, typename CommPolicy<Data>::IndexedTypeFlag>::value,
void>::type
1148 BufferedCommunicator::build(
const Interface& interface)
1150 interfaces_=interface.interfaces();
1151 communicator_=interface.communicator();
1152 typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1153 ::const_iterator const_iterator;
1154 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1155 const const_iterator end = interfaces_.end();
1157 MPI_Comm_rank(communicator_, &lrank);
1162 for(const_iterator interfacePair = interfaces_.begin();
1163 interfacePair != end; ++interfacePair) {
1164 int noSend = MessageSizeCalculator<Data,Flag>() (interfacePair->second.first);
1165 int noRecv = MessageSizeCalculator<Data,Flag>() (interfacePair->second.second);
1166 if (noSend + noRecv > 0)
1167 messageInformation_.insert(std::make_pair(interfacePair->first,
1168 std::make_pair(MessageInformation(bufferSize_[0],
1169 noSend*
sizeof(
typename CommPolicy<Data>::IndexedType)),
1170 MessageInformation(bufferSize_[1],
1171 noRecv*
sizeof(
typename CommPolicy<Data>::IndexedType)))));
1172 bufferSize_[0] += noSend;
1173 bufferSize_[1] += noRecv;
1177 bufferSize_[0] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1178 bufferSize_[1] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1180 buffers_[0] =
new char[bufferSize_[0]];
1181 buffers_[1] =
new char[bufferSize_[1]];
1184 template<
class Data,
class Interface>
1185 void BufferedCommunicator::build(
const Data& source,
const Data& dest,
const Interface& interface)
1188 interfaces_=interface.interfaces();
1189 communicator_=interface.communicator();
1190 typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1191 ::const_iterator const_iterator;
1192 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1193 const const_iterator end = interfaces_.end();
1198 for(const_iterator interfacePair = interfaces_.begin();
1199 interfacePair != end; ++interfacePair) {
1200 int noSend = MessageSizeCalculator<Data,Flag>() (source, interfacePair->second.first);
1201 int noRecv = MessageSizeCalculator<Data,Flag>() (dest, interfacePair->second.second);
1202 if (noSend + noRecv > 0)
1203 messageInformation_.insert(std::make_pair(interfacePair->first,
1204 std::make_pair(MessageInformation(bufferSize_[0],
1205 noSend*
sizeof(
typename CommPolicy<Data>::IndexedType)),
1206 MessageInformation(bufferSize_[1],
1207 noRecv*
sizeof(
typename CommPolicy<Data>::IndexedType)))));
1208 bufferSize_[0] += noSend;
1209 bufferSize_[1] += noRecv;
1212 bufferSize_[0] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1213 bufferSize_[1] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1215 buffers_[0] =
new char[bufferSize_[0]];
1216 buffers_[1] =
new char[bufferSize_[1]];
1219 inline void BufferedCommunicator::free()
1221 messageInformation_.clear();
1223 delete[] buffers_[0];
1226 delete[] buffers_[1];
1227 buffers_[0]=buffers_[1]=0;
1230 inline BufferedCommunicator::~BufferedCommunicator()
1235 template<
class Data>
1236 inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1237 (
const InterfaceInformation& info)
const
1243 template<
class Data>
1244 inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1245 (
const Data&,
const InterfaceInformation& info)
const
1247 return operator()(info);
1251 template<
class Data>
1252 inline int BufferedCommunicator::MessageSizeCalculator<Data, VariableSize>::operator()
1253 (
const Data& data,
const InterfaceInformation& info)
const
1257 for(
size_t i=0; i < info.size(); i++)
1258 entries += CommPolicy<Data>::getSize(data,info[i]);
1264 template<
class Data,
class GatherScatter,
bool FORWARD>
1265 inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,VariableSize>::operator()(
const InterfaceMap& interfaces,
const Data& data, Type* buffer,
size_t bufferSize)
const
1268 typedef typename InterfaceMap::const_iterator
1272 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1273 const const_iterator end = interfaces.end();
1276 for(const_iterator interfacePair = interfaces.begin();
1277 interfacePair != end; ++interfacePair) {
1278 int size = forward ? interfacePair->second.first.size() :
1279 interfacePair->second.second.size();
1281 for(
int i=0; i < size; i++) {
1282 int local = forward ? interfacePair->second.first[i] :
1283 interfacePair->second.second[i];
1284 for(std::size_t j=0; j < CommPolicy<Data>::getSize(data, local); j++, index++) {
1286 #ifdef DUNE_ISTL_WITH_CHECKING
1287 assert(bufferSize>=(index+1)*
sizeof(
typename CommPolicy<Data>::IndexedType));
1289 buffer[index]=GatherScatter::gather(data, local, j);
1298 template<
class Data,
class GatherScatter,
bool FORWARD>
1299 inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,SizeOne>::operator()(
const InterfaceMap& interfaces,
const Data& data, Type* buffer,
size_t bufferSize)
const
1302 typedef typename InterfaceMap::const_iterator
1304 const const_iterator end = interfaces.end();
1308 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1310 for(const_iterator interfacePair = interfaces.begin();
1311 interfacePair != end; ++interfacePair) {
1312 size_t size = FORWARD ? interfacePair->second.first.size() :
1313 interfacePair->second.second.size();
1315 for(
size_t i=0; i < size; i++) {
1317 #ifdef DUNE_ISTL_WITH_CHECKING
1318 assert(bufferSize>=(index+1)*
sizeof(
typename CommPolicy<Data>::IndexedType));
1321 buffer[index++] = GatherScatter::gather(data, FORWARD ? interfacePair->second.first[i] :
1322 interfacePair->second.second[i]);
1329 template<
class Data,
class GatherScatter,
bool FORWARD>
1330 inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,VariableSize>::operator()(
const InterfaceMap& interfaces, Data& data, Type* buffer,
const int& proc)
const
1332 typedef typename InterfaceMap::value_type::second_type::first_type Information;
1333 const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1335 assert(infoPair!=interfaces.end());
1337 const Information& info = FORWARD ? infoPair->second.second :
1338 infoPair->second.first;
1340 for(
size_t i=0, index=0; i < info.size(); i++) {
1341 for(
size_t j=0; j < CommPolicy<Data>::getSize(data, info[i]); j++)
1342 GatherScatter::scatter(data, buffer[index++], info[i], j);
1347 template<
class Data,
class GatherScatter,
bool FORWARD>
1348 inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,SizeOne>::operator()(
const InterfaceMap& interfaces, Data& data, Type* buffer,
const int& proc)
const
1350 typedef typename InterfaceMap::value_type::second_type::first_type Information;
1351 const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1353 assert(infoPair!=interfaces.end());
1355 const Information& info = FORWARD ? infoPair->second.second :
1356 infoPair->second.first;
1358 for(
size_t i=0; i < info.size(); i++) {
1359 GatherScatter::scatter(data, buffer[i], info[i]);
1364 template<
class GatherScatter,
class Data>
1365 void BufferedCommunicator::forward(Data& data)
1367 this->
template sendRecv<GatherScatter,true>(data, data);
1371 template<
class GatherScatter,
class Data>
1372 void BufferedCommunicator::backward(Data& data)
1374 this->
template sendRecv<GatherScatter,false>(data, data);
1378 template<
class GatherScatter,
class Data>
1379 void BufferedCommunicator::forward(
const Data& source, Data& dest)
1381 this->
template sendRecv<GatherScatter,true>(source, dest);
1385 template<
class GatherScatter,
class Data>
1386 void BufferedCommunicator::backward(Data& source,
const Data& dest)
1388 this->
template sendRecv<GatherScatter,false>(dest, source);
1392 template<
class GatherScatter,
bool FORWARD,
class Data>
1393 void BufferedCommunicator::sendRecv(
const Data& source, Data& dest)
1397 MPI_Comm_rank(MPI_COMM_WORLD,&rank);
1398 MPI_Comm_rank(MPI_COMM_WORLD,&lrank);
1400 typedef typename CommPolicy<Data>::IndexedType Type;
1401 Type *sendBuffer, *recvBuffer;
1402 size_t sendBufferSize;
1404 size_t recvBufferSize;
1408 sendBuffer =
reinterpret_cast<Type*
>(buffers_[0]);
1409 sendBufferSize = bufferSize_[0];
1410 recvBuffer =
reinterpret_cast<Type*
>(buffers_[1]);
1412 recvBufferSize = bufferSize_[1];
1415 sendBuffer =
reinterpret_cast<Type*
>(buffers_[1]);
1416 sendBufferSize = bufferSize_[1];
1417 recvBuffer =
reinterpret_cast<Type*
>(buffers_[0]);
1419 recvBufferSize = bufferSize_[0];
1422 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1424 MessageGatherer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, source, sendBuffer, sendBufferSize);
1426 MPI_Request* sendRequests =
new MPI_Request[messageInformation_.size()];
1427 MPI_Request* recvRequests =
new MPI_Request[messageInformation_.size()];
1429 size_t numberOfRealRecvRequests = 0;
1432 typedef typename InformationMap::const_iterator const_iterator;
1434 const const_iterator end = messageInformation_.end();
1436 int* processMap =
new int[messageInformation_.size()];
1438 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i) {
1439 processMap[i]=info->first;
1441 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1442 Dune::dvverb<<rank<<
": receiving "<<info->second.second.size_<<
" from "<<info->first<<std::endl;
1443 if(info->second.second.size_) {
1444 MPI_Irecv(recvBuffer+info->second.second.start_, info->second.second.size_,
1445 MPI_BYTE, info->first, commTag_, communicator_,
1447 numberOfRealRecvRequests += 1;
1450 recvRequests[i]=MPI_REQUEST_NULL;
1453 assert(info->second.first.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= recvBufferSize );
1454 Dune::dvverb<<rank<<
": receiving "<<info->second.first.size_<<
" to "<<info->first<<std::endl;
1455 if(info->second.first.size_) {
1456 MPI_Irecv(recvBuffer+info->second.first.start_, info->second.first.size_,
1457 MPI_BYTE, info->first, commTag_, communicator_,
1459 numberOfRealRecvRequests += 1;
1462 recvRequests[i]=MPI_REQUEST_NULL;
1469 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i)
1471 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1472 Dune::dvverb<<rank<<
": sending "<<info->second.first.size_<<
" to "<<info->first<<std::endl;
1473 assert(info->second.first.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= sendBufferSize );
1474 if(info->second.first.size_)
1475 MPI_Issend(sendBuffer+info->second.first.start_, info->second.first.size_,
1476 MPI_BYTE, info->first, commTag_, communicator_,
1480 sendRequests[i]=MPI_REQUEST_NULL;
1482 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= sendBufferSize );
1483 Dune::dvverb<<rank<<
": sending "<<info->second.second.size_<<
" to "<<info->first<<std::endl;
1484 if(info->second.second.size_)
1485 MPI_Issend(sendBuffer+info->second.second.start_, info->second.second.size_,
1486 MPI_BYTE, info->first, commTag_, communicator_,
1490 sendRequests[i]=MPI_REQUEST_NULL;
1496 int finished = MPI_UNDEFINED;
1500 for(i=0; i< numberOfRealRecvRequests; i++) {
1501 status.MPI_ERROR=MPI_SUCCESS;
1502 MPI_Waitany(messageInformation_.size(), recvRequests, &finished, &status);
1503 assert(finished != MPI_UNDEFINED);
1505 if(status.MPI_ERROR==MPI_SUCCESS) {
1506 int& proc = processMap[finished];
1507 typename InformationMap::const_iterator infoIter = messageInformation_.find(proc);
1508 assert(infoIter != messageInformation_.end());
1510 MessageInformation info = (FORWARD) ? infoIter->second.second : infoIter->second.first;
1511 assert(info.start_+info.size_ <= recvBufferSize);
1513 MessageScatterer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, dest, recvBuffer+info.start_, proc);
1515 std::cerr<<rank<<
": MPI_Error occurred while receiving message from "<<processMap[finished]<<std::endl;
1520 MPI_Status recvStatus;
1523 for(i=0; i< messageInformation_.size(); i++)
1524 if(MPI_SUCCESS!=MPI_Wait(sendRequests+i, &recvStatus)) {
1525 std::cerr<<rank<<
": MPI_Error occurred while sending message to "<<processMap[finished]<<std::endl;
1535 delete[] processMap;
1536 delete[] sendRequests;
1537 delete[] recvRequests;