Actual source code: data_ex.c
1: /*
2: Build a few basic tools to help with partitioned domains.
4: 1)
5: On each processor, have a DomainExchangerTopology.
6: This is a doubly-connected edge list which enumerates the
7: communication paths between connected processors. By numbering
8: these paths we can always uniquely assign message identifers.
10: edge
11: 10
12: proc ---------> proc
13: 0 <-------- 1
14: 11
15: twin
17: Eg: Proc 0 send to proc 1 with message id is 10. To receive the correct
18: message, proc 1 looks for the edge connected to proc 0, and then the
19: message id comes from the twin of that edge
21: 2)
22: A DomainExchangerArrayPacker.
23: A little function which given a piece of data, will memcpy the data into
24: an array (which will be sent to procs) into the correct place.
26: On Proc 1 we sent data to procs 0,2,3. The data is on different lengths.
27: All data gets jammed into single array. Need to "jam" data into correct locations
28: The Packer knows how much is to going to each processor and keeps track of the inserts
29: so as to avoid ever packing TOO much into one slot, and inevatbly corrupting some memory
31: data to 0 data to 2 data to 3
33: |--------|-----------------|--|
35: User has to unpack message themselves. I can get you the pointer for each i
36: entry, but you'll have to cast it to the appropriate data type.
38: Phase A: Build topology
40: Phase B: Define message lengths
42: Phase C: Pack data
44: Phase D: Send data
46: + Constructor
47: DMSwarmDataExCreate()
48: + Phase A
49: DMSwarmDataExTopologyInitialize()
50: DMSwarmDataExTopologyAddNeighbour()
51: DMSwarmDataExTopologyAddNeighbour()
52: DMSwarmDataExTopologyFinalize()
53: + Phase B
54: DMSwarmDataExZeroAllSendCount()
55: DMSwarmDataExAddToSendCount()
56: DMSwarmDataExAddToSendCount()
57: DMSwarmDataExAddToSendCount()
58: + Phase C
59: DMSwarmDataExPackInitialize()
60: DMSwarmDataExPackData()
61: DMSwarmDataExPackData()
62: DMSwarmDataExPackFinalize()
63: +Phase D
64: DMSwarmDataExBegin()
65: ... perform any calculations ...
66: DMSwarmDataExEnd()
68: ... user calls any getters here ...
70: */
71: #include <petscvec.h>
72: #include <petscmat.h>
74: #include "../src/dm/impls/swarm/data_ex.h"
76: const char *status_names[] = {"initialized", "finalized", "unknown"};
78: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerTopologySetup;
79: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerBegin;
80: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerEnd;
81: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerSendCount;
82: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerPack;
84: PetscErrorCode DMSwarmDataExCreate(MPI_Comm comm,const PetscInt count, DMSwarmDataEx *ex)
85: {
86: DMSwarmDataEx d;
88: PetscNew(&d);
89: MPI_Comm_dup(comm,&d->comm);
90: MPI_Comm_rank(d->comm,&d->rank);
92: d->instance = count;
94: d->topology_status = DEOBJECT_STATE_UNKNOWN;
95: d->message_lengths_status = DEOBJECT_STATE_UNKNOWN;
96: d->packer_status = DEOBJECT_STATE_UNKNOWN;
97: d->communication_status = DEOBJECT_STATE_UNKNOWN;
99: d->n_neighbour_procs = -1;
100: d->neighbour_procs = NULL;
102: d->messages_to_be_sent = NULL;
103: d->message_offsets = NULL;
104: d->messages_to_be_recvieved = NULL;
106: d->unit_message_size = (size_t)-1;
107: d->send_message = NULL;
108: d->send_message_length = -1;
109: d->recv_message = NULL;
110: d->recv_message_length = -1;
111: d->total_pack_cnt = -1;
112: d->pack_cnt = NULL;
114: d->send_tags = NULL;
115: d->recv_tags = NULL;
117: d->_stats = NULL;
118: d->_requests = NULL;
119: *ex = d;
120: return 0;
121: }
123: /*
124: This code is horrible, who let it get into main.
126: Should be printing to a viewer, should not be using PETSC_COMM_WORLD
128: */
129: PetscErrorCode DMSwarmDataExView(DMSwarmDataEx d)
130: {
131: PetscMPIInt p;
133: PetscPrintf( PETSC_COMM_WORLD, "DMSwarmDataEx: instance=%D\n",d->instance);
134: PetscPrintf( PETSC_COMM_WORLD, " topology status: %s \n", status_names[d->topology_status]);
135: PetscPrintf( PETSC_COMM_WORLD, " message lengths status: %s \n", status_names[d->message_lengths_status]);
136: PetscPrintf( PETSC_COMM_WORLD, " packer status status: %s \n", status_names[d->packer_status]);
137: PetscPrintf( PETSC_COMM_WORLD, " communication status: %s \n", status_names[d->communication_status]);
139: if (d->topology_status == DEOBJECT_FINALIZED) {
140: PetscPrintf( PETSC_COMM_WORLD, " Topology:\n");
141: PetscSynchronizedPrintf( PETSC_COMM_WORLD, " [%d] neighbours: %d \n", d->rank, d->n_neighbour_procs);
142: for (p=0; p<d->n_neighbour_procs; p++) {
143: PetscSynchronizedPrintf( PETSC_COMM_WORLD, " [%d] neighbour[%d] = %d \n", d->rank, p, d->neighbour_procs[p]);
144: }
145: PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
146: }
148: if (d->message_lengths_status == DEOBJECT_FINALIZED) {
149: PetscPrintf( PETSC_COMM_WORLD, " Message lengths:\n");
150: PetscSynchronizedPrintf( PETSC_COMM_WORLD, " [%d] atomic size: %ld \n", d->rank, (long int)d->unit_message_size);
151: for (p=0; p<d->n_neighbour_procs; p++) {
152: PetscSynchronizedPrintf( PETSC_COMM_WORLD, " [%d] >>>>> ( %D units :: tag = %d) >>>>> [%d] \n", d->rank, d->messages_to_be_sent[p], d->send_tags[p], d->neighbour_procs[p]);
153: }
154: for (p=0; p<d->n_neighbour_procs; p++) {
155: PetscSynchronizedPrintf( PETSC_COMM_WORLD, " [%d] <<<<< ( %D units :: tag = %d) <<<<< [%d] \n", d->rank, d->messages_to_be_recvieved[p], d->recv_tags[p], d->neighbour_procs[p]);
156: }
157: PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
158: }
159: if (d->packer_status == DEOBJECT_FINALIZED) {}
160: if (d->communication_status == DEOBJECT_FINALIZED) {}
161: return 0;
162: }
164: PetscErrorCode DMSwarmDataExDestroy(DMSwarmDataEx d)
165: {
166: MPI_Comm_free(&d->comm);
167: if (d->neighbour_procs) PetscFree(d->neighbour_procs);
168: if (d->messages_to_be_sent) PetscFree(d->messages_to_be_sent);
169: if (d->message_offsets) PetscFree(d->message_offsets);
170: if (d->messages_to_be_recvieved) PetscFree(d->messages_to_be_recvieved);
171: if (d->send_message) PetscFree(d->send_message);
172: if (d->recv_message) PetscFree(d->recv_message);
173: if (d->pack_cnt) PetscFree(d->pack_cnt);
174: if (d->send_tags) PetscFree(d->send_tags);
175: if (d->recv_tags) PetscFree(d->recv_tags);
176: if (d->_stats) PetscFree(d->_stats);
177: if (d->_requests) PetscFree(d->_requests);
178: PetscFree(d);
179: return 0;
180: }
182: /* === Phase A === */
184: PetscErrorCode DMSwarmDataExTopologyInitialize(DMSwarmDataEx d)
185: {
186: d->topology_status = DEOBJECT_INITIALIZED;
187: d->n_neighbour_procs = 0;
188: PetscFree(d->neighbour_procs);
189: PetscFree(d->messages_to_be_sent);
190: PetscFree(d->message_offsets);
191: PetscFree(d->messages_to_be_recvieved);
192: PetscFree(d->pack_cnt);
193: PetscFree(d->send_tags);
194: PetscFree(d->recv_tags);
195: return 0;
196: }
198: PetscErrorCode DMSwarmDataExTopologyAddNeighbour(DMSwarmDataEx d,const PetscMPIInt proc_id)
199: {
200: PetscMPIInt n,found;
201: PetscMPIInt size;
206: /* error on negative entries */
208: /* error on ranks larger than number of procs in communicator */
209: MPI_Comm_size(d->comm,&size);
211: if (d->n_neighbour_procs == 0) PetscMalloc1(1, &d->neighbour_procs);
212: /* check for proc_id */
213: found = 0;
214: for (n = 0; n < d->n_neighbour_procs; n++) {
215: if (d->neighbour_procs[n] == proc_id) {
216: found = 1;
217: }
218: }
219: if (found == 0) { /* add it to list */
220: PetscRealloc(sizeof(PetscMPIInt)*(d->n_neighbour_procs+1), &d->neighbour_procs);
221: d->neighbour_procs[ d->n_neighbour_procs ] = proc_id;
222: d->n_neighbour_procs++;
223: }
224: return 0;
225: }
227: /*
228: counter: the index of the communication object
229: N: the number of processors
230: r0: rank of sender
231: r1: rank of receiver
233: procs = { 0, 1, 2, 3 }
235: 0 ==> 0 e=0
236: 0 ==> 1 e=1
237: 0 ==> 2 e=2
238: 0 ==> 3 e=3
240: 1 ==> 0 e=4
241: 1 ==> 1 e=5
242: 1 ==> 2 e=6
243: 1 ==> 3 e=7
245: 2 ==> 0 e=8
246: 2 ==> 1 e=9
247: 2 ==> 2 e=10
248: 2 ==> 3 e=11
250: 3 ==> 0 e=12
251: 3 ==> 1 e=13
252: 3 ==> 2 e=14
253: 3 ==> 3 e=15
255: If we require that proc A sends to proc B, then the SEND tag index will be given by
256: N * rank(A) + rank(B) + offset
257: If we require that proc A will receive from proc B, then the RECV tag index will be given by
258: N * rank(B) + rank(A) + offset
260: */
261: static void _get_tags(PetscInt counter, PetscMPIInt N, PetscMPIInt r0,PetscMPIInt r1, PetscMPIInt *_st, PetscMPIInt *_rt)
262: {
263: PetscMPIInt st,rt;
265: st = N*r0 + r1 + N*N*counter;
266: rt = N*r1 + r0 + N*N*counter;
267: *_st = st;
268: *_rt = rt;
269: }
271: /*
272: Makes the communication map symmetric
273: */
274: PetscErrorCode _DMSwarmDataExCompleteCommunicationMap(MPI_Comm comm,PetscMPIInt n,PetscMPIInt proc_neighbours[],PetscMPIInt *n_new,PetscMPIInt **proc_neighbours_new)
275: {
276: Mat A;
277: PetscInt i,j,nc;
278: PetscInt n_, *proc_neighbours_;
279: PetscInt rank_;
280: PetscMPIInt size, rank;
281: PetscScalar *vals;
282: const PetscInt *cols;
283: const PetscScalar *red_vals;
284: PetscMPIInt _n_new, *_proc_neighbours_new;
286: n_ = n;
287: PetscMalloc(sizeof(PetscInt) * n_, &proc_neighbours_);
288: for (i = 0; i < n_; ++i) {
289: proc_neighbours_[i] = proc_neighbours[i];
290: }
291: MPI_Comm_size(comm,&size);
292: MPI_Comm_rank(comm,&rank);
293: rank_ = rank;
295: MatCreate(comm,&A);
296: MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,size,size);
297: MatSetType(A,MATAIJ);
298: MatSeqAIJSetPreallocation(A,1,NULL);
299: MatMPIAIJSetPreallocation(A,n_,NULL,n_,NULL);
300: MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
301: /* Build original map */
302: PetscMalloc1(n_, &vals);
303: for (i = 0; i < n_; ++i) {
304: vals[i] = 1.0;
305: }
306: MatSetValues( A, 1,&rank_, n_,proc_neighbours_, vals, INSERT_VALUES);
307: MatAssemblyBegin(A,MAT_FLUSH_ASSEMBLY);
308: MatAssemblyEnd(A,MAT_FLUSH_ASSEMBLY);
309: /* Now force all other connections if they are not already there */
310: /* It's more efficient to do them all at once */
311: for (i = 0; i < n_; ++i) {
312: vals[i] = 2.0;
313: }
314: MatSetValues( A, n_,proc_neighbours_, 1,&rank_, vals, INSERT_VALUES);
315: MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
316: MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
317: /*
318: PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);
319: MatView(A,PETSC_VIEWER_STDOUT_WORLD);
320: PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);
321: */
322: if ((n_new != NULL) && (proc_neighbours_new != NULL)) {
323: MatGetRow(A, rank_, &nc, &cols, &red_vals);
324: _n_new = (PetscMPIInt) nc;
325: PetscMalloc1(_n_new, &_proc_neighbours_new);
326: for (j = 0; j < nc; ++j) {
327: _proc_neighbours_new[j] = (PetscMPIInt)cols[j];
328: }
329: MatRestoreRow( A, rank_, &nc, &cols, &red_vals);
330: *n_new = (PetscMPIInt)_n_new;
331: *proc_neighbours_new = (PetscMPIInt*)_proc_neighbours_new;
332: }
333: MatDestroy(&A);
334: PetscFree(vals);
335: PetscFree(proc_neighbours_);
336: MPI_Barrier(comm);
337: return 0;
338: }
340: PetscErrorCode DMSwarmDataExTopologyFinalize(DMSwarmDataEx d)
341: {
342: PetscMPIInt symm_nn, *symm_procs, r0,n,st,rt, size;
346: PetscLogEventBegin(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
347: /* given information about all my neighbours, make map symmetric */
348: _DMSwarmDataExCompleteCommunicationMap( d->comm,d->n_neighbour_procs,d->neighbour_procs, &symm_nn, &symm_procs);
349: /* update my arrays */
350: PetscFree(d->neighbour_procs);
351: d->n_neighbour_procs = symm_nn;
352: d->neighbour_procs = symm_procs;
353: /* allocates memory */
354: if (!d->messages_to_be_sent) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_sent);
355: if (!d->message_offsets) PetscMalloc1(d->n_neighbour_procs+1, &d->message_offsets);
356: if (!d->messages_to_be_recvieved) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_recvieved);
357: if (!d->pack_cnt) PetscMalloc(sizeof(PetscInt) * d->n_neighbour_procs, &d->pack_cnt);
358: if (!d->_stats) PetscMalloc(sizeof(MPI_Status) * 2*d->n_neighbour_procs, &d->_stats);
359: if (!d->_requests) PetscMalloc(sizeof(MPI_Request) * 2*d->n_neighbour_procs, &d->_requests);
360: if (!d->send_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->send_tags);
361: if (!d->recv_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->recv_tags);
362: /* compute message tags */
363: MPI_Comm_size(d->comm,&size);
364: r0 = d->rank;
365: for (n = 0; n < d->n_neighbour_procs; ++n) {
366: PetscMPIInt r1 = d->neighbour_procs[n];
368: _get_tags( d->instance, size, r0,r1, &st, &rt);
369: d->send_tags[n] = (int)st;
370: d->recv_tags[n] = (int)rt;
371: }
372: d->topology_status = DEOBJECT_FINALIZED;
373: PetscLogEventEnd(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
374: return 0;
375: }
377: /* === Phase B === */
378: PetscErrorCode _DMSwarmDataExConvertProcIdToLocalIndex(DMSwarmDataEx de,PetscMPIInt proc_id,PetscMPIInt *local)
379: {
380: PetscMPIInt i,np;
382: np = de->n_neighbour_procs;
383: *local = -1;
384: for (i = 0; i < np; ++i) {
385: if (proc_id == de->neighbour_procs[i]) {
386: *local = i;
387: break;
388: }
389: }
390: return 0;
391: }
393: PetscErrorCode DMSwarmDataExInitializeSendCount(DMSwarmDataEx de)
394: {
395: PetscMPIInt i;
398: PetscLogEventBegin(DMSWARM_DataExchangerSendCount,0,0,0,0);
399: de->message_lengths_status = DEOBJECT_INITIALIZED;
400: for (i = 0; i < de->n_neighbour_procs; ++i) {
401: de->messages_to_be_sent[i] = 0;
402: }
403: return 0;
404: }
406: /*
407: 1) only allows counters to be set on neighbouring cpus
408: */
409: PetscErrorCode DMSwarmDataExAddToSendCount(DMSwarmDataEx de,const PetscMPIInt proc_id,const PetscInt count)
410: {
411: PetscMPIInt local_val;
416: _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local_val);
419: de->messages_to_be_sent[local_val] = de->messages_to_be_sent[local_val] + count;
420: return 0;
421: }
423: PetscErrorCode DMSwarmDataExFinalizeSendCount(DMSwarmDataEx de)
424: {
427: de->message_lengths_status = DEOBJECT_FINALIZED;
428: PetscLogEventEnd(DMSWARM_DataExchangerSendCount,0,0,0,0);
429: return 0;
430: }
432: /* === Phase C === */
433: /*
434: zero out all send counts
435: free send and recv buffers
436: zeros out message length
437: zeros out all counters
438: zero out packed data counters
439: */
440: PetscErrorCode _DMSwarmDataExInitializeTmpStorage(DMSwarmDataEx de)
441: {
442: PetscMPIInt i, np;
444: np = de->n_neighbour_procs;
445: for (i = 0; i < np; ++i) {
446: /* de->messages_to_be_sent[i] = -1; */
447: de->messages_to_be_recvieved[i] = -1;
448: }
449: PetscFree(de->send_message);
450: PetscFree(de->recv_message);
451: return 0;
452: }
454: /*
455: Zeros out pack data counters
456: Ensures mesaage length is set
457: Checks send counts properly initialized
458: allocates space for pack data
459: */
460: PetscErrorCode DMSwarmDataExPackInitialize(DMSwarmDataEx de,size_t unit_message_size)
461: {
462: PetscMPIInt i,np;
463: PetscInt total;
467: PetscLogEventBegin(DMSWARM_DataExchangerPack,0,0,0,0);
468: de->packer_status = DEOBJECT_INITIALIZED;
469: _DMSwarmDataExInitializeTmpStorage(de);
470: np = de->n_neighbour_procs;
471: de->unit_message_size = unit_message_size;
472: total = 0;
473: for (i = 0; i < np; ++i) {
474: if (de->messages_to_be_sent[i] == -1) {
475: PetscMPIInt proc_neighour = de->neighbour_procs[i];
476: SETERRQ( PETSC_COMM_SELF, PETSC_ERR_ORDER, "Messages_to_be_sent[neighbour_proc=%d] is un-initialised. Call DMSwarmDataExSetSendCount() first", (int)proc_neighour);
477: }
478: total = total + de->messages_to_be_sent[i];
479: }
480: /* create space for the data to be sent */
481: PetscMalloc(unit_message_size * (total + 1), &de->send_message);
482: /* initialize memory */
483: PetscMemzero(de->send_message, unit_message_size * (total + 1));
484: /* set total items to send */
485: de->send_message_length = total;
486: de->message_offsets[0] = 0;
487: total = de->messages_to_be_sent[0];
488: for (i = 1; i < np; ++i) {
489: de->message_offsets[i] = total;
490: total = total + de->messages_to_be_sent[i];
491: }
492: /* init the packer counters */
493: de->total_pack_cnt = 0;
494: for (i = 0; i < np; ++i) {
495: de->pack_cnt[i] = 0;
496: }
497: return 0;
498: }
500: /*
501: Ensures data gets been packed appropriately and no overlaps occur
502: */
503: PetscErrorCode DMSwarmDataExPackData(DMSwarmDataEx de,PetscMPIInt proc_id,PetscInt n,void *data)
504: {
505: PetscMPIInt local;
506: PetscInt insert_location;
507: void *dest;
513: _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local);
516: (int)proc_id, de->messages_to_be_sent[local], n+de->pack_cnt[local]);
518: /* copy memory */
519: insert_location = de->message_offsets[local] + de->pack_cnt[local];
520: dest = ((char*)de->send_message) + de->unit_message_size*insert_location;
521: PetscMemcpy(dest, data, de->unit_message_size * n);
522: /* increment counter */
523: de->pack_cnt[local] = de->pack_cnt[local] + n;
524: return 0;
525: }
527: /*
528: *) Ensures all data has been packed
529: */
530: PetscErrorCode DMSwarmDataExPackFinalize(DMSwarmDataEx de)
531: {
532: PetscMPIInt i,np;
533: PetscInt total;
536: np = de->n_neighbour_procs;
537: for (i = 0; i < np; ++i) {
539: (int)de->neighbour_procs[i], de->messages_to_be_sent[i], de->pack_cnt[i]);
540: }
541: /* init */
542: for (i = 0; i < np; ++i) {
543: de->messages_to_be_recvieved[i] = -1;
544: }
545: /* figure out the recv counts here */
546: for (i = 0; i < np; ++i) {
547: MPI_Isend(&de->messages_to_be_sent[i], 1, MPIU_INT, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
548: }
549: for (i = 0; i < np; ++i) {
550: MPI_Irecv(&de->messages_to_be_recvieved[i], 1, MPIU_INT, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
551: }
552: MPI_Waitall(2*np, de->_requests, de->_stats);
553: /* create space for the data to be recvieved */
554: total = 0;
555: for (i = 0; i < np; ++i) {
556: total = total + de->messages_to_be_recvieved[i];
557: }
558: PetscMalloc(de->unit_message_size * (total + 1), &de->recv_message);
559: /* initialize memory */
560: PetscMemzero(de->recv_message, de->unit_message_size * (total + 1));
561: /* set total items to receive */
562: de->recv_message_length = total;
563: de->packer_status = DEOBJECT_FINALIZED;
564: de->communication_status = DEOBJECT_INITIALIZED;
565: PetscLogEventEnd(DMSWARM_DataExchangerPack,0,0,0,0);
566: return 0;
567: }
569: /* do the actual message passing */
570: PetscErrorCode DMSwarmDataExBegin(DMSwarmDataEx de)
571: {
572: PetscMPIInt i,np;
573: void *dest;
574: PetscInt length;
581: PetscLogEventBegin(DMSWARM_DataExchangerBegin,0,0,0,0);
582: np = de->n_neighbour_procs;
583: /* == NON BLOCKING == */
584: for (i = 0; i < np; ++i) {
585: length = de->messages_to_be_sent[i] * de->unit_message_size;
586: dest = ((char*)de->send_message) + de->unit_message_size * de->message_offsets[i];
587: MPI_Isend( dest, length, MPI_CHAR, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
588: }
589: PetscLogEventEnd(DMSWARM_DataExchangerBegin,0,0,0,0);
590: return 0;
591: }
593: /* do the actual message passing now */
594: PetscErrorCode DMSwarmDataExEnd(DMSwarmDataEx de)
595: {
596: PetscMPIInt i,np;
597: PetscInt total;
598: PetscInt *message_recv_offsets;
599: void *dest;
600: PetscInt length;
604: PetscLogEventBegin(DMSWARM_DataExchangerEnd,0,0,0,0);
605: np = de->n_neighbour_procs;
606: PetscMalloc1(np+1, &message_recv_offsets);
607: message_recv_offsets[0] = 0;
608: total = de->messages_to_be_recvieved[0];
609: for (i = 1; i < np; ++i) {
610: message_recv_offsets[i] = total;
611: total = total + de->messages_to_be_recvieved[i];
612: }
613: /* == NON BLOCKING == */
614: for (i = 0; i < np; ++i) {
615: length = de->messages_to_be_recvieved[i] * de->unit_message_size;
616: dest = ((char*)de->recv_message) + de->unit_message_size * message_recv_offsets[i];
617: MPI_Irecv( dest, length, MPI_CHAR, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
618: }
619: MPI_Waitall( 2*np, de->_requests, de->_stats);
620: PetscFree(message_recv_offsets);
621: de->communication_status = DEOBJECT_FINALIZED;
622: PetscLogEventEnd(DMSWARM_DataExchangerEnd,0,0,0,0);
623: return 0;
624: }
626: PetscErrorCode DMSwarmDataExGetSendData(DMSwarmDataEx de,PetscInt *length,void **send)
627: {
629: *length = de->send_message_length;
630: *send = de->send_message;
631: return 0;
632: }
634: PetscErrorCode DMSwarmDataExGetRecvData(DMSwarmDataEx de,PetscInt *length,void **recv)
635: {
637: *length = de->recv_message_length;
638: *recv = de->recv_message;
639: return 0;
640: }
642: PetscErrorCode DMSwarmDataExTopologyGetNeighbours(DMSwarmDataEx de,PetscMPIInt *n,PetscMPIInt *neigh[])
643: {
644: if (n) {*n = de->n_neighbour_procs;}
645: if (neigh) {*neigh = de->neighbour_procs;}
646: return 0;
647: }