VTK
vtkPPixelTransfer.h
Go to the documentation of this file.
1/*=========================================================================
2
3 Program: Visualization Toolkit
4 Module: vtkPPixelTransfer.h
5
6 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
7 All rights reserved.
8 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
9
10 This software is distributed WITHOUT ANY WARRANTY; without even
11 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
12 PURPOSE. See the above copyright notice for more information.
13
14=========================================================================*/
29#ifndef vtkPPixelTransfer_h
30#define vtkPPixelTransfer_h
31
32#include "vtkPixelTransfer.h"
33#include "vtkRenderingParallelLICModule.h" // for export
34#include "vtkSetGet.h" // for macros
35#include "vtkPixelExtent.h" // for pixel extent
36#include "vtkMPI.h" // for mpi
37#include "vtkMPIPixelTT.h" // for type traits
38#include "vtkMPIPixelView.h" // for mpi subarrays
39
40// included vtkSystemIncludes.h in the base class.
41#include <iostream> // for ostream
42#include <vector> // for vector
43#include <cstring> // for memcpy
44
45// #define vtkPPixelTransferDEBUG
46
47class VTKRENDERINGPARALLELLIC_EXPORT vtkPPixelTransfer : public vtkPixelTransfer
48{
49public:
51 :
52 SrcRank(0),
53 DestRank(0),
54 UseBlockingSend(0),
55 UseBlockingRecv(0)
56 {}
57
63 int srcRank,
64 const vtkPixelExtent &srcWholeExt,
65 const vtkPixelExtent &srcExt,
66 int destRank,
67 const vtkPixelExtent &destWholeExt,
68 const vtkPixelExtent &destExt,
69 int id=0)
70 :
71 Id(id),
72 SrcRank(srcRank),
73 SrcWholeExt(srcWholeExt),
74 SrcExt(srcExt),
75 DestRank(destRank),
76 DestWholeExt(destWholeExt),
77 DestExt(destExt),
78 UseBlockingSend(0),
79 UseBlockingRecv(0)
80 {}
81
87 int srcRank,
88 const vtkPixelExtent &srcWholeExt,
89 const vtkPixelExtent &targetExt,
90 int destRank,
91 const vtkPixelExtent &destWholeExt,
92 int id)
93 :
94 Id(id),
95 SrcRank(srcRank),
96 SrcWholeExt(srcWholeExt),
97 SrcExt(targetExt),
98 DestRank(destRank),
99 DestWholeExt(destWholeExt),
100 DestExt(targetExt),
101 UseBlockingSend(0),
102 UseBlockingRecv(0)
103 {}
104
110 int srcRank,
111 int destRank,
112 const vtkPixelExtent &wholeExt,
113 const vtkPixelExtent &targetExt,
114 int id=0)
115 :
116 Id(id),
117 SrcRank(srcRank),
118 SrcWholeExt(wholeExt),
119 SrcExt(targetExt),
120 DestRank(destRank),
121 DestWholeExt(wholeExt),
122 DestExt(targetExt),
123 UseBlockingSend(0),
124 UseBlockingRecv(0)
125 {}
126
132 int srcRank,
133 int destRank,
134 const vtkPixelExtent &ext,
135 int id=0)
136 :
137 Id(id),
138 SrcRank(srcRank),
139 SrcWholeExt(ext),
140 SrcExt(ext),
141 DestRank(destRank),
142 DestWholeExt(ext),
143 DestExt(ext),
144 UseBlockingSend(0),
145 UseBlockingRecv(0)
146 {}
147
153 int srcRank,
154 const vtkPixelExtent &srcWholeExt,
155 int destRank,
156 const vtkPixelExtent &destWholeExt,
157 int id=0)
158 :
159 Id(id),
160 SrcRank(srcRank),
161 SrcWholeExt(srcWholeExt),
162 SrcExt(srcWholeExt),
163 DestRank(destRank),
164 DestWholeExt(destWholeExt),
165 DestExt(destWholeExt),
166 UseBlockingSend(0),
167 UseBlockingRecv(0)
168 {}
169
176 const vtkPixelExtent &srcWholeExt,
177 const vtkPixelExtent &srcExt,
178 const vtkPixelExtent &destWholeExt,
179 const vtkPixelExtent &destExt)
180 :
181 Id(0),
182 SrcRank(0),
183 SrcWholeExt(srcWholeExt),
184 SrcExt(srcExt),
185 DestRank(0),
186 DestWholeExt(destWholeExt),
187 DestExt(destExt),
188 UseBlockingSend(0),
189 UseBlockingRecv(0)
190 {}
191
193
198 void SetSourceRank(int rank)
199 { this->SrcRank=rank; }
200
201 int GetSourceRank() const
202 { return this->SrcRank; }
203
204 void SetDestinationRank(int rank)
205 { this->DestRank=rank; }
206
208 { return this->DestRank; }
209
215 bool Sender(int rank) const { return (this->SrcRank == rank); }
216 bool Receiver(int rank) const { return (this->DestRank == rank); }
217 bool Local(int rank) const
218 { return (this->Sender(rank) && this->Receiver(rank)); }
219
225 { this->SrcWholeExt=srcExt; }
226
228 { return this->SrcWholeExt; }
229
231 { return this->SrcWholeExt; }
232
238 { this->SrcExt=srcExt; }
239
241 { return this->SrcExt; }
242
244 { return this->SrcExt; }
245
251 { this->DestWholeExt=destExt; }
252
254 { return this->DestWholeExt; }
255
257 { return this->DestWholeExt; }
258
264 { this->DestExt=destExt; }
265
267 { return this->DestExt; }
268
270 { return this->DestExt; }
271
275 void SetTransactionId(int id)
276 { this->Id=id; }
277
279 { return this->Id; }
280
284 void SetUseBlockingSend(int val)
285 { this->UseBlockingSend=val; }
286
288 { return this->UseBlockingSend; }
289
290 void SetUseBlockingRecv(int val)
291 { this->UseBlockingRecv=val; }
292
294 { return this->UseBlockingRecv; }
295
299 template<typename SOURCE_TYPE, typename DEST_TYPE>
300 int Execute(
301 MPI_Comm comm,
302 int rank,
303 int nComps,
304 SOURCE_TYPE *srcData,
305 DEST_TYPE *destData,
306 std::vector<MPI_Request> &reqs,
307 std::deque<MPI_Datatype> &types,
308 int tag);
309
315 MPI_Comm comm,
316 int rank,
317 int nComps,
318 int srcType,
319 void *srcData,
320 int destType,
321 void *destData,
322 std::vector<MPI_Request> &reqs,
323 std::deque<MPI_Datatype> &types,
324 int tag);
325
329 int Blit(
330 int nComps,
331 int srcType,
332 void *srcData,
333 int destType,
334 void *destData);
335
336private:
337 // distpatch helper for vtk data type enum
338 template<typename SOURCE_TYPE>
339 int Execute(
340 MPI_Comm comm,
341 int rank,
342 int nComps,
343 SOURCE_TYPE *srcData,
344 int destType,
345 void *destData,
346 std::vector<MPI_Request> &reqs,
347 std::deque<MPI_Datatype> &types,
348 int tag);
349
350private:
351 int Id; // transaction id
352 int SrcRank; // rank who owns source memory
353 vtkPixelExtent SrcWholeExt; // source extent
354 vtkPixelExtent SrcExt; // source subset to transfer
355 int DestRank; // rank who owns destination memory
356 vtkPixelExtent DestWholeExt; // destination extent
357 vtkPixelExtent DestExt; // destination subset
358 int UseBlockingSend; // controls for non-blocking comm
359 int UseBlockingRecv;
360};
361
362//-----------------------------------------------------------------------------
363template<typename SOURCE_TYPE>
365 MPI_Comm comm,
366 int rank,
367 int nComps,
368 SOURCE_TYPE *srcData,
369 int destType,
370 void *destData,
371 std::vector<MPI_Request> &reqs,
372 std::deque<MPI_Datatype> &types,
373 int tag)
374{
375 // second layer of dispatch
376 switch(destType)
377 {
378 vtkTemplateMacro(
379 return this->Execute(
380 comm,
381 rank,
382 nComps,
383 srcData,
384 (VTK_TT*)destData,
385 reqs,
386 types,
387 tag));
388 }
389 return 0;
390}
391
392//-----------------------------------------------------------------------------
393template<typename SOURCE_TYPE, typename DEST_TYPE>
395 MPI_Comm comm,
396 int rank,
397 int nComps,
398 SOURCE_TYPE *srcData,
399 DEST_TYPE *destData,
400 std::vector<MPI_Request> &reqs,
401 std::deque<MPI_Datatype> &types,
402 int tag)
403{
404 int iErr = 0;
405 if ((comm == MPI_COMM_NULL) || (this->Local(rank)))
406 {
407 // transaction is local, bypass mpi in favor of memcpy
409 this->SrcWholeExt,
410 this->SrcExt,
411 this->DestWholeExt,
412 this->DestExt,
413 nComps,
414 srcData,
415 nComps,
416 destData);
417 }
418
419 if (rank == this->DestRank)
420 {
421 // use mpi to receive the data
422 if (destData == NULL)
423 {
424 return -1;
425 }
426
427 MPI_Datatype subarray;
428 iErr = vtkMPIPixelViewNew<DEST_TYPE>(
429 this->DestWholeExt,
430 this->DestExt,
431 nComps,
432 subarray);
433 if (iErr)
434 {
435 return -4;
436 }
437
438 if (this->UseBlockingRecv)
439 {
440 MPI_Status stat;
441 iErr = MPI_Recv(
442 destData,
443 1,
444 subarray,
445 this->SrcRank,
446 tag,
447 comm,
448 &stat);
449 }
450 else
451 {
452 reqs.push_back(MPI_REQUEST_NULL);
453 iErr = MPI_Irecv(
454 destData,
455 1,
456 subarray,
457 this->SrcRank,
458 tag,
459 comm,
460 &reqs.back());
461 }
462
463 #define HOLD_RECV_TYPES
464 #ifdef HOLD_RECV_YPES
465 types.push_back(subarray);
466 #else
467 MPI_Type_free(&subarray);
468 #endif
469
470 if (iErr)
471 {
472 return -5;
473 }
474 }
475
476 if (rank == this->SrcRank)
477 {
478 // use mpi to send the data
479 if (srcData == NULL)
480 {
481 return -1;
482 }
483
484 MPI_Datatype subarray;
485 iErr = vtkMPIPixelViewNew<SOURCE_TYPE>(
486 this->SrcWholeExt,
487 this->SrcExt,
488 nComps,
489 subarray);
490 if (iErr)
491 {
492 return -2;
493 }
494
495 if (this->UseBlockingSend)
496 {
497 iErr = MPI_Ssend(
498 srcData,
499 1,
500 subarray,
501 this->DestRank,
502 tag,
503 comm);
504 }
505 else
506 {
507 MPI_Request req;
508 iErr = MPI_Isend(
509 srcData,
510 1,
511 subarray,
512 this->DestRank,
513 tag,
514 comm,
515 &req);
516 #define SAVE_SEND_REQS
517 #ifdef SAVE_SEND_REQS
518 reqs.push_back(req);
519 #else
520 MPI_Request_free(&req);
521 #endif
522 }
523
524 #define HOLD_SEND_TYPES
525 #ifdef HOLD_SEND_TYPES
526 types.push_back(subarray);
527 #else
528 MPI_Type_free(&subarray);
529 #endif
530
531 if (iErr)
532 {
533 return -3;
534 }
535 }
536
537 return iErr;
538}
539
540VTKRENDERINGPARALLELLIC_EXPORT
541ostream &operator<<(std::ostream &os, const vtkPPixelTransfer &gt);
542
543#endif
544// VTK-HeaderTest-Exclude: vtkPPixelTransfer.h
class to handle inter-process communication of pixel data from non-contiguous regions of a shared ind...
void SetSourceWholeExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &wholeExt, const vtkPixelExtent &targetExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
void SetUseBlockingRecv(int val)
int GetTransactionId() const
vtkPixelExtent & GetSourceWholeExtent()
int GetUseBlockingRecv() const
bool Receiver(int rank) const
void SetDestinationRank(int rank)
vtkPixelExtent & GetDestinationExtent()
int GetSourceRank() const
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, int destRank, const vtkPixelExtent &destWholeExt, int id=0)
Initialize a transaction from whole extent of source to whole extent of dest, where source and destin...
void SetDestinationWholeExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
int Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE *srcData, DEST_TYPE *destData, std::vector< MPI_Request > &reqs, std::deque< MPI_Datatype > &types, int tag)
Transfer data from source to destination.
const vtkPixelExtent & GetSourceWholeExtent() const
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &ext, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
bool Local(int rank) const
void SetTransactionId(int id)
Set/get the transaction id.
int GetUseBlockingSend() const
vtkPPixelTransfer(const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
const vtkPixelExtent & GetDestinationWholeExtent() const
bool Sender(int rank) const
Tests to determine a given rank's role in this transaction.
vtkPixelExtent & GetSourceExtent()
int Execute(MPI_Comm comm, int rank, int nComps, int srcType, void *srcData, int destType, void *destData, std::vector< MPI_Request > &reqs, std::deque< MPI_Datatype > &types, int tag)
Transfer data from source to destination.
void SetDestinationExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
const vtkPixelExtent & GetSourceExtent() const
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, int destRank, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
vtkPixelExtent & GetDestinationWholeExtent()
const vtkPixelExtent & GetDestinationExtent() const
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &targetExt, int destRank, const vtkPixelExtent &destWholeExt, int id)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are the s...
void SetUseBlockingSend(int val)
Enable/diasable non-blocking communication.
int GetDestinationRank() const
void SetSourceRank(int rank)
Set/Get the MPI rank of source and destination processes.
int Blit(int nComps, int srcType, void *srcData, int destType, void *destData)
Block transfer for local memory to memory transfers, without using mpi.
void SetSourceExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
Representation of a cartesian pixel plane and common operations on it.
pixel extents
static int Blit(const vtkPixelExtent &ext, int nComps, int srcType, void *srcData, int destType, void *destData)
for memory to memory transfers.
VTKRENDERINGPARALLELLIC_EXPORT ostream & operator<<(std::ostream &os, const vtkPPixelTransfer &gt)