vtkPStructuredGridConnectivity.h

Go to the documentation of this file.

// SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
// SPDX-License-Identifier: BSD-3-Clause
#ifndef vtkPStructuredGridConnectivity_h
#define vtkPStructuredGridConnectivity_h

// VTK include directives
#include "vtkDeprecation.h"                   // For VTK_DEPRECATED_IN_9_6_0
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkMPICommunicator.h"               // Needed for vtkMPICommunicator::Request
#include "vtkStructuredGridConnectivity.h"

// C++ include directives
#include <vector> // For STL vector

// Forward declarations
VTK_ABI_NAMESPACE_BEGIN
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
// class vtkMPICommunicator::Request;

class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity
  : public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity, vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent) override;


  virtual void SetController(vtkMultiProcessController*);
  vtkGetObjectMacro(Controller, vtkMultiProcessController);

  void SetNumberOfGrids(unsigned int N) override;

  void RegisterGrid(int gridID, int extents[6], vtkUnsignedCharArray* nodesGhostArray,
    vtkUnsignedCharArray* cellGhostArray, vtkPointData* pointData, vtkCellData* cellData,
    vtkPoints* gridNodes) override;

  int GetNumberOfLocalGrids() { return static_cast<int>(this->GridIds.size()); }

  int GetGridRank(int gridID);

  bool IsGridRemote(int gridID);

  bool IsGridLocal(int gridID);

  void Initialize();

  void ComputeNeighbors() override;

  void CreateGhostLayers(int N = 1) override;

protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity() override;

  vtkMultiProcessController* Controller;
  int Rank;
  bool Initialized;

  std::vector<int> GridRanks; // Corresponding rank for each grid
  std::vector<int> GridIds;   // List of GridIds, owned by this process

  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector<std::vector<vtkPoints*>> RemotePoints;
  std::vector<std::vector<vtkPointData*>> RemotePointData;
  std::vector<std::vector<vtkCellData*>> RemoteCellData;

  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector<std::vector<unsigned int>> SendBufferSizes;
  std::vector<std::vector<unsigned int>> RcvBufferSizes;
  std::vector<std::vector<unsigned char*>> SendBuffers;
  std::vector<std::vector<unsigned char*>> RcvBuffers;

  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;

  // Array of MPI requests
  vtkMPICommunicator::Request* MPIRequests;

  bool GridExtentsAreEqual(int rhs[6], int lhs[6]);

  bool HasPointData(int gridIdx);

  bool HasCellData(int gridIdx);

  bool HasPoints(int gridIdx);

  void InitializeMessageCounters();

  void ClearRemoteData();

  void ClearRawBuffers();

  void RegisterRemoteGrid(int gridID, int extents[6], int process);

  void TransferRemoteNeighborData(int gridIdx, int nei, const vtkStructuredNeighbor& Neighbor);

  void TransferGhostDataFromNeighbors(int gridID) override;

  void PackGhostData();

  void UnpackGhostData();

  void DeserializeBufferSizesForProcess(int* buffersizes, vtkIdType N, int processId);

  void SerializeBufferSizes(int*& sizesbuf, vtkIdType& N);

  void ExchangeBufferSizes();

  void ExchangeGhostDataInit();

  void PostReceives();

  void PostSends();

  void CommunicateGhostData();

  void ExchangeGhostDataPost();

  void ExchangeGhostData();

  void SerializeGhostPoints(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);

  VTK_DEPRECATED_IN_9_6_0("Use vtkMultiProcessStream::Push(vtkDataArray*) instead")
  void SerializeDataArray(vtkDataArray*, vtkMultiProcessStream&);

  void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData* fieldData,
    vtkMultiProcessStream& bytestream);

  void SerializeGhostPointData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);

  void SerializeGhostCellData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);

  void DeserializeGhostPoints(int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);

  VTK_DEPRECATED_IN_9_6_0("Use vtkMultiProcessStream::Pop(vtkDataArray*&) instead")
  void DeserializeDataArray(vtkDataArray*&, int, int, int, vtkMultiProcessStream&);

  VTK_DEPRECATED_IN_9_6_0("Use vtkFieldDataSerializer::Deserialize() instead.")
  void DeserializeFieldData(int ext[6], vtkFieldData* fieldData, vtkMultiProcessStream& bytestream);

  void DeserializeGhostPointData(
    int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);

  void DeserializeGhostCellData(
    int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);

  void SerializeGhostData(
    int sndGridID, int rcvGrid, int sndext[6], unsigned char*& buffer, unsigned int& size);

  void DeserializeGhostData(int gridID, int neiListID, int neiGridIdx, int rcvext[6],
    unsigned char* buffer, unsigned int size);

  void ExchangeGridExtents();

  void SerializeGridExtents(int*& sndbuffer, vtkIdType& N);

  void DeserializeGridExtentForProcess(int* rcvbuffer, vtkIdType& N, int processId);

private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity&) = delete;
  void operator=(const vtkPStructuredGridConnectivity&) = delete;
};

//=============================================================================
//  INLINE METHODS
//=============================================================================

inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(int rhs[6], int lhs[6])
{
  for (int i = 0; i < 6; ++i)
  {
    if (rhs[i] != lhs[i])
    {
      return false;
    }
  }
  return true;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if ((this->GridPointData[gridIdx] != nullptr) &&
    (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if ((this->GridCellData[gridIdx] != nullptr) &&
    (this->GridCellData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));

  if (this->GridPoints[gridIdx] != nullptr)
  {
    return true;
  }
  return false;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs = this->TotalNumberOfRcvs = this->TotalNumberOfSends = 0;
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();

  // STEP 0: Clear send buffers
  for (unsigned int i = 0; i < this->SendBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->SendBuffers[i].size(); ++j)
    {
      delete[] this->SendBuffers[i][j];
    } // END for all neighbors
    this->SendBuffers[i].clear();
  } // END for all grids
  this->SendBuffers.clear();

  // STEP 1: Clear rcv buffers
  for (unsigned int i = 0; i < this->RcvBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RcvBuffers[i].size(); ++j)
    {
      delete[] this->RcvBuffers[i][j];
    } // END for all neighbors
    this->RcvBuffers[i].clear();
  } // END for all grids
  this->RcvBuffers.clear();
}

//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for (unsigned int i = 0; i < this->RemotePoints.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePoints[i].size(); ++j)
    {
      if (this->RemotePoints[i][j] != nullptr)
      {
        this->RemotePoints[i][j]->Delete();
      }
    } // END for all j
    this->RemotePoints[i].clear();
  } // END for all i
  this->RemotePoints.clear();

  // STEP 1: Clear remote point data
  for (unsigned int i = 0; i < this->RemotePointData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePointData[i].size(); ++j)
    {
      if (this->RemotePointData[i][j] != nullptr)
      {
        this->RemotePointData[i][j]->Delete();
      }
    } // END for all j
    this->RemotePointData[i].clear();
  } // END for all i
  this->RemotePointData.clear();

  // STEP 2: Clear remote cell data
  for (unsigned int i = 0; i < this->RemoteCellData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemoteCellData[i].size(); ++j)
    {
      if (this->RemoteCellData[i][j] != nullptr)
      {
        this->RemoteCellData[i][j]->Delete();
      }
    } // END for all j
    this->RemoteCellData[i].clear();
  }
  this->RemoteCellData.clear();
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(int gridID)
{
  return (!this->IsGridLocal(gridID));
}

//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID is out-of-bounds" && (gridID >= 0) &&
    (gridID < static_cast<int>(this->NumberOfGrids)));
  assert(
    "pre: GridRanks is not properly allocated" && this->NumberOfGrids == this->GridRanks.size());
  return ((this->GridRanks[gridID] == this->Rank));
}

//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank(int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID out-of-bounds!" &&
    (gridID >= 0 && gridID < static_cast<int>(this->NumberOfGrids)));
  return (this->GridRanks[gridID]);
}
VTK_ABI_NAMESPACE_END
#endif /* vtkPStructuredGridConnectivity_h */