anonymous_namespace{particleMpi.cpp} Namespace Reference

Functions
void	factor2d (int nproc, double Lx, double Ly, int &nx, int &ny)
void	packGrainKinematics (const data::ModelData &data, const particle::BaseParticle &p, std::vector< double > &buf)
size_t	unpackGrainKinematics (data::ModelData &data, particle::BaseParticle &p, const std::vector< double > &buf, size_t offset)
size_t	ghostRebuildInterval (const data::ModelData &data)
void	rebuildGhostPlan (data::ModelData &data)
void	exchangeCachedKinematics (data::ModelData &data)

Function Documentation

exchangeCachedKinematics()

void anonymous_namespace{particleMpi.cpp}::exchangeCachedKinematics

(

data::ModelData &

data

)

Definition at line 261 of file particleMpi.cpp.

                                                   {
  MPI_Comm comm = util::parallel::mpiComm();
  const int size = util::parallel::mpiSize();
  const auto &grains = data.d_particlesListTypeParticle;
 
  std::vector<std::vector<double>> kin_send(static_cast<size_t>(size));
  for (int r = 0; r < size; ++r) {
    for (int g : data.d_mpiGhostServeTo[static_cast<size_t>(r)])
      packGrainKinematics(data, *grains[static_cast<size_t>(g)],
                          kin_send[static_cast<size_t>(r)]);
  }
 
  std::vector<int> kin_sendcounts(static_cast<size_t>(size), 0);
  std::vector<int> kin_recvcounts(static_cast<size_t>(size), 0);
  for (int r = 0; r < size; ++r)
    kin_sendcounts[static_cast<size_t>(r)] =
        static_cast<int>(kin_send[static_cast<size_t>(r)].size());
  MPI_Alltoall(kin_sendcounts.data(), 1, MPI_INT, kin_recvcounts.data(), 1,
               MPI_INT, comm);
 
  std::vector<int> kin_sdispls(static_cast<size_t>(size), 0);
  std::vector<int> kin_rdispls(static_cast<size_t>(size), 0);
  int kin_send_total = 0;
  int kin_recv_total = 0;
  for (int r = 0; r < size; ++r) {
    kin_sdispls[static_cast<size_t>(r)] = kin_send_total;
    kin_rdispls[static_cast<size_t>(r)] = kin_recv_total;
    kin_send_total += kin_sendcounts[static_cast<size_t>(r)];
    kin_recv_total += kin_recvcounts[static_cast<size_t>(r)];
  }
 
  std::vector<double> kin_sendbuf(static_cast<size_t>(kin_send_total));
  for (int r = 0; r < size; ++r) {
    const auto &v = kin_send[static_cast<size_t>(r)];
    std::copy(v.begin(), v.end(),
              kin_sendbuf.begin() + kin_sdispls[static_cast<size_t>(r)]);
  }
  std::vector<double> kin_recvbuf(static_cast<size_t>(kin_recv_total));
  MPI_Alltoallv(kin_sendbuf.data(), kin_sendcounts.data(), kin_sdispls.data(),
                MPI_DOUBLE, kin_recvbuf.data(), kin_recvcounts.data(),
                kin_rdispls.data(), MPI_DOUBLE, comm);
 
  for (int r = 0; r < size; ++r) {
    size_t off = static_cast<size_t>(kin_rdispls[static_cast<size_t>(r)]);
    for (int g : data.d_mpiGhostNeedFrom[static_cast<size_t>(r)])
      off = unpackGrainKinematics(data, *grains[static_cast<size_t>(g)],
                                  kin_recvbuf, off);
  }
}

References util::parallel::mpiComm(), util::parallel::mpiSize(), packGrainKinematics(), and unpackGrainKinematics().

Referenced by particle::exchangeGhostKinematics().

Here is the call graph for this function:

Here is the caller graph for this function:

factor2d()

void anonymous_namespace{particleMpi.cpp}::factor2d	(	int	nproc,
		double	Lx,
		double	Ly,
		int &	nx,
		int &	ny
	)

Definition at line 52 of file particleMpi.cpp.

                                                                 {
  // nx*ny = nproc; prefer subdomain aspect close to domain aspect.
  nx = 1;
  ny = nproc;
  double best = std::numeric_limits<double>::max();
  for (int i = 1; i <= nproc; ++i) {
    if (nproc % i != 0)
      continue;
    const int j = nproc / i;
    const double ax = Lx / static_cast<double>(i);
    const double ay = Ly / static_cast<double>(j);
    const double score = std::abs(ax - ay);
    if (score < best) {
      best = score;
      nx = i;
      ny = j;
    }
  }
}

Referenced by particle::assignMpiOwners().

Here is the caller graph for this function:

ghostRebuildInterval()

size_t anonymous_namespace{particleMpi.cpp}::ghostRebuildInterval

(

const data::ModelData &

data

)

Definition at line 110 of file particleMpi.cpp.

                                                       {
  size_t interval = data.d_contNeighUpdateInterval;
  if (interval == 0 && data.d_particleDeck_p)
    interval = data.d_particleDeck_p->d_pNeighDeck.d_neighUpdateInterval;
  if (interval == 0)
    interval = 1;
  return interval;
}

Referenced by particle::exchangeGhostKinematics().

Here is the caller graph for this function:

packGrainKinematics()

void anonymous_namespace{particleMpi.cpp}::packGrainKinematics	(	const data::ModelData &	data,
		const particle::BaseParticle &	p,
		std::vector< double > &	buf
	)

Definition at line 72 of file particleMpi.cpp.

                                                 {
  const size_t n = p.getNumNodes();
  const size_t base = buf.size();
  buf.resize(base + 9 * n);
  for (size_t i = 0; i < n; ++i) {
    const size_t g = p.getNodeId(i);
    const auto &x = data.d_x[g];
    const auto &u = data.d_u[g];
    const auto &v = data.d_v[g];
    const size_t o = base + 9 * i;
    buf[o + 0] = x.d_x;
    buf[o + 1] = x.d_y;
    buf[o + 2] = x.d_z;
    buf[o + 3] = u.d_x;
    buf[o + 4] = u.d_y;
    buf[o + 5] = u.d_z;
    buf[o + 6] = v.d_x;
    buf[o + 7] = v.d_y;
    buf[o + 8] = v.d_z;
  }
}

References particle::BaseParticle::getNodeId(), and particle::BaseParticle::getNumNodes().

Referenced by exchangeCachedKinematics().

Here is the call graph for this function:

Here is the caller graph for this function:

rebuildGhostPlan()

void anonymous_namespace{particleMpi.cpp}::rebuildGhostPlan

(

data::ModelData &

data

)

Definition at line 119 of file particleMpi.cpp.

                                           {
  MPI_Comm comm = util::parallel::mpiComm();
  const int size = util::parallel::mpiSize();
  const int rank = util::parallel::mpiRank();
  const auto &grains = data.d_particlesListTypeParticle;
  const size_t n_grains = grains.size();
 
  std::vector<double> local_c(4 * n_grains, 0.);
  for (size_t g = 0; g < n_grains; ++g) {
    const auto c = grains[g]->getXCenter();
    local_c[4 * g + 0] = c.d_x;
    local_c[4 * g + 1] = c.d_y;
    local_c[4 * g + 2] = c.d_z;
    local_c[4 * g + 3] = grains[g]->d_geom_p->boundingRadius();
  }
  std::vector<double> all_c(4 * n_grains * static_cast<size_t>(size), 0.);
  MPI_Allgather(local_c.data(), static_cast<int>(local_c.size()), MPI_DOUBLE,
                all_c.data(), static_cast<int>(local_c.size()), MPI_DOUBLE,
                comm);
 
  std::vector<util::Point> centers(n_grains);
  std::vector<double> radii(n_grains);
  for (size_t g = 0; g < n_grains; ++g) {
    const int own = grains[g]->d_mpiOwner;
    const size_t base = static_cast<size_t>(own) * 4 * n_grains + 4 * g;
    centers[g] = util::Point(all_c[base + 0], all_c[base + 1], all_c[base + 2]);
    radii[g] = all_c[base + 3];
  }
 
  data.d_mpiIncludeInContactCloud.assign(data.d_particlesListTypeAll.size(), 0);
  for (auto *p : data.d_particlesListTypeAll) {
    if (p->isWall() || particle::isLocallyOwned(*p))
      data.d_mpiIncludeInContactCloud[p->getId()] = 1;
  }
 
  // Contact search radius already carries sFactor skin when interval > 1;
  // add a small extra buffer for MPI plan lifetime between rebuilds.
  const double extra =
      std::max(0.25 * data.d_maxContactR, std::max(data.d_hMax, 1.0e-16));
  const double search_r =
      (data.d_contNeighSearchRadius > 1.0e-16)
          ? data.d_contNeighSearchRadius
          : data.d_maxContactR;
  const double cutoff = search_r + extra;
 
  data.d_mpiGhostNeedFrom.assign(static_cast<size_t>(size), {});
  size_t n_ghost = 0;
  auto add_ghost = [&](size_t g) {
    auto *pj = grains[g];
    if (particle::isLocallyOwned(*pj))
      return;
    const int own = pj->d_mpiOwner;
    if (own < 0 || own == rank)
      return;
    auto &vec = data.d_mpiGhostNeedFrom[static_cast<size_t>(own)];
    const int gid = static_cast<int>(g);
    if (std::find(vec.begin(), vec.end(), gid) != vec.end())
      return;
    vec.push_back(gid);
    data.d_mpiIncludeInContactCloud[pj->getId()] = 1;
    ++n_ghost;
  };
 
  for (size_t g = 0; g < n_grains; ++g) {
    bool near = false;
    for (size_t i = 0; i < n_grains && !near; ++i) {
      if (!particle::isLocallyOwned(*grains[i]))
        continue;
      const double lim = radii[i] + radii[g] + cutoff;
      if (centers[i].dist(centers[g]) < lim)
        near = true;
    }
    if (near)
      add_ghost(g);
  }
 
  // Wall contact is assembled on ranks that own wall nodes (rank 0 for
  // particle-MPI). Ghost any grain near a wall so plate/cup searches see them.
  for (auto *w : data.d_particlesListTypeWall) {
    if (!w)
      continue;
    const auto wc = w->getXCenter();
    const double wr = w->d_geom_p->boundingRadius();
    for (size_t g = 0; g < n_grains; ++g) {
      const double lim = wr + radii[g] + cutoff;
      if (wc.dist(centers[g]) < lim)
        add_ghost(g);
    }
  }
 
  std::vector<int> req_sendcounts(static_cast<size_t>(size), 0);
  std::vector<int> req_recvcounts(static_cast<size_t>(size), 0);
  for (int r = 0; r < size; ++r)
    req_sendcounts[static_cast<size_t>(r)] =
        static_cast<int>(data.d_mpiGhostNeedFrom[static_cast<size_t>(r)].size());
  MPI_Alltoall(req_sendcounts.data(), 1, MPI_INT, req_recvcounts.data(), 1,
               MPI_INT, comm);
 
  std::vector<int> req_sdispls(static_cast<size_t>(size), 0);
  std::vector<int> req_rdispls(static_cast<size_t>(size), 0);
  int req_send_total = 0;
  int req_recv_total = 0;
  for (int r = 0; r < size; ++r) {
    req_sdispls[static_cast<size_t>(r)] = req_send_total;
    req_rdispls[static_cast<size_t>(r)] = req_recv_total;
    req_send_total += req_sendcounts[static_cast<size_t>(r)];
    req_recv_total += req_recvcounts[static_cast<size_t>(r)];
  }
 
  std::vector<int> req_sendbuf(static_cast<size_t>(req_send_total));
  for (int r = 0; r < size; ++r) {
    const auto &v = data.d_mpiGhostNeedFrom[static_cast<size_t>(r)];
    std::copy(v.begin(), v.end(),
              req_sendbuf.begin() + req_sdispls[static_cast<size_t>(r)]);
  }
  std::vector<int> req_recvbuf(static_cast<size_t>(req_recv_total));
  MPI_Alltoallv(req_sendbuf.data(), req_sendcounts.data(), req_sdispls.data(),
                MPI_INT, req_recvbuf.data(), req_recvcounts.data(),
                req_rdispls.data(), MPI_INT, comm);
 
  data.d_mpiGhostServeTo.assign(static_cast<size_t>(size), {});
  for (int r = 0; r < size; ++r) {
    const int off = req_rdispls[static_cast<size_t>(r)];
    const int nreq = req_recvcounts[static_cast<size_t>(r)];
    auto &dst = data.d_mpiGhostServeTo[static_cast<size_t>(r)];
    dst.resize(static_cast<size_t>(nreq));
    for (int k = 0; k < nreq; ++k) {
      const int g = req_recvbuf[static_cast<size_t>(off + k)];
      if (g < 0 || static_cast<size_t>(g) >= n_grains)
        throw std::runtime_error("rebuildGhostPlan: bad grain id");
      if (grains[static_cast<size_t>(g)]->d_mpiOwner != rank)
        throw std::runtime_error(
            "rebuildGhostPlan: requested grain not owned here");
      dst[static_cast<size_t>(k)] = g;
    }
  }
 
  data.d_mpiGhostPlanValid = true;
  data.d_mpiGhostStepsSinceRebuild = 0;
  data.setKeyData("mpi_ghost_grain_count", static_cast<double>(n_ghost));
}

References particle::isLocallyOwned(), util::parallel::mpiComm(), util::parallel::mpiRank(), and util::parallel::mpiSize().

Referenced by particle::exchangeGhostKinematics().

Here is the call graph for this function:

Here is the caller graph for this function:

unpackGrainKinematics()

size_t anonymous_namespace{particleMpi.cpp}::unpackGrainKinematics	(	data::ModelData &	data,
		particle::BaseParticle &	p,
		const std::vector< double > &	buf,
		size_t	offset
	)

Definition at line 96 of file particleMpi.cpp.

                                                                          {
  const size_t n = p.getNumNodes();
  for (size_t i = 0; i < n; ++i) {
    const size_t g = p.getNodeId(i);
    const size_t o = offset + 9 * i;
    data.d_x[g] = util::Point(buf[o + 0], buf[o + 1], buf[o + 2]);
    data.d_u[g] = util::Point(buf[o + 3], buf[o + 4], buf[o + 5]);
    data.d_v[g] = util::Point(buf[o + 6], buf[o + 7], buf[o + 8]);
    data.d_vMag[g] = data.d_v[g].length();
  }
  return offset + 9 * n;
}

References particle::BaseParticle::getNodeId(), and particle::BaseParticle::getNumNodes().

Referenced by exchangeCachedKinematics().

Here is the call graph for this function:

Here is the caller graph for this function: