PeriDEM 0.3.0
PeriDEM -- Peridynamics-based high-fidelity model for granular media
Loading...
Searching...
No Matches
Functions
anonymous_namespace{pdMpi.cpp} Namespace Reference

Functions

void uniqueSorted (std::vector< int > &v)
 
void buildGhostPlan (data::ModelData &data)
 
void syncAllOwnedPoints (data::ModelData &data, std::vector< util::Point > &field)
 
void exchangePoints (data::ModelData &data, std::vector< util::Point > &field)
 
void exchangeDoubles (data::ModelData &data, std::vector< double > &field)
 

Function Documentation

◆ buildGhostPlan()

void anonymous_namespace{pdMpi.cpp}::buildGhostPlan ( data::ModelData &  data)

Definition at line 40 of file pdMpi.cpp.

40 {
41 MPI_Comm comm = util::parallel::mpiComm();
42 const int size = util::parallel::mpiSize();
43 const int rank = util::parallel::mpiRank();
44 const size_t n_nodes = data.d_pdNodePartition.size();
45
46 std::vector<std::vector<int>> need_from(static_cast<size_t>(size));
47 for (size_t i = 0; i < n_nodes; ++i) {
48 if (static_cast<int>(data.d_pdNodePartition[i]) != rank)
49 continue;
50 for (size_t j : data.d_neighPd[i]) {
51 const int own = static_cast<int>(data.d_pdNodePartition[j]);
52 if (own != rank)
53 need_from[static_cast<size_t>(own)].push_back(static_cast<int>(j));
54 }
55 }
56 for (auto &v : need_from)
57 uniqueSorted(v);
58
59 std::vector<int> sendcounts(static_cast<size_t>(size), 0);
60 std::vector<int> recvcounts(static_cast<size_t>(size), 0);
61 for (int r = 0; r < size; ++r)
62 sendcounts[static_cast<size_t>(r)] =
63 static_cast<int>(need_from[static_cast<size_t>(r)].size());
64 MPI_Alltoall(sendcounts.data(), 1, MPI_INT, recvcounts.data(), 1, MPI_INT,
65 comm);
66
67 std::vector<int> sdispls(static_cast<size_t>(size), 0);
68 std::vector<int> rdispls(static_cast<size_t>(size), 0);
69 int send_total = 0;
70 int recv_total = 0;
71 for (int r = 0; r < size; ++r) {
72 sdispls[static_cast<size_t>(r)] = send_total;
73 rdispls[static_cast<size_t>(r)] = recv_total;
74 send_total += sendcounts[static_cast<size_t>(r)];
75 recv_total += recvcounts[static_cast<size_t>(r)];
76 }
77
78 std::vector<int> sendbuf(static_cast<size_t>(send_total));
79 for (int r = 0; r < size; ++r) {
80 const auto &v = need_from[static_cast<size_t>(r)];
81 std::copy(v.begin(), v.end(),
82 sendbuf.begin() + sdispls[static_cast<size_t>(r)]);
83 }
84 std::vector<int> recvbuf(static_cast<size_t>(recv_total));
85 MPI_Alltoallv(sendbuf.data(), sendcounts.data(), sdispls.data(), MPI_INT,
86 recvbuf.data(), recvcounts.data(), rdispls.data(), MPI_INT,
87 comm);
88
89 data.d_pdGhostNeedFrom = std::move(need_from);
90 data.d_pdGhostServeTo.assign(static_cast<size_t>(size), {});
91 for (int r = 0; r < size; ++r) {
92 auto &dst = data.d_pdGhostServeTo[static_cast<size_t>(r)];
93 dst.resize(static_cast<size_t>(recvcounts[static_cast<size_t>(r)]));
94 for (int k = 0; k < recvcounts[static_cast<size_t>(r)]; ++k) {
95 const int g = recvbuf[static_cast<size_t>(rdispls[static_cast<size_t>(r)] + k)];
96 if (g < 0 || static_cast<size_t>(g) >= n_nodes)
97 throw std::runtime_error("pdMpi: bad node id in ghost plan");
98 if (static_cast<int>(data.d_pdNodePartition[static_cast<size_t>(g)]) !=
99 rank)
100 throw std::runtime_error("pdMpi: serve node not owned here");
101 dst[static_cast<size_t>(k)] = g;
102 }
103 }
104
105 size_t n_ghost = 0;
106 for (const auto &v : data.d_pdGhostNeedFrom)
107 n_ghost += v.size();
108 data.setKeyData("pd_mpi_ghost_nodes", static_cast<double>(n_ghost));
109}
anonymous_namespace{pdMpi.cpp}::uniqueSorted
void uniqueSorted(std::vector< int > &v)
Definition pdMpi.cpp:35
data
Definition contact.h:20
util::parallel::mpiSize
int mpiSize()
Get size (number) of processors.
Definition parallelUtil.cpp:88
util::parallel::mpiRank
int mpiRank()
get rank (id) of this processor
Definition parallelUtil.cpp:96
util::parallel::mpiComm
MPI_Comm mpiComm()
Get MPI comm.
Definition parallelUtil.cpp:104

References util::parallel::mpiComm(), util::parallel::mpiRank(), util::parallel::mpiSize(), and uniqueSorted().

Referenced by pd::setupDofPartition().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ exchangeDoubles()

void anonymous_namespace{pdMpi.cpp}::exchangeDoubles ( data::ModelData &  data,
std::vector< double > &  field 
)

Definition at line 184 of file pdMpi.cpp.

184 {
185 MPI_Comm comm = util::parallel::mpiComm();
186 const int size = util::parallel::mpiSize();
187
188 std::vector<std::vector<double>> send(static_cast<size_t>(size));
189 for (int r = 0; r < size; ++r) {
190 for (int id : data.d_pdGhostServeTo[static_cast<size_t>(r)])
191 send[static_cast<size_t>(r)].push_back(field[static_cast<size_t>(id)]);
192 }
193
194 std::vector<int> sendcounts(static_cast<size_t>(size), 0);
195 std::vector<int> recvcounts(static_cast<size_t>(size), 0);
196 for (int r = 0; r < size; ++r)
197 sendcounts[static_cast<size_t>(r)] =
198 static_cast<int>(send[static_cast<size_t>(r)].size());
199 MPI_Alltoall(sendcounts.data(), 1, MPI_INT, recvcounts.data(), 1, MPI_INT,
200 comm);
201
202 std::vector<int> sdispls(static_cast<size_t>(size), 0);
203 std::vector<int> rdispls(static_cast<size_t>(size), 0);
204 int send_total = 0;
205 int recv_total = 0;
206 for (int r = 0; r < size; ++r) {
207 sdispls[static_cast<size_t>(r)] = send_total;
208 rdispls[static_cast<size_t>(r)] = recv_total;
209 send_total += sendcounts[static_cast<size_t>(r)];
210 recv_total += recvcounts[static_cast<size_t>(r)];
211 }
212
213 std::vector<double> sendbuf(static_cast<size_t>(send_total));
214 for (int r = 0; r < size; ++r) {
215 const auto &v = send[static_cast<size_t>(r)];
216 std::copy(v.begin(), v.end(),
217 sendbuf.begin() + sdispls[static_cast<size_t>(r)]);
218 }
219 std::vector<double> recvbuf(static_cast<size_t>(recv_total));
220 MPI_Alltoallv(sendbuf.data(), sendcounts.data(), sdispls.data(), MPI_DOUBLE,
221 recvbuf.data(), recvcounts.data(), rdispls.data(), MPI_DOUBLE,
222 comm);
223
224 for (int r = 0; r < size; ++r) {
225 size_t off = static_cast<size_t>(rdispls[static_cast<size_t>(r)]);
226 for (int id : data.d_pdGhostNeedFrom[static_cast<size_t>(r)])
227 field[static_cast<size_t>(id)] = recvbuf[off++];
228 }
229}

References util::parallel::mpiComm(), and util::parallel::mpiSize().

Referenced by pd::exchangeGhostTheta().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ exchangePoints()

void anonymous_namespace{pdMpi.cpp}::exchangePoints ( data::ModelData &  data,
std::vector< util::Point > &  field 
)

Definition at line 130 of file pdMpi.cpp.

130 {
131 MPI_Comm comm = util::parallel::mpiComm();
132 const int size = util::parallel::mpiSize();
133
134 std::vector<std::vector<double>> send(static_cast<size_t>(size));
135 for (int r = 0; r < size; ++r) {
136 for (int id : data.d_pdGhostServeTo[static_cast<size_t>(r)]) {
137 const auto &p = field[static_cast<size_t>(id)];
138 send[static_cast<size_t>(r)].push_back(p.d_x);
139 send[static_cast<size_t>(r)].push_back(p.d_y);
140 send[static_cast<size_t>(r)].push_back(p.d_z);
141 }
142 }
143
144 std::vector<int> sendcounts(static_cast<size_t>(size), 0);
145 std::vector<int> recvcounts(static_cast<size_t>(size), 0);
146 for (int r = 0; r < size; ++r)
147 sendcounts[static_cast<size_t>(r)] =
148 static_cast<int>(send[static_cast<size_t>(r)].size());
149 MPI_Alltoall(sendcounts.data(), 1, MPI_INT, recvcounts.data(), 1, MPI_INT,
150 comm);
151
152 std::vector<int> sdispls(static_cast<size_t>(size), 0);
153 std::vector<int> rdispls(static_cast<size_t>(size), 0);
154 int send_total = 0;
155 int recv_total = 0;
156 for (int r = 0; r < size; ++r) {
157 sdispls[static_cast<size_t>(r)] = send_total;
158 rdispls[static_cast<size_t>(r)] = recv_total;
159 send_total += sendcounts[static_cast<size_t>(r)];
160 recv_total += recvcounts[static_cast<size_t>(r)];
161 }
162
163 std::vector<double> sendbuf(static_cast<size_t>(send_total));
164 for (int r = 0; r < size; ++r) {
165 const auto &v = send[static_cast<size_t>(r)];
166 std::copy(v.begin(), v.end(),
167 sendbuf.begin() + sdispls[static_cast<size_t>(r)]);
168 }
169 std::vector<double> recvbuf(static_cast<size_t>(recv_total));
170 MPI_Alltoallv(sendbuf.data(), sendcounts.data(), sdispls.data(), MPI_DOUBLE,
171 recvbuf.data(), recvcounts.data(), rdispls.data(), MPI_DOUBLE,
172 comm);
173
174 for (int r = 0; r < size; ++r) {
175 size_t off = static_cast<size_t>(rdispls[static_cast<size_t>(r)]);
176 for (int id : data.d_pdGhostNeedFrom[static_cast<size_t>(r)]) {
177 field[static_cast<size_t>(id)] =
178 util::Point(recvbuf[off], recvbuf[off + 1], recvbuf[off + 2]);
179 off += 3;
180 }
181 }
182}
util::Point
A structure to represent 3d vectors.
Definition point.h:30

References util::parallel::mpiComm(), and util::parallel::mpiSize().

Referenced by pd::exchangeGhostDisplacement().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ syncAllOwnedPoints()

void anonymous_namespace{pdMpi.cpp}::syncAllOwnedPoints ( data::ModelData &  data,
std::vector< util::Point > &  field 
)

Gather owned nodal Point fields to every rank (unique Metis owners). Needed for Multi_Particle DOF where contact reaches outside the PD halo.

Definition at line 113 of file pdMpi.cpp.

113 {
114 const int rank = util::parallel::mpiRank();
115 const size_t n = field.size();
116 std::vector<double> buf(3 * n, 0.);
117 for (size_t i = 0; i < n; ++i) {
118 if (static_cast<int>(data.d_pdNodePartition[i]) != rank)
119 continue;
120 buf[3 * i + 0] = field[i].d_x;
121 buf[3 * i + 1] = field[i].d_y;
122 buf[3 * i + 2] = field[i].d_z;
123 }
124 MPI_Allreduce(MPI_IN_PLACE, buf.data(), static_cast<int>(3 * n), MPI_DOUBLE,
125 MPI_SUM, util::parallel::mpiComm());
126 for (size_t i = 0; i < n; ++i)
127 field[i] = util::Point(buf[3 * i], buf[3 * i + 1], buf[3 * i + 2]);
128}

References util::parallel::mpiComm(), and util::parallel::mpiRank().

Referenced by pd::exchangeGhostDisplacement().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ uniqueSorted()

void anonymous_namespace{pdMpi.cpp}::uniqueSorted ( std::vector< int > &  v)

Definition at line 35 of file pdMpi.cpp.

35 {
36 std::sort(v.begin(), v.end());
37 v.erase(std::unique(v.begin(), v.end()), v.end());
38}

Referenced by buildGhostPlan().

Here is the caller graph for this function: