escript  Revision_
blocktools.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2014-2018 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14 * Development from 2019 by School of Earth and Environmental Sciences
15 **
16 *****************************************************************************/
17 
18 #ifndef __RIPLEY_BLOCKTOOLS_H__
19 #define __RIPLEY_BLOCKTOOLS_H__
20 
21 /* This file contains two main classes for dealing with a large 3D region which
22  * has been divided into a 3D Grid of Blocks (usually to be distributed).
23  * Each block is divided into 27 subblocks. The first and last subblocks in
24  * each dimension are cubes.
25  *
26  * class Block manages a single block. It has methods for copying between a
27  * flat array (storing all the values in the 3D block) and buffers storing
28  * individual subblocks. These buffers can be used to transfer individual
29  * subblocks to other blocks (using some external means).
30  *
31  * class BlockGrid deals with the position of a given block in relation to the
32  * rest of the blocks in the grid. (In an MPI setting, there would be one block
33  * per rank.)
34  * It also requires transfers of subblocks into and out of the block in order
35  * to make the whole (global) array consistent.
36  *
37  * Each block has a region "inset" wide in from each edge which is shared with
38  * neighbouring blocks. Where these regions overlap with another block, the
39  * block closest to block 0,0,0 is used.
40  * Or more precisely, values move left->right then lowy->highy and finally
41  * lowz -> highz.
42  *
43  * Please don't mix external calls into this file, it may be useful to separate
44  * it for debugging purposes.
45  *
46  * Types required:
47  * neighbourID_t - Stores the label of a neighbouring block.
48  * In an MPI setting, this will be the type used refer to
49  * ranks
50  * coord_t - Stores a position of a block in the grid of blocks
51  * (it could be within one dimension or overall in a flat
52  * structure. It is not (necessarily) the same as
53  * neighbourID_t because coord_t should be _unsigned_ and
54  * there is no guarantee that neighbourID_t will even be
55  * an integral type.
56  */
57 
58 #include <escript/EsysMPI.h>
59 
60 #include <vector>
61 
62 typedef int neighbourID_t; // This should be the MPI_rank type
63 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
64 
65 typedef std::pair<neighbourID_t, int> neighpair;
66 typedef std::vector<neighpair> neighbourvector;
67 
68 
69 typedef struct
70 {
71 public:
72  neighbourID_t sourceID; // ranks involved in communication
74  int tag;
75  unsigned char srcbuffid; // number of buffer to use for coms
76  unsigned char destbuffid;
77 } message;
78 
79 typedef std::vector<message> messvec;
80 
81 
82 class BlockGrid
83 {
84 public:
85  BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
86 
88 
89 
90  // generate all incoming com messages for this block.
91  // for each subblock (27 of them), there may be an x, y, z direction to search in
92  void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
93 
94 
95  // generate all outgoing com messages for this block
96  void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
97 
98 private:
102 };
103 
104 
105 
106 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
107  They do not exist, such buffers would be:
108  1) big
109  2) unnecessary since the centre sub-block is not sent anywhere
110 
111 Note that this class does not deal with data transfer between blocks
112 Sub-blocks are copied to and from buffers. Other code is required to
113 actually move the data.
114 
115 "dpp" == doubles per point and gives the number of doubles that make up each "point"
116 This is required to calculate offsets and buffer sizes.
117 */
118 
119 class Block
120 {
121 public:
122  // s? specifiy the [local] size (in points) of each dimension
123  Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
124  size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
125 
126  ~Block();
127 
128  // Out buffers are loaded with the contents of the flat array and are
129  // to be sent to other blocks
130  double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
131  double* getOutBuffer(unsigned char bid);
132 
133  // In buffers are populated from external communications
134  // and copied back to the flat array
135  double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
136  double* getInBuffer(unsigned char bid);
137 
138  // return number of doubles in the given block
139  size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
140  size_t getBuffSize(unsigned char bid);
141 
142  // where does the subblock specified start in a source array
143  size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
144 
145  // debug only
146  void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
147 
148  // Copy a 3d region from a flat array into a buffer
149  void copyToBuffer(unsigned char buffid, double* src);
150 
151  // Copy a 3d region from a buffer into a flat array
152  void copyFromBuffer(unsigned char buffid, double* dest);
153 
154 
155  void copyAllToBuffer(double* src);
156 
157  void copyUsedFromBuffer(double* dest);
158 
159  void setUsed(unsigned char buffid);
160 
161 private:
162 
163  // determines the dimensions of each subblock
164  void populateDimsTable();
165  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
166  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
167 
168 
169  double* inbuff;
170  double* outbuff;
171  size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
172  size_t flatoffsets[27]; // starting point of each block within a flat array
173  bool used[27];
174  size_t dims[27][3]; // dimension of each subblock
175  size_t sx;
176  size_t sy;
177  size_t sz;
178  size_t inset;
179  size_t xmidlen;
180  size_t ymidlen;
181  size_t zmidlen;
182  double* inbuffptr[27];
183  double* outbuffptr[27];
184  const unsigned int dpsize; // number of doubles which make up a point
185 };
186 
187 // Returns the MPI message tag to use for a transfer between the two subblocks
188 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez,
189  unsigned char targetx, unsigned char targety, unsigned char targetz);
190 
191 // computes the tag based on the destination and the direction it comes from
192 // the booleans indicate whether a negative shift in that direction is required
193 int getTag(unsigned char destx, unsigned char desty, unsigned char destz,
194  bool deltax, bool deltay, bool deltaz);
195 
196 
197 // the booleans indicate whether a negative shift in that direction is required
198 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty,
199  unsigned char destz, bool deltax, bool deltay,
200  bool deltaz);
201 
202 
203 /* Now the 2D versions */
204 
205 // 2D version
207 {
208 public:
209  BlockGrid2(coord_t maxx, coord_t maxy);
210 
211  neighbourID_t getNID(coord_t x, coord_t y) const;
212 
213  // generate all incoming com messages for this block.
214  // for each subblock (9 of them), there may be an x, y direction to search in
215  void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
216 
217  // generate all outgoing com messages for this block
218  void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
219 
220 private:
223 };
224 
225 // The 2D version - there is no block 4
226 class Block2
227 {
228 public:
229  // s? specifiy the [local] size (in points) of each dimension
230  Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
231  size_t ymidlen, unsigned int dpp=1);
232 
233  ~Block2();
234 
235  // Out buffers are loaded with the contents of the flat array and are
236  // to be sent to other blocks
237  double* getOutBuffer(unsigned char subx, unsigned char suby);
238  double* getOutBuffer(unsigned char bid);
239 
240  // In buffers are populated from external communications
241  // and copied back to the flat array
242  double* getInBuffer(unsigned char subx, unsigned char suby);
243  double* getInBuffer(unsigned char bid);
244 
245  // return number of doubles in the given block
246  size_t getBuffSize(unsigned char subx, unsigned char suby);
247  size_t getBuffSize(unsigned char bid);
248 
249  // where does the subblock specified start in a source array
250  size_t startOffset(unsigned char subx, unsigned char suby);
251 
252  // debug only
253  void displayBlock(unsigned char subx, unsigned char suby, bool out);
254 
255  // Copy a 3d region from a flat array into a buffer
256  void copyToBuffer(unsigned char buffid, double* src);
257 
258  // Copy a 3d region from a buffer into a flat array
259  void copyFromBuffer(unsigned char buffid, double* dest);
260 
261 
262  void copyAllToBuffer(double* src);
263 
264  void copyUsedFromBuffer(double* dest);
265 
266  void setUsed(unsigned char buffid);
267 
268 private:
269 
270  // determines the dimensions of each subblock
271  void populateDimsTable();
272  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
273  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
274 
275 
276  double* inbuff;
277  double* outbuff;
278  size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
279  size_t flatoffsets[9]; // starting point of each block within a flat array
280  bool used[9];
281  size_t dims[9][2]; // dimension of each subblock
282  size_t sx;
283  size_t sy;
284  size_t inset;
285  size_t xmidlen;
286  size_t ymidlen;
287  double* inbuffptr[9];
288  double* outbuffptr[9];
289  const unsigned int dpsize; // number of doubles which make up a point
290 };
291 
292 
293 // Returns the MPI message tag to use for a transfer between the two subblocks
294 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
295 
296 // computes the tag based on the destination and the direction it comes from
297 // the booleans indicate whether a negative shift in that direction is required
298 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
299 
300 
301 // the booleans indicate whether a negative shift in that direction is required
302 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
303 
304 #endif // __RIPLEY_BLOCKTOOLS_H__
305 
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition: blocktools2.cpp:381
std::pair< neighbourID_t, int > neighpair
Definition: blocktools.h:65
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition: blocktools2.cpp:365
unsigned coord_t
Definition: blocktools.h:63
int neighbourID_t
Definition: blocktools.h:62
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:413
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:431
std::vector< message > messvec
Definition: blocktools.h:79
std::vector< neighpair > neighbourvector
Definition: blocktools.h:66
Definition: blocktools.h:227
size_t inset
Definition: blocktools.h:284
double * getOutBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:101
Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, size_t ymidlen, unsigned int dpp=1)
Definition: blocktools2.cpp:247
size_t xmidlen
Definition: blocktools.h:285
void populateDimsTable()
Definition: blocktools2.cpp:166
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:184
void displayBlock(unsigned char subx, unsigned char suby, bool out)
Definition: blocktools2.cpp:289
size_t dims[9][2]
Definition: blocktools.h:281
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:206
const unsigned int dpsize
Definition: blocktools.h:289
size_t flatoffsets[9]
Definition: blocktools.h:279
size_t getBuffSize(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:121
~Block2()
Definition: blocktools2.cpp:160
size_t buffoffsets[9]
Definition: blocktools.h:278
void copyAllToBuffer(double *src)
Definition: blocktools2.cpp:221
size_t sx
Definition: blocktools.h:282
size_t startOffset(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:277
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools2.cpp:318
void copyUsedFromBuffer(double *dest)
Definition: blocktools2.cpp:234
void setUsed(unsigned char buffid)
Definition: blocktools2.cpp:216
double * outbuffptr[9]
Definition: blocktools.h:288
double * inbuffptr[9]
Definition: blocktools.h:287
size_t sy
Definition: blocktools.h:283
double * inbuff
Definition: blocktools.h:276
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools2.cpp:346
double * outbuff
Definition: blocktools.h:277
double * getInBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:111
size_t ymidlen
Definition: blocktools.h:286
bool used[9]
Definition: blocktools.h:280
Definition: blocktools.h:207
BlockGrid2(coord_t maxx, coord_t maxy)
Definition: blocktools2.cpp:24
coord_t xmax
Definition: blocktools.h:221
void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:68
void generateInNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:35
neighbourID_t getNID(coord_t x, coord_t y) const
Definition: blocktools2.cpp:28
coord_t ymax
Definition: blocktools.h:222
Definition: blocktools.h:83
coord_t zmax
Definition: blocktools.h:101
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition: blocktools.cpp:29
coord_t ymax
Definition: blocktools.h:100
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition: blocktools.cpp:25
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:44
coord_t xmax
Definition: blocktools.h:99
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:83
Definition: blocktools.h:120
size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:143
bool used[27]
Definition: blocktools.h:173
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools.cpp:358
~Block()
Definition: blocktools.cpp:182
size_t ymidlen
Definition: blocktools.h:180
size_t dims[27][3]
Definition: blocktools.h:174
double * getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:123
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools.cpp:387
void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out)
Definition: blocktools.cpp:324
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:237
void setUsed(unsigned char buffid)
Definition: blocktools.cpp:247
size_t zmidlen
Definition: blocktools.h:181
double * getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:133
size_t inset
Definition: blocktools.h:178
const unsigned int dpsize
Definition: blocktools.h:184
size_t xmidlen
Definition: blocktools.h:179
size_t sz
Definition: blocktools.h:177
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:215
size_t buffoffsets[27]
Definition: blocktools.h:171
size_t sy
Definition: blocktools.h:176
size_t sx
Definition: blocktools.h:175
size_t flatoffsets[27]
Definition: blocktools.h:172
double * inbuff
Definition: blocktools.h:169
void copyUsedFromBuffer(double *dest)
Definition: blocktools.cpp:265
double * inbuffptr[27]
Definition: blocktools.h:182
size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:310
void copyAllToBuffer(double *src)
Definition: blocktools.cpp:252
Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen, unsigned int dpp=1)
Definition: blocktools.cpp:278
void populateDimsTable()
Definition: blocktools.cpp:188
double * outbuffptr[27]
Definition: blocktools.h:183
double * outbuff
Definition: blocktools.h:170
Definition: blocktools.h:70
unsigned char destbuffid
Definition: blocktools.h:76
unsigned char srcbuffid
Definition: blocktools.h:75
neighbourID_t destID
Definition: blocktools.h:73
int tag
Definition: blocktools.h:74
neighbourID_t sourceID
Definition: blocktools.h:72