8.0
general documentation
cs_halo.h
Go to the documentation of this file.
1 #ifndef __CS_HALO_H__
2 #define __CS_HALO_H__
3 
4 /*============================================================================
5  * Structure and function headers handling with ghost cells
6  *============================================================================*/
7 
8 /*
9  This file is part of code_saturne, a general-purpose CFD tool.
10 
11  Copyright (C) 1998-2023 EDF S.A.
12 
13  This program is free software; you can redistribute it and/or modify it under
14  the terms of the GNU General Public License as published by the Free Software
15  Foundation; either version 2 of the License, or (at your option) any later
16  version.
17 
18  This program is distributed in the hope that it will be useful, but WITHOUT
19  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
20  FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
21  details.
22 
23  You should have received a copy of the GNU General Public License along with
24  this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
25  Street, Fifth Floor, Boston, MA 02110-1301, USA.
26 */
27 
28 /*----------------------------------------------------------------------------*/
29 
30 /*----------------------------------------------------------------------------
31  * Local headers
32  *----------------------------------------------------------------------------*/
33 
34 #include "cs_defs.h"
35 #include "cs_base.h"
36 #include "cs_base_accel.h"
37 #include "cs_interface.h"
38 #include "cs_rank_neighbors.h"
39 
40 #include "fvm_periodicity.h"
41 
42 /*----------------------------------------------------------------------------*/
43 
45 
46 /*============================================================================
47  * Macro definitions
48  *============================================================================*/
49 
50 /*=============================================================================
51  * Type definitions
52  *============================================================================*/
53 
56 typedef enum {
57 
61 
63 
64 /* Halo communication mode */
65 
66 typedef enum {
67 
73 
74 /* Structure for halo management */
75 /* ----------------------------- */
76 
77 typedef struct {
78 
79  int n_c_domains; /* Number of communicating domains. */
80  int n_transforms; /* Number of periodic transformations */
81 
82  int *c_domain_rank; /* List of communicating ranks */
83 
84  const fvm_periodicity_t * periodicity; /* Pointer to periodicity
85  structure describing transforms */
86 
87  int n_rotations; /* Number of periodic transformations
88  involving rotations */
89 
90  cs_lnum_t n_local_elts; /* Number of local elements */
91 
92  /* send_halo features : send to distant ranks */
93 
94  cs_lnum_t n_send_elts[2]; /* Numer of ghost elements in send_list
95  n_elts[0] = standard elements
96  n_elts[1] = extended + standard elements */
97 
98  cs_lnum_t *send_list; /* List of local elements in distant halos
99  (0 to n-1 numbering) */
100 
101  cs_lnum_t *send_index; /* Index on send_list
102  Size = 2*n_c_domains + 1. For each rank, we
103  have an index for standard halo and one
104  for extended halo. */
105 
106  cs_lnum_t *send_perio_lst ; /* For each transformation and for each type of
107  halo on each communicating rank, we store
108  2 values:
109  - start index,
110  - number of elements. */
111 
112  /* halo features : receive from distant ranks */
113 
114  cs_lnum_t n_elts[2]; /* Numer of ghost elements in halo
115  n_elts[0] = standard elements
116  n_elts[1] = extended + standard elements */
117 
118  cs_lnum_t *index; /* Index on halo sections;
119  Size = 2*n_c_domains. For each rank, we
120  have an index for the standard halo and one
121  for the extended halo. */
122 
123  cs_lnum_t *perio_lst; /* For each transformation and for each type of halo
124  on each communicating rank, we store 2 values:
125  - start index,
126  - number of elements. */
127 
128  /* Organisation of perio_lst:
129 
130  -------------------------------------------------
131  T1: | | | | | | | | | | | | |
132  -------------------------------------------------
133  idx n idx n idx n idx n idx n idx n
134  ______ ______ ______ ______ ______ ______
135  std ext std ext std ext
136  ___________ ___________ ___________
137  rank 0 rank 1 rank 2
138 
139  -------------------------------------------------
140  T2: | | | | | | | | | | | | |
141  -------------------------------------------------
142  idx n idx n idx n idx n idx n idx n
143  ______ ______ ______ ______ ______ ______
144  std ext std ext std ext
145  ___________ ___________ ___________
146  rank 0 rank 1 rank 2
147 
148  -------------------------------------------------
149  T3: | | | | | | | | | | | | |
150  -------------------------------------------------
151  idx n idx n idx n idx n idx n idx n
152  ______ ______ ______ ______ ______ ______
153  std ext std ext std ext
154  ___________ ___________ ___________
155  rank 0 rank 1 rank 2
156 
157  etc...
158 
159  */
160 
161 #if defined(HAVE_MPI)
162 
163  MPI_Group c_domain_group; /* Group of connected domains */
164  cs_lnum_t *c_domain_s_shift; /* Target buffer shift for distant
165  ranks using one-sided get */
166 #endif
167 
168 } cs_halo_t;
169 
172 typedef struct _cs_halo_state_t cs_halo_state_t;
173 
174 /*=============================================================================
175  * Global static variables
176  *============================================================================*/
177 
178 /*============================================================================
179  * Public function header for Fortran API
180  *============================================================================*/
181 
182 /*=============================================================================
183  * Public function prototypes
184  *============================================================================*/
185 
186 /*----------------------------------------------------------------------------*/
194 /*----------------------------------------------------------------------------*/
195 
196 cs_halo_t *
198 
199 /*----------------------------------------------------------------------------*/
211 /*----------------------------------------------------------------------------*/
212 
213 void
215 
216 /*----------------------------------------------------------------------------*/
224 /*----------------------------------------------------------------------------*/
225 
226 cs_halo_t *
228 
229 #if defined(HAVE_MPI)
230 
231 /*----------------------------------------------------------------------------*/
255 /*----------------------------------------------------------------------------*/
256 
257 cs_halo_t *
259  cs_lnum_t n_local_elts,
260  cs_lnum_t n_distant_elts,
261  const int elt_rank_id[],
262  const cs_lnum_t elt_id[]);
263 
264 #endif /* HAVE_MPI */
265 
266 /*----------------------------------------------------------------------------*/
272 /*----------------------------------------------------------------------------*/
273 
274 void
275 cs_halo_destroy(cs_halo_t **halo);
276 
277 /*----------------------------------------------------------------------------*/
283 /*----------------------------------------------------------------------------*/
284 
287 
288 /*----------------------------------------------------------------------------*/
295 /*----------------------------------------------------------------------------*/
296 
297 void
299 
300 /*----------------------------------------------------------------------------*/
306 /*----------------------------------------------------------------------------*/
307 
310 
311 /*----------------------------------------------------------------------------*/
321 /*----------------------------------------------------------------------------*/
322 
323 static inline size_t
325  cs_datatype_t data_type,
326  int stride)
327 {
328  size_t elt_size = cs_datatype_size[data_type]*stride;
329  size_t pack_size = halo->n_send_elts[CS_HALO_EXTENDED] * elt_size;
330 
331  return pack_size;
332 }
333 
334 /*----------------------------------------------------------------------------
335  * Apply local cells renumbering to a halo
336  *
337  * parameters:
338  * halo <-- pointer to halo structure
339  * new_cell_id <-- array indicating old -> new cell id (0 to n-1)
340  *---------------------------------------------------------------------------*/
341 
342 void
344  const cs_lnum_t new_cell_id[]);
345 
346 /*----------------------------------------------------------------------------
347  * Apply ghost cells renumbering to a halo
348  *
349  * parameters:
350  * halo <-- pointer to halo structure
351  * old_cell_id <-- array indicating new -> old cell id (0 to n-1)
352  *---------------------------------------------------------------------------*/
353 
354 void
356  const cs_lnum_t old_cell_id[]);
357 
358 /*----------------------------------------------------------------------------*/
381 /*----------------------------------------------------------------------------*/
382 
383 void *
385  cs_halo_type_t sync_mode,
386  cs_datatype_t data_type,
387  int stride,
388  void *send_buf,
389  cs_halo_state_t *hs);
390 
391 /*----------------------------------------------------------------------------*/
410 /*----------------------------------------------------------------------------*/
411 
412 void
413 cs_halo_sync_pack(const cs_halo_t *halo,
414  cs_halo_type_t sync_mode,
415  cs_datatype_t data_type,
416  int stride,
417  void *val,
418  void *send_buf,
419  cs_halo_state_t *hs);
420 
421 #if defined(HAVE_ACCEL)
422 
423 /*----------------------------------------------------------------------------*/
443 /*----------------------------------------------------------------------------*/
444 
445 void
446 cs_halo_sync_pack_d(const cs_halo_t *halo,
447  cs_halo_type_t sync_mode,
448  cs_datatype_t data_type,
449  int stride,
450  void *val,
451  void *send_buf,
452  cs_halo_state_t *hs);
453 
454 #endif /* defined(HAVE_ACCEL) */
455 
456 /*----------------------------------------------------------------------------*/
471 /*----------------------------------------------------------------------------*/
472 
473 void
474 cs_halo_sync_start(const cs_halo_t *halo,
475  void *val,
476  cs_halo_state_t *hs);
477 
478 /*----------------------------------------------------------------------------*/
494 /*----------------------------------------------------------------------------*/
495 
496 void
497 cs_halo_sync_wait(const cs_halo_t *halo,
498  void *val,
499  cs_halo_state_t *hs);
500 
501 /*----------------------------------------------------------------------------*/
515 /*----------------------------------------------------------------------------*/
516 
517 void
518 cs_halo_sync(const cs_halo_t *halo,
519  cs_halo_type_t sync_mode,
520  cs_datatype_t data_type,
521  int stride,
522  void *val);
523 
524 /*----------------------------------------------------------------------------
525  * Update array of any type of halo values in case of parallelism or
526  * periodicity.
527  *
528  * Data is untyped; only its size is given, so this function may also
529  * be used to synchronize interleaved multidimendsional data, using
530  * size = element_size*dim (assuming a homogeneous environment, at least
531  * as far as data encoding goes).
532  *
533  * This function aims at copying main values from local elements
534  * (id between 1 and n_local_elements) to ghost elements on distant ranks
535  * (id between n_local_elements + 1 to n_local_elements_with_halo).
536  *
537  * parameters:
538  * halo <-- pointer to halo structure
539  * sync_mode <-- synchronization mode (standard or extended)
540  * size <-- size of each element
541  * num <-> pointer to local number value array
542  *----------------------------------------------------------------------------*/
543 
544 void
545 cs_halo_sync_untyped(const cs_halo_t *halo,
546  cs_halo_type_t sync_mode,
547  size_t size,
548  void *val);
549 
550 /*----------------------------------------------------------------------------
551  * Update array of integer halo values in case of parallelism or periodicity.
552  *
553  * This function aims at copying main values from local elements
554  * (id between 1 and n_local_elements) to ghost elements on distant ranks
555  * (id between n_local_elements + 1 to n_local_elements_with_halo).
556  *
557  * parameters:
558  * halo <-- pointer to halo structure
559  * sync_mode <-- synchronization mode (standard or extended)
560  * num <-> pointer to local number value array
561  *----------------------------------------------------------------------------*/
562 
563 void
564 cs_halo_sync_num(const cs_halo_t *halo,
565  cs_halo_type_t sync_mode,
566  cs_lnum_t num[]);
567 
568 /*----------------------------------------------------------------------------
569  * Update array of variable (floating-point) halo values in case of
570  * parallelism or periodicity.
571  *
572  * This function aims at copying main values from local elements
573  * (id between 1 and n_local_elements) to ghost elements on distant ranks
574  * (id between n_local_elements + 1 to n_local_elements_with_halo).
575  *
576  * parameters:
577  * halo <-- pointer to halo structure
578  * sync_mode <-- synchronization mode (standard or extended)
579  * var <-> pointer to variable value array
580  *----------------------------------------------------------------------------*/
581 
582 void
583 cs_halo_sync_var(const cs_halo_t *halo,
584  cs_halo_type_t sync_mode,
585  cs_real_t var[]);
586 
587 /*----------------------------------------------------------------------------
588  * Update array of strided variable (floating-point) halo values in case
589  * of parallelism or periodicity.
590  *
591  * This function aims at copying main values from local elements
592  * (id between 1 and n_local_elements) to ghost elements on distant ranks
593  * (id between n_local_elements + 1 to n_local_elements_with_halo).
594  *
595  * parameters:
596  * halo <-- pointer to halo structure
597  * sync_mode <-- synchronization mode (standard or extended)
598  * var <-> pointer to variable value array
599  * stride <-- number of (interlaced) values by entity
600  *----------------------------------------------------------------------------*/
601 
602 void
604  cs_halo_type_t sync_mode,
605  cs_real_t var[],
606  int stride);
607 
608 /*----------------------------------------------------------------------------
609  * Return MPI_Barrier usage flag.
610  *
611  * returns:
612  * true if MPI barriers are used after posting receives and before posting
613  * sends, false otherwise
614  *---------------------------------------------------------------------------*/
615 
616 bool
618 
619 /*----------------------------------------------------------------------------
620  * Set MPI_Barrier usage flag.
621  *
622  * parameters:
623  * use_barrier <-- true if MPI barriers should be used after posting
624  * receives and before posting sends, false otherwise.
625  *---------------------------------------------------------------------------*/
626 
627 void
628 cs_halo_set_use_barrier(bool use_barrier);
629 
630 /*----------------------------------------------------------------------------*/
636 /*----------------------------------------------------------------------------*/
637 
640 
641 /*----------------------------------------------------------------------------*/
647 /*----------------------------------------------------------------------------*/
648 
649 void
651 
652 /*----------------------------------------------------------------------------*/
658 /*----------------------------------------------------------------------------*/
659 
662 
663 /*----------------------------------------------------------------------------*/
669 /*----------------------------------------------------------------------------*/
670 
671 void
673 
674 /*----------------------------------------------------------------------------
675  * Dump a cs_halo_t structure.
676  *
677  * parameters:
678  * halo <-- pointer to cs_halo_t struture
679  * print_level <-- 0 only dimensions and indexes are printed, else (1)
680  * everything is printed
681  *---------------------------------------------------------------------------*/
682 
683 void
684 cs_halo_dump(const cs_halo_t *halo,
685  int print_level);
686 
687 /*----------------------------------------------------------------------------*/
688 
690 
691 #endif /* __CS_HALO_H__ */
cs_alloc_mode_t
Definition: cs_base_accel.h:142
const size_t cs_datatype_size[]
Definition: cs_defs.c:133
cs_datatype_t
Definition: cs_defs.h:272
#define BEGIN_C_DECLS
Definition: cs_defs.h:509
double cs_real_t
Floating-point value.
Definition: cs_defs.h:319
#define END_C_DECLS
Definition: cs_defs.h:510
int cs_lnum_t
local mesh entity id
Definition: cs_defs.h:313
bool cs_halo_get_use_barrier(void)
Definition: cs_halo.c:1997
void cs_halo_sync_pack(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val, void *send_buf, cs_halo_state_t *hs)
Pack halo data to send into dense buffer.
Definition: cs_halo.c:1430
cs_halo_comm_mode_t
Definition: cs_halo.h:66
@ CS_HALO_COMM_P2P
Definition: cs_halo.h:68
@ CS_HALO_COMM_RMA_GET
Definition: cs_halo.h:69
void cs_halo_state_destroy(cs_halo_state_t **halo_state)
Destroy a halo state structure.
Definition: cs_halo.c:1104
void cs_halo_sync_start(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Launch update array of values in case of parallelism or periodicity.
Definition: cs_halo.c:1639
cs_halo_comm_mode_t cs_halo_get_comm_mode(void)
Get default communication mode for halo exchange.
Definition: cs_halo.c:2025
void cs_halo_sync(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val)
Update array of values in case of parallelism or periodicity.
Definition: cs_halo.c:1870
void cs_halo_set_comm_mode(cs_halo_comm_mode_t mode)
Set default communication mode for halo exchange.
Definition: cs_halo.c:2039
cs_alloc_mode_t cs_halo_get_buffer_alloc_mode(void)
Get default host/device allocation mode for message packing arrays.
Definition: cs_halo.c:2054
static size_t cs_halo_pack_size(const cs_halo_t *halo, cs_datatype_t data_type, int stride)
Compute required size for packing send data into dense buffer.
Definition: cs_halo.h:324
void cs_halo_renumber_cells(cs_halo_t *halo, const cs_lnum_t new_cell_id[])
Definition: cs_halo.c:1153
void cs_halo_dump(const cs_halo_t *halo, int print_level)
Definition: cs_halo.c:2083
void cs_halo_sync_wait(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Wait for completion of update array of values in case of parallelism or periodicity.
Definition: cs_halo.c:1786
cs_halo_state_t * cs_halo_state_create(void)
Create a halo state structure.
Definition: cs_halo.c:1061
void cs_halo_set_use_barrier(bool use_barrier)
Definition: cs_halo.c:2011
cs_halo_type_t
Definition: cs_halo.h:56
@ CS_HALO_N_TYPES
Definition: cs_halo.h:60
@ CS_HALO_STANDARD
Definition: cs_halo.h:58
@ CS_HALO_EXTENDED
Definition: cs_halo.h:59
void * cs_halo_sync_pack_init_state(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *send_buf, cs_halo_state_t *hs)
Initialize halo state prior to packing halo data to send.
Definition: cs_halo.c:1339
void cs_halo_create_complete(cs_halo_t *halo)
Ready halo for use.
Definition: cs_halo.c:642
cs_halo_t * cs_halo_create_from_ref(const cs_halo_t *ref)
Create a halo structure, given a reference halo.
Definition: cs_halo.c:702
void cs_halo_sync_untyped(const cs_halo_t *halo, cs_halo_type_t sync_mode, size_t size, void *val)
Definition: cs_halo.c:1913
void cs_halo_set_buffer_alloc_mode(cs_alloc_mode_t mode)
Set default host/device allocation mode for message packing arrays.
Definition: cs_halo.c:2068
void cs_halo_sync_var(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[])
Definition: cs_halo.c:1957
void cs_halo_sync_num(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_lnum_t num[])
Definition: cs_halo.c:1935
void cs_halo_renumber_ghost_cells(cs_halo_t *halo, const cs_lnum_t old_cell_id[])
Definition: cs_halo.c:1177
cs_halo_t * cs_halo_create(const cs_interface_set_t *ifs)
Create a halo structure given an interface set.
Definition: cs_halo.c:501
void cs_halo_sync_var_strided(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[], int stride)
Definition: cs_halo.c:1980
cs_halo_state_t * cs_halo_state_get_default(void)
Get pointer to default halo state structure.
Definition: cs_halo.c:1139
cs_halo_t * cs_halo_create_from_rank_neighbors(const cs_rank_neighbors_t *rn, cs_lnum_t n_local_elts, cs_lnum_t n_distant_elts, const int elt_rank_id[], const cs_lnum_t elt_id[])
Create a halo structure from distant element distant ranks and ids.
Definition: cs_halo.c:792
void cs_halo_destroy(cs_halo_t **halo)
Definition: cs_halo.c:1016
struct _cs_halo_state_t cs_halo_state_t
Definition: cs_halo.h:172
struct _cs_interface_set_t cs_interface_set_t
Definition: cs_interface.h:61
struct _fvm_periodicity_t fvm_periodicity_t
Definition: fvm_periodicity.h:67
Definition: cs_halo.h:77
int n_transforms
Definition: cs_halo.h:80
cs_lnum_t * index
Definition: cs_halo.h:118
cs_lnum_t * send_list
Definition: cs_halo.h:98
cs_lnum_t * send_index
Definition: cs_halo.h:101
int n_rotations
Definition: cs_halo.h:87
cs_lnum_t n_local_elts
Definition: cs_halo.h:90
int n_c_domains
Definition: cs_halo.h:79
int * c_domain_rank
Definition: cs_halo.h:82
cs_lnum_t * perio_lst
Definition: cs_halo.h:123
cs_lnum_t * send_perio_lst
Definition: cs_halo.h:106
const fvm_periodicity_t * periodicity
Definition: cs_halo.h:84
MPI_Group c_domain_group
Definition: cs_halo.h:163
cs_lnum_t * c_domain_s_shift
Definition: cs_halo.h:164
cs_lnum_t n_send_elts[2]
Definition: cs_halo.h:94
Definition: cs_rank_neighbors.h:67