8.2
general documentation
cs_halo.h
Go to the documentation of this file.
1 #ifndef __CS_HALO_H__
2 #define __CS_HALO_H__
3 
4 /*============================================================================
5  * Structure and function headers handling with ghost cells
6  *============================================================================*/
7 
8 /*
9  This file is part of code_saturne, a general-purpose CFD tool.
10 
11  Copyright (C) 1998-2024 EDF S.A.
12 
13  This program is free software; you can redistribute it and/or modify it under
14  the terms of the GNU General Public License as published by the Free Software
15  Foundation; either version 2 of the License, or (at your option) any later
16  version.
17 
18  This program is distributed in the hope that it will be useful, but WITHOUT
19  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
20  FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
21  details.
22 
23  You should have received a copy of the GNU General Public License along with
24  this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
25  Street, Fifth Floor, Boston, MA 02110-1301, USA.
26 */
27 
28 /*----------------------------------------------------------------------------*/
29 
30 /*----------------------------------------------------------------------------
31  * Local headers
32  *----------------------------------------------------------------------------*/
33 
34 #include "cs_defs.h"
35 #include "cs_base.h"
36 #include "cs_base_accel.h"
37 #include "cs_interface.h"
38 #include "cs_rank_neighbors.h"
39 
40 #include "fvm_periodicity.h"
41 
42 /*----------------------------------------------------------------------------*/
43 
45 
46 /*============================================================================
47  * Macro definitions
48  *============================================================================*/
49 
50 /*=============================================================================
51  * Type definitions
52  *============================================================================*/
53 
56 typedef enum {
57 
61 
63 
64 /* Halo communication mode */
65 
66 typedef enum {
67 
73 
74 /* Structure for halo management */
75 /* ----------------------------- */
76 
77 typedef struct {
78 
79  int n_c_domains; /* Number of communicating domains. */
80  int n_transforms; /* Number of periodic transformations */
81 
82  int *c_domain_rank; /* List of communicating ranks */
83 
84  const fvm_periodicity_t * periodicity; /* Pointer to periodicity
85  structure describing transforms */
86 
87  int n_rotations; /* Number of periodic transformations
88  involving rotations */
89 
90  cs_lnum_t n_local_elts; /* Number of local elements */
91 
92  /* send_halo features : send to distant ranks */
93 
94  cs_lnum_t n_send_elts[2]; /* Numer of ghost elements in send_list
95  n_elts[0] = standard elements
96  n_elts[1] = extended + standard elements */
97 
98  cs_lnum_t *send_list; /* List of local elements in distant halos
99  (0 to n-1 numbering) */
100 
101  cs_lnum_t *send_index; /* Index on send_list
102  Size = 2*n_c_domains + 1. For each rank, we
103  have an index for standard halo and one
104  for extended halo. */
105 
106  cs_lnum_t *send_perio_lst; /* For each transformation and for each type of
107  halo on each communicating rank, we store
108  2 values:
109  - start index,
110  - number of elements. */
111 
112  cs_lnum_t std_send_block_size; /* Send block size for packing */
113  cs_lnum_t n_std_send_blocks; /* Number of send blocks for packing with
114  standard exchange */
115  cs_lnum_t *std_send_blocks; /* Block start and end info for packing
116  with standard exchange */
117 
118  /* halo features : receive from distant ranks */
119 
120  cs_lnum_t n_elts[2]; /* Numer of ghost elements in halo
121  n_elts[0] = standard elements
122  n_elts[1] = extended + standard elements */
123 
124  cs_lnum_t *index; /* Index on halo sections;
125  Size = 2*n_c_domains. For each rank, we
126  have an index for the standard halo and one
127  for the extended halo. */
128 
129  cs_lnum_t *perio_lst; /* For each transformation and for each type of halo
130  on each communicating rank, we store 2 values:
131  - start index,
132  - number of elements. */
133 
134  /* Organisation of perio_lst:
135 
136  -------------------------------------------------
137  T1: | | | | | | | | | | | | |
138  -------------------------------------------------
139  idx n idx n idx n idx n idx n idx n
140  ______ ______ ______ ______ ______ ______
141  std ext std ext std ext
142  ___________ ___________ ___________
143  rank 0 rank 1 rank 2
144 
145  -------------------------------------------------
146  T2: | | | | | | | | | | | | |
147  -------------------------------------------------
148  idx n idx n idx n idx n idx n idx n
149  ______ ______ ______ ______ ______ ______
150  std ext std ext std ext
151  ___________ ___________ ___________
152  rank 0 rank 1 rank 2
153 
154  -------------------------------------------------
155  T3: | | | | | | | | | | | | |
156  -------------------------------------------------
157  idx n idx n idx n idx n idx n idx n
158  ______ ______ ______ ______ ______ ______
159  std ext std ext std ext
160  ___________ ___________ ___________
161  rank 0 rank 1 rank 2
162 
163  etc...
164 
165  */
166 
167 #if defined(HAVE_MPI)
168 
169  MPI_Group c_domain_group; /* Group of connected domains */
170  cs_lnum_t *c_domain_s_shift; /* Target buffer shift for distant
171  ranks using one-sided get */
172 #endif
173 
174 } cs_halo_t;
175 
178 typedef struct _cs_halo_state_t cs_halo_state_t;
179 
180 /*=============================================================================
181  * Global static variables
182  *============================================================================*/
183 
184 /*=============================================================================
185  * Public function prototypes
186  *============================================================================*/
187 
188 /*----------------------------------------------------------------------------*/
196 /*----------------------------------------------------------------------------*/
197 
198 cs_halo_t *
200 
201 /*----------------------------------------------------------------------------*/
213 /*----------------------------------------------------------------------------*/
214 
215 void
217 
218 /*----------------------------------------------------------------------------*/
226 /*----------------------------------------------------------------------------*/
227 
228 cs_halo_t *
230 
231 #if defined(HAVE_MPI)
232 
233 /*----------------------------------------------------------------------------*/
257 /*----------------------------------------------------------------------------*/
258 
259 cs_halo_t *
261  cs_lnum_t n_local_elts,
262  cs_lnum_t n_distant_elts,
263  const int elt_rank_id[],
264  const cs_lnum_t elt_id[]);
265 
266 #endif /* HAVE_MPI */
267 
268 /*----------------------------------------------------------------------------*/
274 /*----------------------------------------------------------------------------*/
275 
276 void
277 cs_halo_destroy(cs_halo_t **halo);
278 
279 /*----------------------------------------------------------------------------*/
285 /*----------------------------------------------------------------------------*/
286 
289 
290 /*----------------------------------------------------------------------------*/
297 /*----------------------------------------------------------------------------*/
298 
299 void
301 
302 /*----------------------------------------------------------------------------*/
308 /*----------------------------------------------------------------------------*/
309 
312 
313 /*----------------------------------------------------------------------------*/
323 /*----------------------------------------------------------------------------*/
324 
325 static inline size_t
327  cs_datatype_t data_type,
328  int stride)
329 {
330  size_t elt_size = cs_datatype_size[data_type]*stride;
331  size_t pack_size = halo->n_send_elts[CS_HALO_EXTENDED] * elt_size;
332 
333  return pack_size;
334 }
335 
336 /*----------------------------------------------------------------------------
337  * Apply local cells renumbering to a halo
338  *
339  * parameters:
340  * halo <-- pointer to halo structure
341  * new_cell_id <-- array indicating old -> new cell id (0 to n-1)
342  *---------------------------------------------------------------------------*/
343 
344 void
346  const cs_lnum_t new_cell_id[]);
347 
348 /*----------------------------------------------------------------------------
349  * Apply ghost cells renumbering to a halo
350  *
351  * parameters:
352  * halo <-- pointer to halo structure
353  * old_cell_id <-- array indicating new -> old cell id (0 to n-1)
354  *---------------------------------------------------------------------------*/
355 
356 void
358  const cs_lnum_t old_cell_id[]);
359 
360 /*----------------------------------------------------------------------------*/
383 /*----------------------------------------------------------------------------*/
384 
385 void *
387  cs_halo_type_t sync_mode,
388  cs_datatype_t data_type,
389  int stride,
390  void *send_buf,
391  cs_halo_state_t *hs);
392 
393 /*----------------------------------------------------------------------------*/
412 /*----------------------------------------------------------------------------*/
413 
414 void
415 cs_halo_sync_pack(const cs_halo_t *halo,
416  cs_halo_type_t sync_mode,
417  cs_datatype_t data_type,
418  int stride,
419  void *val,
420  void *send_buf,
421  cs_halo_state_t *hs);
422 
423 #if defined(HAVE_ACCEL)
424 
425 /*----------------------------------------------------------------------------*/
445 /*----------------------------------------------------------------------------*/
446 
447 void
448 cs_halo_sync_pack_d(const cs_halo_t *halo,
449  cs_halo_type_t sync_mode,
450  cs_datatype_t data_type,
451  int stride,
452  void *val,
453  void *send_buf,
454  cs_halo_state_t *hs);
455 
456 #endif /* defined(HAVE_ACCEL) */
457 
458 /*----------------------------------------------------------------------------*/
473 /*----------------------------------------------------------------------------*/
474 
475 void
476 cs_halo_sync_start(const cs_halo_t *halo,
477  void *val,
478  cs_halo_state_t *hs);
479 
480 /*----------------------------------------------------------------------------*/
496 /*----------------------------------------------------------------------------*/
497 
498 void
499 cs_halo_sync_wait(const cs_halo_t *halo,
500  void *val,
501  cs_halo_state_t *hs);
502 
503 /*----------------------------------------------------------------------------*/
517 /*----------------------------------------------------------------------------*/
518 
519 void
520 cs_halo_sync(const cs_halo_t *halo,
521  cs_halo_type_t sync_mode,
522  cs_datatype_t data_type,
523  int stride,
524  void *val);
525 
526 #if defined(HAVE_ACCEL)
527 
528 /*----------------------------------------------------------------------------*/
543 /*----------------------------------------------------------------------------*/
544 
545 void
546 cs_halo_sync_d(const cs_halo_t *halo,
547  cs_halo_type_t sync_mode,
548  cs_datatype_t data_type,
549  int stride,
550  void *val);
551 
552 #endif /* defined(HAVE_ACCEL) */
553 
554 /*----------------------------------------------------------------------------
555  * Update array of any type of halo values in case of parallelism or
556  * periodicity.
557  *
558  * Data is untyped; only its size is given, so this function may also
559  * be used to synchronize interleaved multidimendsional data, using
560  * size = element_size*dim (assuming a homogeneous environment, at least
561  * as far as data encoding goes).
562  *
563  * This function aims at copying main values from local elements
564  * (id between 1 and n_local_elements) to ghost elements on distant ranks
565  * (id between n_local_elements + 1 to n_local_elements_with_halo).
566  *
567  * parameters:
568  * halo <-- pointer to halo structure
569  * sync_mode <-- synchronization mode (standard or extended)
570  * size <-- size of each element
571  * num <-> pointer to local number value array
572  *----------------------------------------------------------------------------*/
573 
574 void
575 cs_halo_sync_untyped(const cs_halo_t *halo,
576  cs_halo_type_t sync_mode,
577  size_t size,
578  void *val);
579 
580 /*----------------------------------------------------------------------------
581  * Update array of integer halo values in case of parallelism or periodicity.
582  *
583  * This function aims at copying main values from local elements
584  * (id between 1 and n_local_elements) to ghost elements on distant ranks
585  * (id between n_local_elements + 1 to n_local_elements_with_halo).
586  *
587  * parameters:
588  * halo <-- pointer to halo structure
589  * sync_mode <-- synchronization mode (standard or extended)
590  * num <-> pointer to local number value array
591  *----------------------------------------------------------------------------*/
592 
593 void
594 cs_halo_sync_num(const cs_halo_t *halo,
595  cs_halo_type_t sync_mode,
596  cs_lnum_t num[]);
597 
598 /*----------------------------------------------------------------------------
599  * Update array of variable (floating-point) halo values in case of
600  * parallelism or periodicity.
601  *
602  * This function aims at copying main values from local elements
603  * (id between 1 and n_local_elements) to ghost elements on distant ranks
604  * (id between n_local_elements + 1 to n_local_elements_with_halo).
605  *
606  * parameters:
607  * halo <-- pointer to halo structure
608  * sync_mode <-- synchronization mode (standard or extended)
609  * var <-> pointer to variable value array
610  *----------------------------------------------------------------------------*/
611 
612 void
613 cs_halo_sync_var(const cs_halo_t *halo,
614  cs_halo_type_t sync_mode,
615  cs_real_t var[]);
616 
617 /*----------------------------------------------------------------------------
618  * Update array of strided variable (floating-point) halo values in case
619  * of parallelism or periodicity.
620  *
621  * This function aims at copying main values from local elements
622  * (id between 1 and n_local_elements) to ghost elements on distant ranks
623  * (id between n_local_elements + 1 to n_local_elements_with_halo).
624  *
625  * parameters:
626  * halo <-- pointer to halo structure
627  * sync_mode <-- synchronization mode (standard or extended)
628  * var <-> pointer to variable value array
629  * stride <-- number of (interlaced) values by entity
630  *----------------------------------------------------------------------------*/
631 
632 void
634  cs_halo_type_t sync_mode,
635  cs_real_t var[],
636  int stride);
637 
638 /*----------------------------------------------------------------------------
639  * Return MPI_Barrier usage flag.
640  *
641  * returns:
642  * true if MPI barriers are used after posting receives and before posting
643  * sends, false otherwise
644  *---------------------------------------------------------------------------*/
645 
646 bool
648 
649 /*----------------------------------------------------------------------------
650  * Set MPI_Barrier usage flag.
651  *
652  * parameters:
653  * use_barrier <-- true if MPI barriers should be used after posting
654  * receives and before posting sends, false otherwise.
655  *---------------------------------------------------------------------------*/
656 
657 void
658 cs_halo_set_use_barrier(bool use_barrier);
659 
660 /*----------------------------------------------------------------------------*/
666 /*----------------------------------------------------------------------------*/
667 
670 
671 /*----------------------------------------------------------------------------*/
677 /*----------------------------------------------------------------------------*/
678 
679 void
681 
682 /*----------------------------------------------------------------------------*/
688 /*----------------------------------------------------------------------------*/
689 
692 
693 /*----------------------------------------------------------------------------*/
699 /*----------------------------------------------------------------------------*/
700 
701 void
703 
704 /*----------------------------------------------------------------------------
705  * Dump a cs_halo_t structure.
706  *
707  * parameters:
708  * halo <-- pointer to cs_halo_t struture
709  * print_level <-- 0 only dimensions and indexes are printed, else (1)
710  * everything is printed
711  *---------------------------------------------------------------------------*/
712 
713 void
714 cs_halo_dump(const cs_halo_t *halo,
715  int print_level);
716 
717 /*----------------------------------------------------------------------------*/
718 
720 
721 #endif /* __CS_HALO_H__ */
cs_alloc_mode_t
Definition: bft_mem.h:50
const size_t cs_datatype_size[]
Definition: cs_defs.c:133
cs_datatype_t
Definition: cs_defs.h:284
#define BEGIN_C_DECLS
Definition: cs_defs.h:528
double cs_real_t
Floating-point value.
Definition: cs_defs.h:332
#define END_C_DECLS
Definition: cs_defs.h:529
int cs_lnum_t
local mesh entity id
Definition: cs_defs.h:325
bool cs_halo_get_use_barrier(void)
Definition: cs_halo.cpp:2111
void cs_halo_sync_pack(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val, void *send_buf, cs_halo_state_t *hs)
Pack halo data to send into dense buffer.
Definition: cs_halo.cpp:1490
cs_halo_comm_mode_t
Definition: cs_halo.h:66
@ CS_HALO_COMM_P2P
Definition: cs_halo.h:68
@ CS_HALO_COMM_RMA_GET
Definition: cs_halo.h:69
void cs_halo_state_destroy(cs_halo_state_t **halo_state)
Destroy a halo state structure.
Definition: cs_halo.cpp:1164
void cs_halo_sync_start(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Launch update array of values in case of parallelism or periodicity.
Definition: cs_halo.cpp:1703
cs_halo_comm_mode_t cs_halo_get_comm_mode(void)
Get default communication mode for halo exchange.
Definition: cs_halo.cpp:2139
void cs_halo_sync(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val)
Update array of values in case of parallelism or periodicity.
Definition: cs_halo.cpp:1939
void cs_halo_set_comm_mode(cs_halo_comm_mode_t mode)
Set default communication mode for halo exchange.
Definition: cs_halo.cpp:2153
cs_alloc_mode_t cs_halo_get_buffer_alloc_mode(void)
Get default host/device allocation mode for message packing arrays.
Definition: cs_halo.cpp:2168
static size_t cs_halo_pack_size(const cs_halo_t *halo, cs_datatype_t data_type, int stride)
Compute required size for packing send data into dense buffer.
Definition: cs_halo.h:326
void cs_halo_renumber_cells(cs_halo_t *halo, const cs_lnum_t new_cell_id[])
Definition: cs_halo.cpp:1213
void cs_halo_dump(const cs_halo_t *halo, int print_level)
Definition: cs_halo.cpp:2197
void cs_halo_sync_wait(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Wait for completion of update array of values in case of parallelism or periodicity.
Definition: cs_halo.cpp:1854
cs_halo_state_t * cs_halo_state_create(void)
Create a halo state structure.
Definition: cs_halo.cpp:1121
void cs_halo_set_use_barrier(bool use_barrier)
Definition: cs_halo.cpp:2125
cs_halo_type_t
Definition: cs_halo.h:56
@ CS_HALO_N_TYPES
Definition: cs_halo.h:60
@ CS_HALO_STANDARD
Definition: cs_halo.h:58
@ CS_HALO_EXTENDED
Definition: cs_halo.h:59
void * cs_halo_sync_pack_init_state(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *send_buf, cs_halo_state_t *hs)
Initialize halo state prior to packing halo data to send.
Definition: cs_halo.cpp:1399
void cs_halo_create_complete(cs_halo_t *halo)
Ready halo for use.
Definition: cs_halo.cpp:642
cs_halo_t * cs_halo_create_from_ref(const cs_halo_t *ref)
Create a halo structure, given a reference halo.
Definition: cs_halo.cpp:750
void cs_halo_sync_untyped(const cs_halo_t *halo, cs_halo_type_t sync_mode, size_t size, void *val)
Definition: cs_halo.cpp:2027
void cs_halo_set_buffer_alloc_mode(cs_alloc_mode_t mode)
Set default host/device allocation mode for message packing arrays.
Definition: cs_halo.cpp:2182
void cs_halo_sync_var(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[])
Definition: cs_halo.cpp:2071
void cs_halo_sync_num(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_lnum_t num[])
Definition: cs_halo.cpp:2049
void cs_halo_renumber_ghost_cells(cs_halo_t *halo, const cs_lnum_t old_cell_id[])
Definition: cs_halo.cpp:1237
cs_halo_t * cs_halo_create(const cs_interface_set_t *ifs)
Create a halo structure given an interface set.
Definition: cs_halo.cpp:497
void cs_halo_sync_var_strided(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[], int stride)
Definition: cs_halo.cpp:2094
cs_halo_state_t * cs_halo_state_get_default(void)
Get pointer to default halo state structure.
Definition: cs_halo.cpp:1199
cs_halo_t * cs_halo_create_from_rank_neighbors(const cs_rank_neighbors_t *rn, cs_lnum_t n_local_elts, cs_lnum_t n_distant_elts, const int elt_rank_id[], const cs_lnum_t elt_id[])
Create a halo structure from distant element distant ranks and ids.
Definition: cs_halo.cpp:846
void cs_halo_destroy(cs_halo_t **halo)
Definition: cs_halo.cpp:1074
struct _cs_halo_state_t cs_halo_state_t
Definition: cs_halo.h:178
struct _cs_interface_set_t cs_interface_set_t
Definition: cs_interface.h:61
struct _fvm_periodicity_t fvm_periodicity_t
Definition: fvm_periodicity.h:67
Definition: cs_halo.h:77
int n_transforms
Definition: cs_halo.h:80
cs_lnum_t * index
Definition: cs_halo.h:124
cs_lnum_t * send_list
Definition: cs_halo.h:98
cs_lnum_t * send_index
Definition: cs_halo.h:101
int n_rotations
Definition: cs_halo.h:87
cs_lnum_t * std_send_blocks
Definition: cs_halo.h:115
cs_lnum_t n_local_elts
Definition: cs_halo.h:90
cs_lnum_t n_std_send_blocks
Definition: cs_halo.h:113
int n_c_domains
Definition: cs_halo.h:79
int * c_domain_rank
Definition: cs_halo.h:82
cs_lnum_t * perio_lst
Definition: cs_halo.h:129
cs_lnum_t std_send_block_size
Definition: cs_halo.h:112
cs_lnum_t * send_perio_lst
Definition: cs_halo.h:106
const fvm_periodicity_t * periodicity
Definition: cs_halo.h:84
MPI_Group c_domain_group
Definition: cs_halo.h:169
cs_lnum_t * c_domain_s_shift
Definition: cs_halo.h:170
cs_lnum_t n_send_elts[2]
Definition: cs_halo.h:94
Definition: cs_rank_neighbors.h:67