8.0
general documentation
Loading...
Searching...
No Matches
cs_halo.h
Go to the documentation of this file.
1#ifndef __CS_HALO_H__
2#define __CS_HALO_H__
3
4/*============================================================================
5 * Structure and function headers handling with ghost cells
6 *============================================================================*/
7
8/*
9 This file is part of code_saturne, a general-purpose CFD tool.
10
11 Copyright (C) 1998-2023 EDF S.A.
12
13 This program is free software; you can redistribute it and/or modify it under
14 the terms of the GNU General Public License as published by the Free Software
15 Foundation; either version 2 of the License, or (at your option) any later
16 version.
17
18 This program is distributed in the hope that it will be useful, but WITHOUT
19 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
20 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
21 details.
22
23 You should have received a copy of the GNU General Public License along with
24 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
25 Street, Fifth Floor, Boston, MA 02110-1301, USA.
26*/
27
28/*----------------------------------------------------------------------------*/
29
30/*----------------------------------------------------------------------------
31 * Local headers
32 *----------------------------------------------------------------------------*/
33
34#include "cs_defs.h"
35#include "cs_base.h"
36#include "cs_base_accel.h"
37#include "cs_interface.h"
38#include "cs_rank_neighbors.h"
39
40#include "fvm_periodicity.h"
41
42/*----------------------------------------------------------------------------*/
43
45
46/*============================================================================
47 * Macro definitions
48 *============================================================================*/
49
50/*=============================================================================
51 * Type definitions
52 *============================================================================*/
53
55
63
64/* Halo communication mode */
65
73
74/* Structure for halo management */
75/* ----------------------------- */
76
77typedef struct {
78
79 int n_c_domains; /* Number of communicating domains. */
80 int n_transforms; /* Number of periodic transformations */
81
82 int *c_domain_rank; /* List of communicating ranks */
83
84 const fvm_periodicity_t * periodicity; /* Pointer to periodicity
85 structure describing transforms */
86
87 int n_rotations; /* Number of periodic transformations
88 involving rotations */
89
90 cs_lnum_t n_local_elts; /* Number of local elements */
91
92 /* send_halo features : send to distant ranks */
93
94 cs_lnum_t n_send_elts[2]; /* Numer of ghost elements in send_list
95 n_elts[0] = standard elements
96 n_elts[1] = extended + standard elements */
97
98 cs_lnum_t *send_list; /* List of local elements in distant halos
99 (0 to n-1 numbering) */
100
101 cs_lnum_t *send_index; /* Index on send_list
102 Size = 2*n_c_domains + 1. For each rank, we
103 have an index for standard halo and one
104 for extended halo. */
105
106 cs_lnum_t *send_perio_lst ; /* For each transformation and for each type of
107 halo on each communicating rank, we store
108 2 values:
109 - start index,
110 - number of elements. */
111
112 /* halo features : receive from distant ranks */
113
114 cs_lnum_t n_elts[2]; /* Numer of ghost elements in halo
115 n_elts[0] = standard elements
116 n_elts[1] = extended + standard elements */
117
118 cs_lnum_t *index; /* Index on halo sections;
119 Size = 2*n_c_domains. For each rank, we
120 have an index for the standard halo and one
121 for the extended halo. */
122
123 cs_lnum_t *perio_lst; /* For each transformation and for each type of halo
124 on each communicating rank, we store 2 values:
125 - start index,
126 - number of elements. */
127
128 /* Organisation of perio_lst:
129
130 -------------------------------------------------
131 T1: | | | | | | | | | | | | |
132 -------------------------------------------------
133 idx n idx n idx n idx n idx n idx n
134 ______ ______ ______ ______ ______ ______
135 std ext std ext std ext
136 ___________ ___________ ___________
137 rank 0 rank 1 rank 2
138
139 -------------------------------------------------
140 T2: | | | | | | | | | | | | |
141 -------------------------------------------------
142 idx n idx n idx n idx n idx n idx n
143 ______ ______ ______ ______ ______ ______
144 std ext std ext std ext
145 ___________ ___________ ___________
146 rank 0 rank 1 rank 2
147
148 -------------------------------------------------
149 T3: | | | | | | | | | | | | |
150 -------------------------------------------------
151 idx n idx n idx n idx n idx n idx n
152 ______ ______ ______ ______ ______ ______
153 std ext std ext std ext
154 ___________ ___________ ___________
155 rank 0 rank 1 rank 2
156
157 etc...
158
159 */
160
161#if defined(HAVE_MPI)
162
163 MPI_Group c_domain_group; /* Group of connected domains */
164 cs_lnum_t *c_domain_s_shift; /* Target buffer shift for distant
165 ranks using one-sided get */
166#endif
167
168} cs_halo_t;
169
171
172typedef struct _cs_halo_state_t cs_halo_state_t;
173
174/*=============================================================================
175 * Global static variables
176 *============================================================================*/
177
178/*============================================================================
179 * Public function header for Fortran API
180 *============================================================================*/
181
182/*=============================================================================
183 * Public function prototypes
184 *============================================================================*/
185
186/*----------------------------------------------------------------------------*/
194/*----------------------------------------------------------------------------*/
195
196cs_halo_t *
198
199/*----------------------------------------------------------------------------*/
211/*----------------------------------------------------------------------------*/
212
213void
215
216/*----------------------------------------------------------------------------*/
224/*----------------------------------------------------------------------------*/
225
226cs_halo_t *
228
229#if defined(HAVE_MPI)
230
231/*----------------------------------------------------------------------------*/
255/*----------------------------------------------------------------------------*/
256
257cs_halo_t *
259 cs_lnum_t n_local_elts,
260 cs_lnum_t n_distant_elts,
261 const int elt_rank_id[],
262 const cs_lnum_t elt_id[]);
263
264#endif /* HAVE_MPI */
265
266/*----------------------------------------------------------------------------*/
272/*----------------------------------------------------------------------------*/
273
274void
276
277/*----------------------------------------------------------------------------*/
283/*----------------------------------------------------------------------------*/
284
287
288/*----------------------------------------------------------------------------*/
295/*----------------------------------------------------------------------------*/
296
297void
299
300/*----------------------------------------------------------------------------*/
306/*----------------------------------------------------------------------------*/
307
310
311/*----------------------------------------------------------------------------*/
321/*----------------------------------------------------------------------------*/
322
323static inline size_t
325 cs_datatype_t data_type,
326 int stride)
327{
328 size_t elt_size = cs_datatype_size[data_type]*stride;
329 size_t pack_size = halo->n_send_elts[CS_HALO_EXTENDED] * elt_size;
330
331 return pack_size;
332}
333
334/*----------------------------------------------------------------------------
335 * Apply local cells renumbering to a halo
336 *
337 * parameters:
338 * halo <-- pointer to halo structure
339 * new_cell_id <-- array indicating old -> new cell id (0 to n-1)
340 *---------------------------------------------------------------------------*/
341
342void
344 const cs_lnum_t new_cell_id[]);
345
346/*----------------------------------------------------------------------------
347 * Apply ghost cells renumbering to a halo
348 *
349 * parameters:
350 * halo <-- pointer to halo structure
351 * old_cell_id <-- array indicating new -> old cell id (0 to n-1)
352 *---------------------------------------------------------------------------*/
353
354void
356 const cs_lnum_t old_cell_id[]);
357
358/*----------------------------------------------------------------------------*/
381/*----------------------------------------------------------------------------*/
382
383void *
385 cs_halo_type_t sync_mode,
386 cs_datatype_t data_type,
387 int stride,
388 void *send_buf,
389 cs_halo_state_t *hs);
390
391/*----------------------------------------------------------------------------*/
410/*----------------------------------------------------------------------------*/
411
412void
413cs_halo_sync_pack(const cs_halo_t *halo,
414 cs_halo_type_t sync_mode,
415 cs_datatype_t data_type,
416 int stride,
417 void *val,
418 void *send_buf,
419 cs_halo_state_t *hs);
420
421#if defined(HAVE_ACCEL)
422
423/*----------------------------------------------------------------------------*/
443/*----------------------------------------------------------------------------*/
444
445void
446cs_halo_sync_pack_d(const cs_halo_t *halo,
447 cs_halo_type_t sync_mode,
448 cs_datatype_t data_type,
449 int stride,
450 void *val,
451 void *send_buf,
452 cs_halo_state_t *hs);
453
454#endif /* defined(HAVE_ACCEL) */
455
456/*----------------------------------------------------------------------------*/
471/*----------------------------------------------------------------------------*/
472
473void
474cs_halo_sync_start(const cs_halo_t *halo,
475 void *val,
476 cs_halo_state_t *hs);
477
478/*----------------------------------------------------------------------------*/
494/*----------------------------------------------------------------------------*/
495
496void
497cs_halo_sync_wait(const cs_halo_t *halo,
498 void *val,
499 cs_halo_state_t *hs);
500
501/*----------------------------------------------------------------------------*/
515/*----------------------------------------------------------------------------*/
516
517void
518cs_halo_sync(const cs_halo_t *halo,
519 cs_halo_type_t sync_mode,
520 cs_datatype_t data_type,
521 int stride,
522 void *val);
523
524/*----------------------------------------------------------------------------
525 * Update array of any type of halo values in case of parallelism or
526 * periodicity.
527 *
528 * Data is untyped; only its size is given, so this function may also
529 * be used to synchronize interleaved multidimendsional data, using
530 * size = element_size*dim (assuming a homogeneous environment, at least
531 * as far as data encoding goes).
532 *
533 * This function aims at copying main values from local elements
534 * (id between 1 and n_local_elements) to ghost elements on distant ranks
535 * (id between n_local_elements + 1 to n_local_elements_with_halo).
536 *
537 * parameters:
538 * halo <-- pointer to halo structure
539 * sync_mode <-- synchronization mode (standard or extended)
540 * size <-- size of each element
541 * num <-> pointer to local number value array
542 *----------------------------------------------------------------------------*/
543
544void
546 cs_halo_type_t sync_mode,
547 size_t size,
548 void *val);
549
550/*----------------------------------------------------------------------------
551 * Update array of integer halo values in case of parallelism or periodicity.
552 *
553 * This function aims at copying main values from local elements
554 * (id between 1 and n_local_elements) to ghost elements on distant ranks
555 * (id between n_local_elements + 1 to n_local_elements_with_halo).
556 *
557 * parameters:
558 * halo <-- pointer to halo structure
559 * sync_mode <-- synchronization mode (standard or extended)
560 * num <-> pointer to local number value array
561 *----------------------------------------------------------------------------*/
562
563void
564cs_halo_sync_num(const cs_halo_t *halo,
565 cs_halo_type_t sync_mode,
566 cs_lnum_t num[]);
567
568/*----------------------------------------------------------------------------
569 * Update array of variable (floating-point) halo values in case of
570 * parallelism or periodicity.
571 *
572 * This function aims at copying main values from local elements
573 * (id between 1 and n_local_elements) to ghost elements on distant ranks
574 * (id between n_local_elements + 1 to n_local_elements_with_halo).
575 *
576 * parameters:
577 * halo <-- pointer to halo structure
578 * sync_mode <-- synchronization mode (standard or extended)
579 * var <-> pointer to variable value array
580 *----------------------------------------------------------------------------*/
581
582void
583cs_halo_sync_var(const cs_halo_t *halo,
584 cs_halo_type_t sync_mode,
585 cs_real_t var[]);
586
587/*----------------------------------------------------------------------------
588 * Update array of strided variable (floating-point) halo values in case
589 * of parallelism or periodicity.
590 *
591 * This function aims at copying main values from local elements
592 * (id between 1 and n_local_elements) to ghost elements on distant ranks
593 * (id between n_local_elements + 1 to n_local_elements_with_halo).
594 *
595 * parameters:
596 * halo <-- pointer to halo structure
597 * sync_mode <-- synchronization mode (standard or extended)
598 * var <-> pointer to variable value array
599 * stride <-- number of (interlaced) values by entity
600 *----------------------------------------------------------------------------*/
601
602void
604 cs_halo_type_t sync_mode,
605 cs_real_t var[],
606 int stride);
607
608/*----------------------------------------------------------------------------
609 * Return MPI_Barrier usage flag.
610 *
611 * returns:
612 * true if MPI barriers are used after posting receives and before posting
613 * sends, false otherwise
614 *---------------------------------------------------------------------------*/
615
616bool
618
619/*----------------------------------------------------------------------------
620 * Set MPI_Barrier usage flag.
621 *
622 * parameters:
623 * use_barrier <-- true if MPI barriers should be used after posting
624 * receives and before posting sends, false otherwise.
625 *---------------------------------------------------------------------------*/
626
627void
628cs_halo_set_use_barrier(bool use_barrier);
629
630/*----------------------------------------------------------------------------*/
636/*----------------------------------------------------------------------------*/
637
640
641/*----------------------------------------------------------------------------*/
647/*----------------------------------------------------------------------------*/
648
649void
651
652/*----------------------------------------------------------------------------*/
658/*----------------------------------------------------------------------------*/
659
662
663/*----------------------------------------------------------------------------*/
669/*----------------------------------------------------------------------------*/
670
671void
673
674/*----------------------------------------------------------------------------
675 * Dump a cs_halo_t structure.
676 *
677 * parameters:
678 * halo <-- pointer to cs_halo_t struture
679 * print_level <-- 0 only dimensions and indexes are printed, else (1)
680 * everything is printed
681 *---------------------------------------------------------------------------*/
682
683void
684cs_halo_dump(const cs_halo_t *halo,
685 int print_level);
686
687/*----------------------------------------------------------------------------*/
688
690
691#endif /* __CS_HALO_H__ */
cs_alloc_mode_t
Definition cs_base_accel.h:142
const size_t cs_datatype_size[]
Definition cs_defs.c:133
cs_datatype_t
Definition cs_defs.h:272
#define BEGIN_C_DECLS
Definition cs_defs.h:509
double cs_real_t
Floating-point value.
Definition cs_defs.h:319
#define END_C_DECLS
Definition cs_defs.h:510
int cs_lnum_t
local mesh entity id
Definition cs_defs.h:313
bool cs_halo_get_use_barrier(void)
Definition cs_halo.c:2008
void cs_halo_sync_pack(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val, void *send_buf, cs_halo_state_t *hs)
Pack halo data to send into dense buffer.
Definition cs_halo.c:1430
cs_halo_state_t * cs_halo_state_create(void)
Create a halo state structure.
Definition cs_halo.c:1061
cs_halo_comm_mode_t
Definition cs_halo.h:66
@ CS_HALO_COMM_P2P
Definition cs_halo.h:68
@ CS_HALO_COMM_RMA_GET
Definition cs_halo.h:69
void cs_halo_state_destroy(cs_halo_state_t **halo_state)
Destroy a halo state structure.
Definition cs_halo.c:1104
void cs_halo_sync_start(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Launch update array of values in case of parallelism or periodicity.
Definition cs_halo.c:1639
cs_halo_comm_mode_t cs_halo_get_comm_mode(void)
Get default communication mode for halo exchange.
Definition cs_halo.c:2036
void cs_halo_sync(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *val)
Update array of values in case of parallelism or periodicity.
Definition cs_halo.c:1881
cs_halo_t * cs_halo_create_from_rank_neighbors(const cs_rank_neighbors_t *rn, cs_lnum_t n_local_elts, cs_lnum_t n_distant_elts, const int elt_rank_id[], const cs_lnum_t elt_id[])
Create a halo structure from distant element distant ranks and ids.
Definition cs_halo.c:792
void cs_halo_set_comm_mode(cs_halo_comm_mode_t mode)
Set default communication mode for halo exchange.
Definition cs_halo.c:2050
cs_alloc_mode_t cs_halo_get_buffer_alloc_mode(void)
Get default host/device allocation mode for message packing arrays.
Definition cs_halo.c:2065
static size_t cs_halo_pack_size(const cs_halo_t *halo, cs_datatype_t data_type, int stride)
Compute required size for packing send data into dense buffer.
Definition cs_halo.h:324
void cs_halo_renumber_cells(cs_halo_t *halo, const cs_lnum_t new_cell_id[])
Definition cs_halo.c:1153
cs_halo_t * cs_halo_create_from_ref(const cs_halo_t *ref)
Create a halo structure, given a reference halo.
Definition cs_halo.c:702
void cs_halo_dump(const cs_halo_t *halo, int print_level)
Definition cs_halo.c:2094
cs_halo_t * cs_halo_create(const cs_interface_set_t *ifs)
Create a halo structure given an interface set.
Definition cs_halo.c:501
void cs_halo_sync_wait(const cs_halo_t *halo, void *val, cs_halo_state_t *hs)
Wait for completion of update array of values in case of parallelism or periodicity.
Definition cs_halo.c:1797
void cs_halo_set_use_barrier(bool use_barrier)
Definition cs_halo.c:2022
cs_halo_type_t
Definition cs_halo.h:56
@ CS_HALO_N_TYPES
Definition cs_halo.h:60
@ CS_HALO_STANDARD
Definition cs_halo.h:58
@ CS_HALO_EXTENDED
Definition cs_halo.h:59
void * cs_halo_sync_pack_init_state(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_datatype_t data_type, int stride, void *send_buf, cs_halo_state_t *hs)
Initialize halo state prior to packing halo data to send.
Definition cs_halo.c:1339
void cs_halo_create_complete(cs_halo_t *halo)
Ready halo for use.
Definition cs_halo.c:642
void cs_halo_sync_untyped(const cs_halo_t *halo, cs_halo_type_t sync_mode, size_t size, void *val)
Definition cs_halo.c:1924
cs_halo_state_t * cs_halo_state_get_default(void)
Get pointer to default halo state structure.
Definition cs_halo.c:1139
void cs_halo_set_buffer_alloc_mode(cs_alloc_mode_t mode)
Set default host/device allocation mode for message packing arrays.
Definition cs_halo.c:2079
void cs_halo_sync_var(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[])
Definition cs_halo.c:1968
void cs_halo_sync_num(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_lnum_t num[])
Definition cs_halo.c:1946
void cs_halo_renumber_ghost_cells(cs_halo_t *halo, const cs_lnum_t old_cell_id[])
Definition cs_halo.c:1177
void cs_halo_sync_var_strided(const cs_halo_t *halo, cs_halo_type_t sync_mode, cs_real_t var[], int stride)
Definition cs_halo.c:1991
void cs_halo_destroy(cs_halo_t **halo)
Definition cs_halo.c:1016
struct _cs_halo_state_t cs_halo_state_t
Definition cs_halo.h:172
struct _cs_interface_set_t cs_interface_set_t
Definition cs_interface.h:61
struct _fvm_periodicity_t fvm_periodicity_t
Definition fvm_periodicity.h:67
Definition cs_halo.h:77
int n_transforms
Definition cs_halo.h:80
cs_lnum_t * index
Definition cs_halo.h:118
cs_lnum_t * send_list
Definition cs_halo.h:98
cs_lnum_t * send_index
Definition cs_halo.h:101
int n_rotations
Definition cs_halo.h:87
cs_lnum_t n_local_elts
Definition cs_halo.h:90
int n_c_domains
Definition cs_halo.h:79
int * c_domain_rank
Definition cs_halo.h:82
cs_lnum_t * perio_lst
Definition cs_halo.h:123
cs_lnum_t * send_perio_lst
Definition cs_halo.h:106
const fvm_periodicity_t * periodicity
Definition cs_halo.h:84
MPI_Group c_domain_group
Definition cs_halo.h:163
cs_lnum_t * c_domain_s_shift
Definition cs_halo.h:164
cs_lnum_t n_send_elts[2]
Definition cs_halo.h:94
cs_lnum_t n_elts[2]
Definition cs_halo.h:114
Definition cs_rank_neighbors.h:67