#!/bin/sh
#============================================================================
#
#                    Code_Saturne version 1.3
#                    ------------------------
#
#
#     This file is part of the Code_Saturne Kernel, element of the
#     Code_Saturne CFD tool.
#
#     Copyright (C) 1998-2008 EDF S.A., France
#
#     contact: saturne-support@edf.fr
#
#     The Code_Saturne Kernel is free software; you can redistribute it
#     and/or modify it under the terms of the GNU General Public License
#     as published by the Free Software Foundation; either version 2 of
#     the License, or (at your option) any later version.
#
#     The Code_Saturne Kernel is distributed in the hope that it will be
#     useful, but WITHOUT ANY WARRANTY; without even the implied warranty
#     of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with the Code_Saturne Kernel; if not, write to the
#     Free Software Foundation, Inc.,
#     51 Franklin St, Fifth Floor,
#     Boston, MA  02110-1301  USA
#
#============================================================================
#
########################################################################
#
# Example of Code_Saturne execution script.
#
########################################################################
########################################################################
#
#                  BATCH FILE FOR THE CCRT (Platine under LSF)
#                  ===========================================
#
#BSUB -n 2
#BSUB -W 00:05  
#BSUB -o t_junctioncas1o.%J 
#BSUB -e t_junctioncas1e.%J 
#BSUB -J t_junctioncas1
#
#  -n : number of processors
#  -W : walltime as hh:mm
#  -o : output file name
#  -e : error file name
#  -J : job name
#
# ------------------------------------------------------------------
#
#                  BATCH FILE FOR THE Chatou CLUSTER (PBS)
#                  =======================================
#
#PBS -l nodes=4:ppn=2,walltime=1:00:00,mem=320mb
#PBS -j eo -N t_junctioncas1
#
#  nodes    : number of nodes
#  ppn      : number of processes per node
#  walltime : wall clock time (hh:mm:ss)
#  mem      : memory
#
#WARNING: when coupling with SYRTHES, 1 processor will be reserved for each
#         instance of SYRTHES. The Kernel will be executed on the remaining
#         processors, so make sure to reserve a sufficiently high number
#         of processors.
#
# ------------------------------------------------------------------
#
#                  BATCH FILE (University of Manchester Cluster)
#                  =============================================
#
# set the name of the job
##$ -N t_junctioncas1
#
# request between 2 and 4 slots
##$ -pe mpich 2-4
#
# Execute the job from the current working directory
# Job output will appear in this directory
##$ -cwd
#   can use -o dirname to redirect stdout
#   can use -e dirname to redirect stderr

#  Export these environment variables
##$ -v MPI_HOME

#set -x
#
# ------------------------------------------------------------------
#
#                  BATCH FILE (AIX, Loadlever)
#                  ===========================
#
#@ shell = /bin/sh
#
#@ job_name = t_junctioncas1
#
#@ job_type = parallel
#@ cpus = 128
#@ node_usage = not_shared
#
#@ network.MPI = csss,shared,US
#@ bulkxfer = oui
#
#@ wall_clock_limit = 00:20:00
#@ account_no = z001
#
#@ output = $(job_name).$(schedd_host).$(jobid).out
#@ error  = $(job_name).$(schedd_host).$(jobid).err
#@ notification = never
#
#@ queue
# suggested environment settings:
#  export MP_EAGER_LIMIT=65536
#  export MP_SHARED_MEMORY=oui
#  export MEMORY_AFFINITY=MCM
#  export MP_TASK_AFFINITY=MCM
#
########################################################################
#
# BEGINNING OF USER MODIFIABLE ZONE FOR STANDARD CALCULATIONS
#
# lance.help gives more details about how to set variables.
#
#                    -------------------------------
#
SOLCOM=0
LONGIA=1000000
LONGRA=5000000
#
# On some systems, some external libraries may require TERM to be defined.
export TERM=xterm
#
ETUDE=T_JUNCTION
CAS=CAS1
PARAM=
MAILLAGE="t_junction.unv"
COMMANDE_RC=
COMMANDE_DF=
COMMANDE_PERIO=
COMMANDE_SYRTHES=
DONNEES_THERMOCHIMIE=
#
# Choose the total number of processors used (if empty, automatic detection
# through the batch system if possible, set to 1 otherwise).
# When coupling with SYRTHES with MODE_COUPLAGE=MPI, the 1st processor is
# used by SYRTHES, so the effective number of processors assigned to the
# Kernel is reduced by 1.
# The processors list is only usable when not running on a batch system
# (as such a system usually already defines a similar list)
NOMBRE_DE_PROCESSEURS=
LISTE_PROCESSEURS=
#
FICHIERS_DONNEES_UTILISATEUR=""
FICHIERS_RESULTATS_UTILISATEUR=""
#
# Working directory (leave empty for automatic default directory)
CS_TMP_PREFIX=
#CS_TMP_PREFIX=/local00/users/`whoami`
#
OPTIMISATION=
OPTION_LIB_EXT=
VALGRIND=
#
ARG_CS_VERIF=""
ARG_CS_OUTPUT=""
ECHOCOMM=""
#
# Adaptation using HOMARD
PILOTAGE_ADAPTATION=
#
resume=resume
REPBASE=/home/dbasu/SATURNE-CODE/Noyau/ncs-1.3.3/doc/TUTORIAL/TEST_CASES/T_JUNCTION/CAS1
DATA=$REPBASE/DATA
RESU=$REPBASE/RESU
FORT=$REPBASE/FORT
SCRIPTS=$REPBASE/SCRIPTS
SUITE_AMONT=$DATA/SUITE
PRE_TRAITEMENT_AMONT=$DATA/PRE_TRAITEMENT
REPMAIL=$REPBASE/../MAILLAGE
#
# The following variables are only used in case of coupling with SYRTHES.
#  DATA_SYR : directory where to find SYRTHES_ENV
#  FORT_SYR : directory where to find possible user source files
DATA_SYR=$REPBASE/DATA_SYR
SYRTHES_ENV=syrthes.env 
FORT_SYR=$REPBASE/FORT_SYR
#
# The possible coupling mode with SYRTHES may be chosen here.
# (be careful of lowercase/uppercase).
#
#  MODE_COUPLAGE=MPI        : should work on most machines.
#  MODE_COUPLAGE=sockets    : should be activated on clusters when the MPI rank
#                             of a process is not determinable or if MPI can
#                             not be used.
#  MODE_COUPLAGE=pipes      : on single node only (as pipes are not visible
#                             across a network).
MODE_COUPLAGE=MPI
#
# We can use 3 execution modes
# (be careful with lowercase/uppercase)
#
#  MODE_EXEC=complet        : default choice (Preprocessor then Kernel)
#  MODE_EXEC=pre_traitement : run Proprocessor only, to generate
#                             preprocessor_output/* files.
#  MODE_EXEC=calcul         : run Kernel only, using pre-generated
#                             preprocessor_output/* files, accessible in
#                             directory $PRE_TRAITEMENT_AMONT.
#
MODE_EXEC=complet
#
#
#
#
########################################################################
#
# END OF USER MODIFIABLE ZONE FOR STANDARD CALCULATIONS
#
########################################################################
#
#  Kernel library
CS_HOME=/home/dbasu/SATURNE-CODE/Noyau/ncs-1.3.3
export CS_HOME
#
# Installation parameters
#
. $CS_HOME/bin/cs_profile
#
# Library compatibility check
if [ ! -f ${CS_HOME}/bin/macros_${NOM_ARCH}.mk ]
then
  echo "Architecture ${NOM_ARCH} not supported by this Code_Saturne version."
  echo ${CS_HOME}
  exit 1
fi
#
########################################################################
#  Mesh Adaptation
#
if [ ! -z  "${PILOTAGE_ADAPTATION}" ] ; then
  if [ "${PILOTAGE_ADAPTATION}" = "-aide" -o "${PILOTAGE_ADAPTATION}" = "-help" ] ; then
    ${CSHOMARD_HOME}/saturne_homard ${PILOTAGE_ADAPTATION}
    exit 0
  else
    Options_HOMARD="  -v"
    ${CSHOMARD_HOME}/saturne_homard -Saturne_Script $0 -Donnees_Calcul $REPBASE \
                    -Pilotage_Adaptation $PILOTAGE_ADAPTATION $Options_HOMARD || exit 1
  fi
fi
#
#########################################################################
#  
#
N_COUPLAGE_SYRTHES=0
N_COUPLAGE_SYRTHES=`echo $COMMANDE_SYRTHES | grep -c syrthes`
if [ $N_COUPLAGE_SYRTHES = 0 ] ; then 
  COUPLAGE_SYRTHES=non
else
  COUPLAGE_SYRTHES=oui
  export SYRTHES
  if [ $N_COUPLAGE_SYRTHES -gt 1 ] ; then
   echo "This script can not handle multiple couplings with SYRTHES"
   echo "Number of SYRTHES couplings requested: $N_COUPLAGE_SYRTHES"
  fi
fi
#
if [ $COUPLAGE_SYRTHES = oui ] ; then 
  if [ $SOLCOM -eq 1 ] ; then
   echo "SYRTHES coupling is not compatible with SOLCOM-type meshes"
   exit 1
  fi
else
  MODE_COUPLAGE=
fi
#
########################################################################
#
# Parameters for execution
#
# General variables
THISSCRIPT=$0
USER=`whoami`
DATE=`date '+%m%d%H%M'`
SUFFIXE=$DATE
EXE=cs13.exe
EXE_SYR=syrthes
#
# Copy lance before changing to the working directory
# (as after that, the relative path will not be up to date).
#

cp $0 $RESU/lance.$SUFFIXE
#
# Execution directory (reachable by all the processors)
#
if [ ! -z "$CS_TMP_PREFIX" ] ; then
  RUN=${CS_TMP_PREFIX}/tmp_Saturne/$ETUDE.$CAS.$DATE
#
else
#  Default if not specified by the user
#
#  On the CCRT, there is no TMPDIR. We work by default in SCRATCHDIR
  if [ "$SCRATCHDIR" != "" ] ; then
    RUN=$SCRATCHDIR/tmp_Saturne/$ETUDE.$CAS.$DATE
#
  elif [ "$TMPDIR" != "" -a "$TMPDIR" != "/tmp" ] ; then
    RUN=$TMPDIR/tmp_Saturne/$ETUDE.$CAS.$DATE
  else
    RUN=$HOME/tmp_Saturne/$ETUDE.$CAS.$DATE
  fi
fi
#
# Create directory if necessary
if [ "$RUN" != "$TMPDIR" ] ; then
  if [ ! -d $RUN ] ; then
    mkdir -p $RUN || exit 1
  else
    echo "RUN=$RUN directory already exists."
    echo "The simulation may not be run."
    exit 1
  fi
fi
#
# Create a temporary file for Salome (equivalent to "ficstp")
if [ "$ARG_CS_OUTPUT" = "--log 0" ] ; then
  echo $RUN > $SCRIPTS/runningstd.$DATE
else
  echo $RUN > $SCRIPTS/runningext.$DATE
fi
#
cd $RUN
#
########################################################################
#
# Set up MPI environment
#
# Processors list
#
if [ -f "$TMPDIR/machines" ] ; then   # with batch system at Univ. Manchester
   MPIHOSTS="$TMPDIR/machines"
elif [ ! -z "$PBS_NODEFILE" ] ; then  # with PBS batch system
   MPIHOSTS="$PBS_NODEFILE"
elif [ ! -z "$LSB_HOSTS" ] ; then     # with LSF batch system
   \rm -f $RUN/hostsfile
   echo $LSB_HOSTS | awk '{ for (ii = 1; ii < NF+1; ++ii) print $ii }' >> $RUN/hostsfile
   MPIHOSTS="$RUN/hostsfile"
elif [ ! -z "$LISTE_PROCESSEURS" ] ; then
   \rm -f $RUN/hostsfile
   localproc=`hostname`
   echo $LISTE_PROCESSEURS | grep -q $localproc || LISTE_PROCESSEURS="$localproc & $LISTE_PROCESSEURS"
   echo $LISTE_PROCESSEURS | awk  -F\& '{ for (ii = 1; ii < NF+1; ++ii) print $ii }' >> $RUN/hostsfile
   MPIHOSTS="$RUN/hostsfile"
fi
# Check number of processors
if [ -z  "${NOMBRE_DE_PROCESSEURS}" ] ; then
  if [ ! -z  "${NSLOTS}" ] ; then
    # for batch sytem at University of Manchester
    NOMBRE_DE_PROCESSEURS=$NSLOTS
  elif [ ! -z "${SLURM_NPROCS}" ] ; then
    # for LSF batch system with Slurm
    NOMBRE_DE_PROCESSEURS=$SLURM_NPROCS
  elif [ ! -z "${LSB_HOSTS}" ] ; then
    # for LSF batch system without Slurm
    NOMBRE_DE_PROCESSEURS=`echo $LSB_HOSTS | wc -w`
  elif [ ! -z  "${MPIHOSTS}" ] ; then
    # for batch sytme on MFEE cluster or whenever MPIHOSTS is defined
    NOMBRE_DE_PROCESSEURS=`cat $MPIHOSTS | wc -l`
  else
    NOMBRE_DE_PROCESSEURS=1
  fi
fi
#
# Check number of nodes if possible
NOMBRE_DE_NOEUDS=1
if [ -z  "${NOMBRE_DE_PROCESSEURS}" ] ; then
  if [ ! -z  "${MPIHOSTS}" ] ; then
    NOMBRE_DE_NOEUDS=`sort $MPIHOSTS | sed -e 's/:/ /' | cut -f 1 -d" " | uniq | wc -l`
  fi
fi
#
# Check for the number of requested processors
nproc_noy=${NOMBRE_DE_PROCESSEURS}
echo
echo
if [ $COUPLAGE_SYRTHES = oui ] ; then
  echo "Coupling of Code_Saturne and SYRTHES on"
  if [ ! -z "$MPIHOSTS" ] ; then
  #  1 processor is reserved for SYRTHES if coupled through MPI.
    if [ "$MODE_COUPLAGE" = "MPI" ] ; then
      (( nproc_noy = nproc_noy - N_COUPLAGE_SYRTHES ))
      if [ $nproc_noy -lt 1 ] ; then 
        echo
        echo
        echo " WARNING, if MODE_COUPLAGE = MPI, SYRTHES reserves 1 processor,"
        echo " so the requested number of processors ($NOMBRE_DE_PROCESSEURS)"
        echo " is too small."
        echo " Increase the number of requested processors."
        echo
        exit 1
      fi
    fi
    # if SYRTHES is used in batch mode, check that everything fits on a node
    if [ "$MODE_COUPLAGE" = "pipes" -a "$NOMBRE_DE_NOEUDS" != 1 ] ; then
      echo
      echo
      echo " Warning, with MODE_COUPLAGE = pipes, SYRTHES-Kernel"
      echo "   communications are done through pipes. "
      echo " Here, multiple nodes (hosts) were selected for the parallel run:"
      cat "$MPIHOSTS"
      echo " Use another value of MODE_COUPLAGE "
      echo
      exit 1
    fi
  else
  # If we are not in batch mode and we use coupling through MPI, we must
  # specify a number of processors greater than 1 to activate MPI.
    if [ "$MODE_COUPLAGE" = "MPI" ] ; then
      (( nproc_noy = nproc_noy - N_COUPLAGE_SYRTHES ))
      if [ $nproc_noy -lt 1 ] ; then 
        echo
        echo
        echo " Warning, if MODE_COUPLAGE = MPI, SYRTHES reserves 1 processor"
        echo " (which need not be a physical processor)."
        echo " ($NOMBRE_DE_PROCESSEURS) must be strictly greater than 1."
        echo " Increase NOMBRE_DE_PROCESSEURS."
        echo
        exit 1
      fi
    fi
  # If were are not in batch mode and are coupling with pipes,
  # we also reduce NOMBRE_DE_PROCESSEURS so as not to need MPI if
  # Code_Saturne and SYRTHES are each running on 1 processor.
    if [ "$MODE_COUPLAGE" = "pipes" ] ; then
      (( nproc_noy = nproc_noy - N_COUPLAGE_SYRTHES ))
      (( NOMBRE_DE_PROCESSEURS = NOMBRE_DE_PROCESSEURS - 1 ))
      if [ $nproc_noy -lt 1 ] ; then 
        echo
        echo
        echo " Warning, with MODE_COUPLAGE = pipes, SYRTHES reserves a processor"
        echo "   (possibly virtual)."
        echo "   The number of processors requested ($NOMBRE_DE_PROCESSEURS)"
        echo "   msut thus be strictly greater than 1."
        echo " Increase NOMBRE_DE_PROCESSEURS."
        echo
        exit 1
      fi
    fi
  fi
fi
if [ $nproc_noy -gt 1 ] ; then
  echo "Parallel Code_Saturne with partitioning in $nproc_noy sub-domains"
  if [ $SOLCOM = 1 ] ; then 
    echo
    echo
    echo " Parallel run impossible with SOLCOM = $SOLCOM "
    echo " Use SOLCOM = 0 "
    echo
    exit 1
  fi
else
  echo "Single processor Code_Saturne simulation"
fi
if [ ! -z "$MPIHOSTS" ] ; then
  echo "Total number of processors: $NOMBRE_DE_PROCESSEURS"
fi
#
# In parallel mode, the number of processors must be specified
# to the Kernel, as well as to the Preprocessor
if [ $nproc_noy -gt 1 ] ; then 
  ARG_ECS_PAR=" -p $nproc_noy"
  ARG_CS_PAR=" -p $nproc_noy"
else
  ARG_ECS_PAR=""
  ARG_CS_PAR=""
fi
#
# System-related parameters
#
if [ $NOMBRE_DE_PROCESSEURS -gt 1 ] ; then 
#
# Default values defined at start
  MPIBOOT=""
  MPIHALT=""
  MPIRUN="mpirun -np ${NOMBRE_DE_PROCESSEURS}"
  if [ -d "$CS_MPI_PATH" ] ; then
    export PATH=${CS_MPI_PATH}:$PATH
  fi
#
  if [ "$NOM_ARCH" = "OSF1" ] ; then 
    MPIRUN="prun -n ${NOMBRE_DE_PROCESSEURS} -stv"
#
  elif [ "$NOM_ARCH" = "Linux_CCRT" ] ; then
    MPIRUN="mpirun -srun"
#
  elif [ "$NOM_ARCH" = "Linux_IA64" ] ; then
    MPIRUN="srun -n ${NOMBRE_DE_PROCESSEURS}"
#
  elif [ "$NOM_ARCH" = "Linux_Ch" ] ; then 
    MPIRUN="mpirun -np ${NOMBRE_DE_PROCESSEURS} -machinefile $PBS_NODEFILE"
#
  else 
    if [ -d "$CS_MPI_PATH" ] ; then 
      if [ -f "${CS_MPI_PATH}/mpdboot" ] ; then          # With MPICH2
        MPIBOOT="${CS_MPI_PATH}/mpdboot"
        MPIHALT="${CS_MPI_PATH}/mpdallexit"
        MPIRUN="${CS_MPI_PATH}/mpiexec -n ${NOMBRE_DE_PROCESSEURS}" 
        if [ ! -z "$MPIHOSTS" ] ; then
          MPIBOOT="$MPIBOOT -n ${NOMBRE_DE_NOEUDS} -v -f $MPIHOSTS"
	fi
      elif [ -f "${CS_MPI_PATH}/ompi_info" ] ; then      # With Open MPI
        MPIRUN="${CS_MPI_PATH}/mpiexec -n ${NOMBRE_DE_PROCESSEURS}" 
        if [ ! -z "$MPIHOSTS" ] ; then
          MPIRUN="$MPIRUN -machinefile $MPIHOSTS"
        fi
      elif [ -f "${CS_MPI_PATH}/lamboot" ] ; then        # With LAM MPI
        MPIBOOT="${CS_MPI_PATH}/lamboot"
        MPIHALT="${CS_MPI_PATH}/lamhalt"
        MPIRUN="${CS_MPI_PATH}/mpirun -np ${NOMBRE_DE_PROCESSEURS}" 
        if [ ! -z "$MPIHOSTS" ] ; then
          MPIBOOT="$MPIBOOT -v $MPIHOSTS"
          MPIHALT="$MPIHALT -v $MPIHOSTS"
        fi
      elif [ -f "${CS_MPI_PATH}/mpichversion" ] ; then   # With MPICH
        MPIRUN="${CS_MPI_PATH}/mpirun -np ${NOMBRE_DE_PROCESSEURS}" 
        if [ ! -z "$TMPDIR/machines" ] ; then   # With batch at Univ. Manchester
          MPIRUN="$MPIRUN -machinefile $TMPDIR/machines"
        elif [ ! -z "$MPIHOSTS" ] ; then
          MPIRUN="$MPIRUN -machinefile $MPIHOSTS"
        fi
      fi
    fi
  fi
#
#   if we do not request multiple processors
#
else
#
  MPIRUN=""
  MPIBOOT=""
  MPIHALT=""
fi
#
########################################################################
#
# Communications
#
# Socket definition
if [ $COUPLAGE_SYRTHES = oui -a "$MODE_COUPLAGE" = "sockets" ] ; then
  SOCKETSYR="-comm-socket"
else
  SOCKETSYR=""
fi
# To help debug coupling problems, make communication more verbose
if [ ! -z "$ECHOCOMM" ] ; then 
  ECHOCOMMNB="$ECHOCOMM"
  ECHOCOMMECS="-echo-comm $ECHOCOMMNB"
  ECHOCOMMSYR="-echo-comm $ECHOCOMMNB"
  ECHOCOMMNOY="--echo-comm $ECHOCOMMNB"
else
  ECHOCOMMNB=""
  ECHOCOMMECS=""
  ECHOCOMMSYR=""
  ECHOCOMMNOY=""
fi
########################################################################
# Greeting message
#
echo ' '
echo '                      Code_Saturne is running '
echo '                      *********************** '
echo ' '
echo ' Working directory (to be periodically cleaned) : ' 
echo '    ' $RUN
#
########################################################################

# Function used to check if a user subroutine is present
#  we keep things minimalistic here
#   (we could check that uslag2 effectively contains uslag2 or
#   that it compiles...) 
presence () {

  presence_sspgm=$1

  if [ ! -f "${presence_sspgm}" ] ; then
    presence_PRESENT="non"
  else
    presence_PRESENT="oui"
  fi
  echo ${presence_PRESENT} 
}
#
########################################################################
#
# Compilation and link
#
# Note: we also check the for the presence of certain user subroutines here.
#
if [ "$CS_HOME" = "" ]; then
   echo
   echo "WARNING: CS_HOME is not set."
   echo
   exit 1
else
  echo
  echo ' Kernel version:          ' $CS_HOME
  echo ' Preprocessor version:    ' $ECS_HOME
fi

source_cas=$FORT

if [ -f compil.log ] ; then
  rm -f compil.log
fi

if [ $MODE_EXEC != pre_traitement ] ; then

  echo
  echo  "  ***************************************************************"
  echo  "   Compilation of user subroutines and linking of Code_Saturne"
  echo  "  ***************************************************************"

  cur_dir=`pwd`

  #
  # Copy of the parameter file
  if [ ! -z "$PARAM" ] ; then 
    var=$DATA/$PARAM
    if [ -f $var ] ; then 
      COMMANDE_PARAM="-param $PARAM"
      cp $var .
    else
      echo ' ' 
      echo ' -- ERROR -- ' 
      echo ' The parametes file ' $var
      echo ' can not be accessed.' 
      exit 1
    fi
  fi

  # Compiler version
  ${CS_HOME}/bin/compiler_version -cshome ${CS_HOME} -nomarch ${NOM_ARCH} -output ${cur_dir}/compil.log

  # Copy of the user source files
  # (no links: the directory is copied later)
  mkdir fort_saturne
  cd fort_saturne
  for fich in ${source_cas}/*.[F,c,h] ; do 
    if [ -f $fich ] ; then
      cp ${fich} .
    fi
  done
#
# Detect presence of modules

# Standard
  USCLIM_PRESENT=`presence "usclim.F"`

# Lagrangien
  USLAG2_PRESENT=`presence "uslag2.F"`

# Radiative transfer
  USRAY2_PRESENT=`presence "usray2.F"`
  if [ ! -z "$PARAM" ] ; then 
    NRAYT=`grep "radiative_transfer\ model" ${cur_dir}/$PARAM | grep -v "off" |wc -l`
    if [ $NRAYT -eq 1 ] ; then
      USRAY2_PRESENT="oui"
    fi
  fi

# Pulverized coal
  USCPCL_PRESENT=`presence "uscpcl.F"`
  if [ ! -z "$PARAM" ] ; then 
    NCHARB=`grep "pulverized_coal\ model" ${cur_dir}/$PARAM | grep -v "off" |wc -l`
    if [ $NCHARB -eq 1 ] ; then
      USCPCL_PRESENT="oui"
    fi
  fi

# Gas combustion with 3 point chemistry
  USD3PC_PRESENT=`presence "usd3pc.F"`

# EBU gas combustion
  USEBUC_PRESENT=`presence "usebuc.F"`

# Combustion gaz LWC
  USLWCC_PRESENT=`presence "uslwcc.F"`

# Electric arcs
  USELCL_PRESENT=`presence "uselcl.F"`

# Compressible
  USCFCL_PRESENT=`presence "uscfcl.F"`

# Fuel
  USFUCL_PRESENT=`presence "usfucl.F"`

# Compatibility tests
  if [ "${USCPCL_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When the pulverized coal model is used,'
      echo '  usclim not be used.'
      echo '  Boundary conditions are defined through'
      echo '  the GUI or uscpcl.'
      echo ' '
	exit 1
    fi
  fi
  if [ "${USD3PC_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When usd3pc is used, usclim must not be'
      echo '  (combustion with 3-point chemistry model)              '
      echo '  Boundary conditions are defined in usd3pc.'
      echo ' '
	exit 1
    fi
  fi
  if [ "${USEBUC_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When usebuc is used, usclim must not be'
      echo '  (EBU combustion model)                              '
      echo '  Boundary conditions are defined in usebuc.'
      echo ' '
	exit 1
    fi
  fi
  if [ "${USLWCC_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When uslwcc is used, usclim must not be'
      echo '  (LWC combustion model)                              '
      echo '  Boundary conditions are defined in uslwcc.'
      echo ' '
	exit 1
    fi
  fi
  if [ "${USELCL_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When uselcl is used, usclim must not be'
      echo '  (Electric arcs model)                              '
      echo '  Boundary conditions are defined in uselcl.'
      echo ' '
	exit 1
    fi
  fi

  if [ "${USCFCL_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When uscfcl is used, usclim must not be'
      echo '  (Compressible model)                              '
      echo '  Boundary conditions are defined in uscfcl.'
      echo ' '
	exit 1
    fi
  fi

  if [ "${USFUCL_PRESENT}" = "oui" ] ; then
    if [ "${USCLIM_PRESENT}" = "oui" ] ; then
      echo ' '
      echo ' -- ERROR -- '
      echo ' When usfucl is used, usclim must not be'
      echo '  (Fuel model)                              '
      echo '  Boundary conditions are defined in usfucl.'
      echo ' '
	exit 1
    fi
  fi

# Translation
  if [ "${USLAG2_PRESENT}" = "oui" ] ; then
    LAGRANGIEN=oui
    echo ' '
    echo ' Use the Lagrangian module'  
    echo ' '
  else
    LAGRANGIEN=non
  fi
  if [ "${USRAY2_PRESENT}" = "oui" ] ; then
    RAYONNEMENT=oui
    echo ' '
    echo ' Use the radiative transfer module'  
    echo ' '
  else
    RAYONNEMENT=non
  fi
  COMBUSTION=non
  if [ "${USCPCL_PRESENT}" = "oui" ] ; then
    COMBUSTION=oui
    COMBUSTION_CPP=oui
    echo ' '
    echo ' Use the pulverized coal combustion module'  
    echo ' '
  else
    COMBUSTION_CPP=non
  fi
  if [ "${USD3PC_PRESENT}" = "oui" ] ; then
    COMBUSTION=oui
    COMBUSTION_C3P=oui
    echo ' '
    echo ' Use the 3-point chemistry combustion module'  
    echo ' '
  else
    COMBUSTION_C3P=non
  fi
  if [ "${USEBUC_PRESENT}" = "oui" ] ; then
    COMBUSTION=oui
    COMBUSTION_EBU=oui
    echo ' '
    echo ' Use the EBU combustion module'  
    echo ' '
  else
    COMBUSTION_EBU=non
  fi
  if [ "${USLWCC_PRESENT}" = "oui" ] ; then
    COMBUSTION=oui
    COMBUSTION_LWC=oui
    echo ' '
    echo ' Use the LWC combustion module'  
    echo ' '
  else
    COMBUSTION_LWC=non
  fi
  if [ "${USELCL_PRESENT}" = "oui" ] ; then
    ELECTRIQUE=oui
    echo ' '
    echo ' Use the electric arc module'  
    echo ' '
  else
    ELECTRIQUE=non
  fi

  if [ "${USCFCL_PRESENT}" = "oui" ] ; then
    COMPRESSIBLE=oui
    echo ' '
    echo ' Use the compressible module'  
    echo ' '
  else
    COMPRESSIBLE=non
  fi

  if [ "${USFUCL_PRESENT}" = "oui" ] ; then
    FUEL=oui
    echo ' '
    echo ' Use the fuel module'
    echo ' '
  else
    FUEL=non
  fi


# No Lagrangian in parallel
  if [ "${LAGRANGIEN}" = "oui" ] ; then 
    if [ ${nproc_noy} -gt 1 ] ; then 
      echo ' '
      echo ' -- ERROR -- '
      echo ' The Lagrangian module is incompatible with'
      echo '  parallel runs as of the current version.'
      echo ' '
	exit 1
    fi
  fi
#
  # Extract main on HP (link with f90) unless it already exists
  # (should be done by Makefile if possible)
  if [ "${NOM_ARCH}" = "HP-UX" ] ; then 
    cheminlib=${CS_HOME}/lib/${NOM_ARCH}
    main=cs_main
    mainc=cs_main.c
    maino=cs_main.o
    if [ ! -f "$mainc" ] ; then 
      libsaturneBASE=libsaturneBASE
      if [ ! -z  "${LISTE_LIB_SAT}" ] ; then 
        for module in ${LISTE_LIB_SAT} ; do 
          if [ `echo ${module} | grep ${libsaturneBASE}` ] ; then 
            ar -xv $cheminlib/${libsaturneBASE}${OPTIMISATION} $maino
          fi
        done 
      else
        ar -xv $cheminlib/${libsaturneBASE}${OPTIMISATION}.a  $maino
      fi 
    fi 
  fi

# Compilation
  ln -s ${CS_HOME}/bin/Makefile .

  make saturne OPT="${OPTIMISATION}" OPTLIBS="${OPTION_LIB_EXT}"  2>>$cur_dir/compil.log 1>&2
  if [ $? -ne 0 ]
  then
    cp $cur_dir/compil.log $RESU/compil.log.$SUFFIXE
    echo COMPILE OR LINK ERROR
    rm -f *.o
    exit 1
  else
    cp $cur_dir/compil.log $RESU/compil.log.$SUFFIXE
  fi

# Cleanup
  make clean
  mv $EXE ${cur_dir}/
  cd ${cur_dir}

  if [ $COUPLAGE_SYRTHES = oui ] ; then 

    echo
    echo  "  **********************************************************"
    echo  "   Compilation of the user subroutines and link of SYRTHES"
    echo  "  **********************************************************"

    cur_dir=`pwd`
    mkdir fort_syrthes 
    cd fort_syrthes
    for fich in $FORT_SYR/*.[F,c,h] ; do 
      if [ -f $fich ] ; then
        cp ${fich} .
      fi
    done
    ln -s $SYRCS_HOME/bin/Makefile .
    make syrthes NOM_ARCH=${NOM_ARCH} 2>>$cur_dir/compil.log 1>&2
    if [ $? -ne 0 ]
    then
      cp $cur_dir/compil.log $RESU/compil.log.$SUFFIXE
      echo COMPILE OR LINK ERROR
      rm -f *.o
      exit 1
    else
      cp $cur_dir/compil.log $RESU/compil.log.$SUFFIXE
    fi
    make clean
    mv syrthes $cur_dir/
    cd $cur_dir

  fi

fi # MODE_EXEC != pre_traitement

#
########################################################################
#
# Data setup
#
echo
echo  "  ********************************************"
echo  "             Preparing calculation            "
echo  "  ********************************************"
echo 
if [ $SOLCOM = 1 ]
then
  ln -s $REPMAIL/$MAILLAGE geomet || exit 1
elif [ $MODE_EXEC != calcul ]
then
  for var in $MAILLAGE ; do
    ln -s $REPMAIL/$var $var || exit 1
    # Special case for meshes in EnSight format: link to .geo file necessary
    # (retrieve name through .case file)
    var2=`basename $var .case`
    if [ $var2 != $var ] ; then
      ficgeo_ensight=`awk '/^model:/ {print $2}' $var`
      ln -s $REPMAIL/$ficgeo_ensight $ficgeo_ensight || FIN
    fi
  done
elif [ $MODE_EXEC = calcul ]
then
    ln -s ${PRE_TRAITEMENT_AMONT} preprocessor_output || exit 1
fi
#
if [ $MODE_EXEC != pre_traitement ] ; then

  for var in ${SUITE_AMONT}/* ; do 
    if [ -f $var ] ; then 
      varb=`basename $var`
      if   [ $varb = suiava ] ; then 
        vara=suiamo
      elif [ $varb = suiavx ] ; then 
        vara=suiamx
      elif [ $varb = vorava ] ; then 
        vara=voramo
      elif [ $varb = t1dava ] ; then 
        vara=t1damo
      elif [ $varb = rayava ] ; then 
        vara=rayamo
      elif [ $varb = lagava ] ; then 
        vara=lagamo
      elif [ $varb = lasava ] ; then 
        vara=lasamo
      else
        vara=$varb
      fi
      ln -s $var $vara
    fi
  done
  #
  if [ "${COMBUSTION}" = "oui" -o "${ELECTRIQUE}" = "oui" -o "${FUEL}" = "oui"  ] ; then 
    var=$DATA/$DONNEES_THERMOCHIMIE
    if [ -f $var ] ; then 
      cp $var dp_tch
      # Copy so as to have correct name upon backup
      if [ "$DONNEES_THERMOCHIMIE" != "dp_tch" ] ; then 
        cp dp_tch $DONNEES_THERMOCHIMIE
      fi
    else
      echo ' ' 
      echo ' -- ERROR -- ' 
      echo ' The thermochemistry file ' $var
      echo ' can not be accessed. ' 
      exit 1
    fi    
    if [ "${COMBUSTION}" = "oui" -o "${FUEL}" = "oui" ] ; then 
      cp $CS_HOME/data/thch/JANAF JANAF
    fi
  fi
  #
  if [ $COUPLAGE_SYRTHES = oui ] ; then 
    #
    cp $DATA_SYR/$SYRTHES_ENV  .
    #
    # Extract information from the syrthes.env file and build a local copy
    # in which all files are local. The initial file is backed-up so that
    # it may be copied with results afterwards.
    #
    cp $SYRTHES_ENV  $SYRTHES_ENV.save
    #
    AMONT_SYR=`grep "AMONT :" $SYRTHES_ENV | awk -F: '{print $NF}' | sed -e "s/ //g"` 
    AVAL_SYR=` grep "AVAL :"  $SYRTHES_ENV | awk -F: '{print $NF}' | sed -e "s/ //g"`
    SUITE_SYR=`grep "SUITE :" $SYRTHES_ENV | awk -F: '{print $NF}' | sed -e "s/ //g"`
    SYRTHES_DATA=`grep "DONNEES DU CALCUL :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_GEOM=`grep "GEOMETRIE SOLIDE :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_SUIT=`grep "SUITE SOLIDE RESU :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_DATA_RAY=`grep "DONNEES POUR LE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_MAIL_RAY=`grep "MAILLAGE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_CORR=`grep "STOCKAGE DES CORRESPONDANTS :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_FFOR_RAY=`grep "FACTEURS DE FORME RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`
    SYRTHES_CORR_RAY=`grep "STOCKAGE DES CORRESPONDANTS RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_RES1=`grep "RESU SYRTHES 1 :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_RES2=`grep "RESU SYRTHES 2 :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_CHR2=`grep "CHRONO SYRTHES 2 :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_HIST=`grep "HISTORIQUE SOLIDE RESULTAT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_MAIL_PEAU_FLUIDE=`grep "MAILLAGE PEAU FLUIDE :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_RESU_PEAU_FLUIDE=`grep "RESULTATS PEAU FLUIDE :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_CHR_PEAU_FLUIDE=`grep "CHRONO PEAU FLUIDE :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_MAIL_RAY=`grep "MAILLAGE DE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_RESU_RAY=`grep "RESULTATS DE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_CHR_RAY=`grep "CHRONO DE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    SYRTHES_AVAL_HIST_RAY=`grep "HISTORIQUE RAYONNEMENT :" $SYRTHES_ENV | awk -F: '{print $NF}'`   
    sed -e "s/:.*\//: /"              $SYRTHES_ENV > $SYRTHES_ENV.mod ; mv -f $SYRTHES_ENV.mod $SYRTHES_ENV
    sed -e "s/AMONT :.*/AMONT : .\//" $SYRTHES_ENV > $SYRTHES_ENV.mod ; mv -f $SYRTHES_ENV.mod $SYRTHES_ENV
    sed -e "s/AVAL :.*/AVAL : .\//"   $SYRTHES_ENV > $SYRTHES_ENV.mod ; mv -f $SYRTHES_ENV.mod $SYRTHES_ENV
    sed -e "s/SUITE :.*/SUITE : .\//" $SYRTHES_ENV > $SYRTHES_ENV.mod ; mv -f $SYRTHES_ENV.mod $SYRTHES_ENV
    #
    # if empty, we use default names for results:
    if [ -z "$SYRTHES_CORR" ] ; then 
      SYRTHES_CORR="corresp"
    fi
    if [ -z "$SYRTHES_CORR_RAY" ] ; then 
      SYRTHES_CORR_RAY="corresp_ray"
    fi 
    if [ -z "$SYRTHES_FFOR_RAY" ] ; then 
      SYRTHES_FFOR_RAY="fdf_ray"
    fi 
    if [ -z "$SYRTHES_AVAL_RES1" ] ; then 
      SYRTHES_AVAL_RES1="geoms"
    fi 
    if [ -z "$SYRTHES_AVAL_RES2" ] ; then 
      SYRTHES_AVAL_RES2="resus"
    fi 
    if [ -z "$SYRTHES_AVAL_CHR2" ] ; then 
      SYRTHES_AVAL_CHR2="chronos"
    fi  
    if [ -z "$SYRTHES_AVAL_HIST" ] ; then 
      SYRTHES_AVAL_HIST="histos"
    fi 
    if [ -z "$SYRTHES_AVAL_MAIL_PEAU_FLUIDE" ] ; then 
      SYRTHES_AVAL_MAIL_PEAU_FLUIDE="geomf_ef"
    fi 
    if [ -z "$SYRTHES_AVAL_RESU_PEAU_FLUIDE" ] ; then 
      SYRTHES_AVAL_RESU_PEAU_FLUIDE="resuf_ef"
    fi 
    if [ -z "$SYRTHES_AVAL_CHR_PEAU_FLUIDE" ] ; then 
      SYRTHES_AVAL_CHR_PEAU_FLUIDE="chronof_ef"
    fi 
    if [ -z "$SYRTHES_AVAL_MAIL_RAY" ] ; then 
      SYRTHES_AVAL_MAIL_RAY="resu_ray.geom"
    fi 
    if [ -z "$SYRTHES_AVAL_RESU_RAY" ] ; then 
      SYRTHES_AVAL_RESU_RAY="resu_ray"
    fi 
    if [ -z "$SYRTHES_AVAL_CHR_RAY" ] ; then 
      SYRTHES_AVAL_CHR_RAY="chrono_ray"
    fi 
    if [ -z "$SYRTHES_AVAL_HIST_RAY" ] ; then 
      SYRTHES_AVAL_HIST_RAY="histor"
    fi 
    #
    #
    # Transfer SYRTHES files
    #
    for var in $SYRTHES_DATA $SYRTHES_GEOM $SYRTHES_SUIT $SYRTHES_DATA_RAY $SYRTHES_MAIL_RAY ; do 
      cp $AMONT_SYR/$var $RUN/`echo $var | sed -e "s/.*\///"`
    done
    for var in $SYRTHES_CORR $SYRTHES_FFOR_RAY $SYRTHES_CORR_RAY ; do 
      cp $SUITE_SYR/$var $RUN/`echo $var | sed -e "s/.*\///"`
    done
    #
  fi
  #
  if [ ! -z "$FICHIERS_DONNEES_UTILISATEUR" ] ; then 
    for fich in $FICHIERS_DONNEES_UTILISATEUR ; do 
      cp $DATA/$fich .
    done
  fi
  #
fi # MODE_EXEC != pre_traitement

########################################################################
# Maximum time for PBS (done here so as to leave time for PBS to
# realize that things have started).
#
if [ "$PBS_JOBID" != "" ] ; then 
  CS_MAXTIME=`qstat -r $PBS_JOBID | grep $PBS_JOBID | sed -e's/ \{1,\}/ /g' | cut -d ' ' -f 9`
  export CS_MAXTIME   
fi
#
########################################################################
#
# Summary: start
#
echo '========================================================'>>$resume
echo '   STARTING TIME    : ' $DATE                            >>$resume
echo '  ----------------------------------------------------'  >>$resume
echo '    CS_HOME         : ' $CS_HOME                         >>$resume
echo '    ECS_HOME        : ' $ECS_HOME                        >>$resume
echo '    ------------------------------------------------  '  >>$resume
cat  $CS_HOME/bin/cs_profile                                   >>$resume
echo '  ----------------------------------------------------'  >>$resume
echo '    USER            : ' $USER                            >>$resume
echo '    ARCHITECTURE    : ' $NOM_ARCH                        >>$resume
echo '========================================================'>>$resume
echo '    MACHINE         : '                                  >>$resume
     uname -a                                                  >>$resume
if [ -z "$NOMBRE_DE_PROCESSEURS" ] ; then 
  echo '    N PROCS         : ' 1                              >>$resume
else
  echo '    N PROCS         : ' $NOMBRE_DE_PROCESSEURS         >>$resume
fi
if [ -z "$LISTE_PROCESSEURS" ] ; then 
  echo '    PROCESSORS      : ' default                        >>$resume
else
  echo '    PROCESSORS      : ' $LISTE_PROCESSEURS             >>$resume
fi
echo '========================================================'>>$resume
echo '  ----------------------------------------------------'  >>$resume
echo '    CASE            : ' $CAS                             >>$resume
echo '      CONFIG.       : ' $CONF                            >>$resume
echo '        DATA        : ' $DATA                            >>$resume
echo '        FORT        : ' $FORT                            >>$resume
echo '        RESU        : ' $RESU                            >>$resume
echo '  ----------------------------------------------------'  >>$resume
echo '    REP. RUN        : ' $RUN                             >>$resume
echo '  ----------------------------------------------------'  >>$resume
echo '    EXECUTABLE      : ' $EXE                             >>$resume
echo '      LIB           : ' $BIB                             >>$resume
echo '      f COMPILER    : ' $COMPF                           >>$resume
echo '      c COMPILER    : ' $COMPC                           >>$resume
echo '      f OPTIONS     : ' $OPTCOMPF1                       >>$resume
echo '  ----------------------------------------------------'  >>$resume
#
# Execution 
echo
echo  "  ********************************************"
echo  "           Code_Saturne is running            "
echo  "  ********************************************"
echo 
#
ERROR=$RUN/error
erreur()
{
echo ERREUR $1 >> $ERROR
}
#
# More settings for SYRTHES coupling
#
if [ $COUPLAGE_SYRTHES = oui ] ; then
  #
  if [ "$MODE_COUPLAGE" = "pipes" ] ; then
    #
    # Pipe creation if necessary
    #
    if [ $nproc_noy -gt 1 ] ; then 
      num=1
      while [ $num -le $nproc_noy ] ; do 
        numf=`echo $num | awk '{printf "%4.4i\n", $1}'`
        for fich in solveur_n${numf}_vers_syrthes.0001 syrthes_vers_solveur_n$numf.0001 ; do
          if [ -f $fich ] ; then
            rm -f $fich
          fi
          mkfifo $fich
        done
        (( num = num + 1 ))
      done
    else
      for fich in solveur_vers_syrthes.0001 syrthes_vers_solveur.0001 ; do
        if [ -f $fich ] ; then 
          rm -f $fich
        fi
        mkfifo $fich
      done
    fi
    #
  # If sockets, change COMMANDE_SYRTHES 
  #   (add -socket after -syrthes)
  #
  elif [ "$MODE_COUPLAGE" = "sockets" ] ; then
    cmdsyr=""
    if [ ! -z "$COMMANDE_SYRTHES" ] ; then 
      for var in $COMMANDE_SYRTHES ; do 
        if [ "$var" = "-syrthes" ] ; then 
          cmdsyr="$cmdsyr $var -socket"
        else 
          cmdsyr="$cmdsyr $var" 
        fi 
      done
      COMMANDE_SYRTHES="$cmdsyr"
    fi
  fi
  #
fi
#
# Preprocessor start
#
if [ $SOLCOM = 0 -a $MODE_EXEC != calcul ] ; then
  #
  ${ECS_HOME}/bin/ecs $ARG_ECS_PAR -mesh $MAILLAGE -case $CAS \
                      $ECHOCOMMECS \
                      $COMMANDE_RC $COMMANDE_PERIO > listpre 2>&1 \
                   || erreur " WHILE RUNNING THE PREPROCESSOR "
  #
  if [ $MODE_EXEC = pre_traitement ] ; then
    #
    PRE_TRAITEMENT_AVAL=$RESU/PRE_TRAITEMENT.$SUFFIXE
    if [ -d ${PRE_TRAITEMENT_AVAL} ] ; then
      echo "The ${PRE_TRAITEMENT_AVAL} already exists."
      echo "Copy of preprocessor_output/n* files impossible"
    else
      cp -r preprocessor_output ${PRE_TRAITEMENT_AVAL}
    fi
  fi
#
fi
#
# Run calculation proper.
#
if [ ! -f $ERROR -a $MODE_EXEC != pre_traitement ] ; then
#
  if [ $COUPLAGE_SYRTHES = oui -a "$MODE_COUPLAGE" = "MPI" ] ; then
    # MPI Communication
    #
    # Make sure to transmit possible additional arguments assigned by mpirun to
    # the executable with some MPI-1 implementations (vanilla MPICH 1.2 sets the
    # parameters needed by MPI_Init through argc/argv): we use $@ to forward
    # arguments passed to localexec to the true executable files.
    #
    localexec=$RUN/localexec
    echo '#!/bin/sh' > $localexec
    echo "MPI_RANK=\`$CS_HOME/bin/rang_mpi.sh \$@\`" >> $localexec
    echo cd $RUN >> $localexec 
    echo "if [ \$MPI_RANK -eq 0 ] ; then" >> $localexec
    echo "  $RUN/$EXE_SYR \$@ -comm-mpi 1 -nbr_sat $nproc_noy $ECHOCOMMSYR > listsyr 2>&1" >> $localexec
    echo "else" >> $localexec
    echo "  $VALGRIND $RUN/$EXE \$@ $ARG_CS_PAR $COMMANDE_SYRTHES -proc 0 $COMMANDE_DF "\
         "$ARG_CS_VERIF $ARG_CS_OUTPUT $ECHOCOMMNOY -iasize $LONGIA -rasize $LONGRA "\
         "$COMMANDE_PARAM " >> $localexec 
    echo "fi" >> $localexec
    echo "retour=\$?" >> $localexec
    echo "exit \$retour" >> $localexec
    chmod 700 $localexec
    #
    $MPIBOOT
    $MPIRUN $localexec || erreur " WHILE RUNNING THE KERNEL (SOLVER) "
    $MPIHALT
    #
  elif [ $COUPLAGE_SYRTHES = oui  ] ; then
    # Pipes or socket communications
    #
    # Start Code_Saturne
    # Be careful to forward the optional additional arguments passed by
    # mpirun to the executable (for the standard MPICH 1.2 case, as mpirun
    # passes parameters used later by MPI_Init through argc/argv): we
    # use $@ to forward arguments passed to localexec
    localexec=$RUN/localexec
    echo '#!/bin/sh' > $localexec 
    echo cd $RUN >> $localexec 
    echo "$VALGRIND $RUN/$EXE \$@ $ARG_CS_PAR $COMMANDE_SYRTHES $COMMANDE_DF "\
                 " $ARG_CS_VERIF $ARG_CS_OUTPUT $ECHOCOMMNOY -iasize $LONGIA "\
                 " -rasize $LONGRA $COMMANDE_PARAM " >> $localexec 
    echo "retour=\$?" >> $localexec
    echo "exit \$retour" >> $localexec
    chmod 700 $localexec
    $MPIBOOT
    $MPIRUN $localexec || erreur " WHILE RUNNING THE KERNEL (SOLVER) " &
    #
    if [ "$MODE_COUPLAGE" = "sockets" ] ; then
      # We must obtain the machine and port information from the log;
      #   we sleep to leave enought time to the Kernel to start and print
      #   this information; we avoid while which could loop endlessly
      sleep 5
      MACHPORT=""
      MACHPORT=`grep 'Communication possible sur' listing|sed -e "s/Communication possible sur //g"|sed -e "s/, port /:/g"`
      SOCKETSYR="$SOCKETSYR $MACHPORT"
    fi
    #
    # Start SYRTHES
    if [ $COUPLAGE_SYRTHES = oui ] ; then
      ./$EXE_SYR -nbr_sat $nproc_noy $ECHOCOMMSYR $SOCKETSYR > listsyr 2>&1 
    fi
    #
    $MPIHALT
    #
  elif [ $SOLCOM = 0 ] ; then
    #
    if [ ! -f $ERROR ] ; then 
      localexec=$RUN/localexec
      echo '#!/bin/sh' > $localexec 
      echo cd $RUN >> $localexec 
      echo "$VALGRIND $RUN/$EXE \$@ $ARG_CS_PAR "\
           " $COMMANDE_DF $ARG_CS_VERIF $ARG_CS_OUTPUT $ECHOCOMMNOY "\
	   "-iasize $LONGIA  -rasize $LONGRA $COMMANDE_PARAM "  >> $localexec 
      echo "retour=\$?" >> $localexec
      echo "exit \$retour" >> $localexec
      chmod 700 $localexec
      $MPIBOOT
      $MPIRUN $localexec || erreur " WHILE RUNNING THE KERNEL (SOLVER) "
      $MPIHALT
    fi
    #
  else # cas SOLCOM = 1
    #
    $VALGRIND $RUN/$EXE -solcom \
          -iasize $LONGIA -rasize $LONGRA $ARG_CS_VERIF $ARG_CS_OUTPUT $COMMANDE_PARAM \
          || erreur " WHILE RUNNING THE KERNEL (SOLVER) "
    #
  fi
#
fi
#
# Remove the Salome temporary files
rm -f $SCRIPTS/running*.$DATE
#
########################################################################
#
# Treatment of the ouput files:
#   Starts with the restart files
#   (in case of full disk, increases chances of being able to continue).
#
if [ $MODE_EXEC != pre_traitement ] ; then
 
  SUITE_AVAL=$RESU/SUITE.$SUFFIXE
  iok=1
  mkdir ${SUITE_AVAL} || iok=0
  if [ $iok = 1 ] ; then 
    for fich in suiava suiavx t1dava vorava rayava lagava* lasava* ; do 
      if [ -f $fich ] ; then 
        cp $fich ${SUITE_AVAL}
      fi
    done
  else
    for fich in suiava suiavx t1dava vorava rayava lagava* lasava* ; do 
      if [ -f $fich ] ; then 
        cp $fich $RESU/$fich.$SUFFIXE
      fi
    done
  fi

  resuser=0
  for fich in ${FICHIERS_RESULTATS_UTILISATEUR} ; do 
    if [ -f $fich ] ; then 
      resuser=1
    fi
  done
  if [ ${resuser} = 1 ] ; then 
    RES_USER=$RESU/RES_USER.$SUFFIXE
    iok=1
    mkdir ${RES_USER} || iok=0
    if [ $iok = 1 ] ; then 
      for fich in ${FICHIERS_RESULTATS_UTILISATEUR} ; do 
        if [ -f $fich ] ; then 
          cp $fich ${RES_USER}
        fi
      done
    else
      for fich in ${FICHIERS_RESULTATS_UTILISATEUR} ; do 
        if [ -f $fich ] ; then 
          cp $fich $RESU/$fich.$SUFFIXE
        fi
      done
    fi
  fi

  for fich in $PARAM $DONNEES_THERMOCHIMIE ; do 
    if [ -f $fich ] ; then 
      cp $fich $RESU/$fich.$SUFFIXE
    fi
  done

  for fich in *.hst ; do
    if [ -f $fich ] ; then
      if [ ! -d $RESU/HIST.$SUFFIXE ] ; then 
        mkdir $RESU/HIST.$SUFFIXE
      fi
      cp $fich $RESU/HIST.$SUFFIXE
    fi
  done
  for fich in ush* ; do
    if [ -f $fich ] ; then
      if [ ! -d $RESU/HIST.$SUFFIXE ] ; then 
        mkdir $RESU/HIST.$SUFFIXE
      fi
      cp $fich $RESU/HIST.$SUFFIXE
    fi
  done

fi  # output files

for fich in list* erreur* *.med *.cgns ; do 
  if [ -f $fich ] ; then 
    cp $fich $RESU/$fich.$SUFFIXE
  fi
done

# Treatment of EnSight and MED files
#   The $dir (=*.ensight and/or *.med) directories are copied
#   to $DIR.$SUFFIXE

#   We place directories $dir (=*.ensight and/or *.med)
#   in $DIR.$SUFFIXE

cas=`echo $CAS |tr "[:upper:]" "[:lower:]"`

for dir in *.ensight *.med ; do
  if [ -d $dir ] ; then
    DIR=`echo $dir |tr "[:lower:]" "[:upper:]"`
    mkdir $RESU/$DIR.$SUFFIXE
    if [ $? -ne 0 ] ; then
      echo Creating $RESU/$DIR.$SUFFIXE failed
    else
      for fich in $dir/*  ; do 
        if [ -f $fich ] ; then    
          cp -R ${fich} $RESU/$DIR.$SUFFIXE/.
        fi
      done
    fi
  fi
done

if [ $MODE_EXEC != pre_traitement ] ; then
 
  if [ "${RAYONNEMENT}" = "oui" ] ; then 
    for fich in bord* ; do
      if [ -f $fich ] ; then  
        if [ ! -d $RESU/CHR.$SUFFIXE ] ; then 
          mkdir $RESU/CHR.$SUFFIXE 
        fi
        cp $fich $RESU/CHR.$SUFFIXE/.
      fi
    done
  fi


  if [ "${LAGRANGIEN}" = "oui" ] ; then 
    mkdir $RESU/LAGR.$SUFFIXE
    for fich in debug* deplacement* trajectoire* frontiere* ; do 
      if [ -f $fich ] ; then 
        cp $fich $RESU/LAGR.$SUFFIXE
      fi
    done
  fi

  # Matisse output files
  if [ -f ${RUN}/resuMatisse ] ; then
    matisse=`grep -i matisse $DATA/$PARAM`
    if [ ! -z "$matisse" ] ; then
  # The date is added to the first line of resuMatisse
      AFDATE="Date of the case                                       : "$DATE
      sed  "1i\ ${AFDATE}" ${RUN}/resuMatisse >> ${RUN}/resuMatisse.mod
      mv ${RUN}/resuMatisse.mod ${RUN}/resuMatisse  
    fi
    cp ${RUN}/resuMatisse ${RESU}/resuMatisse.$SUFFIXE
  fi

  for dir in fort_saturne ; do
    if [ -d $dir ] ; then
      mkdir $RESU/FORT.$SUFFIXE
      if [ $? -ne 0 ] ; then
        echo Failure creating $RESU/FORT.$SUFFIXE
      else
        for fich in $dir/*.[f,F,c,h] ; do
          if [ -f ${fich} ] ; then 
            cp -R ${fich} $RESU/FORT.$SUFFIXE/. 
            fichbase=`basename ${fich}`
            chmod a-w $RESU/FORT.$SUFFIXE/${fichbase}
          fi
        done
      fi
    fi
  done 

  if [ $COUPLAGE_SYRTHES = oui ] ; then 
    mkdir $RESU/RESU_SYR.$SUFFIXE 
    if [ $? -ne 0 ] ; then
      echo Failure creating $RESU/RESU_SYR.$SUFFIXE
    else
      for var in   $SYRTHES_ENV.save $SYRTHES_DATA $SYRTHES_DATA_RAY \
                   $SYRTHES_CORR $SYRTHES_CORR_RAY $SYRTHES_FFOR_RAY \
                   $SYRTHES_AVAL_RES1 $SYRTHES_AVAL_RES2 $SYRTHES_AVAL_CHR2 \
                   $SYRTHES_AVAL_HIST \
                   $SYRTHES_AVAL_MAIL_PEAU_FLUIDE $SYRTHES_AVAL_RESU_PEAU_FLUIDE \
                   $SYRTHES_AVAL_CHR_PEAU_FLUIDE \
                   $SYRTHES_AVAL_MAIL_RAY $SYRTHES_AVAL_RESU_RAY \
                   $SYRTHES_AVAL_CHR_RAY \
                   $SYRTHES_AVAL_HIST_RAY   ; do
          fich=$RUN/`echo $var | sed -e "s/.*\///"`
          if [ -f $fich ] ; then 	
            cp $fich $RESU/RESU_SYR.$SUFFIXE/.
          fi
      done
    fi
    #
    for dir in fort_syrthes ; do
      if [ -d $dir ] ; then
        mkdir $RESU/FORT_SYR.$SUFFIXE
        if [ $? -ne 0 ] ; then
          echo Failure creating $RESU/FORT_SYR.$SUFFIXE
        else
          for fich in $dir/*.[f,F,c,h] ; do
            if [ -f ${fich} ] ; then 
              cp -R ${fich} $RESU/FORT_SYR.$SUFFIXE/.
              fichbase=`basename ${fich}`
              chmod a-w $RESU/FORT_SYR.$SUFFIXE/${fichbase}
            fi
          done
        fi
      fi
    done 
  fi

fi  # input data and outputs
#
########################################################################
#
# Summary: end
#
DATEFIN=`date '+%m%d%H%M'`
echo '    End date        : ' $DATEFIN                         >>$resume
echo '========================================================'>>$resume
#
cp $resume  $RESU/$resume.$SUFFIXE
#
########################################################################
#
#
# Fin
#
if [ -f $ERROR ] ; then 
#
  echo
  echo  "  ********************************************"
  cat                        $ERROR
  echo  "  ********************************************"
  exit 1
#
else
#
  echo
  echo  "  ********************************************"
  echo  "           Normal simulation finish"
  echo  "  ********************************************"
  exit 0
#
fi
#
########################################################################
#
########################################################################