# Use the Bash Shell
#PBS -S /bin/bash

# Put a unique job name here
# so that you can monitor it in the queue
#PBS -N pipe_v4.3.0

# Define the queue you're submitting to
#PBS -q production.q

# Notifying user by email at the beginning,end,abort,suspensions of the job run
#PBS -M fafan@clarkson.edu
#PBS -m eab

# The max walltime for this job Hours:Minutes:Seconds
# # Set this longer than required for run!!!
# # Set number of nodes/ppn to number of cores required: 
#PBS -l walltime=120:00:00,nodes=1:ppn=12
## #PBS -l walltime=120:00:00,nodes=compute-0-0.local:ppn=5+compute-0-4.local:ppn=5+compute-0-5.local:ppn=5+compute-0-9.local:ppn=5
#

# Export paths here if necessary or recommended.
export PATH="/home/fafan/Code_Saturne/4.3.0/code_saturne-4.3.0/arch/Linux_x86_64/bin":$PATH
export LD_LIBRARY_PATH="/home/fafan/Code_Saturne/4.3.0/code_saturne-4.3.0/arch/Linux_x86_64/lib":$LD_LIBRARY_PATH
#
cd "$PBS_O_WORKDIR"
#
# Redirecting standard output / error to files
# named "output" and "errors"
#PBS -o output
#PBS -e errors

local_dir=$(pwd | cut -c 23-)
echo "This job was started in ${local_dir}"
echo "The start time was $(date)"
echo "The job id is $JOB_ID"
echo ${NSLOTS}
echo ${PE}

# Executing the program
mpiexec -np 12 /home/fafan/Code_Saturne/4.3.0/code_saturne-4.3.0/arch/Linux_x86_64/libexec/code_saturne/cs_solver --param pipe.xml --mpi $@
export CS_RET=$?

exit $CS_RET