#!/bin/bash
@@{PARTITION}
@@{CONSTRAINTS}
#SBATCH --nodes=@@{NNODES}-@@{NNODES}
#SBATCH --ntasks-per-node=@@{PPN}
#SBATCH --time=@@{WALLTIME}
#SBATCH -J JOB_@@{JOBID}_@@{INSTANCEID}
#SBATCH -o slurm_@@{RUNNAME}_@@{INSTANCEID}.stdout
#SBATCH -e slurm_@@{RUNNAME}_@@{INSTANCEID}.stderr
@@{REMOTEBATCHACCOUNT}
@@{SUBMISSIONSCRIPTCOMMANDS}

@@{ENVIRONMENT}

# RemoteBatchPBS:makeParallelTemplate
# SubmissionScripts/Distributor/Batch/SLURM/parallel

trap cleanup HUP INT QUIT ABRT TERM

cleanup()
{
   echo "Abnormal termination by signal"
   if [ ! -s @@{TS_FINISH} ] ; then
      date +"%s" > @@{TS_FINISH}
   fi
   exit 1
}

export PATH=${SLURM_SUBMIT_DIR}:${PATH}
# Change to directory where job was submitted.
cd ${SLURM_SUBMIT_DIR}

timeMeasured=0

@@{QUOTA}

date +"%s" > @@{TS_START}

@@{PREMANAGERCOMMANDS}
TIMEPATH=
if [ -n "@@{TIMEPATHS}" ] ; then
   for timePath in @@{TIMEPATHS} ; do
      if [ -x ${timePath} ] ; then
         TIMEPATH=${timePath}
         break
      fi
   done
fi

if [ -n "${TIMEPATH}" ] ; then
   ${TIMEPATH} --format "Command exited with status %x\nreal %e\nuser %U\nsys %S" -o @@{TIME_RESULTS} \
      @@{MANAGERCOMMAND} @@{EXECUTABLE} @@{ARGUMENTS}  < @@{STDIN} \
                                                       > @@{RUNNAME}_@@{INSTANCEID}.stdout \
                                                      2> @@{RUNNAME}_@@{INSTANCEID}.stderr
else
   { time -p @@{MANAGERCOMMAND} @@{EXECUTABLE} @@{ARGUMENTS}  < @@{STDIN} \
                                                              > @@{RUNNAME}_@@{INSTANCEID}.stdout \
                                                             2> @@{RUNNAME}_@@{INSTANCEID}.stderr ; } \
                 > @@{TIME_RESULTS} 2>&1
   echo "Command exited with status $?" >> @@{TIME_RESULTS}
fi
@@{POSTMANAGERCOMMANDS}

date +"%s" > @@{TS_FINISH}

if [ ${timeMeasured} -eq 0 ] ; then
   mv @@{TIME_RESULTS} @@{TIME_RESULTS}.status
   grep status @@{TIME_RESULTS}.status > @@{TIME_RESULTS}
   startTime=`cat @@{TS_START}`
   finishTime=`cat @@{TS_FINISH}`
   let wallTime=finishTime-startTime
   cnt=0
   while [ "${cnt}" != "@@{NPROCESSORS}" ] ; do
      cat >> @@{TIME_RESULTS} << EOF
real ${wallTime}.00
user ${wallTime}.00
sys 0.00
EOF
      cnt=`expr ${cnt} + 1`
   done
   rm -f @@{TIME_RESULTS}.status
fi
