#!/bin/bash

# Parallel launcher for the various PEs on Legion.

# Dr Owain Kenway, UCL

# For licensing terms, see LICENSE.txt

# This variable should be set by MPI modules:

# GERUN_LAUNCHER

set -e

# See if GERUN_PATH is set, otherwise guess it out from $0.

export GERUN_PATH="${GERUN_PATH:-$(dirname $(readlink -f $(which $0)))}"

# Load function definitions.

source $GERUN_PATH/gerunlib.sh

stderrmsg

stderrmsg "Note: Lines like this one prefixed with \"GERun:\" are for debugging"

stderrmsg " purposes only and you do not need to report them to rc-support"

stderrmsg " unless your job fails for other reasons."

stderrmsg

# Make it possible to disable all output from GERun.

# This is to make it work with some testing regimes.

# To disable output, set GERUN_SILENT=yes

GERUN_SILENT="${GERUN_SILENT:-no}"

# To lowercase (only works in bash 4+)

GERUN_SILENT="${GERUN_SILENT,,}"

# Make sure user hasn't done something else odd.

if [[ "$GERUN_SILENT" == "true" ]]; then

GERUN_SILENT=yes

fi

# Make sure it's available in launcher scripts if need be.

export GERUN_SILENT

# Check to see if we have a customised message and if so use it rather

if [[ "$GERUN_SILENT" != "yes" ]]; then

if [[ -s "$GERUN_PATH/local-message" ]]

then

stderrcat "$GERUN_PATH/local-message"

else

stderrcat "$GERUN_PATH/gerun-message"

fi

fi

# We need to support mixed mode code on new Legion.

OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"

NSLOTS="${NSLOTS:-1}"

GERUN_PROCS="$((NSLOTS/OMP_NUM_THREADS))"

export OMP_NUM_THREADS

export GERUN_PROCS

if [[ "$GERUN_SILENT" != "yes" ]]; then

stderrmsg

stderrmsg "Using environment: $GERUN_LAUNCHER"

stderrmsg "Running on $NSLOTS slots:"

stderrprintf " %3d MPI tasks\n" "$GERUN_PROCS"

stderrprintf " %3d threads per task\n" "$OMP_NUM_THREADS"

stderrmsg "TMPDIR=$TMPDIR"

stderrmsg

fi

if [[ -s "$TMPDIR/machines" ]]; then

if [[ "$GERUN_SILENT" != "yes" ]]; then

stderrmsg "Contents of machinefile:"

stderrcat "$TMPDIR/machines"

stderrmsg

fi

# Create a sorted, unique machine file for mpiruns which work better with that.

sort -u "$TMPDIR/machines" > "$TMPDIR/machines.unique"

"$GERUN_PATH/gerun-$GERUN_LAUNCHER" "$@"

else

if [[ "$GERUN_SILENT" != "yes" ]]; then

if [[ "${PE:0:3}" == "smp" ]]; then

stderrmsg "WARNING: SMP environment detected."

stderrmsg "You appear to be trying to run an MPI package inside the smp parallel"

stderrmsg "environment (i.e. you have requested -pe smp in your job script)."

stderrmsg "There is no machine file available - running on only one node."

stderrmsg "Falling back to basic mpirun."

stderrmsg

else

stderrmsg "WARNING: No machine file found in $TMPDIR."

stderrmsg "You appear to be running outside of a Grid Engine scheduled environment."

stderrmsg "Falling back to basic mpirun."

stderrmsg

fi

fi

mpirun "$@"

fi