#!/bin/sh
#
echo "This is a SAMPLE run script.  Change it to reflect the correct number"
echo "of CPUs/threads, number of nodes, MPI processes per node, etc.."
#

# USAGE NOTES: This script should be uploaded to MIC and executed there, NOT on host CPU.
# Before running this script on MIC, please upload/copy the following necessary files to MIC:
# 
# 1. HPL input file     (scp HPL_hybrid.dat mic0:/tmp)
# 2. HPL binary for MIC (scp xhpl_hybrid_mic_dynamic mic0:/tmp)
# 3. OpenMP run-time library for MIC (libiomp5). Change the path set to LD_LIBRARY_PATH below accordingly.
# 4. mpiexec.hydra process manager and MPI run-time libraries (Please follow the instructions documented at <intel mpi installdir>/doc/ for installing Intel MPI on MIC)

export LD_LIBRARY_PATH=/tmp

# These settings are useful for setting the desired number of OpenMP threads.
# Both parameters (MKL_NUM_THREADS/OMP_NUM_THREADS) should be used and set to the same value.
#          
export OMP_NUM_THREADS=$(cat /proc/cpuinfo|grep proc|wc -l)
export MKL_NUM_THREADS=$(cat /proc/cpuinfo|grep proc|wc -l)
export KMP_AFFINITY=explicit,granularity=fine,proclist=[1-$(($(cat /proc/cpuinfo|grep proc|wc -l)-1)),0]

echo 3584 > /proc/sys/vm/nr_hugepages

export OUT=xhpl_hybrid_mic_dynamic_outputs.txt

cp HPL_hybrid.dat HPL.dat

echo -n "This run was done on: "
date

# Capture some meaningful data for future reference:
echo -n "This run was done on: " >> $OUT
date >> $OUT
echo "HPL.dat: " >> $OUT
cat HPL.dat >> $OUT
echo "Binary name: " >> $OUT
ls -l xhpl_hybrid_mic_dynamic >> $OUT
echo "This script: " >> $OUT
cat runme_hybrid_mic_dynamic >> $OUT
echo "Environment variables: " >> $OUT
env >> $OUT
echo "Actual run: " >> $OUT

# Environment variables can also be also be set on the Intel MPI command line
# using the -genv option (to appear before the -np 1):
#
# You can find description of all Intel MPI parameters in the Intel MPI Reference Manual.
# See <intel mpi installdir>/doc/Reference_manual.pdf
#

mpiexec.hydra -n 1 -wdir /tmp -host mic0 /tmp/xhpl_hybrid_mic_dynamic | tee -a $OUT

echo -n "Done: " >> $OUT
date >> $OUT

echo -n "Done: "
date
