Step 1: Get an interactive job
$ qsub -I -ATRN001 -lwalltime=00:30:00,nodes=5

Step 2: Downloading the source code
$ curl -o sph.tar https://users.nccs.gov/~atj/sph.tar
$ tar xf sph.tar

Step 3: Load vampir modules
$ module load cudatoolkit
$ module load vampirtrace/5.14-chester

Step 4: Modify the compilers and flags in the makefile
CC=cc changes to CC=vtcc
CFLAGS= -O3 -omp changes to CFLAGS= -vt:hyb -g -omp

Step 5: Make directories in scratch
$ cd /tmp/work/$USER
$ mkdir vampir
$ mkdir vampir/traces

Step 6: Create filter file
$ cat /tmp/work/$USER/vampir/filter
*– 10000

Step 7: Set environment variables
$ export VT_FILTER_SPEC=/tmp/work/$USER/vampir/filter
$ export VT_PFORM_GDIR=/tmp/work/$USER/vampir/traces
$ export VT_BUFFER_SIZE=200M
$ export VT_MAX_FLUSHES=10
$ export VT_BINDIR=/tmp/work/$USER/.vt/pgi/bin

Step 8: Run the job
$ cd /tmp/work/$USER
$ export OMP_NUM_THREADS=16
$ aprun -n5 -d16 ./sph.out

Step 9: View the traces with vampir
$ module load vampir/8.0.0-121113-chester
$ vampir