exp

#! /bin/bash

set -e

export TMPFS_PATH=$BENCH_PATH/tmpfs

export PERF_BASE_PREFIX="perf record -a -c 1 -e sched:sched_thread_placement,sched:sched_switch,sched:sched_load_update"

export PINTHREADS_PATH=$HOME/bench/tools/PinThreads
export PINTHREADS_BIN=$PINTHREADS_PATH/pinthreads

export PARSEC_DIR=$BENCH_PATH/benchmarks/parsec
export PHOENIX_DIR=$BENCH_PATH/benchmarks/phoenix

date=$(date +'%y.%m.%d-%H.%M.%S')
export TMP_INPUTS_DIR=$TMPFS_PATH/inputs
export TMP_RESULTS_DIR=$TMPFS_PATH/results/$date
export RESULTS_DIR=$BENCH_PATH/tmp/results/$date
export RESULTS_LAST=$BENCH_PATH/tmp/results/last

msg() {
    if [[ $verbose == 'true' ]]; then
	echo $@
    fi
}

usage() {
  cat <<HD
Usage: $script [-v] <programs> <pinning> <nodes> <use_perf> <autogroup> <repeat>
     -v             toggle verbose mode

     <programs>     list of programs to run
     <pinning>      list of values among { cores, nodes, init, subset, none }
     <nodes>        list of numbers of nodes
     <use_perf>     whether ('y') or not ('n') perf should be used
     <autogroup>    whether ('y') or not ('n') autogroup should be enabled
     <repeat>       the number of time to run the application for each parameter combinations

Run volrend on 64 cores with and without pinning one time each:
     >> $0 volrend "cores none" 8 n n 1
Run volrend and wordcount
with pinning and without pinning,
on 2,3,4,5,6,7,8 nodes
with and without perf,
with and without autogroup,
five times each:
     >> $0 "volrend word_count" "cores none" "\$(seq 2 8)" "y n" "y n" 5
HD
}

prepare_bench() {
    msg "# Preparing bench"
    if [[ $verbose == 'true' ]]; then
	make -C$BENCH_PATH
    else
	if ! output=$(make -C$BENCH_PATH); then
	    echo "!!! Make failed"
	    echo "$output"
	    exit 1
	fi
    fi
}

prepare_tmpfs() {
    # Everything is done in a tmpfs filesystem, i.e.: in RAM,
    # so that performance are not bound to disk access time
    msg "# Preparing tmpfs"

    mkdir -p $TMPFS_PATH
    if ! mount | grep -q $TMPFS_PATH 2>&1; then
        sudo mount -t tmpfs -o size=100G,mpol=interleave \
                tmpfs $TMPFS_PATH 2>/dev/null
    fi
}

prepare_programs() {
    msg "# Preparing programs"

    for program in $programs; do
	source $BENCH_PATH/exp/run.d/$program.sh
	prepare_$program
    done
}

prepare_directories() {
    msg "# Preparing directories"

    rm -rf $TMP_INPUT_DIR
    mkdir -p $TMP_INPUTS_DIR
    mkdir -p $TMP_RESULTS_DIR
    mkdir -p $(dirname $RESULTS_DIR)
    rm -rf $RESULTS_LAST
    ln -s $RESULTS_DIR $RESULTS_LAST
}

prepare_system() {
    msg "# Preparing system"

    sudo sysctl -w kernel.perf_event_paranoid=-1 >/dev/null
    sudo sysctl -w kernel.numa_balancing=0       >/dev/null
    if sudo sysctl kernel.sched_schedstats >/dev/null 2>&1; then
	sudo sysctl kernel.sched_schedstats=1 >/dev/null
    fi
}

dump_system() {
    msg "# Dumping system"

    local system_dir=$RESULTS_DIR/system
    local sched_features=/sys/kernel/debug/sched_features
    mkdir -p $system_dir

    sudo cat /etc/os-release >>$system_dir/os-release     2>/dev/null
    sudo uname -a            >>$system_dir/uname          2>/dev/null
    sudo cat /proc/cmdline   >>$system_dir/cmdline        2>/dev/null
    sudo uptime              >>$system_dir/uptime         2>/dev/null
    sudo mount               >>$system_dir/mount          2>/dev/null
    sudo sysctl -a           >>$system_dir/sysctl         2>/dev/null
    sudo cat $sched_features >>$system_dir/sched_features 2>/dev/null
    sudo dpkg -l             >>$system_dir/packages       2>/dev/null
    sudo systemctl           >>$system_dir/systemctl      2>/dev/null
    sudo top -n1 -b          >>$system_dir/top            2>/dev/null
    env                      >>$system_dir/env            2>/dev/null
    sudo lscpu               >>$system_dir/topology       2>/dev/null
    sudo hwloc-distances     >>$system_dir/topology       2>/dev/null
}

dump_versions() {
    msg "# Dumping versions"

    local version_dir=$RESULTS_DIR/version
    mkdir -p $version_dir

    git -C $BENCH_PATH rev-parse HEAD         >>$version_dir/bench-hash      2>/dev/null
    git submodule foreach git rev-parse HEAD  >>$version_dir/submodules-hash 2>/dev/null

    git -C $BENCH_PATH diff HEAD              >>$version_dir/bench-diff      2>/dev/null
    git submodule foreach git diff HEAD       >>$version_dir/submodules-diff 2>/dev/null

}

dump_args() {
    msg "# Dumping args"

    local args_file=$RESULTS_DIR/args

    echo "PROGRAM   = $programs"    >>$args_file
    echo "PINNING   = $pinnings"    >>$args_file
    echo "NODES     = $nodes_list"  >>$args_file
    echo "USE_PERF  = $use_perfs"   >>$args_file
    echo "AUTOGROUP = $autogroups"  >>$args_file
    echo "REPEAT    = $repeat"      >>$args_file
}

dump_programs() {
    msg "# Dumping programs"

    local dump_program_dir=$RESULTS_DIR/program
    mkdir -p $dump_program_dir

    for program in $programs; do
	dump_$program
    done
}

run_iterations() {
    local program=$1; shift
    local pinning=$1; shift
    local nodes=$1; shift
    local use_perf=$1; shift
    local autogroup=$1; shift

    local threads=$(($nodes * 8))
    if [[ $pinning == 'cores' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1))"
    elif [[ $pinning == 'nodes' ]]; then
        local pin_prefix="$PINTHREADS_BIN -n 0-$(($nodes-1)) -N"
    elif [[ $pinning == 'init' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1)) -i"
    elif [[ $pinning == 'subset' ]]; then
        local pin_prefix="numactl -N 0-$(($nodes-1))"
    elif [[ $pinning == 'none' ]]; then
        local pin_prefix=''
    else
        echo "Unknown pinning $pinning"
        exit 34
    fi

    if [[ $autogroup == "y" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=1
    elif [[ $autogroup == "n" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=0
    else
        echo "Unknown autogroup $autogroup"
        exit 35
    fi

    printf '[%14s][%6s][%2d][%s][%s] ' $program $pinning $nodes $use_perf $autogroup

    local exp_dir=$TMP_RESULTS_DIR/outputs/$program-$pinning-$nodes-$use_perf-$autogroup

    for iteration in $(seq 1 $repeat); do
        local iteration_dir=$exp_dir/$iteration
        mkdir -p $iteration_dir

        local time_file=$iteration_dir/time
        local time_prefix="/usr/bin/time -f%e -o $iteration_dir/time -a"

        local perf_prefix=""
        if [[ $use_perf == "y" ]]; then
            perf_file="$iteration_dir/perf.data"
            perf_prefix="$PERF_BASE_PREFIX -o $perf_file"
        fi

        local output=$iteration_dir/output
        local errors=$iteration_dir/error

        local cmd_prefix="$perf_prefix $time_prefix $pin_prefix"
        local cmd="$(command_$program "$cmd_prefix" "$threads")"
        echo "#### sudo $cmd" >$iteration_dir/command

        cat /proc/schedstat >$iteration_dir/schedstat.before
        eval "sudo $cmd" 1>>$output 2>>$errors
        echo "$?" >$iteration_dir/exit_status

        cat /proc/schedstat >$iteration_dir/schedstat.after

        printf '.'
    done

    avg=$(awk '{ sum += $1 } END { if (NR > 0) print sum / NR; else print NaN }'     $exp_dir/*/time)
    sd=$(awk '{ sum += $1; sum2 += $0^2 } END { print sqrt(sum2 / NR - (sum/NR)^2)}' $exp_dir/*/time)
    printf ' | Avg: %6.2f  ~%3.2f\n' $avg $sd
}

run_all() {
    msg "# Running exp"

    for program in $programs; do
    for pinning in $pinnings; do
    for nodes in $nodes_list; do
    for use_perf in $use_perfs; do
    for autogroup in $autogroups; do
        run_iterations $program $pinning $nodes $use_perf $autogroup
    done
    done
    done
    done
    done
}

check_errors() {
    msg "# Checking errors"

    if grep -v 0 $TMP_RESULTS_DIR/outputs/*/*/exit_status; then
        exit 1
    fi
}

move_results() {
    msg "# Moving results"

    sudo chown -R $(id -un):$(id -gn) $TMP_RESULTS_DIR
    mv $TMP_RESULTS_DIR/* $RESULTS_DIR/
    rm -rf $TMP_RESULTS_DIR

    echo
    echo " => Results written in $RESULTS_DIR"
}

cleanup_system() {
    if sysctl kernel.sched_schedstats >/dev/null 2>&1; then
        sudo sysctl kernel.sched_schedstats=0 >/dev/null
    fi
}

script=$0

verbose=''
while [[ $1 == -* ]]; do
    case $1 in
    -v)
	    verbose=true
	    shift
	  ;;
    *)
	    echo "Unkown flag $0"
	    usage
	    exit 126
	  ;;
    esac
done

if [[ $# != 6 ]]; then
    usage
    exit 125
fi

programs=$1; shift
pinnings=$1; shift
nodes_list=$1; shift
use_perfs=$1; shift
autogroups=$1; shift
repeat=$1; shift

prepare_bench
prepare_tmpfs
prepare_directories
prepare_system
prepare_programs

dump_system
dump_versions
dump_args

run_all
check_errors

move_results

cleanup_system