exp 8.91 KB
Newer Older
aumgn's avatar
aumgn committed
1
2
3
4
5
6
#! /bin/bash

set -e

export TMPFS_PATH=$BENCH_PATH/tmpfs

7
export PERF_BASE_PREFIX="perf record -a -c 1 -e sched:sched_thread_placement,sched:sched_switch,sched:sched_load_update"
aumgn's avatar
aumgn committed
8
9
10
11

export PINTHREADS_PATH=$HOME/bench/tools/PinThreads
export PINTHREADS_BIN=$PINTHREADS_PATH/pinthreads

aumgn's avatar
aumgn committed
12
export PARSEC_DIR=$BENCH_PATH/benchmarks/parsec
aumgn's avatar
aumgn committed
13
export PHOENIX_DIR=$BENCH_PATH/benchmarks/phoenix
aumgn's avatar
aumgn committed
14
15

date=$(date +'%y.%m.%d-%H.%M.%S')
16
export TMP_INPUTS_DIR=$TMPFS_PATH/inputs
aumgn's avatar
aumgn committed
17
export TMP_RESULTS_DIR=$TMPFS_PATH/results/$date
aumgn's avatar
aumgn committed
18
19
20
21
22
23
24
25
export RESULTS_DIR=$BENCH_PATH/tmp/results/$date
export RESULTS_LAST=$BENCH_PATH/tmp/results/last

msg() {
    if [[ $verbose == 'true' ]]; then
	echo $@
    fi
}
aumgn's avatar
aumgn committed
26
27
28

usage() {
  cat <<HD
29
Usage: $script [-v] <programs> <pinning> <nodes> <use_perf> <autogroup> <lb_bias> <repeat>
aumgn's avatar
aumgn committed
30
31
     -v             toggle verbose mode

aumgn's avatar
aumgn committed
32
     <programs>     list of programs to run
aumgn's avatar
aumgn committed
33
     <pinning>      list of values among { cores, nodes, init, subset, none }
aumgn's avatar
aumgn committed
34
     <nodes>        list of numbers of nodes
35
     <use_perf>     whether ('y') or not ('n') perf should be used
aumgn's avatar
aumgn committed
36
     <autogroup>    whether ('y') or not ('n') autogroup should be enabled
37
     <lb_bias>      whether ('y') or not ('n') load_balancing should be biased with exponential smoothing
aumgn's avatar
aumgn committed
38
39
40
     <repeat>       the number of time to run the application for each parameter combinations

Run volrend on 64 cores with and without pinning one time each:
41
42
43
44
45
46
47
48
49
     >> $0 volrend "cores none" 8 n n n 1
Run volrend and wordcount
with pinning and without pinning,
on 2,3,4,5,6,7,8 nodes
with and without perf,
with and without autogroup,
with and without lb_bias,
five times each:
     >> $0 "volrend word_count" "cores none" "\$(seq 2 8)" "y n" "y n" "y n" 5
aumgn's avatar
aumgn committed
50
51
52
HD
}

aumgn's avatar
aumgn committed
53
prepare_bench() {
aumgn's avatar
aumgn committed
54
55
56
57
58
59
60
61
62
    msg "# Preparing bench"
    if [[ $verbose == 'true' ]]; then
	make -C$BENCH_PATH
    else
	if ! output=$(make -C$BENCH_PATH); then
	    echo "!!! Make failed"
	    echo "$output"
	    exit 1
	fi
aumgn's avatar
aumgn committed
63
    fi
aumgn's avatar
aumgn committed
64
65
}

aumgn's avatar
aumgn committed
66
prepare_tmpfs() {
aumgn's avatar
aumgn committed
67
68
    # Everything is done in a tmpfs filesystem, i.e.: in RAM,
    # so that performance are not bound to disk access time
aumgn's avatar
aumgn committed
69
    msg "# Preparing tmpfs"
aumgn's avatar
aumgn committed
70
71
72
73
74
75

    mkdir -p $TMPFS_PATH
    if ! mount | grep -q $TMPFS_PATH 2>&1; then
        sudo mount -t tmpfs -o size=100G,mpol=interleave \
                tmpfs $TMPFS_PATH 2>/dev/null
    fi
aumgn's avatar
aumgn committed
76
77
78
}

prepare_programs() {
aumgn's avatar
aumgn committed
79
80
    msg "# Preparing programs"

aumgn's avatar
aumgn committed
81
    for program in $programs; do
aumgn's avatar
aumgn committed
82
	source $BENCH_PATH/exp/run.d/$program.sh
aumgn's avatar
aumgn committed
83
84
85
86
87
	prepare_$program
    done
}

prepare_directories() {
aumgn's avatar
aumgn committed
88
89
    msg "# Preparing directories"

aumgn's avatar
aumgn committed
90
91
92
93
94
95
96
97
    rm -rf $TMP_INPUT_DIR
    mkdir -p $TMP_INPUTS_DIR
    mkdir -p $TMP_RESULTS_DIR
    mkdir -p $(dirname $RESULTS_DIR)
    rm -rf $RESULTS_LAST
    ln -s $RESULTS_DIR $RESULTS_LAST
}

aumgn's avatar
aumgn committed
98
prepare_system() {
aumgn's avatar
aumgn committed
99
100
    msg "# Preparing system"

101
102
103
    sudo sysctl -w kernel.perf_event_paranoid=-1 >/dev/null
    sudo sysctl -w kernel.numa_balancing=0       >/dev/null
    if sudo sysctl kernel.sched_schedstats >/dev/null 2>&1; then
104
105
	sudo sysctl kernel.sched_schedstats=1 >/dev/null
    fi
aumgn's avatar
aumgn committed
106
107
}

aumgn's avatar
aumgn committed
108
dump_system() {
aumgn's avatar
aumgn committed
109
110
    msg "# Dumping system"

aumgn's avatar
aumgn committed
111
    local system_dir=$RESULTS_DIR/system
aumgn's avatar
aumgn committed
112
    local sched_features=/sys/kernel/debug/sched_features
aumgn's avatar
aumgn committed
113
114
    mkdir -p $system_dir

aumgn's avatar
aumgn committed
115
116
    sudo cat /etc/os-release >>$system_dir/os-release     2>/dev/null
    sudo uname -a            >>$system_dir/uname          2>/dev/null
aumgn's avatar
aumgn committed
117
    sudo cat /proc/cmdline   >>$system_dir/cmdline        2>/dev/null
aumgn's avatar
aumgn committed
118
119
120
121
122
123
124
125
126
127
    sudo uptime              >>$system_dir/uptime         2>/dev/null
    sudo mount               >>$system_dir/mount          2>/dev/null
    sudo sysctl -a           >>$system_dir/sysctl         2>/dev/null
    sudo cat $sched_features >>$system_dir/sched_features 2>/dev/null
    sudo dpkg -l             >>$system_dir/packages       2>/dev/null
    sudo systemctl           >>$system_dir/systemctl      2>/dev/null
    sudo top -n1 -b          >>$system_dir/top            2>/dev/null
    env                      >>$system_dir/env            2>/dev/null
    sudo lscpu               >>$system_dir/topology       2>/dev/null
    sudo hwloc-distances     >>$system_dir/topology       2>/dev/null
aumgn's avatar
aumgn committed
128
129
}

aumgn's avatar
aumgn committed
130
dump_versions() {
aumgn's avatar
aumgn committed
131
132
    msg "# Dumping versions"

aumgn's avatar
aumgn committed
133
    local version_dir=$RESULTS_DIR/version
aumgn's avatar
aumgn committed
134
135
    mkdir -p $version_dir

136
    git -C $BENCH_PATH rev-parse HEAD >>$version_dir/bench      2>/dev/null
aumgn's avatar
aumgn committed
137
    git -C $BENCH_PATH submodule      >>$version_dir/submodules 2>/dev/null
aumgn's avatar
aumgn committed
138
139
140
}

dump_args() {
aumgn's avatar
aumgn committed
141
142
    msg "# Dumping args"

aumgn's avatar
aumgn committed
143
144
    local args_file=$RESULTS_DIR/args

145
146
147
    echo "PROGRAM   = $programs"    >>$args_file
    echo "PINNING   = $pinnings"    >>$args_file
    echo "NODES     = $nodes_list"  >>$args_file
aumgn's avatar
aumgn committed
148
    echo "USE_PERF  = $use_perfs"   >>$args_file
149
150
151
    echo "AUTOGROUP = $autogroups"  >>$args_file
    echo "LB_BIAS   = $lb_biases"   >>$args_file
    echo "REPEAT    = $repeat"      >>$args_file
aumgn's avatar
aumgn committed
152
153
}

aumgn's avatar
aumgn committed
154
155
156
157
158
159
160
161
162
163
164
dump_programs() {
    msg "# Dumping programs"

    local dump_program_dir=$RESULTS_DIR/program
    mkdir -p $dump_program_dir

    for program in $programs; do
	dump_$program
    done
}

aumgn's avatar
aumgn committed
165
run_iterations() {
166
167
168
169
170
171
    local program=$1; shift
    local pinning=$1; shift
    local nodes=$1; shift
    local use_perf=$1; shift
    local autogroup=$1; shift
    local lb_bias=$1; shift
aumgn's avatar
aumgn committed
172
173
174
175
176
177

    local threads=$(($nodes * 8))
    if [[ $pinning == 'cores' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1))"
    elif [[ $pinning == 'nodes' ]]; then
        local pin_prefix="$PINTHREADS_BIN -n 0-$(($nodes-1)) -N"
178
179
    elif [[ $pinning == 'init' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1)) -i"
aumgn's avatar
aumgn committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    elif [[ $pinning == 'subset' ]]; then
        local pin_prefix="numactl -N 0-$(($nodes-1))"
    elif [[ $pinning == 'none' ]]; then
        local pin_prefix=''
    else
        echo "Unknown pinning $pinning"
        exit 34
    fi

    if [[ $autogroup == "y" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=1
    elif [[ $autogroup == "n" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=0
    else
        echo "Unknown autogroup $autogroup"
        exit 35
    fi

198
199
200
201
202
203
204
205
    if [[ $lb_bias == "y" ]]; then
        echo 'LB_BIAS' | sudo tee /sys/kernel/debug/sched_features >/dev/null
    elif [[ $lb_bias == "n" ]]; then
        echo 'NO_LB_BIAS' | sudo tee /sys/kernel/debug/sched_features >/dev/null
    else
        echo "Unknown lb_bias $lb_bias (y/n)"
        exit 35
    fi
aumgn's avatar
aumgn committed
206

207
    printf '[%14s][%6s][%2d][%s][%s][%s] ' $program $pinning $nodes $use_perf $autogroup $lb_bias
208

209
    local exp_dir=$TMP_RESULTS_DIR/outputs/$program-$pinning-$nodes-$use_perf-$autogroup-$lb_bias
aumgn's avatar
aumgn committed
210
211

    for iteration in $(seq 1 $repeat); do
212
213
        local iteration_dir=$exp_dir/$iteration
        mkdir -p $iteration_dir
aumgn's avatar
aumgn committed
214

215
216
        local time_file=$iteration_dir/time
        local time_prefix="/usr/bin/time -f%e -o $iteration_dir/time -a"
aumgn's avatar
aumgn committed
217
218

        local perf_prefix=""
219
        if [[ $use_perf == "y" ]]; then
aumgn's avatar
aumgn committed
220
221
222
223
            perf_file="$iteration_dir/perf.data"
            perf_prefix="$PERF_BASE_PREFIX -o $perf_file"
        fi

224
225
        local output=$iteration_dir/output
        local errors=$iteration_dir/error
aumgn's avatar
aumgn committed
226

227
        local cmd_prefix="$perf_prefix $time_prefix $pin_prefix"
aumgn's avatar
aumgn committed
228
        local cmd="$(command_$program "$cmd_prefix" "$threads")"
229
        echo "#### sudo $cmd" >$iteration_dir/command
aumgn's avatar
aumgn committed
230
231

        cat /proc/schedstat >$iteration_dir/schedstat.before
232
        eval "sudo $cmd" 1>>$output 2>>$errors
233
        echo "$?" >$iteration_dir/exit_status
aumgn's avatar
aumgn committed
234

aumgn's avatar
aumgn committed
235
236
237
238
239
        cat /proc/schedstat >$iteration_dir/schedstat.after

        printf '.'
    done

240
241
242
    avg=$(awk '{ sum += $1 } END { if (NR > 0) print sum / NR; else print NaN }'     $exp_dir/*/time)
    sd=$(awk '{ sum += $1; sum2 += $0^2 } END { print sqrt(sum2 / NR - (sum/NR)^2)}' $exp_dir/*/time)
    printf ' | Avg: %6.2f  ~%3.2f\n' $avg $sd
aumgn's avatar
aumgn committed
243
244
245
}

run_all() {
aumgn's avatar
aumgn committed
246
247
    msg "# Running exp"

aumgn's avatar
aumgn committed
248
249
250
    for program in $programs; do
    for pinning in $pinnings; do
    for nodes in $nodes_list; do
251
    for use_perf in $use_perfs; do
aumgn's avatar
aumgn committed
252
    for autogroup in $autogroups; do
253
    for lb_bias in $lb_biases; do
254
255
        run_iterations $program $pinning $nodes $use_perf $autogroup $lb_bias
    done
256
    done
aumgn's avatar
aumgn committed
257
258
259
260
261
262
    done
    done
    done
    done
}

aumgn's avatar
aumgn committed
263
check_errors() {
aumgn's avatar
aumgn committed
264
265
    msg "# Checking errors"

aumgn's avatar
aumgn committed
266
    if grep -v 0 $TMP_RESULTS_DIR/outputs/*/*/exit_status; then
267
        exit 1
aumgn's avatar
aumgn committed
268
    fi
aumgn's avatar
aumgn committed
269
270
}

aumgn's avatar
aumgn committed
271
move_results() {
aumgn's avatar
aumgn committed
272
273
    msg "# Moving results"

274
    sudo chown -R $(id -un):$(id -gn) $TMP_RESULTS_DIR
aumgn's avatar
aumgn committed
275
    mv $TMP_RESULTS_DIR/* $RESULTS_DIR/
aumgn's avatar
aumgn committed
276
    rm -rf $TMP_RESULTS_DIR
aumgn's avatar
aumgn committed
277
278
279
280
281

    echo
    echo " => Results written in $RESULTS_DIR"
}

aumgn's avatar
aumgn committed
282
cleanup_system() {
283
    if sysctl kernel.sched_schedstats >/dev/null 2>&1; then
284
        sudo sysctl kernel.sched_schedstats=0 >/dev/null
285
    fi
aumgn's avatar
aumgn committed
286
287
}

aumgn's avatar
aumgn committed
288
289
290
291
292
script=$0

verbose=''
while [[ $1 == -* ]]; do
    case $1 in
293
    -v)
aumgn's avatar
aumgn committed
294
295
	    verbose=true
	    shift
296
297
	  ;;
    *)
aumgn's avatar
aumgn committed
298
299
300
	    echo "Unkown flag $0"
	    usage
	    exit 126
301
	  ;;
aumgn's avatar
aumgn committed
302
303
304
    esac
done

305
if [[ $# != 7 ]]; then
aumgn's avatar
aumgn committed
306
307
    usage
    exit 125
aumgn's avatar
aumgn committed
308
309
fi

aumgn's avatar
aumgn committed
310
311
312
programs=$1; shift
pinnings=$1; shift
nodes_list=$1; shift
313
use_perfs=$1; shift
aumgn's avatar
aumgn committed
314
autogroups=$1; shift
315
lb_biases=$1; shift
aumgn's avatar
aumgn committed
316
repeat=$1; shift
aumgn's avatar
aumgn committed
317
318

prepare_bench
aumgn's avatar
aumgn committed
319
320
prepare_tmpfs
prepare_directories
aumgn's avatar
aumgn committed
321
322
prepare_system
prepare_programs
aumgn's avatar
aumgn committed
323

aumgn's avatar
aumgn committed
324
325
326
dump_system
dump_versions
dump_args
aumgn's avatar
aumgn committed
327

aumgn's avatar
aumgn committed
328
329
run_all
check_errors
aumgn's avatar
aumgn committed
330
331
332

move_results

aumgn's avatar
aumgn committed
333
cleanup_system