exp 8.5 KB
Newer Older
aumgn's avatar
aumgn committed
1
2
3
4
5
6
#! /bin/bash

set -e

export TMPFS_PATH=$BENCH_PATH/tmpfs

7
export PERF_BASE_PREFIX="perf record -a -c 1 -e sched:sched_thread_placement,sched:sched_switch,sched:sched_load_update"
aumgn's avatar
aumgn committed
8
9
10
11

export PINTHREADS_PATH=$HOME/bench/tools/PinThreads
export PINTHREADS_BIN=$PINTHREADS_PATH/pinthreads

aumgn's avatar
aumgn committed
12
export PARSEC_DIR=$BENCH_PATH/benchmarks/parsec
aumgn's avatar
aumgn committed
13
export PHOENIX_DIR=$BENCH_PATH/benchmarks/phoenix
aumgn's avatar
aumgn committed
14
15

date=$(date +'%y.%m.%d-%H.%M.%S')
16
export TMP_INPUTS_DIR=$TMPFS_PATH/inputs
aumgn's avatar
aumgn committed
17
export TMP_RESULTS_DIR=$TMPFS_PATH/results/$date
aumgn's avatar
aumgn committed
18
19
20
21
22
23
24
25
export RESULTS_DIR=$BENCH_PATH/tmp/results/$date
export RESULTS_LAST=$BENCH_PATH/tmp/results/last

msg() {
    if [[ $verbose == 'true' ]]; then
	echo $@
    fi
}
aumgn's avatar
aumgn committed
26
27
28

usage() {
  cat <<HD
aumgn's avatar
aumgn committed
29
Usage: $script [-v] <programs> <pinning> <nodes> <use_perf> <autogroup> <repeat>
aumgn's avatar
aumgn committed
30
31
     -v             toggle verbose mode

aumgn's avatar
aumgn committed
32
     <programs>     list of programs to run
aumgn's avatar
aumgn committed
33
     <pinning>      list of values among { cores, nodes, init, subset, none }
aumgn's avatar
aumgn committed
34
     <nodes>        list of numbers of nodes
35
     <use_perf>     whether ('y') or not ('n') perf should be used
aumgn's avatar
aumgn committed
36
37
38
39
     <autogroup>    whether ('y') or not ('n') autogroup should be enabled
     <repeat>       the number of time to run the application for each parameter combinations

Run volrend on 64 cores with and without pinning one time each:
40
41
42
43
44
45
46
     >> $0 volrend "cores none" 8 n n n 1
Run volrend and wordcount
with pinning and without pinning,
on 2,3,4,5,6,7,8 nodes
with and without perf,
with and without autogroup,
five times each:
aumgn's avatar
aumgn committed
47
     >> $0 "volrend word_count" "cores none" "\$(seq 2 8)" "y n" "y n" 5
aumgn's avatar
aumgn committed
48
49
50
HD
}

aumgn's avatar
aumgn committed
51
prepare_bench() {
aumgn's avatar
aumgn committed
52
53
54
55
56
57
58
59
60
    msg "# Preparing bench"
    if [[ $verbose == 'true' ]]; then
	make -C$BENCH_PATH
    else
	if ! output=$(make -C$BENCH_PATH); then
	    echo "!!! Make failed"
	    echo "$output"
	    exit 1
	fi
aumgn's avatar
aumgn committed
61
    fi
aumgn's avatar
aumgn committed
62
63
}

aumgn's avatar
aumgn committed
64
prepare_tmpfs() {
aumgn's avatar
aumgn committed
65
66
    # Everything is done in a tmpfs filesystem, i.e.: in RAM,
    # so that performance are not bound to disk access time
aumgn's avatar
aumgn committed
67
    msg "# Preparing tmpfs"
aumgn's avatar
aumgn committed
68
69
70
71
72
73

    mkdir -p $TMPFS_PATH
    if ! mount | grep -q $TMPFS_PATH 2>&1; then
        sudo mount -t tmpfs -o size=100G,mpol=interleave \
                tmpfs $TMPFS_PATH 2>/dev/null
    fi
aumgn's avatar
aumgn committed
74
75
76
}

prepare_programs() {
aumgn's avatar
aumgn committed
77
78
    msg "# Preparing programs"

aumgn's avatar
aumgn committed
79
    for program in $programs; do
aumgn's avatar
aumgn committed
80
	source $BENCH_PATH/exp/run.d/$program.sh
aumgn's avatar
aumgn committed
81
82
83
84
85
	prepare_$program
    done
}

prepare_directories() {
aumgn's avatar
aumgn committed
86
87
    msg "# Preparing directories"

aumgn's avatar
aumgn committed
88
89
90
91
92
93
94
95
    rm -rf $TMP_INPUT_DIR
    mkdir -p $TMP_INPUTS_DIR
    mkdir -p $TMP_RESULTS_DIR
    mkdir -p $(dirname $RESULTS_DIR)
    rm -rf $RESULTS_LAST
    ln -s $RESULTS_DIR $RESULTS_LAST
}

aumgn's avatar
aumgn committed
96
prepare_system() {
aumgn's avatar
aumgn committed
97
98
    msg "# Preparing system"

99
100
101
    sudo sysctl -w kernel.perf_event_paranoid=-1 >/dev/null
    sudo sysctl -w kernel.numa_balancing=0       >/dev/null
    if sudo sysctl kernel.sched_schedstats >/dev/null 2>&1; then
102
103
	sudo sysctl kernel.sched_schedstats=1 >/dev/null
    fi
aumgn's avatar
aumgn committed
104
105
}

aumgn's avatar
aumgn committed
106
dump_system() {
aumgn's avatar
aumgn committed
107
108
    msg "# Dumping system"

aumgn's avatar
aumgn committed
109
    local system_dir=$RESULTS_DIR/system
aumgn's avatar
aumgn committed
110
    local sched_features=/sys/kernel/debug/sched_features
aumgn's avatar
aumgn committed
111
112
    mkdir -p $system_dir

aumgn's avatar
aumgn committed
113
114
    sudo cat /etc/os-release >>$system_dir/os-release     2>/dev/null
    sudo uname -a            >>$system_dir/uname          2>/dev/null
aumgn's avatar
aumgn committed
115
    sudo cat /proc/cmdline   >>$system_dir/cmdline        2>/dev/null
aumgn's avatar
aumgn committed
116
117
118
119
120
121
122
123
124
125
    sudo uptime              >>$system_dir/uptime         2>/dev/null
    sudo mount               >>$system_dir/mount          2>/dev/null
    sudo sysctl -a           >>$system_dir/sysctl         2>/dev/null
    sudo cat $sched_features >>$system_dir/sched_features 2>/dev/null
    sudo dpkg -l             >>$system_dir/packages       2>/dev/null
    sudo systemctl           >>$system_dir/systemctl      2>/dev/null
    sudo top -n1 -b          >>$system_dir/top            2>/dev/null
    env                      >>$system_dir/env            2>/dev/null
    sudo lscpu               >>$system_dir/topology       2>/dev/null
    sudo hwloc-distances     >>$system_dir/topology       2>/dev/null
aumgn's avatar
aumgn committed
126
127
}

aumgn's avatar
aumgn committed
128
dump_versions() {
aumgn's avatar
aumgn committed
129
130
    msg "# Dumping versions"

aumgn's avatar
aumgn committed
131
    local version_dir=$RESULTS_DIR/version
aumgn's avatar
aumgn committed
132
133
    mkdir -p $version_dir

aumgn's avatar
aumgn committed
134
135
136
137
138
139
    git -C $BENCH_PATH rev-parse HEAD         >>$version_dir/bench-hash      2>/dev/null
    git submodule foreach git rev-parse HEAD  >>$version_dir/submodules-hash 2>/dev/null

    git -C $BENCH_PATH diff HEAD              >>$version_dir/bench-diff      2>/dev/null
    git submodule foreach git diff HEAD       >>$version_dir/submodules-diff 2>/dev/null

aumgn's avatar
aumgn committed
140
141
142
}

dump_args() {
aumgn's avatar
aumgn committed
143
144
    msg "# Dumping args"

aumgn's avatar
aumgn committed
145
146
    local args_file=$RESULTS_DIR/args

147
148
149
    echo "PROGRAM   = $programs"    >>$args_file
    echo "PINNING   = $pinnings"    >>$args_file
    echo "NODES     = $nodes_list"  >>$args_file
aumgn's avatar
aumgn committed
150
    echo "USE_PERF  = $use_perfs"   >>$args_file
151
152
    echo "AUTOGROUP = $autogroups"  >>$args_file
    echo "REPEAT    = $repeat"      >>$args_file
aumgn's avatar
aumgn committed
153
154
}

aumgn's avatar
aumgn committed
155
156
157
158
159
160
161
162
163
164
165
dump_programs() {
    msg "# Dumping programs"

    local dump_program_dir=$RESULTS_DIR/program
    mkdir -p $dump_program_dir

    for program in $programs; do
	dump_$program
    done
}

aumgn's avatar
aumgn committed
166
run_iterations() {
167
168
169
170
171
    local program=$1; shift
    local pinning=$1; shift
    local nodes=$1; shift
    local use_perf=$1; shift
    local autogroup=$1; shift
aumgn's avatar
aumgn committed
172
173
174
175
176
177

    local threads=$(($nodes * 8))
    if [[ $pinning == 'cores' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1))"
    elif [[ $pinning == 'nodes' ]]; then
        local pin_prefix="$PINTHREADS_BIN -n 0-$(($nodes-1)) -N"
178
179
    elif [[ $pinning == 'init' ]]; then
        local pin_prefix="$PINTHREADS_BIN -c 0-$((threads-1)) -i"
aumgn's avatar
aumgn committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    elif [[ $pinning == 'subset' ]]; then
        local pin_prefix="numactl -N 0-$(($nodes-1))"
    elif [[ $pinning == 'none' ]]; then
        local pin_prefix=''
    else
        echo "Unknown pinning $pinning"
        exit 34
    fi

    if [[ $autogroup == "y" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=1
    elif [[ $autogroup == "n" ]]; then
        sudo sysctl -q kernel.sched_autogroup_enabled=0
    else
        echo "Unknown autogroup $autogroup"
        exit 35
    fi

aumgn's avatar
aumgn committed
198
    printf '[%14s][%6s][%2d][%s][%s] ' $program $pinning $nodes $use_perf $autogroup
aumgn's avatar
aumgn committed
199

aumgn's avatar
aumgn committed
200
    local exp_dir=$TMP_RESULTS_DIR/outputs/$program-$pinning-$nodes-$use_perf-$autogroup
aumgn's avatar
aumgn committed
201
202

    for iteration in $(seq 1 $repeat); do
203
204
        local iteration_dir=$exp_dir/$iteration
        mkdir -p $iteration_dir
aumgn's avatar
aumgn committed
205

206
207
        local time_file=$iteration_dir/time
        local time_prefix="/usr/bin/time -f%e -o $iteration_dir/time -a"
aumgn's avatar
aumgn committed
208
209

        local perf_prefix=""
210
        if [[ $use_perf == "y" ]]; then
aumgn's avatar
aumgn committed
211
212
213
214
            perf_file="$iteration_dir/perf.data"
            perf_prefix="$PERF_BASE_PREFIX -o $perf_file"
        fi

215
216
        local output=$iteration_dir/output
        local errors=$iteration_dir/error
aumgn's avatar
aumgn committed
217

218
        local cmd_prefix="$perf_prefix $time_prefix $pin_prefix"
aumgn's avatar
aumgn committed
219
        local cmd="$(command_$program "$cmd_prefix" "$threads")"
220
        echo "#### sudo $cmd" >$iteration_dir/command
aumgn's avatar
aumgn committed
221
222

        cat /proc/schedstat >$iteration_dir/schedstat.before
223
        eval "sudo $cmd" 1>>$output 2>>$errors
224
        echo "$?" >$iteration_dir/exit_status
aumgn's avatar
aumgn committed
225

aumgn's avatar
aumgn committed
226
227
228
229
230
        cat /proc/schedstat >$iteration_dir/schedstat.after

        printf '.'
    done

231
232
233
    avg=$(awk '{ sum += $1 } END { if (NR > 0) print sum / NR; else print NaN }'     $exp_dir/*/time)
    sd=$(awk '{ sum += $1; sum2 += $0^2 } END { print sqrt(sum2 / NR - (sum/NR)^2)}' $exp_dir/*/time)
    printf ' | Avg: %6.2f  ~%3.2f\n' $avg $sd
aumgn's avatar
aumgn committed
234
235
236
}

run_all() {
aumgn's avatar
aumgn committed
237
238
    msg "# Running exp"

aumgn's avatar
aumgn committed
239
240
241
    for program in $programs; do
    for pinning in $pinnings; do
    for nodes in $nodes_list; do
242
    for use_perf in $use_perfs; do
aumgn's avatar
aumgn committed
243
    for autogroup in $autogroups; do
aumgn's avatar
aumgn committed
244
        run_iterations $program $pinning $nodes $use_perf $autogroup
245
    done
aumgn's avatar
aumgn committed
246
247
248
249
250
251
    done
    done
    done
    done
}

aumgn's avatar
aumgn committed
252
check_errors() {
aumgn's avatar
aumgn committed
253
254
    msg "# Checking errors"

aumgn's avatar
aumgn committed
255
    if grep -v 0 $TMP_RESULTS_DIR/outputs/*/*/exit_status; then
256
        exit 1
aumgn's avatar
aumgn committed
257
    fi
aumgn's avatar
aumgn committed
258
259
}

aumgn's avatar
aumgn committed
260
move_results() {
aumgn's avatar
aumgn committed
261
262
    msg "# Moving results"

263
    sudo chown -R $(id -un):$(id -gn) $TMP_RESULTS_DIR
aumgn's avatar
aumgn committed
264
    mv $TMP_RESULTS_DIR/* $RESULTS_DIR/
aumgn's avatar
aumgn committed
265
    rm -rf $TMP_RESULTS_DIR
aumgn's avatar
aumgn committed
266
267
268
269
270

    echo
    echo " => Results written in $RESULTS_DIR"
}

aumgn's avatar
aumgn committed
271
cleanup_system() {
272
    if sysctl kernel.sched_schedstats >/dev/null 2>&1; then
273
        sudo sysctl kernel.sched_schedstats=0 >/dev/null
274
    fi
aumgn's avatar
aumgn committed
275
276
}

aumgn's avatar
aumgn committed
277
278
279
280
281
script=$0

verbose=''
while [[ $1 == -* ]]; do
    case $1 in
282
    -v)
aumgn's avatar
aumgn committed
283
284
	    verbose=true
	    shift
285
286
	  ;;
    *)
aumgn's avatar
aumgn committed
287
288
289
	    echo "Unkown flag $0"
	    usage
	    exit 126
290
	  ;;
aumgn's avatar
aumgn committed
291
292
293
    esac
done

294
if [[ $# != 7 ]]; then
aumgn's avatar
aumgn committed
295
296
    usage
    exit 125
aumgn's avatar
aumgn committed
297
298
fi

aumgn's avatar
aumgn committed
299
300
301
programs=$1; shift
pinnings=$1; shift
nodes_list=$1; shift
302
use_perfs=$1; shift
aumgn's avatar
aumgn committed
303
304
autogroups=$1; shift
repeat=$1; shift
aumgn's avatar
aumgn committed
305
306

prepare_bench
aumgn's avatar
aumgn committed
307
308
prepare_tmpfs
prepare_directories
aumgn's avatar
aumgn committed
309
310
prepare_system
prepare_programs
aumgn's avatar
aumgn committed
311

aumgn's avatar
aumgn committed
312
313
314
dump_system
dump_versions
dump_args
aumgn's avatar
aumgn committed
315

aumgn's avatar
aumgn committed
316
317
run_all
check_errors
aumgn's avatar
aumgn committed
318
319
320

move_results

aumgn's avatar
aumgn committed
321
cleanup_system