Commit 02c015c3 authored by aumgn's avatar aumgn
Browse files

Add linear_regression and pca from phoenix

parent 383f44aa
......@@ -6,9 +6,15 @@ PARSECMGMT=benchmarks/parsec/bin/parsecmgmt
WORD_COUNT_INPUTS_URL=http://csl.stanford.edu/~christos/data/word_count.tar.gz
WORD_COUNT_INPUTS_ARCHIVE=tmp/word_count_input.tar.gz
HISTOGRAM_INPUT=tmp/histogram_input.bmp
LIN_REG_INPUTS_URL=http://csl.stanford.edu/~christos/data/linear_regression.tar.gz
LIN_REG_INPUTS_ARCHIVE=tmp/lin_reg_input.tar.gz
.PHONY: install
install: submodules $(PARSEC_INPUTS) $(WORD_COUNT_INPUTS_ARCHIVE) $(HISTOGRAM_INPUT)
install: submodules \
$(PARSEC_INPUTS) \
$(WORD_COUNT_INPUTS_ARCHIVE) \
$(HISTOGRAM_INPUT) \
$(LIN_REG_INPUTS_ARCHIVE)
.PHONY: submodules
submodules:
......@@ -60,6 +66,11 @@ $(HISTOGRAM_INPUT):
@echo "# Generating input for histogram (can take a while)"
convert -size 30000x30000 xc:gray +noise random $(HISTOGRAM_INPUT)
$(LIN_REG_INPUTS_ARCHIVE):
mkdir -p tmp/
@echo "### Downloading input for word_count (can take a while)"
wget $(LIN_REG_INPUTS_URL) -O $(LIN_REG_INPUTS_ARCHIVE)
.PHONY: clean-parsec
clean-parsec: clean-bodytrack clean-streamcluster clean-volrend clean-vips
......
......@@ -5,9 +5,7 @@ set -e
export BENCH_PATH=$HOME/bench
export TMPFS_PATH=$BENCH_PATH/tmpfs
export TRACE_BASE_PREFIX="sudo trace-cmd record -e sched"
export PERF_BASE_PREFIX="sudo perf record -e sched:sched_migrate_task,sched:sched_switch --call-graph dwarf -F 99"
export SCHED_PROFILER_MOD_PATH=tools/wastedcores/tools/visualizations_4.1/sched_profiler/sched_profiler.ko
export PERF_BASE_PREFIX="perf record -a -c 1 -e sched:sched_thread_placement"
export PINTHREADS_PATH=$HOME/bench/tools/PinThreads
export PINTHREADS_BIN=$PINTHREADS_PATH/pinthreads
......@@ -16,7 +14,7 @@ export PARSEC_DIR=$BENCH_PATH/benchmarks/parsec
export PHOENIX_DIR=$BENCH_PATH/benchmarks/phoenix
date=$(date +'%y.%m.%d-%H.%M.%S')
export TMP_INPUTS_DIR=$TMPFS_PATH/inputs/
export TMP_INPUTS_DIR=$TMPFS_PATH/inputs
export TMP_RESULTS_DIR=$TMPFS_PATH/results/$date
export RESULTS_DIR=$BENCH_PATH/tmp/results/$date
export RESULTS_LAST=$BENCH_PATH/tmp/results/last
......@@ -29,22 +27,27 @@ msg() {
usage() {
cat <<HD
Usage: $script [-v] <programs> <pinning> <nodes> <autogroup> <lb_bias> <repeat>
Usage: $script [-v] <programs> <pinning> <nodes> <use_perf> <autogroup> <lb_bias> <repeat>
-v toggle verbose mode
<programs> list of programs to run
<pinning> list of values among { cores, nodes, subset, none }
<nodes> list of numbers of nodes
<use_perf> whether ('y') or not ('n') perf should be used
<autogroup> whether ('y') or not ('n') autogroup should be enabled
<lb_bias> whether ('y') or not ('n') load_balancing should be biased with exponential smoothing
<repeat> the number of time to run the application for each parameter combinations
Run volrend on 64 cores with and without pinning one time each:
>> $0 volrend "cores none" 8 n n 1
Run volrend and wordcount on 1,2,3,4,5,6,7,8 nodes
with pinning and without pinning, with and without
autogroup, five times each:
>> $0 "volrend word_count" "cores none" "\$(seq 1 8)" "y n" "y n" 5
>> $0 volrend "cores none" 8 n n n 1
Run volrend and wordcount
with pinning and without pinning,
on 2,3,4,5,6,7,8 nodes
with and without perf,
with and without autogroup,
with and without lb_bias,
five times each:
>> $0 "volrend word_count" "cores none" "\$(seq 2 8)" "y n" "y n" "y n" 5
HD
}
......@@ -96,8 +99,9 @@ prepare_directories() {
prepare_system() {
msg "# Preparing system"
sudo sysctl kernel.numa_balancing=0 >/dev/null
if sysctl kernel.sched_schedstats >/dev/null 2>&1; then
sudo sysctl -w kernel.perf_event_paranoid=-1 >/dev/null
sudo sysctl -w kernel.numa_balancing=0 >/dev/null
if sudo sysctl kernel.sched_schedstats >/dev/null 2>&1; then
sudo sysctl kernel.sched_schedstats=1 >/dev/null
fi
}
......@@ -161,11 +165,12 @@ dump_programs() {
}
run_iterations() {
local program=$1
local pinning=$2
local nodes=$3
local autogroup=$4
local lb_bias=$5
local program=$1; shift
local pinning=$1; shift
local nodes=$1; shift
local use_perf=$1; shift
local autogroup=$1; shift
local lb_bias=$1; shift
local threads=$(($nodes * 8))
if [[ $pinning == 'cores' ]]; then
......@@ -201,9 +206,9 @@ run_iterations() {
exit 35
fi
printf '[%10s][%6s][%2d][%s][%s] ' $program $pinning $nodes $autogroup $lb_bias
printf '[%14s][%6s][%2d][%s][%s][%s] ' $program $pinning $nodes $use_perf $autogroup $lb_bias
local exp_dir=$TMP_RESULTS_DIR/outputs/$program-$pinning-$nodes-$autogroup-$lb_bias
local exp_dir=$TMP_RESULTS_DIR/outputs/$program-$pinning-$nodes-$use_perf-$autogroup-$lb_bias
for iteration in $(seq 1 $repeat); do
local iteration_dir=$exp_dir/$iteration
......@@ -212,14 +217,8 @@ run_iterations() {
local time_file=$iteration_dir/time
local time_prefix="/usr/bin/time -f%e -o $iteration_dir/time -a"
local trace_prefix=""
if [[ -n $WITH_TRACE ]]; then
trace_file="$iteration_dir/trace"
trace_prefix="$TRACE_BASE_PREFIX -o $trace_file"
fi
local perf_prefix=""
if [[ -n $WITH_PERF ]]; then
if [[ $use_perf == "y" ]]; then
perf_file="$iteration_dir/perf.data"
perf_prefix="$PERF_BASE_PREFIX -o $perf_file"
fi
......@@ -232,15 +231,8 @@ run_iterations() {
echo "#### $cmd" >$iteration_dir/command
cat /proc/schedstat >$iteration_dir/schedstat.before
if [[ -n $WITH_WC_VISU ]]; then
sudo insmod $SCHED_PROFILER_MOD_PATH
fi
eval "$cmd" 1>>$output 2>>$errors
echo "$?" >$iteration_dir/exit_status
if [[ -n $WITH_WC_VISU ]]; then
cat /proc/sched_profiler > $iteration_dir/sched_profiler.output
sudo rmmod $SCHED_PROFILER_MOD_PATH
fi
cat /proc/schedstat >$iteration_dir/schedstat.after
......@@ -258,9 +250,11 @@ run_all() {
for program in $programs; do
for pinning in $pinnings; do
for nodes in $nodes_list; do
for use_perf in $use_perfs; do
for autogroup in $autogroups; do
for lb_bias in $lb_biases; do
run_iterations $program $pinning $nodes $autogroup $lb_bias
run_iterations $program $pinning $nodes $use_perf $autogroup $lb_bias
done
done
done
done
......@@ -309,7 +303,7 @@ while [[ $1 == -* ]]; do
esac
done
if [[ $# != 6 ]]; then
if [[ $# != 7 ]]; then
usage
exit 125
fi
......@@ -317,6 +311,7 @@ fi
programs=$1; shift
pinnings=$1; shift
nodes_list=$1; shift
use_perfs=$1; shift
autogroups=$1; shift
lb_biases=$1; shift
repeat=$1; shift
......
HISTOGRAM_DIR=$PHOENIX_DIR/phoenix-2.0/tests/histogram
HISTOGRAM_BIN=$HISTOGRAM_DIR/histogram
HISTOGRAM_INPUT=$BENCH_PATH/tmp/histogram_input.bmp
HISTOGRAM_INPUTS_DIR=$TMP_INPUTS_DIR/histogram
prepare_histogram() {
if [ ! -f $HISTOGRAM_INPUTS_DIR/input ]; then
mkdir -p $HISTOGRAM_INPUTS_DIR
mv $HISTOGRAM_INPUT $HISTOGRAM_INPUTS_DIR/input
fi
}
dump_volrend() {
sha1sum $HISTOGRAM_BIN >>$dump_program_dir/histogram
sha1sum $HISTOGRAM_INPUTS_DIR/input >>$dump_program_dir/histogram
}
command_histogram() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $HISTOGRAM_BIN $HISTOGRAM_INPUTS_DIR/input
}
HISTOGRAM_MR_DIR=$PHOENIX_DIR/phoenix-2.0/tests/histogram
HISTOGRAM_MR_BIN=$HISTOGRAM_MR_DIR/histogram
HISTOGRAM_MR_INPUT=$BENCH_PATH/tmp/histogram_input.bmp
HISTOGRAM_MR_INPUTS_DIR=$TMP_INPUTS_DIR/histogram
prepare_histogram_mr() {
if [ ! -f $HISTOGRAM_MR_INPUTS_DIR/input ]; then
mkdir -p $HISTOGRAM_MR_INPUTS_DIR
mv $HISTOGRAM_MR_INPUT $HISTOGRAM_MR_INPUTS_DIR/input
fi
}
dump_volrend() {
sha1sum $HISTOGRAM_MR_BIN >>$dump_program_dir/histogram_mr
sha1sum $HISTOGRAM_MR_INPUTS_DIR/input >>$dump_program_dir/histogram_mr
}
command_histogram_mr() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $HISTOGRAM_MR_BIN $HISTOGRAM_MR_INPUTS_DIR/input
}
HISTOGRAM_PT_DIR=$PHOENIX_DIR/phoenix-2.0/tests/histogram
HISTOGRAM_PT_BIN=$HISTOGRAM_PT_DIR/histogram
HISTOGRAM_PT_INPUT=$BENCH_PATH/tmp/histogram_input.bmp
HISTOGRAM_PT_INPUTS_DIR=$TMP_INPUTS_DIR/histogram
prepare_histogram_pt() {
if [ ! -f $HISTOGRAM_PT_INPUTS_DIR/input ]; then
mkdir -p $HISTOGRAM_PT_INPUTS_DIR
mv $HISTOGRAM_PT_INPUT $HISTOGRAM_PT_INPUTS_DIR/input
fi
}
dump_volrend() {
sha1sum $HISTOGRAM_PT_BIN >>$dump_program_dir/histogram_pt
sha1sum $HISTOGRAM_PT_INPUTS_DIR/input >>$dump_program_dir/histogram_pt
}
command_histogram_pt() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $HISTOGRAM_PT_BIN $HISTOGRAM_PT_INPUTS_DIR/input
}
LIN_REG_MR_DIR=$PHOENIX_DIR/phoenix-2.0/tests/linear_regression
LIN_REG_MR_BIN=$LIN_REG_MR_DIR/linear_regression
LIN_REG_MR_INPUTS_ARCHIVE=$BENCH_PATH/tmp/lin_reg_input.tar.gz
LIN_REG_MR_INPUTS_DIR=$TMP_INPUTS_DIR/lin_reg
prepare_lin_reg_mr() {
if [[ ! -f $LIN_REG_MR_INPUTS_DIR/word_100MB.txt ]]; then
mkdir -p $LIN_REG_MR_INPUTS_DIR
tar -xzf $LIN_REG_MR_INPUTS_ARCHIVE -C $LIN_REG_MR_INPUTS_DIR --strip-components 1 linear_regression_datafiles/key_file_500MB.txt
fi
if [ ! -f $LIN_REG_MR_INPUTS_DIR/input ]; then
for i in $(seq 1 6); do
cat $LIN_REG_MR_INPUTS_DIR/key_file_500MB.txt >> $LIN_REG_MR_INPUTS_DIR/input
done
fi
}
dump_lin_reg_mr() {
sha1sum $LIN_REG_MR_BIN >>$dump_program_dir/lin_reg_mr
sha1sum $LIN_REG_MR_INPUTS_DIR/input >>$dump_program_dir/lin_reg_mr
}
command_lin_reg_mr() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $LIN_REG_MR_BIN $LIN_REG_MR_INPUTS_DIR/input
}
LIN_REG_PT_DIR=$PHOENIX_DIR/phoenix-2.0/tests/linear_regression
LIN_REG_PT_BIN=$LIN_REG_PT_DIR/linear_regression-pthread
LIN_REG_PT_INPUTS_ARCHIVE=$BENCH_PATH/tmp/lin_reg_input.tar.gz
LIN_REG_PT_INPUTS_DIR=$TMP_INPUTS_DIR/lin_reg
prepare_lin_reg_pt() {
if [[ ! -f $LIN_REG_PT_INPUTS_DIR/word_100MB.txt ]]; then
mkdir -p $LIN_REG_PT_INPUTS_DIR
tar -xzf $LIN_REG_PT_INPUTS_ARCHIVE -C $LIN_REG_PT_INPUTS_DIR --strip-components 1 linear_regression_datafiles/key_file_500MB.txt
fi
if [ ! -f $LIN_REG_PT_INPUTS_DIR/input ]; then
for i in $(seq 1 6); do
cat $LIN_REG_PT_INPUTS_DIR/key_file_500MB.txt >> $LIN_REG_PT_INPUTS_DIR/input
done
fi
}
dump_lin_reg_pt() {
sha1sum $LIN_REG_PT_BIN >>$dump_program_dir/lin_reg_pt
sha1sum $LIN_REG_PT_INPUTS_DIR/input >>$dump_program_dir/lin_reg_pt
}
command_lin_reg_pt() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $LIN_REG_PT_BIN $LIN_REG_PT_INPUTS_DIR/input
}
export PCA_MR_DIR=$PHOENIX_DIR/phoenix-2.0/tests/pca
export PCA_MR_BIN=$PCA_MR_DIR/pca
prepare_pca_mr() {
true
}
dump_pca_mr() {
sha1sum $PCA_MR_BIN >>$dump_program_dir/pca_mr
}
command_pca_mr() {
echo MR_NUMPROCS=$threads $cmd_prefix $PCA_MR_BIN -r 4000 -c 4000
}
export PCA_PT_DIR=$PHOENIX_DIR/phoenix-2.0/tests/pca
export PCA_PT_BIN=$PCA_PT_DIR/pca-pthread
prepare_pca_pt() {
true
}
dump_pca_pt() {
sha1sum $PCA_PT_BIN >>$dump_program_dir/pca_pt
}
command_pca_pt() {
echo MR_NUMPROCS=$threads $cmd_prefix $PCA_PT_BIN -r 4000 -c 4000
}
WORD_COUNT_DIR=$PHOENIX_DIR/phoenix-2.0/tests/word_count
WORD_COUNT_BIN=$WORD_COUNT_DIR/word_count
WORD_COUNT_INPUTS_ARCHIVE=$BENCH_PATH/tmp/word_count_input.tar.gz
WORD_COUNT_INPUTS_DIR=$TMP_INPUTS_DIR/word_count
prepare_word_count() {
if [[ ! -f $WORD_COUNT_INPUTS_DIR/word_100MB.txt ]]; then
mkdir -p $WORD_COUNT_INPUTS_DIR
tar -xzf $WORD_COUNT_INPUTS_ARCHIVE -C $WORD_COUNT_INPUTS_DIR --strip-components 1
fi
if [ ! -f $WORD_COUNT_INPUTS_DIR/input ]; then
for i in $(seq 1 15); do
cat $WORD_COUNT_INPUTS_DIR/word_100MB.txt >> $WORD_COUNT_INPUTS_DIR/input
done
fi
}
dump_word_count() {
sha1sum $WORD_COUNT_BIN >>$dump_program_dir/word_count
sha1sum $WORD_COUNT_INPUTS_DIR/input >>$dump_program_dir/word_count
}
command_word_count() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $WORD_COUNT_BIN $WORD_COUNT_INPUTS_DIR/input
}
WORD_COUNT_MR_DIR=$PHOENIX_DIR/phoenix-2.0/tests/word_count
WORD_COUNT_MR_BIN=$WORD_COUNT_MR_DIR/word_count
WORD_COUNT_MR_INPUTS_ARCHIVE=$BENCH_PATH/tmp/word_count_input.tar.gz
WORD_COUNT_MR_INPUTS_DIR=$TMP_INPUTS_DIR/word_count
prepare_word_count_mr() {
if [[ ! -f $WORD_COUNT_MR_INPUTS_DIR/word_100MB.txt ]]; then
mkdir -p $WORD_COUNT_MR_INPUTS_DIR
tar -xzf $WORD_COUNT_MR_INPUTS_ARCHIVE -C $WORD_COUNT_MR_INPUTS_DIR --strip-components 1
fi
if [ ! -f $WORD_COUNT_MR_INPUTS_DIR/input ]; then
for i in $(seq 1 15); do
cat $WORD_COUNT_MR_INPUTS_DIR/word_100MB.txt >> $WORD_COUNT_MR_INPUTS_DIR/input
done
fi
}
dump_word_count_mr() {
sha1sum $WORD_COUNT_MR_BIN >>$dump_program_dir/word_count_mr
sha1sum $WORD_COUNT_MR_INPUTS_DIR/input >>$dump_program_dir/word_count_mr
}
command_word_count_mr() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $WORD_COUNT_MR_BIN $WORD_COUNT_MR_INPUTS_DIR/input
}
WORD_COUNT_PT_DIR=$PHOENIX_DIR/phoenix-2.0/tests/word_count
WORD_COUNT_PT_BIN=$WORD_COUNT_PT_DIR/word_count
WORD_COUNT_PT_INPUTS_ARCHIVE=$BENCH_PATH/tmp/word_count_input.tar.gz
WORD_COUNT_PT_INPUTS_DIR=$TMP_INPUTS_DIR/word_count
prepare_word_count_pt() {
if [[ ! -f $WORD_COUNT_PT_INPUTS_DIR/word_100MB.txt ]]; then
mkdir -p $WORD_COUNT_PT_INPUTS_DIR
tar -xzf $WORD_COUNT_PT_INPUTS_ARCHIVE -C $WORD_COUNT_PT_INPUTS_DIR --strip-components 1
fi
if [ ! -f $WORD_COUNT_PT_INPUTS_DIR/input ]; then
for i in $(seq 1 15); do
cat $WORD_COUNT_PT_INPUTS_DIR/word_100MB.txt >> $WORD_COUNT_PT_INPUTS_DIR/input
done
fi
}
dump_word_count_pt() {
sha1sum $WORD_COUNT_PT_BIN >>$dump_program_dir/word_count_pt
sha1sum $WORD_COUNT_PT_INPUTS_DIR/input >>$dump_program_dir/word_count_pt
}
command_word_count_pt() {
echo MR_NUMPROCS=$threads MAPRED_NO_BINDING=1 $cmd_prefix $WORD_COUNT_PT_BIN $WORD_COUNT_PT_INPUTS_DIR/input
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment