Commit 6e1e44fd authored by Millian Poquet's avatar Millian Poquet
Browse files

[code] mapping should now work + test

parent 7f72a178
......@@ -300,3 +300,9 @@ add_test(pybatsim_tests
${CMAKE_SOURCE_DIR}/test/pybatsim_tests.yaml
-bod /tmp/pybatsim_tests
-bwd ${CMAKE_SOURCE_DIR})
add_test(fewer_resources
${CMAKE_SOURCE_DIR}/tools/experiments/execute_instances.py
${CMAKE_SOURCE_DIR}/test/test_fewer_resources.yaml
-bod /tmp/fewer_resources
-bwd ${CMAKE_SOURCE_DIR})
Subproject commit 52216b1972ef11126468b46ab13dbfb95f969295
Subproject commit c2bbc1f2a9ab12856ea782a30321f0b22c5b1a13
......@@ -753,8 +753,6 @@ void export_jobs_to_csv(const std::string &filename, const BatsimContext *contex
f << buf;
free(buf);
xbt_assert((int)job->allocation.size() == job->required_nb_res);
f << job->allocation.to_string_hyphen(" ") << "\n";
}
}
......
......@@ -525,7 +525,7 @@ void JsonProtocolReader::handle_execute_job(int event_number,
}
else
{
resource = std::stoi(key_value.GetString());
resource = std::stoi(value_value.GetString());
}
}
catch (const std::exception &)
......@@ -543,13 +543,13 @@ void JsonProtocolReader::handle_execute_job(int event_number,
int nb_inserted = 0;
xbt_assert(mit->first == nb_inserted, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): no resource associated to executor %d.", event_number, nb_inserted);
xbt_assert(mit->second >= 0 && mit->second < nb_allocated_resources, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): the %d-th resource within the allocation is told to be used, but there are only %d allocated resources.", event_number, mit->second, nb_allocated_resources);
xbt_assert(mit->second >= 0 && mit->second < nb_allocated_resources, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): executor %d should use the %d-th resource within the allocation, but there are only %d allocated resources.", event_number, mit->first, mit->second, nb_allocated_resources);
message->allocation->mapping.push_back(mit->second);
for (++mit, ++nb_inserted; mit != mapping_map.end(); ++mit, ++nb_inserted)
{
xbt_assert(mit->first == nb_inserted, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): no resource associated to executor %d.", event_number, nb_inserted);
xbt_assert(mit->second >= 0 && mit->second < nb_allocated_resources, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): the %d-th resource within the allocation is told to be used, but there are only %d allocated resources.", event_number, mit->second, nb_allocated_resources);
xbt_assert(mit->second >= 0 && mit->second < nb_allocated_resources, "Invalid JSON message: Invalid 'mapping' object of event %d (EXECUTE_JOB): executor %d should use the %d-th resource within the allocation, but there are only %d allocated resources.", event_number, mit->first, mit->second, nb_allocated_resources);
message->allocation->mapping.push_back(mit->second);
}
......
base_output_directory: /tmp/batsim_tests/fewer_resources
base_variables:
batsim_dir: ${base_working_directory}
agg_dir: ${base_output_directory}/aggregated
implicit_instances:
implicit:
sweep:
platform :
- {"name":"homo128", "filename":"${batsim_dir}/platforms/energy_platform_homogeneous_no_net_128.xml"}
workload :
- {"name":"one_job", "filename":"${batsim_dir}/workload_profiles/one_computation_job.json"}
algo:
- {"name":"filler", "algo_name":"filler"}
resource_fraction: [0.25, 0.5, 0.75, 1.0]
generic_instance:
timeout: 10
working_directory: ${base_working_directory}
output_directory: ${base_output_directory}/results/${instance_id}
batsim_command: batsim -p ${platform[filename]} -w ${workload[filename]} -e ${output_directory}/${instance_id} --mmax-workload --redis-prefix ${instance_id}
sched_command: batsched -v ${algo[algo_name]} --variant_options_filepath ${output_directory}/sched_input.json
commands_before_execution:
# Generate sched options
- |
#!/usr/bin/env bash
# Since bash associative arrays are not exported, the variables.bash
# is sourced here.
source ${output_directory}/variables.bash
# Let's generate an input file for the scheduler
cat > ${output_directory}/sched_input.json << EOF
{
"fraction_of_machines_to_use": ${resource_fraction}
}
EOF
commands_before_instances:
- ${batsim_dir}/test/is_batsim_dir.py ${base_working_directory}
- ${batsim_dir}/test/clean_output_dir.py ${base_output_directory}
- rm -rf ${agg_dir}
- mkdir -p ${agg_dir}
commands_after_instances:
# Let's merge the sched output csv files so they include who they are!
- |
#!/usr/bin/env bash
cat > ${agg_dir}/analysis.py <<EOF
#!/usr/bin/env python3
import glob
import re
import shutil
import os
import pandas as pd
import subprocess
########################################
# Let's merge the schedules' overviews #
########################################
instances_info_df = pd.read_csv('${base_output_directory}/instances/instances_info.csv')
filenames = glob.glob('${base_output_directory}/results/*/*_schedule.csv')
df_list = []
for filename in filenames:
m = re.search('${base_output_directory}/results/.*/(.*)_schedule\.csv', filename)
instance_id = m.group(1)
schedule_df = pd.read_csv(filename)
schedule_df['instance_id'] = instance_id
df_list.append(schedule_df)
aggregated_df = pd.concat(df_list, ignore_index=True)
joined_df = pd.merge(aggregated_df, instances_info_df, on='instance_id')
joined_df.to_csv('${agg_dir}/schedules_aggregated.csv',
index=False, na_rep = 'NA')
###################################################
# Let's make sure the makespan are those expected #
###################################################
joined_df.sort_values(by='resource_fraction', inplace=True)
joined_df.reset_index(inplace=True)
base_makespan = 10
assert(len(joined_df) == 4), "Wrong number of instances (expected 4, got {})".format(len(joined_df))
# Only one machine (resource_fraction == 0.25).
# The machine is used 4 times more than usual.
assert(joined_df['resource_fraction'][0] == 0.25), "Wrong resource_fraction (expected {}, got {})".format(0.25, joined_df['resource_fraction'][0])
assert(joined_df['makespan'][0] == base_makespan * 4), "Wrong makespan (expected {}, got {})".format(base_makespan*4, joined_df['makespan'][0])
# Only two machines (resource_fraction == 0.5).
# The two machines are used 2 times more than usual.
assert(joined_df['resource_fraction'][1] == 0.5), "Wrong resource_fraction (expected {}, got {})".format(0.5, joined_df['resource_fraction'][1])
assert(joined_df['makespan'][1] == base_makespan * 2), "Wrong makespan (expected {}, got {})".format(base_makespan*2, joined_df['makespan'][1])
# Only three machines (resource_fraction == 0.75).
# One machine is used 2 times more than usual. The used is used as usual.
assert(joined_df['resource_fraction'][2] == 0.75), "Wrong resource_fraction (expected {}, got {})".format(0.75, joined_df['resource_fraction'][2])
assert(joined_df['makespan'][2] == base_makespan * 2), "Wrong makespan (expected {}, got {})".format(base_makespan*2, joined_df['makespan'][2])
# Three machines (resource_fraction == 1).
# All machines are used as usual.
assert(joined_df['resource_fraction'][3] == 1), "Wrong resource_fraction (expected {}, got {})".format(1, joined_df['resource_fraction'][3])
assert(joined_df['makespan'][3] == base_makespan), "Wrong makespan (expected {}, got {})".format(base_makespan, joined_df['makespan'][3])
print('All the instances have been executed with expected makespan!')
EOF
- chmod +x ${agg_dir}/analysis.py
- ${agg_dir}/analysis.py
{
"nb_res": 4,
"jobs": [
{"id":1, "subtime":0, "walltime": 100, "res": 4, "profile": "msg_hg_10"}
],
"profiles": {
"msg_hg_10": {
"type": "msg_par_hg",
"cpu": 1e9,
"com": 0
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment