Commit 9442e0bb authored by Millian Poquet's avatar Millian Poquet
Browse files

Workload names are now compressed.

In order to have deterministic workload names, I chose to use a hash function
rather than an uuid. The workload names are now the first 6 characters of the
hexadecimal representation of the SHA-1 of the absolute file name of the
workload file.
parent 0948aba3
......@@ -57,6 +57,10 @@ include_directories(${Boost_INCLUDE_DIR})
find_package(rapidjson REQUIRED)
include_directories(${RAPIDJSON_INCLUDE_DIRS})
## OpenSSL dependency
find_package(OpenSSL REQUIRED)
include_directories(${OPENSSL_INCLUDE_DIR})
## Redox dependency
find_package(redox REQUIRED)
include_directories(${REDOX_INCLUDE_DIR})
......@@ -87,6 +91,7 @@ target_link_libraries(batsim
${Boost_LOCALE_LIBRARY_DEBUG}
${Boost_REGEX_LIBRARY_DEBUG}
${Boost_SYSTEM_LIBRARY_DEBUG}
${OPENSSL_LIBRARIES}
${REDOX_LIBRARY}
${LIBEV_LIBRARY}
${HIREDIS_LIBRARY})
......
......@@ -3,12 +3,13 @@
* @brief Batsim's entry point
*/
#include <string>
#include <sys/types.h>
#include <stdio.h>
#include <argp.h>
#include <unistd.h>
#include <string>
#include <simgrid/msg.h>
#include <smpi/smpi.h>
#include <simgrid/plugins/energy.h>
......@@ -17,6 +18,8 @@
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
#include <openssl/sha.h>
#include "batsim.hpp"
#include "context.hpp"
#include "export.hpp"
......@@ -35,6 +38,33 @@ using namespace std;
XBT_LOG_NEW_DEFAULT_CATEGORY(batsim, "batsim"); //!< Logging
string generate_sha1_string(std::string string_to_hash, int output_length)
{
static_assert(sizeof(unsigned char) == sizeof(char), "sizeof(unsigned char) should equals to sizeof(char)");
xbt_assert(output_length > 0);
xbt_assert(output_length < SHA_DIGEST_LENGTH);
unsigned char sha1_buf[SHA_DIGEST_LENGTH];
SHA1((const unsigned char *)string_to_hash.c_str(), string_to_hash.size(), sha1_buf);
char * output_buf = (char *) calloc(SHA_DIGEST_LENGTH * 2 + 1, sizeof(char));
xbt_assert(output_buf != 0, "Couldn't allocate memory");
for (int i = 0; i < SHA_DIGEST_LENGTH; ++i)
{
int nb_printed_char = snprintf(output_buf + 2*i, 3, "%02x", sha1_buf[i]);
xbt_assert(nb_printed_char == 2, "Fix me :(");
}
XBT_DEBUG("SHA-1 of string '%s' is '%s'\n", string_to_hash.c_str(), output_buf);
string result(output_buf, output_length);
xbt_assert((int)result.size() == output_length);
free(output_buf);
return result;
}
/**
* @brief Used to parse the main function parameters
* @param[in] key The current key
......@@ -60,12 +90,20 @@ int parse_opt (int key, char *arg, struct argp_state *state)
}
case 'w':
{
main_args->workload_filenames.push_back(arg);
if (access(((string) arg).c_str(), R_OK) == -1)
{
main_args->abort = true;
main_args->abortReason += "\n invalid WORKLOAD_FILE argument: file '" + string(arg) + "' cannot be read";
}
else
{
MainArguments::WorkloadDescription desc;
desc.filename = absolute_filename(arg);
desc.name = generate_sha1_string(arg);
XBT_INFO("Workload '%s' corresponds to workload file '%s'.", desc.name.c_str(), desc.filename.c_str());
main_args->workload_descriptions.push_back(desc);
}
break;
}
case 'W':
......@@ -88,7 +126,13 @@ int parse_opt (int key, char *arg, struct argp_state *state)
if (parts.size() == 2)
workflow_start_time = std::stod(parts.at(1));
main_args->workflow_descriptions.push_back({workflow_filename, workflow_start_time});
MainArguments::WorkflowDescription desc;
desc.filename = absolute_filename(workflow_filename);
desc.name = generate_sha1_string(desc.filename);
desc.start_time = workflow_start_time;
XBT_INFO("Workflow '%s' corresponds to workflow file '%s'.", desc.name.c_str(), desc.filename.c_str());
main_args->workflow_descriptions.push_back(desc);
}
break;
}
......@@ -302,32 +346,29 @@ void load_workloads_and_workflows(const MainArguments & main_args, BatsimContext
int max_nb_machines_in_workloads = -1;
// Let's create the workloads
for (const auto & workload_filename : main_args.workload_filenames)
for (const MainArguments::WorkloadDescription & desc : main_args.workload_descriptions)
{
string workload_name = workload_filename; // TODO: compress names?
Workload * workload = new Workload(workload_name);
Workload * workload = new Workload(desc.name);
int nb_machines_in_workload = -1;
workload->load_from_json(workload_filename, nb_machines_in_workload);
workload->load_from_json(desc.filename, nb_machines_in_workload);
max_nb_machines_in_workloads = max(max_nb_machines_in_workloads, nb_machines_in_workload);
context->workloads.insert_workload(workload_name, workload);
context->workloads.insert_workload(desc.name, workload);
}
// Let's create the workflows
for (const auto & desc : main_args.workflow_descriptions)
for (const MainArguments::WorkflowDescription & desc : main_args.workflow_descriptions)
{
string workflow_name = desc.workflow_filename; // TODO: compress names?
string workflow_workload_name = workflow_name;
Workload * workload = new Workload(workflow_workload_name); // Is creating the Workload now necessary? Workloads::add_job_if_not_exists may be enough
Workload * workload = new Workload(desc.workload_name); // Is creating the Workload now necessary? Workloads::add_job_if_not_exists may be enough
workload->jobs = new Jobs;
workload->profiles = new Profiles;
context->workloads.insert_workload(workflow_workload_name, workload);
context->workloads.insert_workload(desc.workload_name, workload);
Workflow * workflow = new Workflow(workflow_name);
workflow->start_time = desc.workflow_start_time;
workflow->load_from_xml(desc.workflow_filename);
context->workflows.insert_workflow(workflow_name, workflow);
Workflow * workflow = new Workflow(desc.name);
workflow->start_time = desc.start_time;
workflow->load_from_xml(desc.filename);
context->workflows.insert_workflow(desc.name, workflow);
}
// Let's compute how the number of machines to use should be limited
......@@ -353,31 +394,28 @@ void start_initial_simulation_processes(const MainArguments & main_args, BatsimC
const Machine * master_machine = context->machines.master_machine();
// Let's run a static_job_submitter process for each workload
for (const auto & workload_filename : main_args.workload_filenames)
for (const MainArguments::WorkloadDescription & desc : main_args.workload_descriptions)
{
XBT_DEBUG("Creating a workload_submitter process...");
string workload_name = workload_filename;
JobSubmitterProcessArguments * submitter_args = new JobSubmitterProcessArguments;
submitter_args->context = context;
submitter_args->workload_name = workload_name;
submitter_args->workload_name = desc.name;
string submitter_instance_name = "workload_submitter_" + workload_name;
string submitter_instance_name = "workload_submitter_" + desc.name;
MSG_process_create(submitter_instance_name.c_str(), static_job_submitter_process, (void*)submitter_args, master_machine->host);
XBT_INFO("The process '%s' has been created.", submitter_instance_name.c_str());
}
// Let's run a workflow_submitter process for each workflow
for (const auto & desc : main_args.workflow_descriptions)
for (const MainArguments::WorkflowDescription & desc : main_args.workflow_descriptions)
{
XBT_DEBUG("Creating a workflow_submitter process...");
string workflow_name = desc.workflow_filename;
WorkflowSubmitterProcessArguments * submitter_args = new WorkflowSubmitterProcessArguments;
submitter_args->context = context;
submitter_args->workflow_name = workflow_name;
submitter_args->workflow_name = desc.name;
string submitter_instance_name = "workflow_submitter_" + workflow_name;
string submitter_instance_name = "workflow_submitter_" + desc.name;
MSG_process_create(submitter_instance_name.c_str(), workflow_submitter_process, (void*)submitter_args, master_machine->host);
XBT_INFO("The process '%s' has been created.", submitter_instance_name.c_str());
}
......
#pragma once
#include <string>
#include <list>
/**
* @brief Batsim verbosity level
*/
......@@ -11,23 +14,44 @@ enum class VerbosityLevel
,DEBUG //!< Debug informations should be displayed too
};
/**
* @brief Generates a uuid string
* @param[in] string_to_hash T
* @param[in] output_length The maximum length of the output string
* @pre output_length > 0 && output_length < 20
* @post result.size() == output_length
* @return The generated hash (truncated to output_length characters if needed)
*/
std::string generate_sha1_string(std::string string_to_hash, int output_length = 6);
/**
* @brief Stores Batsim's arguments, a.k.a. the main function arguments
*/
struct MainArguments
{
/**
* @brief Stores the command-line description of a workload
*/
struct WorkloadDescription
{
std::string filename; //!< The name of the workload file
std::string name; //!< The name of the workload
};
/**
* @brief Stores the command-line description of a workflow
*/
struct WorkflowDescription
{
std::string workflow_filename; //!< The name of the workflow file
double workflow_start_time; //!< The moment in time at which the workflow should be started
std::string filename; //!< The name of the workflow file
std::string name; //!< The name of the workflow
std::string workload_name; //!< The name of the workload associated with the workflow
double start_time; //!< The moment in time at which the workflow should be started
};
std::string platform_filename; //!< The SimGrid platform filename
std::list<std::string> workload_filenames; //!< The JSON workload filename
std::list<WorkflowDescription> workflow_descriptions; //!< The workflow descriptions
std::list<WorkloadDescription> workload_descriptions; //!< The workloads' descriptions
std::list<WorkflowDescription> workflow_descriptions; //!< The workflows' descriptions
std::string socket_filename = "/tmp/bat_socket"; //!< The Unix Domain Socket filename
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment