Commit 1e8b2fd5 authored by Millian Poquet's avatar Millian Poquet
Browse files

[tools] execN whole lotta codacy cosmetics

parent 0a4e6498
......@@ -6,7 +6,11 @@ from sortedcontainers import SortedSet
def take_first_resources(available_res, nb_res_to_take):
"""Take the first nb_res_to_take resources into available_res."""
assert len(available_res) >= nb_res_to_take, 'Invalid schedule. Want to use {} resources whereas only {} are available ({})'.format(nb_res_to_take, len(available_res), available_res)
if nb_res_to_take > len(available_res):
raise Exception('Invalid schedule. Want to use {} resources whereas '
'only {} are available({})'.format(nb_res_to_take,
len(available_res),
available_res))
allocation = SortedSet()
l = list(available_res)
......@@ -18,6 +22,7 @@ def take_first_resources(available_res, nb_res_to_take):
if len(available_res) > 0:
min_available_res = min(available_res)
for res in allocation:
assert res < min_available_res, "Invalid sort"
if res >= min_available_res:
raise Exception('Invalid sort')
return (available_res, allocation)
This diff is collapsed.
......@@ -37,8 +37,8 @@ def find_info_from_batsim_command(batsim_command):
"""Get information from the Batsim command."""
split_command = shlex.split(batsim_command)
assert len(split_command) > 1, "Invalid Batsim command: '{}'".format(
batsim_command)
if len(split_command) <= 1:
raise Exception('Invalid Batsim command: "{}"'.format(batsim_command))
if 'batsim' not in split_command[0]:
logger.warning("Batsim does not seem to be executed directly. "
......@@ -137,7 +137,8 @@ def delete_file_if_exists(filename):
def random_string(length=16):
"""Generate a alphanum string."""
assert(length > 0)
if length <= 0:
raise Exception('Bad length. Expected 1 or more.')
alphanum = "abcdefghijklmnopqrstuvwxyz0123456789"
s = ''
for i in range(length):
......@@ -161,14 +162,16 @@ def retrieve_info_from_instance(variables,
r = random_string()
script_filename = '{wd}/{rand}_script.sh'.format(wd=working_directory,
rand=r)
output_dir_filename = '{wd}/{rand}_out_dir'.format(wd=working_directory,
rand=r)
working_dir_filename = '{wd}/{rand}_working_dir'.format(wd=working_directory,
rand=r)
output_dir_filename = '{wd}/{rand}_out_dir'.format(
wd=working_directory, rand=r)
working_dir_filename = '{wd}/{rand}_working_dir'.format(
wd=working_directory, rand=r)
command_filename = '{wd}/{rand}_command'.format(wd=working_directory,
rand=r)
filename_ok = not os.path.exists(script_filename) and not os.path.exists(
output_dir_filename) and not os.path.exists(working_dir_filename) and not os.path.exists(command_filename)
filename_ok = (not os.path.exists(script_filename) and
not os.path.exists(output_dir_filename) and
not os.path.exists(working_dir_filename) and
not os.path.exists(command_filename))
put_variables_in_file(
variables, variables_declaration_order, script_filename)
......@@ -190,7 +193,8 @@ def retrieve_info_from_instance(variables,
# Let's execute the script
p = subprocess.run('bash {f}'.format(f=script_filename),
shell=True, stdout=subprocess.PIPE)
assert(p.returncode == 0)
if p.returncode != 0:
raise Exception('Script returned %d (expected 0)'.format(p.returncode))
# Let's get the working directory
f = open(working_dir_filename, 'r')
......@@ -218,6 +222,7 @@ def retrieve_info_from_instance(variables,
def find_all_values(var_value):
"""Return all subvalues of a variable value."""
ret = set()
if isinstance(var_value, dict):
......@@ -290,8 +295,10 @@ def check_variables(variables):
undeclared_variables = set(
[var_name for var_name in variables]) - declared_variables
undeclared_variables = list(undeclared_variables)
logger.error("Invalid variables: Could not find a declaration order that allows to declare these variables: {}. All variables : {}".format(
undeclared_variables, variables))
logger.error("Invalid variables: Could not find a declaration order "
"that allows to declare these variables: {}. "
"All variables : {}".format(undeclared_variables,
variables))
return (False, [])
......@@ -337,7 +344,8 @@ def put_variables_in_file(variables,
text_to_write += "\n# Variables definition\n"
for var_name in variables_declaration_order:
assert(var_name in variables)
if var_name not in variables:
raise Exception('Using undeclared variable {}'.format(var_name))
text_to_write += variable_to_text(variables, var_name)
# Let's export all variables
......@@ -383,7 +391,8 @@ def display_process_output_on_error(process_name, stdout_file, stderr_file,
# Let's retrieve the file length (ugly.)
cmd_wc = "wc -l {}".format(filename)
p = subprocess.run(cmd_wc, shell=True, stdout=subprocess.PIPE)
assert(p.returncode == 0)
if p.returncode != 0:
raise Exception('wc returned %d'.format(p.returncode))
nb_lines = int(str(p.stdout.decode('utf-8')).split(' ')[0])
if nb_lines > 0:
......@@ -402,13 +411,20 @@ def display_process_output_on_error(process_name, stdout_file, stderr_file,
p_tail = subprocess.run(cmd_tail, shell=True,
stdout=subprocess.PIPE)
assert(p_head.returncode == 0)
assert(p_tail.returncode == 0)
logger.error('{} {}:\n{}\n...\n...\n... (truncated... whole log in {})\n...\n...\n{}'.format(
process_name,
fname, str(p_head.stdout.decode('utf-8')), filename,
str(p_tail.stdout.decode('utf-8'))))
if p_head.returncode != 0:
raise Exception(
'head returned %d'.format(p_head.returncode))
if p_tail.returncode != 0:
raise Exception(
'head returned %d'.format(p_tail.returncode))
logger.error('{} {}:\n{}\n...\n...\n... '
'(truncated... whole log in {})\n...\n...\n'
'{}'.format(
process_name, fname,
str(p_head.stdout.decode('utf-8')),
filename,
str(p_tail.stdout.decode('utf-8'))))
async def execute_command_inner(command, stdout_file=None, stderr_file=None,
timeout=None, process_name=None,
......@@ -435,13 +451,14 @@ async def run_wait_any(batsim_command, sched_command,
sched_stdout_file, sched_stderr_file,
timeout=None, where_to_append_proc=None):
"""Asynchronously waits for the termination of Batsim or Sched."""
done, pending = await asyncio.wait([execute_command_inner(batsim_command,
batsim_stdout_file, batsim_stderr_file,
timeout, "Batsim", where_to_append_proc),
execute_command_inner(sched_command,
sched_stdout_file, sched_stderr_file,
timeout, "Sched", where_to_append_proc)],
return_when=asyncio.FIRST_COMPLETED)
done, pending = await asyncio.wait([
execute_command_inner(batsim_command,
batsim_stdout_file, batsim_stderr_file,
timeout, "Batsim", where_to_append_proc),
execute_command_inner(sched_command,
sched_stdout_file, sched_stderr_file,
timeout, "Sched", where_to_append_proc)],
return_when=asyncio.FIRST_COMPLETED)
return done, pending
......@@ -456,20 +473,22 @@ def kill_processes_and_all_descendants(proc_set):
"""Kill a set of processes and all their children."""
pids_to_kill = set()
for proc in proc_set:
cmd = "pstree {} -pal | cut -d',' -f2 | cut -d' ' -f1 | cut -d')' -f1".format(
proc.pid)
cmd = "pstree {} -pal | cut -d',' -f2 | cut -d' ' -f1 |" \
" cut -d')' -f1".format(proc.pid)
p = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)
for pid in [int(pid_str) for pid_str in p.stdout.decode('utf-8').replace('\n', ' ').strip().split(' ')]:
for pid in [int(pid_str) for pid_str in p.stdout.decode('utf-8')
.replace('\n', ' ').strip().split(' ')]:
pids_to_kill.add(pid)
assert(os.getpid() not in pids_to_kill)
if os.getpid() in pids_to_kill:
raise Exception('Trying to commit suicide with "kill"...')
logger.error("Killing remaining processes {}".format(pids_to_kill))
cmd = "kill -9 {}".format(' '.join([str(pid) for pid in pids_to_kill]))
p = subprocess.run(cmd, shell=True)
def execute_batsim_alone(batsim_command, batsim_stdout_file, batsim_stderr_file,
timeout=None):
def execute_batsim_alone(batsim_command, batsim_stdout_file,
batsim_stderr_file, timeout=None):
"""Execute Batsim and wait for its termination."""
loop = asyncio.get_event_loop()
proc_set = set()
......@@ -519,7 +538,8 @@ def execute_batsim_and_sched(batsim_command, sched_command,
sched_stdout_file, sched_stderr_file,
timeout, proc_set))
finished_tuples = [finished.result() for finished in done]
assert(len(finished_tuples) in {1, 2})
if len(finished_tuples) not in {1, 2}:
raise Exception('Unknown finished tuple received!')
if (len(done) == 2):
logger.info("Both processes finished at the same time!")
......@@ -530,8 +550,8 @@ def execute_batsim_and_sched(batsim_command, sched_command,
logger.error("{} finished (returncode={})".format(
finished_tuple[1],
finished_tuple[0].returncode))
display_process_output_on_error(finished_tuple[1],
*out_files[finished_tuple[1]])
display_process_output_on_error(
finished_tuple[1], *out_files[finished_tuple[1]])
else:
logger.info("{} finished".format(finished_tuple[1]))
return all_ok
......@@ -596,22 +616,22 @@ def execute_command(command,
make_file_executable(output_subscript_filename)
fake_command = os.path.abspath(output_subscript_filename)
create_file_from_command(fake_command,
output_filename=output_script_filename,
variables_definition_filename=variables_filename)
create_file_from_command(
fake_command, output_filename=output_script_filename,
variables_definition_filename=variables_filename)
else:
create_file_from_command(command=command,
output_filename=output_script_filename,
variables_definition_filename=variables_filename)
create_file_from_command(
command=command, output_filename=output_script_filename,
variables_definition_filename=variables_filename)
create_dir_if_not_exists(output_script_output_dir)
# Let's prepare the real command call
cmd = 'bash {f}'.format(f=output_script_filename)
stdout_file = open('{out}/{name}.stdout'.format(out=output_script_output_dir,
name=command_name), 'wb')
stderr_file = open('{out}/{name}.stderr'.format(out=output_script_output_dir,
name=command_name), 'wb')
stdout_file = open('{out}/{name}.stdout'.format(
out=output_script_output_dir, name=command_name), 'wb')
stderr_file = open('{out}/{name}.stderr'.format(
out=output_script_output_dir, name=command_name), 'wb')
# Let's run the command
logger.info("Executing command '{}'".format(command_name))
......@@ -689,9 +709,9 @@ def execute_one_instance(working_directory,
if not is_batexec:
logger.info('Sched command: "{}"'.format(sched_command))
create_file_from_command(command=sched_command,
output_filename=sched_script_filename,
variables_definition_filename=variables_filename)
create_file_from_command(
command=sched_command, output_filename=sched_script_filename,
variables_definition_filename=variables_filename)
if do_not_execute is True:
logger.info('Skipping the execution of the instance because '
......@@ -716,8 +736,8 @@ def execute_one_instance(working_directory,
# If batsim's socket is still opened, let's wait for it to close
socket_wait_timeout = 5
socket_usable = wait_for_batsim_socket_to_be_usable(sock=batsim_socket,
timeout=socket_wait_timeout)
socket_usable = wait_for_batsim_socket_to_be_usable(
sock=batsim_socket, timeout=socket_wait_timeout)
if not socket_usable:
logger.error("Socket {} is still busy (after timeout={})".format(
batsim_socket, socket_wait_timeout))
......@@ -798,7 +818,7 @@ Examples of such input files can be found in subdirectory instance_examples.
# Let's read the YAML file content to get the real parameters
desc_file = open(args.instance_description_filename, 'r')
desc_data = yaml.load(desc_file)
desc_data = yaml.safe_load(desc_file)
working_directory = os.getcwd()
output_directory = os.getcwd()
......@@ -807,8 +827,10 @@ Examples of such input files can be found in subdirectory instance_examples.
variables = {}
timeout = None
assert 'batsim_command' in desc_data
assert 'sched_command' in desc_data
if 'batsim_command' not in desc_data:
raise Exception('Cannot find "batsim_command"!')
if 'sched_command' not in desc_data:
raise Exception('Cannot find "sched_command"!')
batsim_command = str(desc_data['batsim_command'])
sched_command = str(desc_data['sched_command'])
......@@ -848,10 +870,8 @@ Examples of such input files can be found in subdirectory instance_examples.
sys.exit(-2)
# Let's correctly interpret the working_dir and output_dir values
(wd, od, interpreted_batsim_command) = retrieve_info_from_instance(variables,
var_decl_order,
"/tmp",
batsim_command)
(wd, od, interpreted_batsim_command) = retrieve_info_from_instance(
variables, var_decl_order, "/tmp", batsim_command)
# Let's update those values
variables['working_directory'] = working_directory = wd
......@@ -897,24 +917,26 @@ Examples of such input files can be found in subdirectory instance_examples.
commands_dir=pre_commands_dir,
name=command_name)
if not execute_command(command=commands_before_execution[command_id],
working_directory=working_directory,
variables_filename=variables_filename,
output_script_filename=output_command_filename,
output_subscript_filename=output_subscript_filename,
output_script_output_dir=pre_commands_output_dir,
command_name=command_name):
if not execute_command(
command=commands_before_execution[command_id],
working_directory=working_directory,
variables_filename=variables_filename,
output_script_filename=output_command_filename,
output_subscript_filename=output_subscript_filename,
output_script_output_dir=pre_commands_output_dir,
command_name=command_name):
sys.exit(1)
# Instance execution
if not execute_one_instance(working_directory=working_directory,
output_directory=output_directory,
batsim_command=batsim_command,
interpreted_batsim_command=interpreted_batsim_command,
sched_command=sched_command,
variables_filename=variables_filename,
timeout=timeout,
do_not_execute=do_not_execute):
if not execute_one_instance(
working_directory=working_directory,
output_directory=output_directory,
batsim_command=batsim_command,
interpreted_batsim_command=interpreted_batsim_command,
sched_command=sched_command,
variables_filename=variables_filename,
timeout=timeout,
do_not_execute=do_not_execute):
sys.exit(2)
# Commands after instance execution
......@@ -939,13 +961,14 @@ Examples of such input files can be found in subdirectory instance_examples.
commands_dir=post_commands_dir,
name=command_name)
if not execute_command(command=commands_after_execution[command_id],
working_directory=working_directory,
variables_filename=variables_filename,
output_script_filename=output_command_filename,
output_subscript_filename=output_subscript_filename,
output_script_output_dir=post_commands_output_dir,
command_name=command_name):
if not execute_command(
command=commands_after_execution[command_id],
working_directory=working_directory,
variables_filename=variables_filename,
output_script_filename=output_command_filename,
output_subscript_filename=output_subscript_filename,
output_script_output_dir=post_commands_output_dir,
command_name=command_name):
sys.exit(3)
# Everything went succesfully, let's return 0
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment