Commit 8ed5b001 authored by Guillaume Huard's avatar Guillaume Huard
Browse files

New execution method, has filesize limit and timeouts, to be tested.

parent 4dd7ad4c
......@@ -77,7 +77,7 @@ sub get_value($$$) {
my $description = shift;
my $name = shift;
my $default = shift;
if (exists($description->{$name})) {
if ($description && exists($description->{$name})) {
return $description->{$name};
} else {
return $default;
......@@ -103,22 +103,31 @@ sub run_command($$$) {
my $test_name = shift;
my $command = shift;
my $test = shift;
my @arguments = ( $command );
push @arguments, @{$test->{args}} if $test && exists($test->{args});
my $arguments = " ";
chmod(0755, "$command") || error("Cannot make $command executable : $!".
(exists($test->{compile})?"":" (and no 'compile' directive found)"));
my ($child_in, $child_out, $child_err);
$child_err = gensym;
my $pid = open3($child_in, $child_out, $child_err, @arguments)
|| error("Cannot run program $command for test $test_name");
print $child_in $test->{input} if $test && exists($test->{input});
close($child_in);
(exists($test->{compile})?"":" (and no 'compile' directive found)"));
if (exists($test->{args})) {
for my $arg (@{$test->{args}}) {
$arg =~ s/"/\\"/g;
$arguments .= '"'.$arg.'" ';
}
}
my $data = {};
$data->{output} = read_from_fd($child_out);
$data->{error} = read_from_fd($child_err);
waitpid $pid, 0;
$data->{code} = $?;
my $timeout = get_value($test, "timeout", 10);
my $IOlimit = get_value($test, "IOlimit", 10);
my $command_line = "ulimit -f$IOlimit;timeout -s9 $timeout $command$arguments>.output.txt 2>.error.txt";
if (exists($test->{input})) {
write_file(".input.txt", $test->{input});
$command_line .= " <.input.txt";
}
debug("Executing: $command_line\n");
# One should use the POSIX macros WIFEXITED, WEXITSTATUS, and so on, but it's too cumbersome
$data->{failure} = system("$command_line");
$data->{signal} = $data->{failure} & 0xFF;
$data->{code} = $data->{failure} >> 8;
$data->{output} = read_file(".output.txt");
$data->{error} = read_file(".error.txt");
return $data;
}
......@@ -182,7 +191,7 @@ sub run_special($$$$) {
debug("Compiling $name with @{$special->{files}}\n");
setup_env($name, $special);
my $data = run_special($name, '.compile', $special->{compile}, $special);
if ($data->{code}) {
if ($data->{failure}) {
dispatch_error("Cannot compile $name", $data, $soft_fail);
$data->{compilation_error} = 1;
return $data;
......@@ -201,7 +210,7 @@ sub run_special($$$$) {
} else {
debug("Executing $name\n");
$result = run_command($test_name, "./$name", $test);
if ($result->{code}) {
if ($result->{failure}) {
dispatch_error("Cannot execute $name", $result, $soft_fail);
}
}
......@@ -277,6 +286,17 @@ sub preformat($) {
return $line;
}
sub format_error($) {
my $data = shift;
my $result = "";
$result .= "code $data->{code}" if $data->{code};
if ($data->{signal}) {
$result .= " and " if $result;
$result .= "signal $data->{signal}";
}
return $result;
}
# runs all tests for a given testcase, returns the score (percentage)
sub perform_tests($$) {
my $program = shift;
......@@ -302,7 +322,8 @@ sub perform_tests($$) {
foreach my $part ('output', 'error') {
print STDERR "Setup $part :\n$setup->{$part}" if length($setup->{$part});
}
print STDERR "*** POSSIBLE ERROR : Setup exited with code $setup->{code}\n" if $setup->{code};
print STDERR "*** POSSIBLE ERROR : Setup exited with ".format_error($setup)."\n"
if $setup->{failure};
}
# runs program
......@@ -330,11 +351,11 @@ sub perform_tests($$) {
} else {
$match = [ 'output' ];
}
foreach my $part ('output', 'error', 'code') {
foreach my $part ('output', 'error', 'code', 'signal') {
if (scalar(grep(/$part/, @$match))) {
$test->{$part} = $answer->{$part};
} else {
if ($part eq 'code') {
if (($part eq 'code') || ($part eq 'signal')) {
debug("Solution $part :\n$answer->{$part}\n") if $answer->{$part};
} else {
debug("Solution $part :\n$answer->{$part}\n") if length($answer->{$part});
......@@ -353,7 +374,7 @@ sub perform_tests($$) {
ensure_write(".expected_$part.txt", $test->{$part});
}
my $valid = run_special($test_name, '.validator', $test->{validator}, $test);
$result = ($valid->{code} == 0);
$result = ($valid->{failure} == 0);
foreach my $part ('output', 'error') {
$details .= "Validator $part is:\n".preformat($valid->{$part})."\n" if length($valid->{$part});
}
......@@ -365,11 +386,12 @@ sub perform_tests($$) {
}
# compare produced output to expected one for each category
foreach my $part ('output', 'error', 'code') {
foreach my $part ('output', 'error', 'code', 'signal') {
my $description = {
output => "produced on standard output",
error => "produced on standard error",
code => "returned the code" };
code => "returned the code",
signal => "received the signal" };
if (defined($test->{$part}) || ($data->{$part})) {
$details .= "Your program $description->{$part}:\n".preformat($data->{$part});
if (defined($test->{$part})) {
......@@ -385,6 +407,13 @@ sub perform_tests($$) {
} else {
$details .= "\n";
}
if (!$result && ($part eq 'signal')) {
if ($data->{signal} == 9) {
$details .= "Looks like a timeout, did you write an infinite loop ?\n"
} elsif ($data->{signal} == 25) {
$details .= "Filesize limit exceeded, did you write an infinite loop ?\n";
}
}
}
}
if (!defined($result)) {
......
......@@ -130,5 +130,5 @@ else
internal_error "vpl_evaluate.pl missing"
fi
chmod +x vpl_evaluate.pl || internal_error "can't make vpl_evaluate.pl executable"
exec ./vpl_evaluate.pl $* || internal_error "can't exec vpl_evaluate.pl"
exec ./vpl_evaluate.pl $* $EVALUATE_MODE || internal_error "can't exec vpl_evaluate.pl"
fi
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment