diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index 1dba8fef..edeea4b9 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import random import re import string @@ -129,8 +128,7 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None: for flags_str in all_flags: flags = flags_str.split() for executable in executables: - status, _ = executable.run(junk_file, args=flags, work_dir=work_dir) - if os.WEXITSTATUS(status) != 42: + if executable.run(junk_file, args=flags, work_dir=work_dir).exitcode != 42: break else: diag.warning(f'No validator rejects {desc} with flags "{" ".join(flags)}"') @@ -149,8 +147,7 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab for flags_str in all_flags: flags = flags_str.split() for executable in executables: - status, _ = executable.run(modified_file, args=flags, work_dir=work_dir) - if os.WEXITSTATUS(status) != 42: + if executable.run(modified_file, args=flags, work_dir=work_dir).exitcode != 42: # expected behavior; validator rejects modified input return False @@ -192,18 +189,14 @@ def _run_input_validators(validators: InputValidators, testcase: TestCase, work_ if executable is None: continue - with tempfile.NamedTemporaryFile(dir=work_dir) as outfile, tempfile.NamedTemporaryFile(dir=work_dir) as errfile: - status, _ = executable.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir) - if not os.WIFEXITED(status): - emsg = f'Input format validator {val} crashed on input {testcase.infile}' - elif os.WEXITSTATUS(status) != 42: - emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {os.WEXITSTATUS(status)}' - else: - continue - validator_stdout = outfile.read().decode('utf-8', 'replace') - validator_stderr = errfile.read().decode('utf-8', 'replace') - validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out) - errors.append((emsg, validator_output)) + result = executable.run_capture(work_dir, testcase.infile, args=flags, work_dir=work_dir) + if result.exitcode is None: + emsg = f'Input format validator {val} crashed on input {testcase.infile} ({result})' + elif result.exitcode != 42: + emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {result.exitcode}' + else: + continue + errors.append((emsg, result.output)) return errors diff --git a/problemtools/judge/execute.py b/problemtools/judge/execute.py index 10776182..69a2f5fa 100644 --- a/problemtools/judge/execute.py +++ b/problemtools/judge/execute.py @@ -27,7 +27,7 @@ from ..diagnostics import Diagnostics from ..metadata import Metadata from ..model import TestCase -from ..run import Executable, get_tool +from ..run import Executable, RunResult, get_tool from .result import SubmissionResult from .validate import _parse_validator_result, _validate_output @@ -36,24 +36,12 @@ _INTERACTIVE = get_tool('interactive').executable -def _is_TLE(status: int, may_signal_with_usr1: bool = False) -> bool: - return os.WIFSIGNALED(status) and ( - os.WTERMSIG(status) == signal.SIGXCPU or (may_signal_with_usr1 and os.WTERMSIG(status) == signal.SIGUSR1) - ) - - -def _is_RTE(status: int) -> bool: - return not os.WIFEXITED(status) or bool(os.WEXITSTATUS(status)) +def _is_TLE(result: RunResult, may_signal_with_usr1: bool = False) -> bool: + return result.signal == signal.SIGXCPU or (may_signal_with_usr1 and result.signal == signal.SIGUSR1) -def _rte_reason(status: int) -> str: - if os.WIFSIGNALED(status): - sig = os.WTERMSIG(status) - try: - return signal.Signals(sig).name - except ValueError: - return f'SIG{sig}' - return f'Exit {os.WEXITSTATUS(status)}' +def _is_RTE(result: RunResult) -> bool: + return result.exitcode != 0 def _read_safe(path: Path) -> str | None: @@ -76,7 +64,7 @@ def _run_normal( """Run a submission once (non-interactive)""" outfile = execution_dir / 'submission_stdout' errfile = execution_dir / 'submission_stderr' - status, runtime = sub.run( + sub_result = sub.run( infile=infile, outfile=outfile, errfile=errfile, @@ -84,13 +72,13 @@ def _run_normal( memlim=metadata.limits.memory, work_dir=sub.build_dir, ) - if _is_TLE(status) or runtime > timelim: + if _is_TLE(sub_result) or sub_result.runtime > timelim: result = SubmissionResult('TLE') - elif _is_RTE(status): - result = SubmissionResult('RTE', reason=_rte_reason(status), additional_info=_read_safe(errfile)) + elif _is_RTE(sub_result): + result = SubmissionResult('RTE', reason=str(sub_result), additional_info=_read_safe(errfile)) else: result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, diag, infile=infile) - result.runtime = runtime + result.runtime = sub_result.runtime return result @@ -112,7 +100,7 @@ def _run_interactive( feedback_dir = execution_dir / 'feedback' interactive_out = execution_dir / 'interactive_output' - i_status, _ = _INTERACTIVE.run( + i_result = _INTERACTIVE.run( outfile=interactive_out, args=( ['1', str(math.ceil(2 * timelim))] @@ -125,9 +113,9 @@ def _run_interactive( work_dir=sub.build_dir, ) - if _is_RTE(i_status): - diag.error(f'Interactive runner crashed, status {i_status}') - return SubmissionResult('JE', reason=f'Interactive runner crashed, status {i_status}') + if _is_RTE(i_result): + diag.error(f'Interactive runner failed ({i_result})') + return SubmissionResult('JE', reason=f'Interactive runner failed ({i_result})') output = interactive_out.read_text() diag.debug(f'Interactive output: "{output}"') @@ -136,26 +124,26 @@ def _run_interactive( diag.error(f'Interactive runner produced unexpected output: "{output}"') return SubmissionResult('JE', reason=f'Interactive runner produced unexpected output: "{output}"') - val_status_str, _, sub_status_str, sub_runtime_str, first = output.split() - val_status = int(val_status_str) - sub_status = int(sub_status_str) - sub_runtime = float(sub_runtime_str) + val_status_str, val_runtime_str, sub_status_str, sub_runtime_str, first = output.split() + val_result = RunResult(int(val_status_str), float(val_runtime_str)) + sub_result = RunResult(int(sub_status_str), float(sub_runtime_str)) + sub_runtime = sub_result.runtime - val_JE = not os.WIFEXITED(val_status) or os.WEXITSTATUS(val_status) not in [42, 43] - val_WA = os.WIFEXITED(val_status) and os.WEXITSTATUS(val_status) == 43 + val_JE = val_result.exitcode not in [42, 43] + val_WA = val_result.exitcode == 43 if val_JE or (val_WA and first == 'validator'): # Validator crashed or exited first with WA — follow validator verdict. # Cap runtime, as the submission can behave erratically and time out # after the validator exited. - result = _parse_validator_result(output_validator, val_status, feedback_dir, metadata) + result = _parse_validator_result(output_validator, val_result, feedback_dir, metadata) sub_runtime = min(sub_runtime, timelim) - elif _is_TLE(sub_status, may_signal_with_usr1=True) or sub_runtime > timelim: + elif _is_TLE(sub_result, may_signal_with_usr1=True) or sub_runtime > timelim: result = SubmissionResult('TLE') - elif _is_RTE(sub_status): - result = SubmissionResult('RTE', reason=_rte_reason(sub_status)) + elif _is_RTE(sub_result): + result = SubmissionResult('RTE', reason=str(sub_result)) else: - result = _parse_validator_result(output_validator, val_status, feedback_dir, metadata) + result = _parse_validator_result(output_validator, val_result, feedback_dir, metadata) result.runtime = sub_runtime result.validator_first = first == 'validator' diff --git a/problemtools/judge/grade.py b/problemtools/judge/grade.py index 2608f256..73fdac6a 100644 --- a/problemtools/judge/grade.py +++ b/problemtools/judge/grade.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import tempfile from pathlib import Path @@ -40,31 +39,20 @@ def grade_group( with tempfile.TemporaryDirectory(dir=base_dir) as tmpdir: infile = Path(tmpdir) / 'grader_in' - outfile = Path(tmpdir) / 'grader_out' - errfile = Path(tmpdir) / 'grader_err' infile.write_text(grader_input) + result = grader_executable.run_capture(Path(tmpdir), infile, args=grader_flags) - status, _runtime = grader_executable.run(infile, outfile, errfile, args=grader_flags) - - grader_output = outfile.read_text(errors='replace') if outfile.exists() else '' - stderr_content = errfile.read_text(errors='replace') if errfile.exists() else '' - - if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: - if not os.WIFEXITED(status): - diag.error(f'Judge error: {grader} crashed') - else: - diag.error(f'Judge error: exit code {os.WEXITSTATUS(status)} for grader {grader}, expected 0') - if stderr_content: - diag.error(f'Grader stderr:\n{stderr_content}') + if result.exitcode != 0: + diag.error(f'Judge error: grader {grader} failed ({result}), expected exit code 0', result.stderr) diag.debug(f'Grader input:\n{grader_input}') return ('JE', None) - if not _GRADER_OUTPUT_RE.match(grader_output): + if not _GRADER_OUTPUT_RE.match(result.stdout): diag.error('Judge error: invalid format of grader output') diag.debug(f'Output must match: "{_GRADER_OUTPUT_RE.pattern}"') - diag.debug(f'Output was: "{grader_output}"') + diag.debug(f'Output was: "{result.stdout}"') return ('JE', None) - verdict_str, score_str = grader_output.split() + verdict_str, score_str = result.stdout.split() diag.debug(f'Grader result: {verdict_str} ({score_str})') return cast(Verdict, verdict_str), float(score_str) diff --git a/problemtools/judge/validate.py b/problemtools/judge/validate.py index 85d749dc..3529cb9c 100644 --- a/problemtools/judge/validate.py +++ b/problemtools/judge/validate.py @@ -7,7 +7,7 @@ from ..diagnostics import Diagnostics from ..metadata import Metadata from ..model import TestCase -from ..run import Executable +from ..run import Executable, RunResult from .result import SubmissionResult @@ -28,28 +28,20 @@ def _get_feedback(feedback_dir: Path) -> str | None: def _parse_validator_result( val: Executable, - status: int, + val_result: RunResult, feedback_dir: Path, metadata: Metadata, ) -> SubmissionResult: - if not os.WIFEXITED(status): + if val_result.exitcode not in [42, 43]: return SubmissionResult( 'JE', - reason=f'output validator {val} crashed, status {status}', - additional_info=_get_feedback(feedback_dir), - ) - - ret = os.WEXITSTATUS(status) - if ret not in [42, 43]: - return SubmissionResult( - 'JE', - reason=f'output validator {val} exited with status {ret}', + reason=f'output validator {val} failed ({val_result})', additional_info=_get_feedback(feedback_dir), ) score_file = feedback_dir / 'score.txt' - if ret == 43: + if val_result.exitcode == 43: if score_file.is_file(): return SubmissionResult('JE', reason='validator produced "score.txt" on a WA verdict') return SubmissionResult('WA', additional_info=_get_feedback(feedback_dir)) @@ -95,7 +87,7 @@ def _validate_output( val_stdout = execution_dir / 'val_stdout' val_stderr = execution_dir / 'val_stderr' - status, _ = output_validator.run( + val_result = output_validator.run( infile=submission_output, args=[str(effective_infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + flags, timelim=val_timelim, @@ -109,7 +101,7 @@ def _validate_output( diag.debug(f'Validator {label}: {content}') except OSError as e: diag.info(f'Failed to read validator output: {e}') - return _parse_validator_result(output_validator, status, feedback_dir, metadata) + return _parse_validator_result(output_validator, val_result, feedback_dir, metadata) def validate_output( diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index fdf92f0f..71dac6c6 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -4,7 +4,9 @@ from .buildrun import BuildRun as BuildRun from .checktestdata import Checktestdata as Checktestdata +from .executable import CapturedRunResult as CapturedRunResult from .executable import Executable as Executable +from .executable import RunResult as RunResult from .program import CompileResult as CompileResult from .program import Program as Program from .source import SourceCode as SourceCode diff --git a/problemtools/run/checktestdata.py b/problemtools/run/checktestdata.py index 750bbeef..16f90691 100644 --- a/problemtools/run/checktestdata.py +++ b/problemtools/run/checktestdata.py @@ -2,7 +2,6 @@ verification language (https://github.com/DOMjudge/checktestdata) """ -import os import sys from pathlib import Path @@ -31,8 +30,8 @@ def __init__(self, path: Path) -> None: def _do_compile(self, work_dir: Path) -> CompileResult: """Syntax-check the Checktestdata script""" - (status, _) = self._executable.run() + result = self._executable.run_capture(work_dir) # Checktestdata accepting (42 after swapping) or rejecting (1) the empty input both mean the syntax is fine. - if os.WIFEXITED(status) and os.WEXITSTATUS(status) in [42, 1]: + if result.exitcode in [42, 1]: return CompileResult(executable=self._executable) - return CompileResult(errmsg='Checktestdata syntax check failed') + return CompileResult(errmsg=f'Checktestdata syntax check failed:\n{result.output}') diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index fafaf0c0..aa8d45fd 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -4,6 +4,8 @@ import os import resource import signal +import tempfile +from dataclasses import dataclass from pathlib import Path from . import limit @@ -13,6 +15,50 @@ DEV_NULL = Path(os.devnull) +@dataclass(frozen=True) +class RunResult: + """Outcome of running a program.""" + + status: int # Raw wait status of the process, as returned by os.wait4 + runtime: float # User+sys runtime of the process, in seconds + + @property + def exitcode(self) -> int | None: + """Exit code of the process, or None if it did not exit normally (e.g. was killed by a signal).""" + return os.WEXITSTATUS(self.status) if os.WIFEXITED(self.status) else None + + @property + def signal(self) -> int | None: + """Signal that terminated the process, or None if it was not killed by a signal.""" + return os.WTERMSIG(self.status) if os.WIFSIGNALED(self.status) else None + + def __str__(self) -> str: + if (sig := self.signal) is not None: + try: + return signal.Signals(sig).name + except ValueError: + return f'SIG{sig}' + if self.exitcode is not None: + return f'Exit {self.exitcode}' + # Should never happen: we only wait for terminated processes + return f'status {self.status}' + + +@dataclass(frozen=True) +class CapturedRunResult(RunResult): + """Outcome of running a program, including its output.""" + + stdout: str + stderr: str + + @property + def output(self) -> str: + """stdout followed by stderr, separated by a newline if both are non-empty.""" + if self.stdout and self.stderr: + return f'{self.stdout}\n{self.stderr}' + return self.stdout or self.stderr + + class Executable: """Something that can be run: the result of compiling a Program.""" @@ -86,7 +132,7 @@ def run( timelim: int = 1000, memlim: int = 1024, work_dir: Path | None = None, - ) -> tuple[int, float]: + ) -> RunResult: """Run the program. Args: @@ -99,9 +145,8 @@ def run( work_dir: directory to run the program in Returns: - pair (status, runtime): - status: exit status of the process - runtime: user+sys runtime of the process, in seconds + the RunResult of the process. If swap_exit_codes is set, exit + codes 0 and 42 are swapped in it. """ argv = self.get_runcmd(memlim=memlim) + (args if args is not None else []) if self._infile_as_arg: @@ -116,7 +161,40 @@ def run( status = 42 << 8 elif os.WEXITSTATUS(status) == 42: status = 0 - return status, runtime + return RunResult(status, runtime) + + def run_capture( + self, + tmp_dir: Path, + infile: Path = DEV_NULL, + args: list[str] | None = None, + timelim: int = 1000, + memlim: int = 1024, + work_dir: Path | None = None, + ) -> CapturedRunResult: + """Run the program, capturing its stdout and stderr. + + Args: + tmp_dir: directory in which to store the output while the + program runs. A uniquely named subdirectory is created in + it and removed afterwards, so it is safe to share between + concurrent runs. + other arguments: as for run() + + Returns: + the CapturedRunResult of the process, with output decoded as + UTF-8 (invalid bytes replaced). + """ + with tempfile.TemporaryDirectory(dir=tmp_dir) as capture_dir: + outfile = Path(capture_dir) / 'stdout' + errfile = Path(capture_dir) / 'stderr' + result = self.run(infile, outfile, errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir) + return CapturedRunResult( + result.status, + result.runtime, + stdout=outfile.read_text(encoding='utf-8', errors='replace'), + stderr=errfile.read_text(encoding='utf-8', errors='replace'), + ) def __run_wait( self, diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index 92f9bd17..a6c78bef 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -2,7 +2,6 @@ verification language (http://viva.vanb.org/). """ -import os from pathlib import Path from .executable import Executable @@ -38,8 +37,8 @@ def _do_compile(self, work_dir: Path) -> CompileResult: swap_exit_codes=True, infile_as_arg=True, ) - (status, _) = executable.run() + result = executable.run_capture(work_dir) # Without an input file, VIVA only checks the script, and accepts (42 after swapping) if the syntax is fine. - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: + if result.exitcode == 42: return CompileResult(executable=executable) - return CompileResult(errmsg='VIVA syntax check failed') + return CompileResult(errmsg=f'VIVA syntax check failed:\n{result.output}') diff --git a/tests/test_executable.py b/tests/test_executable.py new file mode 100644 index 00000000..b3055c03 --- /dev/null +++ b/tests/test_executable.py @@ -0,0 +1,44 @@ +from pathlib import Path + +from problemtools.run import Executable + + +def _sh(script: str, swap_exit_codes: bool = False) -> Executable: + return Executable('sh', Path('/bin/sh'), args=['-c', script], swap_exit_codes=swap_exit_codes) + + +def test_run_capture(tmp_path: Path) -> None: + result = _sh('echo out; echo err >&2; exit 3').run_capture(tmp_path) + assert result.exitcode == 3 + assert result.stdout == 'out\n' + assert result.stderr == 'err\n' + assert result.output == 'out\n\nerr\n' + assert list(tmp_path.iterdir()) == [] + + +def test_run_capture_output_single_stream(tmp_path: Path) -> None: + assert _sh('echo out').run_capture(tmp_path).output == 'out\n' + assert _sh('echo err >&2').run_capture(tmp_path).output == 'err\n' + + +def test_run_capture_reads_infile(tmp_path: Path) -> None: + infile = tmp_path / 'input' + infile.write_text('hello\n') + result = _sh('cat').run_capture(tmp_path, infile) + assert result.exitcode == 0 + assert result.stdout == 'hello\n' + + +def test_run_capture_swaps_exit_codes(tmp_path: Path) -> None: + assert _sh('exit 0', swap_exit_codes=True).run_capture(tmp_path).exitcode == 42 + + +def test_run_capture_signal(tmp_path: Path) -> None: + result = _sh('echo partial; kill -TERM $$').run_capture(tmp_path) + assert result.exitcode is None + assert str(result) == 'SIGTERM' + assert result.stdout == 'partial\n' + + +def test_run_capture_invalid_utf8(tmp_path: Path) -> None: + assert _sh(r"printf '\377'").run_capture(tmp_path).stdout == '�'