Improve error messages for ctd and viva (and more). Refactor run result handling

problemtools/checks/validators.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 import random
 import re
 import string
@@ -129,8 +128,7 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None:
             for flags_str in all_flags:
                 flags = flags_str.split()
                 for executable in executables:
-                    status, _ = executable.run(junk_file, args=flags, work_dir=work_dir)
-                    if os.WEXITSTATUS(status) != 42:
+                    if executable.run(junk_file, args=flags, work_dir=work_dir).exitcode != 42:
                         break
                 else:
                     diag.warning(f'No validator rejects {desc} with flags "{" ".join(flags)}"')
@@ -149,8 +147,7 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab
                 for flags_str in all_flags:
                     flags = flags_str.split()
                     for executable in executables:
-                        status, _ = executable.run(modified_file, args=flags, work_dir=work_dir)
-                        if os.WEXITSTATUS(status) != 42:
+                        if executable.run(modified_file, args=flags, work_dir=work_dir).exitcode != 42:
                             # expected behavior; validator rejects modified input
                             return False
 
@@ -192,18 +189,14 @@ def _run_input_validators(validators: InputValidators, testcase: TestCase, work_
         if executable is None:
             continue
 
-        with tempfile.NamedTemporaryFile(dir=work_dir) as outfile, tempfile.NamedTemporaryFile(dir=work_dir) as errfile:
-            status, _ = executable.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir)
-            if not os.WIFEXITED(status):
-                emsg = f'Input format validator {val} crashed on input {testcase.infile}'
-            elif os.WEXITSTATUS(status) != 42:
-                emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {os.WEXITSTATUS(status)}'
-            else:
-                continue
-            validator_stdout = outfile.read().decode('utf-8', 'replace')
-            validator_stderr = errfile.read().decode('utf-8', 'replace')
-            validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out)
-            errors.append((emsg, validator_output))
+        result = executable.run_capture(work_dir, testcase.infile, args=flags, work_dir=work_dir)
+        if result.exitcode is None:
+            emsg = f'Input format validator {val} crashed on input {testcase.infile} ({result})'
+        elif result.exitcode != 42:
+            emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {result.exitcode}'
+        else:
+            continue
+        errors.append((emsg, result.output))
     return errors
 
 

problemtools/judge/execute.py

@@ -27,7 +27,7 @@
 from ..diagnostics import Diagnostics
 from ..metadata import Metadata
 from ..model import TestCase
-from ..run import Executable, get_tool
+from ..run import Executable, RunResult, get_tool
 from .result import SubmissionResult
 from .validate import _parse_validator_result, _validate_output
 
@@ -36,24 +36,12 @@
 _INTERACTIVE = get_tool('interactive').executable
 
 
-def _is_TLE(status: int, may_signal_with_usr1: bool = False) -> bool:
-    return os.WIFSIGNALED(status) and (
-        os.WTERMSIG(status) == signal.SIGXCPU or (may_signal_with_usr1 and os.WTERMSIG(status) == signal.SIGUSR1)
-    )
-
-
-def _is_RTE(status: int) -> bool:
-    return not os.WIFEXITED(status) or bool(os.WEXITSTATUS(status))
+def _is_TLE(result: RunResult, may_signal_with_usr1: bool = False) -> bool:
+    return result.signal == signal.SIGXCPU or (may_signal_with_usr1 and result.signal == signal.SIGUSR1)
 
 
-def _rte_reason(status: int) -> str:
-    if os.WIFSIGNALED(status):
-        sig = os.WTERMSIG(status)
-        try:
-            return signal.Signals(sig).name
-        except ValueError:
-            return f'SIG{sig}'
-    return f'Exit {os.WEXITSTATUS(status)}'
+def _is_RTE(result: RunResult) -> bool:
+    return result.exitcode != 0
 
 
 def _read_safe(path: Path) -> str | None:
@@ -76,21 +64,21 @@ def _run_normal(
     """Run a submission once (non-interactive)"""
     outfile = execution_dir / 'submission_stdout'
     errfile = execution_dir / 'submission_stderr'
-    status, runtime = sub.run(
+    sub_result = sub.run(
         infile=infile,
         outfile=outfile,
         errfile=errfile,
         timelim=math.ceil(timelim) + 1,
         memlim=metadata.limits.memory,
         work_dir=sub.build_dir,
     )
-    if _is_TLE(status) or runtime > timelim:
+    if _is_TLE(sub_result) or sub_result.runtime > timelim:
         result = SubmissionResult('TLE')
-    elif _is_RTE(status):
-        result = SubmissionResult('RTE', reason=_rte_reason(status), additional_info=_read_safe(errfile))
+    elif _is_RTE(sub_result):
+        result = SubmissionResult('RTE', reason=str(sub_result), additional_info=_read_safe(errfile))
     else:
         result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, diag, infile=infile)
-    result.runtime = runtime
+    result.runtime = sub_result.runtime
     return result
 
 
@@ -112,7 +100,7 @@ def _run_interactive(
     feedback_dir = execution_dir / 'feedback'
     interactive_out = execution_dir / 'interactive_output'
 
-    i_status, _ = _INTERACTIVE.run(
+    i_result = _INTERACTIVE.run(
         outfile=interactive_out,
         args=(
             ['1', str(math.ceil(2 * timelim))]
@@ -125,9 +113,9 @@ def _run_interactive(
         work_dir=sub.build_dir,
     )
 
-    if _is_RTE(i_status):
-        diag.error(f'Interactive runner crashed, status {i_status}')
-        return SubmissionResult('JE', reason=f'Interactive runner crashed, status {i_status}')
+    if _is_RTE(i_result):
+        diag.error(f'Interactive runner failed ({i_result})')
+        return SubmissionResult('JE', reason=f'Interactive runner failed ({i_result})')
 
     output = interactive_out.read_text()
     diag.debug(f'Interactive output: "{output}"')
@@ -136,26 +124,26 @@ def _run_interactive(
         diag.error(f'Interactive runner produced unexpected output: "{output}"')
         return SubmissionResult('JE', reason=f'Interactive runner produced unexpected output: "{output}"')
 
-    val_status_str, _, sub_status_str, sub_runtime_str, first = output.split()
-    val_status = int(val_status_str)
-    sub_status = int(sub_status_str)
-    sub_runtime = float(sub_runtime_str)
+    val_status_str, val_runtime_str, sub_status_str, sub_runtime_str, first = output.split()
+    val_result = RunResult(int(val_status_str), float(val_runtime_str))
+    sub_result = RunResult(int(sub_status_str), float(sub_runtime_str))
+    sub_runtime = sub_result.runtime
 
-    val_JE = not os.WIFEXITED(val_status) or os.WEXITSTATUS(val_status) not in [42, 43]
-    val_WA = os.WIFEXITED(val_status) and os.WEXITSTATUS(val_status) == 43
+    val_JE = val_result.exitcode not in [42, 43]
+    val_WA = val_result.exitcode == 43
 
     if val_JE or (val_WA and first == 'validator'):
         # Validator crashed or exited first with WA — follow validator verdict.
         # Cap runtime, as the submission can behave erratically and time out
         # after the validator exited.
-        result = _parse_validator_result(output_validator, val_status, feedback_dir, metadata)
+        result = _parse_validator_result(output_validator, val_result, feedback_dir, metadata)
         sub_runtime = min(sub_runtime, timelim)
-    elif _is_TLE(sub_status, may_signal_with_usr1=True) or sub_runtime > timelim:
+    elif _is_TLE(sub_result, may_signal_with_usr1=True) or sub_runtime > timelim:
         result = SubmissionResult('TLE')
-    elif _is_RTE(sub_status):
-        result = SubmissionResult('RTE', reason=_rte_reason(sub_status))
+    elif _is_RTE(sub_result):
+        result = SubmissionResult('RTE', reason=str(sub_result))
     else:
-        result = _parse_validator_result(output_validator, val_status, feedback_dir, metadata)
+        result = _parse_validator_result(output_validator, val_result, feedback_dir, metadata)
 
     result.runtime = sub_runtime
     result.validator_first = first == 'validator'

problemtools/judge/grade.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 import re
 import tempfile
 from pathlib import Path
@@ -40,31 +39,20 @@ def grade_group(
 
     with tempfile.TemporaryDirectory(dir=base_dir) as tmpdir:
         infile = Path(tmpdir) / 'grader_in'
-        outfile = Path(tmpdir) / 'grader_out'
-        errfile = Path(tmpdir) / 'grader_err'
         infile.write_text(grader_input)
+        result = grader_executable.run_capture(Path(tmpdir), infile, args=grader_flags)
 
-        status, _runtime = grader_executable.run(infile, outfile, errfile, args=grader_flags)
-
-        grader_output = outfile.read_text(errors='replace') if outfile.exists() else ''
-        stderr_content = errfile.read_text(errors='replace') if errfile.exists() else ''
-
-    if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
-        if not os.WIFEXITED(status):
-            diag.error(f'Judge error: {grader} crashed')
-        else:
-            diag.error(f'Judge error: exit code {os.WEXITSTATUS(status)} for grader {grader}, expected 0')
-        if stderr_content:
-            diag.error(f'Grader stderr:\n{stderr_content}')
+    if result.exitcode != 0:
+        diag.error(f'Judge error: grader {grader} failed ({result}), expected exit code 0', result.stderr)
         diag.debug(f'Grader input:\n{grader_input}')
         return ('JE', None)
 
-    if not _GRADER_OUTPUT_RE.match(grader_output):
+    if not _GRADER_OUTPUT_RE.match(result.stdout):
         diag.error('Judge error: invalid format of grader output')
         diag.debug(f'Output must match: "{_GRADER_OUTPUT_RE.pattern}"')
-        diag.debug(f'Output was: "{grader_output}"')
+        diag.debug(f'Output was: "{result.stdout}"')
         return ('JE', None)
 
-    verdict_str, score_str = grader_output.split()
+    verdict_str, score_str = result.stdout.split()
     diag.debug(f'Grader result: {verdict_str} ({score_str})')
     return cast(Verdict, verdict_str), float(score_str)

problemtools/judge/validate.py

@@ -7,7 +7,7 @@
 from ..diagnostics import Diagnostics
 from ..metadata import Metadata
 from ..model import TestCase
-from ..run import Executable
+from ..run import Executable, RunResult
 from .result import SubmissionResult
 
 
@@ -28,28 +28,20 @@ def _get_feedback(feedback_dir: Path) -> str | None:
 
 def _parse_validator_result(
     val: Executable,
-    status: int,
+    val_result: RunResult,
     feedback_dir: Path,
     metadata: Metadata,
 ) -> SubmissionResult:
-    if not os.WIFEXITED(status):
+    if val_result.exitcode not in [42, 43]:
         return SubmissionResult(
             'JE',
-            reason=f'output validator {val} crashed, status {status}',
-            additional_info=_get_feedback(feedback_dir),
-        )
-
-    ret = os.WEXITSTATUS(status)
-    if ret not in [42, 43]:
-        return SubmissionResult(
-            'JE',
-            reason=f'output validator {val} exited with status {ret}',
+            reason=f'output validator {val} failed ({val_result})',
             additional_info=_get_feedback(feedback_dir),
         )
 
     score_file = feedback_dir / 'score.txt'
 
-    if ret == 43:
+    if val_result.exitcode == 43:
         if score_file.is_file():
             return SubmissionResult('JE', reason='validator produced "score.txt" on a WA verdict')
         return SubmissionResult('WA', additional_info=_get_feedback(feedback_dir))
@@ -95,7 +87,7 @@ def _validate_output(
 
     val_stdout = execution_dir / 'val_stdout'
     val_stderr = execution_dir / 'val_stderr'
-    status, _ = output_validator.run(
+    val_result = output_validator.run(
         infile=submission_output,
         args=[str(effective_infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + flags,
         timelim=val_timelim,
@@ -109,7 +101,7 @@ def _validate_output(
                 diag.debug(f'Validator {label}: {content}')
         except OSError as e:
             diag.info(f'Failed to read validator output: {e}')
-    return _parse_validator_result(output_validator, status, feedback_dir, metadata)
+    return _parse_validator_result(output_validator, val_result, feedback_dir, metadata)
 
 
 def validate_output(

problemtools/run/__init__.py

@@ -4,7 +4,9 @@
 
 from .buildrun import BuildRun as BuildRun
 from .checktestdata import Checktestdata as Checktestdata
+from .executable import CapturedRunResult as CapturedRunResult
 from .executable import Executable as Executable
+from .executable import RunResult as RunResult
 from .program import CompileResult as CompileResult
 from .program import Program as Program
 from .source import SourceCode as SourceCode

problemtools/run/checktestdata.py

@@ -2,7 +2,6 @@
 verification language (https://github.com/DOMjudge/checktestdata)
 """
 
-import os
 import sys
 from pathlib import Path
 
@@ -31,8 +30,8 @@ def __init__(self, path: Path) -> None:
 
     def _do_compile(self, work_dir: Path) -> CompileResult:
         """Syntax-check the Checktestdata script"""
-        (status, _) = self._executable.run()
+        result = self._executable.run_capture(work_dir)
         # Checktestdata accepting (42 after swapping) or rejecting (1) the empty input both mean the syntax is fine.
-        if os.WIFEXITED(status) and os.WEXITSTATUS(status) in [42, 1]:
+        if result.exitcode in [42, 1]:
             return CompileResult(executable=self._executable)
-        return CompileResult(errmsg='Checktestdata syntax check failed')
+        return CompileResult(errmsg=f'Checktestdata syntax check failed:\n{result.output}')