From b707c9439d6cee53a3910df2eb6f01cee64fecbc Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 13:38:17 +0200 Subject: [PATCH 1/6] Separate Program's concerns of building and execution into separate classes When compiling, one now gets back an Executable, which can be run. Program is now an immutable representation of a program (with a small, thread-safe cache of compilation results). The old class Executable was no longer needed, so its name was stolen for the new class. --- problemtools/checks/validators.py | 20 ++-- problemtools/judge/execute.py | 21 ++-- problemtools/judge/grade.py | 5 +- problemtools/judge/validate.py | 5 +- problemtools/model/graders.py | 5 +- problemtools/model/validators.py | 5 +- problemtools/run/__init__.py | 2 + problemtools/run/buildrun.py | 44 +++---- problemtools/run/checktestdata.py | 61 +++------- problemtools/run/executable.py | 186 ++++++++++++++++++++++++++---- problemtools/run/program.py | 178 +++++----------------------- problemtools/run/source.py | 108 +++++++++-------- problemtools/run/tools.py | 2 +- problemtools/run/viva.py | 67 +++-------- 14 files changed, 339 insertions(+), 370 deletions(-) diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index 528591b2..a04e9eb7 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -20,7 +20,7 @@ from ..judge import SubmissionResult, validate_output from ..metadata import Metadata from ..model import InputValidators, OutputValidators, TestCase, TestDataGroup -from ..run import ProgramError, SourceCode +from ..run import Executable, ProgramError, SourceCode # Junk data. The validator should reject these cases _JUNK_CASES: list[tuple[str, bytes]] = [ @@ -102,11 +102,14 @@ def check_input_validators(validators: InputValidators, testdata: TestDataGroup, if len(validators.validators) == 0: diag.error('No input format validators found') + executables: list[Executable] = [] for val in validators.validators: try: result = val.compile(work_dir) - if not result.success: + if result.executable is None: diag.error(f'Compile error for {val}', result.errmsg) + else: + executables.append(result.executable) except ProgramError as e: diag.error(str(e)) @@ -128,8 +131,8 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None: with _temp_file_with(case, work_dir) as junk_file: for flags_str in all_flags: flags = flags_str.split() - for val in validators.validators: - status, _ = val.run(junk_file, args=flags, work_dir=work_dir) + for executable in executables: + status, _ = executable.run(junk_file, args=flags, work_dir=work_dir) if os.WEXITSTATUS(status) != 42: break else: @@ -148,8 +151,8 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab with _temp_file_with(modifier(infile_data).encode('utf8'), work_dir) as modified_file: for flags_str in all_flags: flags = flags_str.split() - for val in validators.validators: - status, _ = val.run(modified_file, args=flags, work_dir=work_dir) + for executable in executables: + status, _ = executable.run(modified_file, args=flags, work_dir=work_dir) if os.WEXITSTATUS(status) != 42: # expected behavior; validator rejects modified input return False @@ -188,11 +191,12 @@ def _run_input_validators(validators: InputValidators, testcase: TestCase, work_ for val in validators.validators: # A validator that failed to compile was already reported by check_input_validators; skip it. - if not val.compile(work_dir).success: + executable = val.compile(work_dir).executable + if executable is None: continue with tempfile.NamedTemporaryFile(dir=work_dir) as outfile, tempfile.NamedTemporaryFile(dir=work_dir) as errfile: - status, _ = val.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir) + status, _ = executable.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir) if not os.WIFEXITED(status): emsg = f'Input format validator {val} crashed on input {testcase.infile}' elif os.WEXITSTATUS(status) != 42: diff --git a/problemtools/judge/execute.py b/problemtools/judge/execute.py index 4f0d281f..9e379ef2 100644 --- a/problemtools/judge/execute.py +++ b/problemtools/judge/execute.py @@ -75,14 +75,15 @@ def _run_normal( """Run a submission once (non-interactive)""" outfile = execution_dir / 'submission_stdout' errfile = execution_dir / 'submission_stderr' - sub_path = sub.compile(base_dir).path - status, runtime = sub.run( + sub_executable = sub.compile(base_dir).executable + assert sub_executable is not None, f'{sub} must be successfully compiled before running' + status, runtime = sub_executable.run( infile=infile, outfile=outfile, errfile=errfile, timelim=math.ceil(timelim) + 1, memlim=metadata.limits.memory, - work_dir=sub_path, + work_dir=sub_executable.build_dir, ) if _is_TLE(status) or runtime > timelim: result = SubmissionResult('TLE') @@ -111,10 +112,12 @@ def _run_interactive( diag.error('Could not locate interactive runner') return SubmissionResult('JE', reason='Could not locate interactive runner') - if not output_validator.compile(base_dir).success: + validator_executable = output_validator.compile(base_dir).executable + if validator_executable is None: return SubmissionResult('JE', reason=f'output validator {output_validator} failed to compile') - sub_path = sub.compile(base_dir).path + sub_executable = sub.compile(base_dir).executable + assert sub_executable is not None, f'{sub} must be successfully compiled before running' feedback_dir = execution_dir / 'feedback' interactive_out = execution_dir / 'interactive_output' @@ -123,13 +126,13 @@ def _run_interactive( outfile=interactive_out, args=( ['1', str(math.ceil(2 * timelim))] - + output_validator.get_runcmd(memlim=metadata.limits.validation_memory) + + validator_executable.get_runcmd(memlim=metadata.limits.validation_memory) + [str(infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + testcase.output_validator_flags + [';'] - + sub.get_runcmd(memlim=metadata.limits.memory) + + sub_executable.get_runcmd(memlim=metadata.limits.memory) ), - work_dir=sub_path, + work_dir=sub_executable.build_dir, ) if _is_RTE(i_status): @@ -224,7 +227,7 @@ def execute_testcase( base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: - """Run sub on a single testcase.""" + """Run sub on a single testcase. sub must already have been successfully compiled to base_dir.""" with tempfile.TemporaryDirectory(dir=base_dir) as exec_dir: execution_dir = Path(exec_dir) (execution_dir / 'feedback').mkdir() diff --git a/problemtools/judge/grade.py b/problemtools/judge/grade.py index 1669b446..2608f256 100644 --- a/problemtools/judge/grade.py +++ b/problemtools/judge/grade.py @@ -29,7 +29,8 @@ def grade_group( if not sub_results: return ('AC', 0.0) - if not grader.compile(base_dir).success: + grader_executable = grader.compile(base_dir).executable + if grader_executable is None: diag.error(f'Failed to compile grader {grader}') return ('JE', None) @@ -43,7 +44,7 @@ def grade_group( errfile = Path(tmpdir) / 'grader_err' infile.write_text(grader_input) - status, _runtime = grader.run(infile, outfile, errfile, args=grader_flags) + status, _runtime = grader_executable.run(infile, outfile, errfile, args=grader_flags) grader_output = outfile.read_text(errors='replace') if outfile.exists() else '' stderr_content = errfile.read_text(errors='replace') if errfile.exists() else '' diff --git a/problemtools/judge/validate.py b/problemtools/judge/validate.py index 14a8512a..072d069e 100644 --- a/problemtools/judge/validate.py +++ b/problemtools/judge/validate.py @@ -94,11 +94,12 @@ def _validate_output( 'OLE', reason=f'output ({output_size:.1f} MiB) exceeds output limit ({metadata.limits.output} MiB)' ) - if not output_validator.compile(base_dir).success: + validator_executable = output_validator.compile(base_dir).executable + if validator_executable is None: return SubmissionResult('JE', reason=f'output validator {output_validator} failed to compile') val_stdout = execution_dir / 'val_stdout' val_stderr = execution_dir / 'val_stderr' - status, _ = output_validator.run( + status, _ = validator_executable.run( infile=submission_output, args=[str(effective_infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + flags, timelim=val_timelim, diff --git a/problemtools/model/graders.py b/problemtools/model/graders.py index 526e90d4..6c4545ed 100644 --- a/problemtools/model/graders.py +++ b/problemtools/model/graders.py @@ -4,10 +4,11 @@ from pathlib import Path from ..languages import Languages -from ..run import BuildRun, SourceCode, as_source_or_buildrun, find_programs, get_tool +from ..run import BuildRun, Prebuilt, SourceCode, as_source_or_buildrun, find_programs, get_tool from .paths import resolve -DEFAULT_GRADER = get_tool('default_grader') +_default_grader = get_tool('default_grader') +DEFAULT_GRADER = Prebuilt(_default_grader) if _default_grader is not None else None @dataclass(frozen=True) diff --git a/problemtools/model/validators.py b/problemtools/model/validators.py index 3330ff16..d07799e5 100644 --- a/problemtools/model/validators.py +++ b/problemtools/model/validators.py @@ -6,11 +6,12 @@ from ..formatversion import FormatVersion from ..languages import Languages from ..metadata import Metadata -from ..run import BuildRun, Program, SourceCode, as_source_or_buildrun, find_programs, get_tool +from ..run import BuildRun, Prebuilt, Program, SourceCode, as_source_or_buildrun, find_programs, get_tool from .paths import resolve #: The problemtools-provided validator used when a problem doesn't ship a custom output validator. -DEFAULT_VALIDATOR = get_tool('default_validator') +_default_validator = get_tool('default_validator') +DEFAULT_VALIDATOR = Prebuilt(_default_validator) if _default_validator is not None else None @dataclass(frozen=True) diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index c4f18067..20d66471 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -11,6 +11,8 @@ from .buildrun import BuildRun from .checktestdata import Checktestdata from .errors import ProgramError as ProgramError +from .executable import Executable as Executable +from .prebuilt import Prebuilt as Prebuilt from .program import CompileResult as CompileResult from .program import Program from .source import SourceCode diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index 45edb13c..15e8f284 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING from .errors import ProgramError +from .executable import Executable from .program import CompileResult, Program if TYPE_CHECKING: @@ -30,50 +31,33 @@ def __init__(self, name: str, files: 'ProgramFiles') -> None: super().__init__(name=name) self.files = files - def do_compile(self, work_dir: Path) -> CompileResult: + def _do_compile(self, work_dir: Path) -> CompileResult: """Set up the compile work-space (writing the build script and friends into work_dir) and run the build script.""" name = self.name - run_path = work_dir / name - if run_path.exists(): - run_path = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) + build_dir = work_dir / name + if build_dir.exists(): + build_dir = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) else: - run_path.mkdir(parents=True) - self._path = run_path + build_dir.mkdir(parents=True) try: - self.files.materialize(self.path) + self.files.materialize(build_dir) except OSError as e: - return CompileResult(False, f'Failed to write program files: {e}', self.path) + return CompileResult(errmsg=f'Failed to write program files: {e}') - build = self.path / 'build' + build = build_dir / 'build' if not build.is_file(): raise ProgramError(f'{self.name} does not have a build script') if not os.access(build, os.X_OK): raise ProgramError(f'{self.name}/build is not executable') try: - subprocess.check_output(['./build'], stderr=subprocess.STDOUT, cwd=self.path) + subprocess.check_output(['./build'], stderr=subprocess.STDOUT, cwd=build_dir) except subprocess.CalledProcessError as err: - return CompileResult(False, err.output.decode('utf8', 'replace'), self.path) + return CompileResult(errmsg=err.output.decode('utf8', 'replace')) - run = self.path / 'run' + run = build_dir / 'run' if not run.is_file() or not os.access(run, os.X_OK): - return CompileResult(False, 'build script did not produce an executable called "run"', self.path) - return CompileResult(True, None, self.path) - - def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: - """Run command for the program. - - Must not be called until compile() has been called. - - Args: - cwd: if not None, the run command is provided - relative to cwd (otherwise absolute paths are given). - """ - path = self.path if cwd is None else Path(os.path.relpath(self.path, cwd)) - return [str(path / 'run')] - - def should_skip_memory_rlimit(self) -> bool: - """Ugly hack (see program.py for details).""" - return True + return CompileResult(errmsg='build script did not produce an executable called "run"') + return CompileResult(executable=Executable(self.name, run, build_dir=build_dir, skip_memory_rlimit=True)) diff --git a/problemtools/run/checktestdata.py b/problemtools/run/checktestdata.py index 38f53856..750bbeef 100644 --- a/problemtools/run/checktestdata.py +++ b/problemtools/run/checktestdata.py @@ -7,10 +7,10 @@ from pathlib import Path from .executable import Executable -from .program import DEV_NULL, CompileResult +from .program import CompileResult, Program -class Checktestdata(Executable): +class Checktestdata(Program): """Wrapper class for running Checktestdata scripts.""" def __init__(self, path: Path) -> None: @@ -19,47 +19,20 @@ def __init__(self, path: Path) -> None: Args: path: path to .ctd source file """ - super().__init__(Path(sys.executable), args=['-m', 'checktestdata', str(path)], name=path.name) + super().__init__(name=path.name) + # Checktestdata exits with 0 on accept, so swap that with our accept exit status 42. + self._executable = Executable( + path.name, + Path(sys.executable), + args=['-m', 'checktestdata', str(path)], + skip_memory_rlimit=True, + swap_exit_codes=True, + ) - def do_compile(self, work_dir: Path) -> CompileResult: + def _do_compile(self, work_dir: Path) -> CompileResult: """Syntax-check the Checktestdata script""" - (status, _) = super().run() - success = os.WIFEXITED(status) and os.WEXITSTATUS(status) in [0, 1] - return CompileResult(success, None, self.path) - - def run( - self, - infile: Path = DEV_NULL, - outfile: Path = DEV_NULL, - errfile: Path = DEV_NULL, - args: list[str] | None = None, - timelim: int = 1000, - memlim: int = 1024, - work_dir: Path | None = None, - ) -> tuple[int, float]: - """Run the Checktestdata script to validate an input file. - - Args: - infile: input file to validate - outfile: file to save stdout of Checktestdata in - errfile: file to save stderr of Checktestdata in - args: additional command-line arguments to pass to Checktestdata - timelim: time limit for the Checktestdata process in seconds - - Returns: - tuple (status, runtime): - status: exit status of the validator. - WEXITSTATUS(status) will be 42 if and only if - Checktestdata accepted the input file. - runtime: runtime of the Checktestdata process in seconds - """ - (status, runtime) = super().run( - infile=infile, outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir - ) - # This is ugly, switches the accept exit status and our accept - # exit status 42. - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42 << 8, runtime) - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: - return (0, runtime) - return (status, runtime) + (status, _) = self._executable.run() + # Checktestdata accepting (42 after swapping) or rejecting (1) the empty input both mean the syntax is fine. + if os.WIFEXITED(status) and os.WEXITSTATUS(status) in [42, 1]: + return CompileResult(executable=self._executable) + return CompileResult(errmsg='Checktestdata syntax check failed') diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index 338051e0..6b499d54 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -1,36 +1,180 @@ -""" -Implementation of programs provided by an executable file. -""" +"""A compiled program, ready to be run.""" +import logging import os +import resource +import signal from pathlib import Path +from . import limit from .errors import ProgramError -from .program import Program +log = logging.getLogger(__name__) -class Executable(Program): - """Class for executable files.""" +DEV_NULL = Path(os.devnull) - def __init__(self, path: Path, args: list[str] | None = None, name: str | None = None) -> None: + +class Executable: + """Something that can be run: the result of compiling a Program.""" + + name: str # Human-readable name of the program + build_dir: Path | None # Directory the program was built in, if any + + def __init__( + self, + name: str, + binary: Path, + args: list[str] | None = None, + build_dir: Path | None = None, + skip_memory_rlimit: bool = False, + swap_exit_codes: bool = False, + infile_as_arg: bool = False, + ) -> None: """Instantiate executable object. Args: - path: path to the executable file. Must be a file, - and must be executable. - args: list of additional command line arguments that - should be passed to the program every time it is executed. - name: name to use for the program. Defaults to the basename of path. + name: human-readable name of the program. + binary: path to the file to execute. + args: additional command line arguments passed to the program + every time it is executed. + build_dir: directory the program was built in, if any. + skip_memory_rlimit: do not apply a memory rlimit when running + the program. Ugly workaround to accommodate Java -- the JVM + will crash and burn if there is a memory rlimit applied and + this will probably not change anytime soon [time of writing + this: 2017-02-05], see e.g.: + https://bugs.openjdk.java.net/browse/JDK-8071445 + 2019-02-22: Turns out sbcl for Common Lisp also wants to roam + free and becomes sad when reined in by a memory rlimit. + swap_exit_codes: swap exit codes 0 and 42, for programs which + exit with 0 on accept (e.g. Checktestdata and VIVA). + infile_as_arg: pass infile as the last command line argument + instead of on stdin. """ - if not path.is_file() or not os.access(path, os.X_OK): - raise ProgramError(f'{path} is not an executable program') - super().__init__(name=name if name is not None else path.name, path=path) - self.args = args if args is not None else [] + self.name = name + self.build_dir = build_dir + self._binary = binary + self._args = args if args is not None else [] + self._skip_memory_rlimit = skip_memory_rlimit + self._swap_exit_codes = swap_exit_codes + self._infile_as_arg = infile_as_arg + + def __str__(self) -> str: + return self.name def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: - """Command to run the program.""" - return [str(self.path)] + self.args + """Command to run the program. + + Args: + cwd: if not None, the run command is provided + relative to cwd (otherwise absolute paths are given). + Only properly supported for programs compiled from + SourceCode or BuildRun, with cwd a parent of build_dir: + paths in args (e.g. a Checktestdata script) are not made + relative. + memlim: memory limit in MiB (only relevant for + programs where memory limit is passed on command line) + """ + binary = self._binary if cwd is None else Path(os.path.relpath(self._binary, cwd)) + return [str(binary)] + self._args + + def run( + self, + infile: Path = DEV_NULL, + outfile: Path = DEV_NULL, + errfile: Path = DEV_NULL, + args: list[str] | None = None, + timelim: int = 1000, + memlim: int = 1024, + work_dir: Path | None = None, + ) -> tuple[int, float]: + """Run the program. + + Args: + infile: file to pass on stdin + outfile: file to send stdout to + errfile: file to send stderr to + args: additional command-line arguments to pass to the program + timelim: CPU time limit in seconds + memlim: memory limit in MiB + work_dir: directory to run the program in + + Returns: + pair (status, runtime): + status: exit status of the process + runtime: user+sys runtime of the process, in seconds + """ + runcmd = self.get_runcmd(memlim=memlim) + if runcmd == []: + raise ProgramError(f'Could not figure out how to run {self}') + argv = runcmd + (args if args is not None else []) + if self._infile_as_arg: + if infile != DEV_NULL: + argv.append(str(infile)) + infile = DEV_NULL + + status, runtime = self.__run_wait(argv, infile, outfile, errfile, timelim, memlim, work_dir) + + if self._swap_exit_codes and os.WIFEXITED(status): + if os.WEXITSTATUS(status) == 0: + status = 42 << 8 + elif os.WEXITSTATUS(status) == 42: + status = 0 + return status, runtime + + def __run_wait( + self, + argv: list[str], + infile: Path, + outfile: Path, + errfile: Path, + timelim: int, + memlim: int, + work_dir: Path | None, + ) -> tuple[int, float]: + log.debug('run "%s < %s > %s 2> %s"', ' '.join(argv), infile, outfile, errfile) + pid = os.fork() + if pid == 0: # child + try: + # The Python interpreter internally sets some signal dispositions + # to SIG_IGN (notably SIGPIPE), and unless we reset them manually + # this leaks through to the program we exec. That can has some + # funny side effects, like programs not crashing as expected when + # trying to write to an interactive validator that has terminated + # and closed the read end of a pipe. + # + # This *shouldn't* cause any verdict changes given the setup for + # interactive problems, but reset them anyway, for sanity. + if hasattr(signal, 'SIGPIPE'): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + if hasattr(signal, 'SIGXFZ'): + signal.signal(signal.SIGXFZ, signal.SIG_DFL) + if hasattr(signal, 'SIGXFSZ'): + signal.signal(signal.SIGXFSZ, signal.SIG_DFL) + + limit.try_limit(resource.RLIMIT_CPU, timelim, timelim + 1) + if not self._skip_memory_rlimit: + limit.try_limit(resource.RLIMIT_AS, memlim * (1024**2), resource.RLIM_INFINITY) + limit.try_limit(resource.RLIMIT_STACK, resource.RLIM_INFINITY, resource.RLIM_INFINITY) + + Executable.__setfd(0, infile, os.O_RDONLY) + Executable.__setfd(1, outfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + Executable.__setfd(2, errfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + if work_dir is not None: + os.chdir(work_dir) + os.execvp(argv[0], argv) + except Exception as exc: + print('Oops. Fatal error in child process:') + print(exc) + os.kill(os.getpid(), signal.SIGTERM) + # Unreachable + log.error('Unreachable part of run_wait reached') + os.kill(os.getpid(), signal.SIGTERM) + (pid, status, rusage) = os.wait4(pid, 0) + return status, rusage.ru_utime + rusage.ru_stime - def should_skip_memory_rlimit(self) -> bool: - """Ugly hack (see program.py for details).""" - return True + @staticmethod + def __setfd(fd: int, filename: Path, flag: int) -> None: + tmpfd = os.open(filename, flag) + os.dup2(tmpfd, fd) + os.close(tmpfd) diff --git a/problemtools/run/program.py b/problemtools/run/program.py index 163b2704..5c7bb562 100644 --- a/problemtools/run/program.py +++ b/problemtools/run/program.py @@ -1,193 +1,67 @@ """Abstract base class for programs.""" import dataclasses -import logging -import os -import resource -import signal import threading from abc import ABC, abstractmethod from pathlib import Path -from . import limit -from .errors import ProgramError - -log = logging.getLogger(__name__) - -DEV_NULL = Path(os.devnull) +from .executable import Executable @dataclasses.dataclass(frozen=True) class CompileResult: """Result of compiling a Program. - `path` is always set (even on a failed compile): it's the resolved path to - the program, established as a side effect of compiling. It's the only - reliable way to learn a program's path -- `Program.path` raises if read - before compile() has been called.""" + Exactly one of `executable` (on success) and `errmsg` (on failure) is set.""" + + executable: Executable | None = None + errmsg: str | None = None - success: bool - errmsg: str | None - path: Path + def __post_init__(self) -> None: + assert (self.executable is None) != (self.errmsg is None), 'CompileResult needs exactly one of executable and errmsg' + + @property + def success(self) -> bool: + return self.executable is not None class Program(ABC): """Abstract base class for programs.""" - def __init__(self, name: str, path: Path | None = None) -> None: + name: str # Human-readable name of the program + + def __init__(self, name: str) -> None: """Instantiate program object. Args: name: human-readable name of the program. - path: full path to the program, if already known (possibly in a - temporary directory). Subclasses for which the path isn't known - until compile time (e.g. source code that gets copied into a - work directory) should leave this unset and assign `self._path` - as part of `do_compile()`. """ - self._path = path self.name = name self._compile_lock = threading.Lock() - self._compile_result: CompileResult | None = None - - @property - def path(self) -> Path: - if self._path is None: - raise ProgramError(f'{self} has not been compiled yet') - return self._path + self._compile_results: dict[Path, CompileResult] = {} def __str__(self) -> str: return self.name - @abstractmethod - def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: - pass - - def run( - self, - infile: Path = DEV_NULL, - outfile: Path = DEV_NULL, - errfile: Path = DEV_NULL, - args: list[str] | None = None, - timelim: int = 1000, - memlim: int = 1024, - work_dir: Path | None = None, - ) -> tuple[int, float]: - """Run the program. - - Args: - infile: file to pass on stdin - outfile: file to send stdout to - errfile: file to send stderr to - args: additional command-line arguments to pass to the program - timelim: CPU time limit in seconds - memlim: memory limit in MiB - - Returns: - pair (status, runtime): - status: exit status of the process - runtime: user+sys runtime of the process, in seconds - """ - runcmd = self.get_runcmd(memlim=memlim) - if runcmd == []: - raise ProgramError(f'Could not figure out how to run {self}') - if args is None: - args = [] - - status, runtime = self.__run_wait(runcmd + args, infile, outfile, errfile, timelim, memlim, work_dir) - - return status, runtime - def compile(self, work_dir: Path) -> CompileResult: """Compile the program, if needed, and return the result. - Only the first call actually compiles; later calls (even with a different - work_dir) return the cached result. work_dir is only used by subclasses that - need a place to set up a compile workspace (i.e. source code); others ignore it. + Results are cached per work_dir, so compiling to a work_dir the + program has already been compiled to returns the earlier result. + work_dir is only used by subclasses that need a place to set up a + compile workspace (i.e. source code); others ignore it. """ + key = work_dir.resolve() with self._compile_lock: - if self._compile_result is None: - self._compile_result = self.do_compile(work_dir) - return self._compile_result + if key not in self._compile_results: + self._compile_results[key] = self._do_compile(work_dir) + return self._compile_results[key] - def do_compile(self, work_dir: Path) -> CompileResult: - """Actually compile the program, if needed. Subclasses should override this method. - Do not call this manually -- use compile() instead.""" - return CompileResult(True, None, self.path) + @abstractmethod + def _do_compile(self, work_dir: Path) -> CompileResult: + """Actually compile the program. Subclasses implement this, callers use compile().""" def code_size(self) -> int: """Subclasses should override this method with the total size of the source code.""" return 0 - - def should_skip_memory_rlimit(self) -> bool: - """Ugly workaround to accommodate Java -- the JVM will crash and burn - if there is a memory rlimit applied and this will probably not - change anytime soon [time of writing this: 2017-02-05], see - e.g.: https://bugs.openjdk.java.net/browse/JDK-8071445 - - Subclasses of Program where the associated program is (or may - be) a Java program need to override this method and return - True (which will cause the memory rlimit to not be applied). - - 2019-02-22: Turns out sbcl for Common Lisp also wants to roam - free and becomes sad when reined in by a memory rlimit. - """ - return False - - def __run_wait( - self, - argv: list[str], - infile: Path, - outfile: Path, - errfile: Path, - timelim: int, - memlim: int, - work_dir: Path | None, - ) -> tuple[int, float]: - log.debug('run "%s < %s > %s 2> %s"', ' '.join(argv), infile, outfile, errfile) - pid = os.fork() - if pid == 0: # child - try: - # The Python interpreter internally sets some signal dispositions - # to SIG_IGN (notably SIGPIPE), and unless we reset them manually - # this leaks through to the program we exec. That can has some - # funny side effects, like programs not crashing as expected when - # trying to write to an interactive validator that has terminated - # and closed the read end of a pipe. - # - # This *shouldn't* cause any verdict changes given the setup for - # interactive problems, but reset them anyway, for sanity. - if hasattr(signal, 'SIGPIPE'): - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - if hasattr(signal, 'SIGXFZ'): - signal.signal(signal.SIGXFZ, signal.SIG_DFL) - if hasattr(signal, 'SIGXFSZ'): - signal.signal(signal.SIGXFSZ, signal.SIG_DFL) - - limit.try_limit(resource.RLIMIT_CPU, timelim, timelim + 1) - if not self.should_skip_memory_rlimit(): - limit.try_limit(resource.RLIMIT_AS, memlim * (1024**2), resource.RLIM_INFINITY) - limit.try_limit(resource.RLIMIT_STACK, resource.RLIM_INFINITY, resource.RLIM_INFINITY) - - Program.__setfd(0, infile, os.O_RDONLY) - Program.__setfd(1, outfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - Program.__setfd(2, errfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - if work_dir is not None: - os.chdir(work_dir) - os.execvp(argv[0], argv) - except Exception as exc: - print('Oops. Fatal error in child process:') - print(exc) - os.kill(os.getpid(), signal.SIGTERM) - # Unreachable - log.error('Unreachable part of run_wait reached') - os.kill(os.getpid(), signal.SIGTERM) - (pid, status, rusage) = os.wait4(pid, 0) - return status, rusage.ru_utime + rusage.ru_stime - - @staticmethod - def __setfd(fd: int, filename: Path, flag: int) -> None: - tmpfd = os.open(filename, flag) - os.dup2(tmpfd, fd) - os.close(tmpfd) diff --git a/problemtools/run/source.py b/problemtools/run/source.py index 392e1b85..70bb9a06 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -2,6 +2,7 @@ Implementation of programs provided by source code. """ +import dataclasses import logging import os import subprocess @@ -10,7 +11,7 @@ from typing import TYPE_CHECKING from ..languages import CommandSubstitution, Language -from .errors import ProgramError +from .executable import Executable from .program import CompileResult, Program if TYPE_CHECKING: @@ -23,6 +24,7 @@ class SourceCode(Program): """Class representing a program provided by source code.""" files: 'ProgramFiles' # The program's own source files, not including any include files + language: Language # The programming language of the code def __init__(self, name: str, files: 'ProgramFiles', language: Language, includes: 'LanguageIncludes') -> None: """Instantiate SourceCode object @@ -49,89 +51,99 @@ def __init__(self, name: str, files: 'ProgramFiles', language: Language, include def code_size(self) -> int: return self.files.size() - def do_compile(self, work_dir: Path) -> CompileResult: + def _do_compile(self, work_dir: Path) -> CompileResult: """Set up the compile work-space (writing source and includes into work_dir) and compile the source code.""" name = self.name # Set up work-space - run_path = work_dir / name - if run_path.exists(): - run_path = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) + build_dir = work_dir / name + if build_dir.exists(): + build_dir = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) else: - run_path.mkdir(parents=True) - self._path = run_path + build_dir.mkdir(parents=True) all_files = self.files.merged(self._includes.files) try: - all_files.materialize(self.path) + all_files.materialize(build_dir) except OSError as e: - return CompileResult(False, f'Failed to write program files: {e}', self.path) + return CompileResult(errmsg=f'Failed to write program files: {e}') - self.src = [self.path / f for f in self.language.get_source_files([f.path for f in all_files.files])] - if len(self.src) == 0: - raise ProgramError(f'No source files found for language {self.language.lang_id} in {self.name}') + src = self.language.get_source_files([f.path for f in all_files.files]) + if len(src) == 0: + return CompileResult(errmsg=f'No source files found for language {self.language.lang_id}') if self._includes.mainfile is not None: - self.mainfile = self.path / self._includes.mainfile + mainfile: Path = self._includes.mainfile else: - candidates = self.language.mainfile_candidates(self.src) - self.mainfile = candidates[0] if candidates else self.src[0] - - self.mainclass = self.mainfile.stem - self.Mainclass = self.mainclass[0].upper() + self.mainclass[1:] - - self.binary = self.path / 'run' + candidates = self.language.mainfile_candidates(src) + mainfile = candidates[0] if candidates else src[0] + + mainclass = mainfile.stem + subs = CommandSubstitution( + path=str(build_dir), + files=' '.join(str(build_dir / f) for f in src), + binary=str(build_dir / 'run'), + mainfile=str(build_dir / mainfile), + mainclass=mainclass, + Mainclass=mainclass[0].upper() + mainclass[1:], + ) not_installed = self.language.check_installed() if not_installed is not None: - return CompileResult(False, not_installed, self.path) + return CompileResult(errmsg=not_installed) - command = self.language.get_compile_command(self.__get_substitution()) + executable = SourceExecutable(str(self), self.language, subs) + command = self.language.get_compile_command(subs) if command is None: - return CompileResult(True, None, self.path) + return CompileResult(executable=executable) log.debug('compile command: %s', command) try: subprocess.check_output(command, stderr=subprocess.STDOUT) - return CompileResult(True, None, self.path) + return CompileResult(executable=executable) except subprocess.CalledProcessError as err: - return CompileResult(False, err.output.decode('utf8', 'replace'), self.path) + return CompileResult(errmsg=err.output.decode('utf8', 'replace')) + + def __str__(self) -> str: + """String representation""" + return f'{self.name} ({self.language.name})' + + +class SourceExecutable(Executable): + """A compiled SourceCode program, run using its language's run command.""" + + def __init__(self, name: str, language: Language, subs: CommandSubstitution) -> None: + """Instantiate SourceExecutable object + + Args: + name: name of the program. + language: language definition for the programming language of the code. + subs: values to substitute into the language's run command (memlim + is overridden when running). + """ + super().__init__( + name, + Path(subs.binary), + build_dir=Path(subs.path), + skip_memory_rlimit=language.name in ['Java', 'Scala', 'Kotlin', 'Common Lisp'], + ) + self._language = language + self._subs = subs def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: """Run command for the program. - Must not be called until compile() has been called. - Args: cwd: if not None, the run command is provided relative to cwd (otherwise absolute paths are given). memlim: memory limit in MiB (only relevant for languages where memory limit is passed on command line) """ - subs = self.__get_substitution(memlim) + subs = dataclasses.replace(self._subs, memlim=memlim) if cwd is not None: subs.path = os.path.relpath(subs.path, cwd) subs.binary = os.path.relpath(subs.binary, cwd) subs.mainfile = os.path.relpath(subs.mainfile, cwd) - return self.language.get_run_command(subs) - - def should_skip_memory_rlimit(self) -> bool: - """Ugly hack (see program.py for details).""" - return self.language.name in ['Java', 'Scala', 'Kotlin', 'Common Lisp'] - - def __str__(self) -> str: - """String representation""" - return f'{self.name} ({self.language.name})' - - def __get_substitution(self, memlim: int = 1024) -> CommandSubstitution: - return CommandSubstitution( - path=str(self.path), - files=' '.join(str(f) for f in self.src), - memlim=memlim, - mainfile=str(self.mainfile), - mainclass=self.mainclass, - Mainclass=self.Mainclass, - binary=str(self.binary), - ) + return self._language.get_run_command(subs) diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index be8bfc1d..3a2e7224 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -35,7 +35,7 @@ def get_tool(name: str) -> Executable | None: the tool was not found. """ path = get_tool_path(name) - return Executable(path) if path is not None else None + return Executable(path.name, path, skip_memory_rlimit=True) if path is not None else None def __locate_executable(candidate_paths: list[Path]) -> Path | None: diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index 04d0d611..3ed37da3 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -7,11 +7,11 @@ from .errors import ProgramError from .executable import Executable -from .program import DEV_NULL, CompileResult +from .program import CompileResult, Program from .tools import get_tool_path -class Viva(Executable): +class Viva(Program): """Wrapper class for running VIVA scripts.""" _VIVA_PATH = get_tool_path('viva.sh') @@ -24,53 +24,22 @@ def __init__(self, path: Path) -> None: """ if Viva._VIVA_PATH is None: raise ProgramError(f'Could not locate the VIVA program to run {path}') - super().__init__(Viva._VIVA_PATH, args=[str(path)], name=path.name) + super().__init__(name=path.name) + # VIVA takes input as argument and not on stdin, and exits with 0 on + # accept, so swap that with our accept exit status 42. + self._executable = Executable( + path.name, + Viva._VIVA_PATH, + args=[str(path)], + skip_memory_rlimit=True, + swap_exit_codes=True, + infile_as_arg=True, + ) - def do_compile(self, work_dir: Path) -> CompileResult: + def _do_compile(self, work_dir: Path) -> CompileResult: """Syntax-check the VIVA script""" - (status, _) = super().run() - success = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 - return CompileResult(success, None, self.path) - - def run( - self, - infile: Path = DEV_NULL, - outfile: Path = DEV_NULL, - errfile: Path = DEV_NULL, - args: list[str] | None = None, - timelim: int = 1000, - memlim: int = 1024, - work_dir: Path | None = None, - ) -> tuple[int, float]: - """Run the VIVA script to validate an input file. - - Args: - infile: input file to validate - outfile: file to save stdout of VIVA in - errfile: file to save stderr of VIVA in - args: additional command-line arguments to pass to VIVA - timelim: time limit for the VIVA process in seconds - - Returns: - tuple (status, runtime): - status: exit status of the validator. - WEXITSTATUS(status) will be 42 if and only if VIVA - accepted the input file. - runtime: runtime of the VIVA process in seconds - """ - if args is None: - args = [] - # VIVA takes input as argument and not on stdin - if infile != DEV_NULL: - args = args + [str(infile)] - - (status, runtime) = super().run( - outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir - ) - # This is ugly, switches the accept exit status and our accept - # exit status 42. - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42 << 8, runtime) + (status, _) = self._executable.run() + # Without an input file, VIVA only checks the script, and accepts (42 after swapping) if the syntax is fine. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: - return (0, runtime) - return (status, runtime) + return CompileResult(executable=self._executable) + return CompileResult(errmsg='VIVA syntax check failed') From 22af97e05dc292147df7c835dfa73c446008daba Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 14:26:34 +0200 Subject: [PATCH 2/6] Drop ProgramError - it's all just normal compile errors now --- problemtools/checks/validators.py | 24 +++++++++--------------- problemtools/run/__init__.py | 1 - problemtools/run/buildrun.py | 5 ++--- problemtools/run/errors.py | 7 ------- problemtools/run/executable.py | 6 +----- problemtools/run/viva.py | 22 +++++++++++----------- 6 files changed, 23 insertions(+), 42 deletions(-) delete mode 100644 problemtools/run/errors.py diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index a04e9eb7..183d3e62 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -20,7 +20,7 @@ from ..judge import SubmissionResult, validate_output from ..metadata import Metadata from ..model import InputValidators, OutputValidators, TestCase, TestDataGroup -from ..run import Executable, ProgramError, SourceCode +from ..run import Executable, SourceCode # Junk data. The validator should reject these cases _JUNK_CASES: list[tuple[str, bytes]] = [ @@ -104,14 +104,11 @@ def check_input_validators(validators: InputValidators, testdata: TestDataGroup, executables: list[Executable] = [] for val in validators.validators: - try: - result = val.compile(work_dir) - if result.executable is None: - diag.error(f'Compile error for {val}', result.errmsg) - else: - executables.append(result.executable) - except ProgramError as e: - diag.error(str(e)) + result = val.compile(work_dir) + if result.executable is None: + diag.error(f'Compile error for {val}', result.errmsg) + else: + executables.append(result.executable) # Only sanity check input validators if they all actually compiled if diag.errors != errors_before: @@ -295,12 +292,9 @@ def check_output_validators( elif not validators.uses_default(format_version, metadata) and not validators.validators: diag.fatal('problem.yaml specifies custom validator but no validator programs found') - try: - result = selected.compile(work_dir) - if not result.success: - diag.fatal(f'Compile error for output validator {selected}', result.errmsg) - except ProgramError as e: - diag.fatal(f'Compile error for output validator {selected}', str(e)) + result = selected.compile(work_dir) + if not result.success: + diag.fatal(f'Compile error for output validator {selected}', result.errmsg) # Only sanity check output validators if they all actually compiled if diag.errors != errors_before: diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 20d66471..7208060e 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -10,7 +10,6 @@ from . import rutil from .buildrun import BuildRun from .checktestdata import Checktestdata -from .errors import ProgramError as ProgramError from .executable import Executable as Executable from .prebuilt import Prebuilt as Prebuilt from .program import CompileResult as CompileResult diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index 15e8f284..c0ae3d2a 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import TYPE_CHECKING -from .errors import ProgramError from .executable import Executable from .program import CompileResult, Program @@ -48,9 +47,9 @@ def _do_compile(self, work_dir: Path) -> CompileResult: build = build_dir / 'build' if not build.is_file(): - raise ProgramError(f'{self.name} does not have a build script') + return CompileResult(errmsg='no build script') if not os.access(build, os.X_OK): - raise ProgramError(f'{self.name}/build is not executable') + return CompileResult(errmsg='build script is not executable') try: subprocess.check_output(['./build'], stderr=subprocess.STDOUT, cwd=build_dir) diff --git a/problemtools/run/errors.py b/problemtools/run/errors.py deleted file mode 100644 index a44c2abf..00000000 --- a/problemtools/run/errors.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Error handling. -""" - - -class ProgramError(Exception): - """Base exception class for errors within the run package.""" diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index 6b499d54..fafaf0c0 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -7,7 +7,6 @@ from pathlib import Path from . import limit -from .errors import ProgramError log = logging.getLogger(__name__) @@ -104,10 +103,7 @@ def run( status: exit status of the process runtime: user+sys runtime of the process, in seconds """ - runcmd = self.get_runcmd(memlim=memlim) - if runcmd == []: - raise ProgramError(f'Could not figure out how to run {self}') - argv = runcmd + (args if args is not None else []) + argv = self.get_runcmd(memlim=memlim) + (args if args is not None else []) if self._infile_as_arg: if infile != DEV_NULL: argv.append(str(infile)) diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index 3ed37da3..92f9bd17 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -5,7 +5,6 @@ import os from pathlib import Path -from .errors import ProgramError from .executable import Executable from .program import CompileResult, Program from .tools import get_tool_path @@ -22,24 +21,25 @@ def __init__(self, path: Path) -> None: Args: path: path to .viva source file """ - if Viva._VIVA_PATH is None: - raise ProgramError(f'Could not locate the VIVA program to run {path}') super().__init__(name=path.name) + self._path = path + + def _do_compile(self, work_dir: Path) -> CompileResult: + """Syntax-check the VIVA script""" + if Viva._VIVA_PATH is None: + return CompileResult(errmsg='Could not locate the VIVA program') # VIVA takes input as argument and not on stdin, and exits with 0 on # accept, so swap that with our accept exit status 42. - self._executable = Executable( - path.name, + executable = Executable( + self.name, Viva._VIVA_PATH, - args=[str(path)], + args=[str(self._path)], skip_memory_rlimit=True, swap_exit_codes=True, infile_as_arg=True, ) - - def _do_compile(self, work_dir: Path) -> CompileResult: - """Syntax-check the VIVA script""" - (status, _) = self._executable.run() + (status, _) = executable.run() # Without an input file, VIVA only checks the script, and accepts (42 after swapping) if the syntax is fine. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: - return CompileResult(executable=self._executable) + return CompileResult(executable=executable) return CompileResult(errmsg='VIVA syntax check failed') From a0297fe2fadcdca112915666e95cd5a8dee2dd82 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 14:35:57 +0200 Subject: [PATCH 3/6] Adjust interface of get_tool to get rid of some clumsy None checks --- problemtools/checks/testdata.py | 4 +--- problemtools/judge/execute.py | 2 +- problemtools/model/graders.py | 5 ++--- problemtools/model/validators.py | 10 ++++------ problemtools/run/__init__.py | 2 +- problemtools/run/tools.py | 33 +++++++++++++++++++++++--------- problemtools/verifyproblem.py | 5 +---- 7 files changed, 34 insertions(+), 27 deletions(-) diff --git a/problemtools/checks/testdata.py b/problemtools/checks/testdata.py index 06bb22ee..a5889d2d 100644 --- a/problemtools/checks/testdata.py +++ b/problemtools/checks/testdata.py @@ -42,8 +42,6 @@ def check_testdata( ) -> None: """Run all checks on a problem's test data.""" output_validator = output_validators.select(format_version, metadata) - if output_validator is None: - diag.fatal('Unable to locate default validator') all_testcases = testdata.get_all_testcases() included_testcases = [tc for tc in all_testcases if tc.matches_filter(context.data_filter)] @@ -55,7 +53,7 @@ def check_testdata( diag.msg(msg) has_custom_grader = graders.grader is not None - has_default_grader = DEFAULT_GRADER is not None + has_default_grader = DEFAULT_GRADER.compile(work_dir).success if metadata.is_scoring(): _warn_reject_score(testdata, diag) diff --git a/problemtools/judge/execute.py b/problemtools/judge/execute.py index 9e379ef2..88a81956 100644 --- a/problemtools/judge/execute.py +++ b/problemtools/judge/execute.py @@ -107,7 +107,7 @@ def _run_interactive( diag: Diagnostics, ) -> SubmissionResult: """Run a submission once (interactive)""" - interactive = get_tool('interactive') + interactive = get_tool('interactive').compile(base_dir).executable if interactive is None: diag.error('Could not locate interactive runner') return SubmissionResult('JE', reason='Could not locate interactive runner') diff --git a/problemtools/model/graders.py b/problemtools/model/graders.py index 6c4545ed..526e90d4 100644 --- a/problemtools/model/graders.py +++ b/problemtools/model/graders.py @@ -4,11 +4,10 @@ from pathlib import Path from ..languages import Languages -from ..run import BuildRun, Prebuilt, SourceCode, as_source_or_buildrun, find_programs, get_tool +from ..run import BuildRun, SourceCode, as_source_or_buildrun, find_programs, get_tool from .paths import resolve -_default_grader = get_tool('default_grader') -DEFAULT_GRADER = Prebuilt(_default_grader) if _default_grader is not None else None +DEFAULT_GRADER = get_tool('default_grader') @dataclass(frozen=True) diff --git a/problemtools/model/validators.py b/problemtools/model/validators.py index d07799e5..d00b1f3c 100644 --- a/problemtools/model/validators.py +++ b/problemtools/model/validators.py @@ -6,12 +6,11 @@ from ..formatversion import FormatVersion from ..languages import Languages from ..metadata import Metadata -from ..run import BuildRun, Prebuilt, Program, SourceCode, as_source_or_buildrun, find_programs, get_tool +from ..run import BuildRun, Program, SourceCode, as_source_or_buildrun, find_programs, get_tool from .paths import resolve #: The problemtools-provided validator used when a problem doesn't ship a custom output validator. -_default_validator = get_tool('default_validator') -DEFAULT_VALIDATOR = Prebuilt(_default_validator) if _default_validator is not None else None +DEFAULT_VALIDATOR = get_tool('default_validator') @dataclass(frozen=True) @@ -45,9 +44,8 @@ def uses_default(self, format_version: FormatVersion, metadata: Metadata) -> boo return metadata.legacy_validation == 'default' return not self.validators - def select(self, format_version: FormatVersion, metadata: Metadata) -> Program | None: - """The output validator that will actually be used, or None if the default validator - is required but not available on this problemtools install.""" + def select(self, format_version: FormatVersion, metadata: Metadata) -> Program: + """The output validator that will actually be used.""" if self.uses_default(format_version, metadata) or not self.validators: return DEFAULT_VALIDATOR return self.validators[0] diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 7208060e..6f2ef2bb 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -11,10 +11,10 @@ from .buildrun import BuildRun from .checktestdata import Checktestdata from .executable import Executable as Executable -from .prebuilt import Prebuilt as Prebuilt from .program import CompileResult as CompileResult from .program import Program from .source import SourceCode +from .tools import Tool as Tool from .tools import get_tool as get_tool from .tools import get_tool_path as get_tool_path from .viva import Viva diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index 3a2e7224..ebbdfcff 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -2,16 +2,36 @@ from pathlib import Path from .executable import Executable +from .program import CompileResult, Program _PACKAGE_DIR = Path(__file__).parent.parent +class Tool(Program): + """One of problemtools' external tools. Compiling it fails if the tool was not found.""" + + def __init__(self, name: str, binary: Path | None) -> None: + """Instantiate tool object. + + Args: + name: name of the tool. + binary: path to the tool, or None if it was not found. + """ + super().__init__(name=name) + self._binary = binary + + def _do_compile(self, work_dir: Path) -> CompileResult: + if self._binary is None: + return CompileResult(errmsg=f'Could not locate {self.name}') + return CompileResult(executable=Executable(self.name, self._binary, skip_memory_rlimit=True)) + + def get_tool_path(name: str) -> Path | None: """Find the path to one of problemtools' external tools. Args: name: which tool is wanted (one of [default_grader, - default_validator, interactive, checktestdata, viva.sh]) + default_validator, interactive, viva.sh]) Returns: path to the tool, or None if the tool was not found. @@ -24,18 +44,13 @@ def get_tool_path(name: str) -> Path | None: ) -def get_tool(name: str) -> Executable | None: - """Get an Executable instance for one of problemtools' external tools. +def get_tool(name: str) -> Tool: + """Get a Tool instance for one of problemtools' external tools. Args: name: same as for get_tool_path - - Returns: - problemtools.run.Executable object for the tool, or None if - the tool was not found. """ - path = get_tool_path(name) - return Executable(path.name, path, skip_memory_rlimit=True) if path is not None else None + return Tool(name, get_tool_path(name)) def __locate_executable(candidate_paths: list[Path]) -> Path | None: diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 1eb540e2..22161939 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -88,10 +88,7 @@ def set_timelim(value: float) -> None: @functools.cache def get_output_validator() -> run.Program: - validator = problem.output_validators.select(problem.format_version, problem.metadata) - if validator is None: - diag.fatal('Unable to locate default validator') - return validator + return problem.output_validators.select(problem.format_version, problem.metadata) try: if not re.match('^[a-z0-9]+$', problem.shortname): From 81877437332a9107d080d2313b01d67c999d9c35 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 14:43:53 +0200 Subject: [PATCH 4/6] Check that programs only contain files + directories after compilation Symlinks in compilation artifacts is not well supported in judge systems like Kattis. A build step creating more esoteric file objects is likely a bug (or exploitation attempt). --- problemtools/run/buildrun.py | 4 ++++ problemtools/run/rutil.py | 23 +++++++++++++++++++++++ problemtools/run/source.py | 6 +++++- tests/test_rutil.py | 22 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tests/test_rutil.py diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index c0ae3d2a..14020a9c 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING +from . import rutil from .executable import Executable from .program import CompileResult, Program @@ -56,6 +57,9 @@ def _do_compile(self, work_dir: Path) -> CompileResult: except subprocess.CalledProcessError as err: return CompileResult(errmsg=err.output.decode('utf8', 'replace')) + if (errmsg := rutil.check_build_dir(build_dir)) is not None: + return CompileResult(errmsg=errmsg) + run = build_dir / 'run' if not run.is_file() or not os.access(run, os.X_OK): return CompileResult(errmsg='build script did not produce an executable called "run"') diff --git a/problemtools/run/rutil.py b/problemtools/run/rutil.py index 7cc062b7..ce4a4a00 100644 --- a/problemtools/run/rutil.py +++ b/problemtools/run/rutil.py @@ -1,6 +1,7 @@ """Some utility functions for the run module.""" import os +import stat from pathlib import Path @@ -15,3 +16,25 @@ def list_files_recursive(root: Path) -> list[Path]: for path, _, files in os.walk(root): ret.extend(Path(path) / filename for filename in files) return ret + + +def check_build_dir(build_dir: Path) -> str | None: + """Check that a build only produced regular files and directories. + + Symlinks are not followed. + + Returns: + an error message listing everything else (symlinks, fifos, sockets, + devices, ...) found in build_dir, or None if there is nothing else. + """ + special = [] + for path, dirnames, filenames in os.walk(build_dir): + for name in dirnames + filenames: + entry = Path(path) / name + mode = entry.lstat().st_mode + if not stat.S_ISREG(mode) and not stat.S_ISDIR(mode): + special.append(entry.relative_to(build_dir)) + if not special: + return None + names = ', '.join(str(path) for path in sorted(special)) + return f'Build produced something other than regular files and directories: {names}' diff --git a/problemtools/run/source.py b/problemtools/run/source.py index 70bb9a06..2ad97d58 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from ..languages import CommandSubstitution, Language +from . import rutil from .executable import Executable from .program import CompileResult, Program @@ -102,10 +103,13 @@ def _do_compile(self, work_dir: Path) -> CompileResult: try: subprocess.check_output(command, stderr=subprocess.STDOUT) - return CompileResult(executable=executable) except subprocess.CalledProcessError as err: return CompileResult(errmsg=err.output.decode('utf8', 'replace')) + if (errmsg := rutil.check_build_dir(build_dir)) is not None: + return CompileResult(errmsg=errmsg) + return CompileResult(executable=executable) + def __str__(self) -> str: """String representation""" return f'{self.name} ({self.language.name})' diff --git a/tests/test_rutil.py b/tests/test_rutil.py new file mode 100644 index 00000000..19fef3e1 --- /dev/null +++ b/tests/test_rutil.py @@ -0,0 +1,22 @@ +import os +from pathlib import Path + +from problemtools.run.rutil import check_build_dir + + +def test_regular_files_and_directories_ok(tmp_path: Path) -> None: + (tmp_path / 'sub').mkdir() + (tmp_path / 'sub' / 'main.o').write_bytes(b'') + (tmp_path / 'run').write_bytes(b'') + assert check_build_dir(tmp_path) is None + + +def test_symlinks_and_fifos_are_errors(tmp_path: Path) -> None: + (tmp_path / 'sub').mkdir() + (tmp_path / 'run').write_bytes(b'') + (tmp_path / 'sub' / 'link').symlink_to('../run') + (tmp_path / 'dirlink').symlink_to('sub') + os.mkfifo(tmp_path / 'pipe') + errmsg = check_build_dir(tmp_path) + assert errmsg is not None + assert errmsg.endswith(f': dirlink, pipe, {Path("sub/link")}') From 93abd46a93327bda64309b8142d3d0dc62edd4d0 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 14:56:24 +0200 Subject: [PATCH 5/6] Stricten type of submissions - they now have to be SourceCode --- problemtools/model/submissions.py | 6 +-- problemtools/run/__init__.py | 64 +++++++++++++++++++++---------- problemtools/run/program.py | 5 --- problemtools/run/source.py | 1 + 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/problemtools/model/submissions.py b/problemtools/model/submissions.py index 1fd7aa95..4ffc3f2e 100644 --- a/problemtools/model/submissions.py +++ b/problemtools/model/submissions.py @@ -6,7 +6,7 @@ from ..languages import Languages from ..metadata import Metadata -from ..run import Program, find_programs +from ..run import SourceCode, find_source_programs from ..submission_rules import FinalVerdictRule, PermittedRule, Rule, ScoreRule from .includes import Includes from .paths import RelativePath, relpath, resolve @@ -20,7 +20,7 @@ class Submission: `path` is relative to the submissions directory, e.g. for submissions/accepted/hello.java, path is accepted/hello.java.""" - program: Program + program: SourceCode path: RelativePath def __post_init__(self) -> None: @@ -110,7 +110,7 @@ def load_submissions(probdir: Path, language_config: Languages, includes: Includ submissions = [] for entry in sorted(subs_root.iterdir()): if entry.is_dir(): - for program in find_programs(entry, language_config=language_config, includes=includes): + for program in find_source_programs(entry, language_config=language_config, includes=includes): submissions.append(Submission(program=program, path=relpath(Path(entry.name) / program.name))) return Submissions(submissions=submissions) diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 6f2ef2bb..afe66990 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -26,7 +26,6 @@ def find_programs( path: Path, language_config: Languages, - includes: 'Includes | None' = None, allow_validation_script: bool = False, ) -> list[Program]: """Find all programs in a directory. @@ -38,10 +37,6 @@ def find_programs( programming language of source code and providing info on how to compile and run the source code. - includes: include files to add to programs found, resolved - per-program based on its detected language (see - Includes.get_includes_for_language). - allow_validation_script: if true, also looks for validation scripts in the Checktestdata and VIVA formats. @@ -56,7 +51,6 @@ def find_programs( run = get_program( fullpath, language_config=language_config, - includes=includes, allow_validation_script=allow_validation_script, ) if run is not None: @@ -67,7 +61,6 @@ def find_programs( def get_program( path: Path, language_config: Languages, - includes: 'Includes | None' = None, allow_validation_script: bool = False, ) -> Program | None: """Get a Program object for a program @@ -81,10 +74,6 @@ def get_program( programming language of source code and providing info on how to compile and run the source code. - includes: include files to add to the program, resolved per - the program's detected language (see - Includes.get_includes_for_language). Defaults to no includes. - allow_validation_script: if true, also looks for validation scripts in the Checktestdata and VIVA formats. @@ -94,10 +83,7 @@ def get_program( """ # Imported lazily (rather than at module scope) since `model` depends on `run` # (e.g. for `run.find_programs`), so importing it here avoids a circular import. - from ..model import Includes, load_program_files - - if includes is None: - includes = Includes() + from ..model import load_program_files if path.is_file(): if allow_validation_script: @@ -105,17 +91,55 @@ def get_program( return Viva(path) if path.suffix == '.ctd': return Checktestdata(path) - files = [path] else: build = path / 'build' if build.is_file() and os.access(build, os.X_OK): return BuildRun(path.name, load_program_files(path)) - files = rutil.list_files_recursive(path) + return get_source_program(path, language_config) + + +def find_source_programs(path: Path, language_config: Languages, includes: 'Includes') -> list[SourceCode]: + """Find all programs provided as source code in a directory. + + Like find_programs, but never gives a BuildRun: a directory with a build + script is treated like any other source code directory. + """ + if not path.is_dir(): + return [] + ret = [] + for fullpath in sorted(path.iterdir()): + program = get_source_program(fullpath, language_config=language_config, includes=includes) + if program is not None: + ret.append(program) + return ret + + +def get_source_program(path: Path, language_config: Languages, includes: 'Includes | None' = None) -> SourceCode | None: + """Get a SourceCode object for a program. + + Args: + path, language_config: see get_program. + + includes: include files to add to the program, resolved per + the program's detected language (see + Includes.get_includes_for_language). Defaults to no includes. + + Returns: + a SourceCode instance, or None if no source code in a known + language was found at the given path. + """ + # Imported lazily, see get_program + from ..model import Includes, load_program_files + + if includes is None: + includes = Includes() + + files = [path] if path.is_file() else rutil.list_files_recursive(path) lang = language_config.detect_language(files) - if lang is not None: - return SourceCode(path.name, load_program_files(path), lang, includes=includes.get_includes_for_language(lang.lang_id)) - return None + if lang is None: + return None + return SourceCode(path.name, load_program_files(path), lang, includes=includes.get_includes_for_language(lang.lang_id)) def as_source_or_buildrun(programs: list[Program]) -> list[SourceCode | BuildRun]: diff --git a/problemtools/run/program.py b/problemtools/run/program.py index 5c7bb562..90f15f1c 100644 --- a/problemtools/run/program.py +++ b/problemtools/run/program.py @@ -60,8 +60,3 @@ def compile(self, work_dir: Path) -> CompileResult: @abstractmethod def _do_compile(self, work_dir: Path) -> CompileResult: """Actually compile the program. Subclasses implement this, callers use compile().""" - - def code_size(self) -> int: - """Subclasses should override this method with the total size of the - source code.""" - return 0 diff --git a/problemtools/run/source.py b/problemtools/run/source.py index 2ad97d58..86d421e6 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -50,6 +50,7 @@ def __init__(self, name: str, files: 'ProgramFiles', language: Language, include self._includes = includes def code_size(self) -> int: + """Total size of the program's own source files, not including any include files.""" return self.files.size() def _do_compile(self, work_dir: Path) -> CompileResult: From cd3d9f0c5693589e89d0f362fb14943d8ed98a34 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Fri, 9 Oct 2026 15:26:16 +0200 Subject: [PATCH 6/6] Change some uses of Program to Executable, moving some compile error checks earlier --- problemtools/checks/submissions.py | 7 ++- problemtools/checks/testdata.py | 8 +++- problemtools/checks/validators.py | 11 ++--- problemtools/judge/execute.py | 66 +++++++++++--------------- problemtools/judge/submission_judge.py | 14 +++--- problemtools/judge/validate.py | 16 +++---- problemtools/run/tools.py | 8 ++-- 7 files changed, 63 insertions(+), 67 deletions(-) diff --git a/problemtools/checks/submissions.py b/problemtools/checks/submissions.py index 82d6676a..2a9440bb 100644 --- a/problemtools/checks/submissions.py +++ b/problemtools/checks/submissions.py @@ -47,9 +47,14 @@ def check_submissions( if not has_testcases: diag.warning('Found no test cases to run on. Did you filter them all out?') + compile_result = output_validator.compile(work_dir) + if compile_result.executable is None: + diag.error(f'Compile error for output validator {output_validator}', compile_result.errmsg) + return + submissions_judge = context.submissions_judge_factory( root=testdata, - output_validator=output_validator, + output_validator=compile_result.executable, metadata=metadata, base_dir=work_dir, context=context, diff --git a/problemtools/checks/testdata.py b/problemtools/checks/testdata.py index a5889d2d..ad4872ce 100644 --- a/problemtools/checks/testdata.py +++ b/problemtools/checks/testdata.py @@ -406,6 +406,12 @@ def _check_answers( if metadata.is_interactive() or metadata.is_multi_pass(): return + compile_result = output_validator.compile(work_dir) + executable = compile_result.executable + if executable is None: + diag.error(f'Compile error for output validator {output_validator}', compile_result.errmsg) + return + testcases = [tc for tc in testdata.get_all_testcases() if tc.matches_filter(context.data_filter)] sample = [tc for tc in testcases if tc.is_in_sample_group()] secret = [tc for tc in testcases if not tc.is_in_sample_group()] @@ -414,7 +420,7 @@ def validate(testcase: TestCase) -> SubmissionResult: return validate_output( testcase=testcase, submission_output=testcase.ansfile, - output_validator=output_validator, + output_validator=executable, metadata=metadata, base_dir=work_dir, diag=diag, diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index 183d3e62..1dba8fef 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -274,8 +274,6 @@ def check_output_validators( format_version, diag, f'Support for multiple output validators has been dropped. will only use {selected}' ) - if selected is None: - diag.fatal('Unable to locate default validator') diag.msg('Checking output validator') safe_output_validator_languages = {'c', 'cpp', 'python3'} @@ -292,9 +290,10 @@ def check_output_validators( elif not validators.uses_default(format_version, metadata) and not validators.validators: diag.fatal('problem.yaml specifies custom validator but no validator programs found') - result = selected.compile(work_dir) - if not result.success: - diag.fatal(f'Compile error for output validator {selected}', result.errmsg) + compile_result = selected.compile(work_dir) + executable = compile_result.executable + if executable is None: + diag.fatal(f'Compile error for output validator {selected}', compile_result.errmsg) # Only sanity check output validators if they all actually compiled if diag.errors != errors_before: @@ -307,7 +306,7 @@ def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[TestCase] result = validate_output( testcase=testcase, submission_output=junk_file, - output_validator=selected, + output_validator=executable, metadata=metadata, base_dir=work_dir, diag=diag, diff --git a/problemtools/judge/execute.py b/problemtools/judge/execute.py index 88a81956..10776182 100644 --- a/problemtools/judge/execute.py +++ b/problemtools/judge/execute.py @@ -27,12 +27,14 @@ from ..diagnostics import Diagnostics from ..metadata import Metadata from ..model import TestCase -from ..run import Program, get_tool +from ..run import Executable, get_tool from .result import SubmissionResult from .validate import _parse_validator_result, _validate_output _INTERACTIVE_OUTPUT_RE = re.compile(r'\d+ \d+\.\d+ \d+ \d+\.\d+ (validator|submission)') +_INTERACTIVE = get_tool('interactive').executable + def _is_TLE(status: int, may_signal_with_usr1: bool = False) -> bool: return os.WIFSIGNALED(status) and ( @@ -64,33 +66,30 @@ def _read_safe(path: Path) -> str | None: def _run_normal( infile: Path, testcase: TestCase, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, timelim: float, execution_dir: Path, - base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: """Run a submission once (non-interactive)""" outfile = execution_dir / 'submission_stdout' errfile = execution_dir / 'submission_stderr' - sub_executable = sub.compile(base_dir).executable - assert sub_executable is not None, f'{sub} must be successfully compiled before running' - status, runtime = sub_executable.run( + status, runtime = sub.run( infile=infile, outfile=outfile, errfile=errfile, timelim=math.ceil(timelim) + 1, memlim=metadata.limits.memory, - work_dir=sub_executable.build_dir, + work_dir=sub.build_dir, ) if _is_TLE(status) or runtime > timelim: result = SubmissionResult('TLE') elif _is_RTE(status): result = SubmissionResult('RTE', reason=_rte_reason(status), additional_info=_read_safe(errfile)) else: - result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, base_dir, diag, infile=infile) + result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, diag, infile=infile) result.runtime = runtime return result @@ -98,41 +97,32 @@ def _run_normal( def _run_interactive( infile: Path, testcase: TestCase, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, timelim: float, execution_dir: Path, - base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: """Run a submission once (interactive)""" - interactive = get_tool('interactive').compile(base_dir).executable - if interactive is None: + if _INTERACTIVE is None: diag.error('Could not locate interactive runner') return SubmissionResult('JE', reason='Could not locate interactive runner') - validator_executable = output_validator.compile(base_dir).executable - if validator_executable is None: - return SubmissionResult('JE', reason=f'output validator {output_validator} failed to compile') - - sub_executable = sub.compile(base_dir).executable - assert sub_executable is not None, f'{sub} must be successfully compiled before running' - feedback_dir = execution_dir / 'feedback' interactive_out = execution_dir / 'interactive_output' - i_status, _ = interactive.run( + i_status, _ = _INTERACTIVE.run( outfile=interactive_out, args=( ['1', str(math.ceil(2 * timelim))] - + validator_executable.get_runcmd(memlim=metadata.limits.validation_memory) + + output_validator.get_runcmd(memlim=metadata.limits.validation_memory) + [str(infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + testcase.output_validator_flags + [';'] - + sub_executable.get_runcmd(memlim=metadata.limits.memory) + + sub.get_runcmd(memlim=metadata.limits.memory) ), - work_dir=sub_executable.build_dir, + work_dir=sub.build_dir, ) if _is_RTE(i_status): @@ -175,35 +165,33 @@ def _run_interactive( def _run_pass( infile: Path, testcase: TestCase, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, timelim: float, execution_dir: Path, - base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: """Run a submission once (the common case, or one pass for a multi-pass problem)""" if metadata.is_interactive(): - return _run_interactive(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag) - return _run_normal(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag) + return _run_interactive(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag) + return _run_normal(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag) def _run_multipass( testcase: TestCase, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, timelim: float, execution_dir: Path, - base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: infile: Path = testcase.infile slowest = 0.0 feedback_dir = execution_dir / 'feedback' for _ in range(metadata.limits.validation_passes): - result = _run_pass(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag) + result = _run_pass(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag) slowest = max(slowest, result.runtime) result.runtime = slowest nextpass = feedback_dir / 'nextpass.in' @@ -220,21 +208,21 @@ def _run_multipass( def execute_testcase( testcase: TestCase, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, timelim: float, base_dir: Path, diag: Diagnostics, ) -> SubmissionResult: - """Run sub on a single testcase. sub must already have been successfully compiled to base_dir.""" + """Run sub on a single testcase.""" with tempfile.TemporaryDirectory(dir=base_dir) as exec_dir: execution_dir = Path(exec_dir) (execution_dir / 'feedback').mkdir() if metadata.is_multi_pass(): - result = _run_multipass(testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag) + result = _run_multipass(testcase, sub, output_validator, metadata, timelim, execution_dir, diag) else: - result = _run_pass(testcase.infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag) + result = _run_pass(testcase.infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag) result.test_node = testcase result.runtime_testcase = testcase return result diff --git a/problemtools/judge/submission_judge.py b/problemtools/judge/submission_judge.py index e7096964..a160fdc4 100644 --- a/problemtools/judge/submission_judge.py +++ b/problemtools/judge/submission_judge.py @@ -10,7 +10,7 @@ from ..diagnostics import Diagnostics from ..metadata import Metadata from ..model import DEFAULT_GRADER, Graders, Submission, TestCase, TestDataGroup -from ..run import CompileResult, Program +from ..run import CompileResult, Executable, Program from .cache import ResultStore from .execute import execute_testcase from .grade import grade_group @@ -55,8 +55,8 @@ class SubmissionJudge: def __init__( self, - sub: Program, - output_validator: Program, + sub: Executable, + output_validator: Executable, metadata: Metadata, root: TestDataGroup, base_dir: Path, @@ -234,7 +234,7 @@ class SubmissionsJudge: def __init__( self, root: TestDataGroup, - output_validator: Program, + output_validator: Executable, metadata: Metadata, base_dir: Path, context: Context, @@ -262,10 +262,10 @@ def precompute(self, submissions: list[Submission], timelim: float) -> dict[Subm for sub in submissions: result = sub.program.compile(self._base_dir) outcomes[sub] = result - if result.success: + if result.executable is not None: assert sub not in self._judges, f'precompute() called more than once for submission {sub}' judge = SubmissionJudge( - sub=sub.program, + sub=result.executable, output_validator=self._output_validator, metadata=self._metadata, root=self._root, @@ -291,7 +291,7 @@ class SubmissionsJudgeFactory(Protocol): def __call__( self, root: TestDataGroup, - output_validator: Program, + output_validator: Executable, metadata: Metadata, base_dir: Path, context: Context, diff --git a/problemtools/judge/validate.py b/problemtools/judge/validate.py index 072d069e..85d749dc 100644 --- a/problemtools/judge/validate.py +++ b/problemtools/judge/validate.py @@ -7,7 +7,7 @@ from ..diagnostics import Diagnostics from ..metadata import Metadata from ..model import TestCase -from ..run import Program +from ..run import Executable from .result import SubmissionResult @@ -27,7 +27,7 @@ def _get_feedback(feedback_dir: Path) -> str | None: def _parse_validator_result( - val: Program, + val: Executable, status: int, feedback_dir: Path, metadata: Metadata, @@ -75,10 +75,9 @@ def _parse_validator_result( def _validate_output( testcase: TestCase, submission_output: Path, - output_validator: Program, + output_validator: Executable, metadata: Metadata, execution_dir: Path, - base_dir: Path, diag: Diagnostics, infile: Path | None = None, ) -> SubmissionResult: @@ -94,12 +93,9 @@ def _validate_output( 'OLE', reason=f'output ({output_size:.1f} MiB) exceeds output limit ({metadata.limits.output} MiB)' ) - validator_executable = output_validator.compile(base_dir).executable - if validator_executable is None: - return SubmissionResult('JE', reason=f'output validator {output_validator} failed to compile') val_stdout = execution_dir / 'val_stdout' val_stderr = execution_dir / 'val_stderr' - status, _ = validator_executable.run( + status, _ = output_validator.run( infile=submission_output, args=[str(effective_infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + flags, timelim=val_timelim, @@ -119,7 +115,7 @@ def _validate_output( def validate_output( testcase: TestCase, submission_output: Path, - output_validator: Program, + output_validator: Executable, metadata: Metadata, base_dir: Path, diag: Diagnostics, @@ -127,4 +123,4 @@ def validate_output( with tempfile.TemporaryDirectory(dir=base_dir) as exec_dir: execution_dir = Path(exec_dir) (execution_dir / 'feedback').mkdir() - return _validate_output(testcase, submission_output, output_validator, metadata, execution_dir, base_dir, diag) + return _validate_output(testcase, submission_output, output_validator, metadata, execution_dir, diag) diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index ebbdfcff..54edd43a 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -10,6 +10,8 @@ class Tool(Program): """One of problemtools' external tools. Compiling it fails if the tool was not found.""" + executable: Executable | None # The tool, or None if it was not found. Needs no compiling + def __init__(self, name: str, binary: Path | None) -> None: """Instantiate tool object. @@ -18,12 +20,12 @@ def __init__(self, name: str, binary: Path | None) -> None: binary: path to the tool, or None if it was not found. """ super().__init__(name=name) - self._binary = binary + self.executable = Executable(name, binary, skip_memory_rlimit=True) if binary is not None else None def _do_compile(self, work_dir: Path) -> CompileResult: - if self._binary is None: + if self.executable is None: return CompileResult(errmsg=f'Could not locate {self.name}') - return CompileResult(executable=Executable(self.name, self._binary, skip_memory_rlimit=True)) + return CompileResult(executable=self.executable) def get_tool_path(name: str) -> Path | None: