Separate concerns in Program - compilation artifacts are now Executable

problemtools/checks/submissions.py

@@ -47,9 +47,14 @@ def check_submissions(
     if not has_testcases:
         diag.warning('Found no test cases to run on. Did you filter them all out?')
 
+    compile_result = output_validator.compile(work_dir)
+    if compile_result.executable is None:
+        diag.error(f'Compile error for output validator {output_validator}', compile_result.errmsg)
+        return
+
     submissions_judge = context.submissions_judge_factory(
         root=testdata,
-        output_validator=output_validator,
+        output_validator=compile_result.executable,
         metadata=metadata,
         base_dir=work_dir,
         context=context,

problemtools/checks/testdata.py

@@ -42,8 +42,6 @@ def check_testdata(
 ) -> None:
     """Run all checks on a problem's test data."""
     output_validator = output_validators.select(format_version, metadata)
-    if output_validator is None:
-        diag.fatal('Unable to locate default validator')
 
     all_testcases = testdata.get_all_testcases()
     included_testcases = [tc for tc in all_testcases if tc.matches_filter(context.data_filter)]
@@ -55,7 +53,7 @@ def check_testdata(
     diag.msg(msg)
 
     has_custom_grader = graders.grader is not None
-    has_default_grader = DEFAULT_GRADER is not None
+    has_default_grader = DEFAULT_GRADER.compile(work_dir).success
 
     if metadata.is_scoring():
         _warn_reject_score(testdata, diag)
@@ -408,6 +406,12 @@ def _check_answers(
     if metadata.is_interactive() or metadata.is_multi_pass():
         return
 
+    compile_result = output_validator.compile(work_dir)
+    executable = compile_result.executable
+    if executable is None:
+        diag.error(f'Compile error for output validator {output_validator}', compile_result.errmsg)
+        return
+
     testcases = [tc for tc in testdata.get_all_testcases() if tc.matches_filter(context.data_filter)]
     sample = [tc for tc in testcases if tc.is_in_sample_group()]
     secret = [tc for tc in testcases if not tc.is_in_sample_group()]
@@ -416,7 +420,7 @@ def validate(testcase: TestCase) -> SubmissionResult:
         return validate_output(
             testcase=testcase,
             submission_output=testcase.ansfile,
-            output_validator=output_validator,
+            output_validator=executable,
             metadata=metadata,
             base_dir=work_dir,
             diag=diag,

problemtools/checks/validators.py

@@ -20,7 +20,7 @@
 from ..judge import SubmissionResult, validate_output
 from ..metadata import Metadata
 from ..model import InputValidators, OutputValidators, TestCase, TestDataGroup
-from ..run import ProgramError, SourceCode
+from ..run import Executable, SourceCode
 
 # Junk data. The validator should reject these cases
 _JUNK_CASES: list[tuple[str, bytes]] = [
@@ -102,13 +102,13 @@ def check_input_validators(validators: InputValidators, testdata: TestDataGroup,
     if len(validators.validators) == 0:
         diag.error('No input format validators found')
 
+    executables: list[Executable] = []
     for val in validators.validators:
-        try:
-            result = val.compile(work_dir)
-            if not result.success:
-                diag.error(f'Compile error for {val}', result.errmsg)
-        except ProgramError as e:
-            diag.error(str(e))
+        result = val.compile(work_dir)
+        if result.executable is None:
+            diag.error(f'Compile error for {val}', result.errmsg)
+        else:
+            executables.append(result.executable)
 
     # Only sanity check input validators if they all actually compiled
     if diag.errors != errors_before:
@@ -128,8 +128,8 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None:
         with _temp_file_with(case, work_dir) as junk_file:
             for flags_str in all_flags:
                 flags = flags_str.split()
-                for val in validators.validators:
-                    status, _ = val.run(junk_file, args=flags, work_dir=work_dir)
+                for executable in executables:
+                    status, _ = executable.run(junk_file, args=flags, work_dir=work_dir)
                     if os.WEXITSTATUS(status) != 42:
                         break
                 else:
@@ -148,8 +148,8 @@ def modified_input_validates(applicable: Callable[[str], bool], modifier: Callab
             with _temp_file_with(modifier(infile_data).encode('utf8'), work_dir) as modified_file:
                 for flags_str in all_flags:
                     flags = flags_str.split()
-                    for val in validators.validators:
-                        status, _ = val.run(modified_file, args=flags, work_dir=work_dir)
+                    for executable in executables:
+                        status, _ = executable.run(modified_file, args=flags, work_dir=work_dir)
                         if os.WEXITSTATUS(status) != 42:
                             # expected behavior; validator rejects modified input
                             return False
@@ -188,11 +188,12 @@ def _run_input_validators(validators: InputValidators, testcase: TestCase, work_
 
     for val in validators.validators:
         # A validator that failed to compile was already reported by check_input_validators; skip it.
-        if not val.compile(work_dir).success:
+        executable = val.compile(work_dir).executable
+        if executable is None:
             continue
 
         with tempfile.NamedTemporaryFile(dir=work_dir) as outfile, tempfile.NamedTemporaryFile(dir=work_dir) as errfile:
-            status, _ = val.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir)
+            status, _ = executable.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir)
             if not os.WIFEXITED(status):
                 emsg = f'Input format validator {val} crashed on input {testcase.infile}'
             elif os.WEXITSTATUS(status) != 42:
@@ -273,8 +274,6 @@ def check_output_validators(
             format_version, diag, f'Support for multiple output validators has been dropped. will only use {selected}'
         )
 
-    if selected is None:
-        diag.fatal('Unable to locate default validator')
     diag.msg('Checking output validator')
 
     safe_output_validator_languages = {'c', 'cpp', 'python3'}
@@ -291,12 +290,10 @@ def check_output_validators(
     elif not validators.uses_default(format_version, metadata) and not validators.validators:
         diag.fatal('problem.yaml specifies custom validator but no validator programs found')
 
-    try:
-        result = selected.compile(work_dir)
-        if not result.success:
-            diag.fatal(f'Compile error for output validator {selected}', result.errmsg)
-    except ProgramError as e:
-        diag.fatal(f'Compile error for output validator {selected}', str(e))
+    compile_result = selected.compile(work_dir)
+    executable = compile_result.executable
+    if executable is None:
+        diag.fatal(f'Compile error for output validator {selected}', compile_result.errmsg)
 
     # Only sanity check output validators if they all actually compiled
     if diag.errors != errors_before:
@@ -309,7 +306,7 @@ def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[TestCase]
                 result = validate_output(
                     testcase=testcase,
                     submission_output=junk_file,
-                    output_validator=selected,
+                    output_validator=executable,
                     metadata=metadata,
                     base_dir=work_dir,
                     diag=diag,

problemtools/judge/execute.py

@@ -27,12 +27,14 @@
 from ..diagnostics import Diagnostics
 from ..metadata import Metadata
 from ..model import TestCase
-from ..run import Program, get_tool
+from ..run import Executable, get_tool
 from .result import SubmissionResult
 from .validate import _parse_validator_result, _validate_output
 
 _INTERACTIVE_OUTPUT_RE = re.compile(r'\d+ \d+\.\d+ \d+ \d+\.\d+ (validator|submission)')
 
+_INTERACTIVE = get_tool('interactive').executable
+
 
 def _is_TLE(status: int, may_signal_with_usr1: bool = False) -> bool:
     return os.WIFSIGNALED(status) and (
@@ -64,62 +66,53 @@ def _read_safe(path: Path) -> str | None:
 def _run_normal(
     infile: Path,
     testcase: TestCase,
-    sub: Program,
-    output_validator: Program,
+    sub: Executable,
+    output_validator: Executable,
     metadata: Metadata,
     timelim: float,
     execution_dir: Path,
-    base_dir: Path,
     diag: Diagnostics,
 ) -> SubmissionResult:
     """Run a submission once (non-interactive)"""
     outfile = execution_dir / 'submission_stdout'
     errfile = execution_dir / 'submission_stderr'
-    sub_path = sub.compile(base_dir).path
     status, runtime = sub.run(
         infile=infile,
         outfile=outfile,
         errfile=errfile,
         timelim=math.ceil(timelim) + 1,
         memlim=metadata.limits.memory,
-        work_dir=sub_path,
+        work_dir=sub.build_dir,
     )
     if _is_TLE(status) or runtime > timelim:
         result = SubmissionResult('TLE')
     elif _is_RTE(status):
         result = SubmissionResult('RTE', reason=_rte_reason(status), additional_info=_read_safe(errfile))
     else:
-        result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, base_dir, diag, infile=infile)
+        result = _validate_output(testcase, outfile, output_validator, metadata, execution_dir, diag, infile=infile)
     result.runtime = runtime
     return result
 
 
 def _run_interactive(
     infile: Path,
     testcase: TestCase,
-    sub: Program,
-    output_validator: Program,
+    sub: Executable,
+    output_validator: Executable,
     metadata: Metadata,
     timelim: float,
     execution_dir: Path,
-    base_dir: Path,
     diag: Diagnostics,
 ) -> SubmissionResult:
     """Run a submission once (interactive)"""
-    interactive = get_tool('interactive')
-    if interactive is None:
+    if _INTERACTIVE is None:
         diag.error('Could not locate interactive runner')
         return SubmissionResult('JE', reason='Could not locate interactive runner')
 
-    if not output_validator.compile(base_dir).success:
-        return SubmissionResult('JE', reason=f'output validator {output_validator} failed to compile')
-
-    sub_path = sub.compile(base_dir).path
-
     feedback_dir = execution_dir / 'feedback'
     interactive_out = execution_dir / 'interactive_output'
 
-    i_status, _ = interactive.run(
+    i_status, _ = _INTERACTIVE.run(
         outfile=interactive_out,
         args=(
             ['1', str(math.ceil(2 * timelim))]
@@ -129,7 +122,7 @@ def _run_interactive(
             + [';']
             + sub.get_runcmd(memlim=metadata.limits.memory)
         ),
-        work_dir=sub_path,
+        work_dir=sub.build_dir,
     )
 
     if _is_RTE(i_status):
@@ -172,35 +165,33 @@ def _run_interactive(
 def _run_pass(
     infile: Path,
     testcase: TestCase,
-    sub: Program,
-    output_validator: Program,
+    sub: Executable,
+    output_validator: Executable,
     metadata: Metadata,
     timelim: float,
     execution_dir: Path,
-    base_dir: Path,
     diag: Diagnostics,
 ) -> SubmissionResult:
     """Run a submission once (the common case, or one pass for a multi-pass problem)"""
     if metadata.is_interactive():
-        return _run_interactive(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag)
-    return _run_normal(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag)
+        return _run_interactive(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag)
+    return _run_normal(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag)
 
 
 def _run_multipass(
     testcase: TestCase,
-    sub: Program,
-    output_validator: Program,
+    sub: Executable,
+    output_validator: Executable,
     metadata: Metadata,
     timelim: float,
     execution_dir: Path,
-    base_dir: Path,
     diag: Diagnostics,
 ) -> SubmissionResult:
     infile: Path = testcase.infile
     slowest = 0.0
     feedback_dir = execution_dir / 'feedback'
     for _ in range(metadata.limits.validation_passes):
-        result = _run_pass(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag)
+        result = _run_pass(infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag)
         slowest = max(slowest, result.runtime)
         result.runtime = slowest
         nextpass = feedback_dir / 'nextpass.in'
@@ -217,8 +208,8 @@ def _run_multipass(
 
 def execute_testcase(
     testcase: TestCase,
-    sub: Program,
-    output_validator: Program,
+    sub: Executable,
+    output_validator: Executable,
     metadata: Metadata,
     timelim: float,
     base_dir: Path,
@@ -229,9 +220,9 @@ def execute_testcase(
         execution_dir = Path(exec_dir)
         (execution_dir / 'feedback').mkdir()
         if metadata.is_multi_pass():
-            result = _run_multipass(testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag)
+            result = _run_multipass(testcase, sub, output_validator, metadata, timelim, execution_dir, diag)
         else:
-            result = _run_pass(testcase.infile, testcase, sub, output_validator, metadata, timelim, execution_dir, base_dir, diag)
+            result = _run_pass(testcase.infile, testcase, sub, output_validator, metadata, timelim, execution_dir, diag)
     result.test_node = testcase
     result.runtime_testcase = testcase
     return result

problemtools/judge/grade.py

@@ -29,7 +29,8 @@ def grade_group(
     if not sub_results:
         return ('AC', 0.0)
 
-    if not grader.compile(base_dir).success:
+    grader_executable = grader.compile(base_dir).executable
+    if grader_executable is None:
         diag.error(f'Failed to compile grader {grader}')
         return ('JE', None)
 
@@ -43,7 +44,7 @@ def grade_group(
         errfile = Path(tmpdir) / 'grader_err'
         infile.write_text(grader_input)
 
-        status, _runtime = grader.run(infile, outfile, errfile, args=grader_flags)
+        status, _runtime = grader_executable.run(infile, outfile, errfile, args=grader_flags)
 
         grader_output = outfile.read_text(errors='replace') if outfile.exists() else ''
         stderr_content = errfile.read_text(errors='replace') if errfile.exists() else ''

problemtools/judge/submission_judge.py

@@ -10,7 +10,7 @@
 from ..diagnostics import Diagnostics
 from ..metadata import Metadata
 from ..model import DEFAULT_GRADER, Graders, Submission, TestCase, TestDataGroup
-from ..run import CompileResult, Program
+from ..run import CompileResult, Executable, Program
 from .cache import ResultStore
 from .execute import execute_testcase
 from .grade import grade_group
@@ -55,8 +55,8 @@ class SubmissionJudge:
 
     def __init__(
         self,
-        sub: Program,
-        output_validator: Program,
+        sub: Executable,
+        output_validator: Executable,
         metadata: Metadata,
         root: TestDataGroup,
         base_dir: Path,
@@ -234,7 +234,7 @@ class SubmissionsJudge:
     def __init__(
         self,
         root: TestDataGroup,
-        output_validator: Program,
+        output_validator: Executable,
         metadata: Metadata,
         base_dir: Path,
         context: Context,
@@ -262,10 +262,10 @@ def precompute(self, submissions: list[Submission], timelim: float) -> dict[Subm
         for sub in submissions:
             result = sub.program.compile(self._base_dir)
             outcomes[sub] = result
-            if result.success:
+            if result.executable is not None:
                 assert sub not in self._judges, f'precompute() called more than once for submission {sub}'
                 judge = SubmissionJudge(
-                    sub=sub.program,
+                    sub=result.executable,
                     output_validator=self._output_validator,
                     metadata=self._metadata,
                     root=self._root,
@@ -291,7 +291,7 @@ class SubmissionsJudgeFactory(Protocol):
     def __call__(
         self,
         root: TestDataGroup,
-        output_validator: Program,
+        output_validator: Executable,
         metadata: Metadata,
         base_dir: Path,
         context: Context,