diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py index 0402ef7c..f888d7f9 100644 --- a/problemtools/checks/validators.py +++ b/problemtools/checks/validators.py @@ -114,8 +114,9 @@ def collect_flags(group: TestDataGroup, flags: set[str]) -> None: collect_flags(testdata, all_flags) - fd, file_name = tempfile.mkstemp() + fd, tmp_name = tempfile.mkstemp() os.close(fd) + file_name = Path(tmp_name) for desc, case in _JUNK_CASES: with open(file_name, 'wb') as f: f.write(case) @@ -189,7 +190,7 @@ def _run_input_validators(validators: InputValidators, testcase: TestCase, work_ continue with tempfile.NamedTemporaryFile() as outfile, tempfile.NamedTemporaryFile() as errfile: - status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=work_dir) + status, _ = val.run(testcase.infile, Path(outfile.name), Path(errfile.name), args=flags, work_dir=work_dir) if not os.WIFEXITED(status): emsg = f'Input format validator {val} crashed on input {testcase.infile}' elif os.WEXITSTATUS(status) != 42: diff --git a/problemtools/judge/execute.py b/problemtools/judge/execute.py index fe366f0e..4f0d281f 100644 --- a/problemtools/judge/execute.py +++ b/problemtools/judge/execute.py @@ -77,9 +77,9 @@ def _run_normal( errfile = execution_dir / 'submission_stderr' sub_path = sub.compile(base_dir).path status, runtime = sub.run( - infile=str(infile), - outfile=str(outfile), - errfile=str(errfile), + infile=infile, + outfile=outfile, + errfile=errfile, timelim=math.ceil(timelim) + 1, memlim=metadata.limits.memory, work_dir=sub_path, @@ -120,7 +120,7 @@ def _run_interactive( interactive_out = execution_dir / 'interactive_output' i_status, _ = interactive.run( - outfile=str(interactive_out), + outfile=interactive_out, args=( ['1', str(math.ceil(2 * timelim))] + output_validator.get_runcmd(memlim=metadata.limits.validation_memory) diff --git a/problemtools/judge/grade.py b/problemtools/judge/grade.py index 5b18a693..1669b446 100644 --- a/problemtools/judge/grade.py +++ b/problemtools/judge/grade.py @@ -43,7 +43,7 @@ def grade_group( errfile = Path(tmpdir) / 'grader_err' infile.write_text(grader_input) - status, _runtime = grader.run(str(infile), str(outfile), str(errfile), args=grader_flags) + status, _runtime = grader.run(infile, outfile, errfile, args=grader_flags) grader_output = outfile.read_text(errors='replace') if outfile.exists() else '' stderr_content = errfile.read_text(errors='replace') if errfile.exists() else '' diff --git a/problemtools/judge/validate.py b/problemtools/judge/validate.py index c1520466..14a8512a 100644 --- a/problemtools/judge/validate.py +++ b/problemtools/judge/validate.py @@ -99,12 +99,12 @@ def _validate_output( val_stdout = execution_dir / 'val_stdout' val_stderr = execution_dir / 'val_stderr' status, _ = output_validator.run( - infile=str(submission_output), + infile=submission_output, args=[str(effective_infile), str(testcase.ansfile), str(feedback_dir) + os.sep] + flags, timelim=val_timelim, memlim=val_memlim, - outfile=str(val_stdout), - errfile=str(val_stderr), + outfile=val_stdout, + errfile=val_stderr, ) for label, path in [('stdout', val_stdout), ('stderr', val_stderr)]: try: diff --git a/problemtools/model/graders.py b/problemtools/model/graders.py index 2480511c..526e90d4 100644 --- a/problemtools/model/graders.py +++ b/problemtools/model/graders.py @@ -27,5 +27,5 @@ def grader(self) -> SourceCode | BuildRun | None: def load_graders(probdir: Path, language_config: Languages) -> Graders: probdir = resolve(probdir) - graders = as_source_or_buildrun(find_programs(str(probdir / 'graders'), language_config=language_config)) + graders = as_source_or_buildrun(find_programs(probdir / 'graders', language_config=language_config)) return Graders(graders=graders) diff --git a/problemtools/model/submissions.py b/problemtools/model/submissions.py index 39691c50..82745f9f 100644 --- a/problemtools/model/submissions.py +++ b/problemtools/model/submissions.py @@ -85,6 +85,6 @@ def load_submissions(probdir: Path, language_config: Languages, includes: Includ submissions = [] for entry in sorted(subs_root.iterdir()): if entry.is_dir(): - for program in find_programs(str(entry), language_config=language_config, includes=includes): + for program in find_programs(entry, language_config=language_config, includes=includes): submissions.append(Submission(program=program, path=relpath(Path(entry.name) / program.name))) return Submissions(submissions=submissions) diff --git a/problemtools/model/validators.py b/problemtools/model/validators.py index 996e89b2..3330ff16 100644 --- a/problemtools/model/validators.py +++ b/problemtools/model/validators.py @@ -27,7 +27,7 @@ def load_input_validators(probdir: Path, language_config: Languages) -> InputVal validators = [ program for directory in ('input_format_validators', 'input_validators') - for program in find_programs(str(probdir / directory), language_config=language_config, allow_validation_script=True) + for program in find_programs(probdir / directory, language_config=language_config, allow_validation_script=True) ] return InputValidators(validators=validators) @@ -55,6 +55,6 @@ def select(self, format_version: FormatVersion, metadata: Metadata) -> Program | def load_output_validators(probdir: Path, format_version: FormatVersion, language_config: Languages) -> OutputValidators: probdir = resolve(probdir) validators = as_source_or_buildrun( - find_programs(str(probdir / format_version.output_validator_directory), language_config=language_config) + find_programs(probdir / format_version.output_validator_directory, language_config=language_config) ) return OutputValidators(validators=validators) diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index b348971b..d94ac042 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -3,6 +3,7 @@ """ import os +from pathlib import Path from typing import TYPE_CHECKING from ..languages import Languages @@ -22,7 +23,7 @@ def find_programs( - path: str, + path: Path, language_config: Languages, includes: 'Includes | None' = None, allow_validation_script: bool = False, @@ -47,11 +48,10 @@ def find_programs( list of Program instances, all programs found in path. """ - if not os.path.isdir(path): + if not path.is_dir(): return [] ret = [] - for name in sorted(os.listdir(path)): - fullpath = os.path.join(path, name) + for fullpath in sorted(path.iterdir()): run = get_program( fullpath, language_config=language_config, @@ -64,7 +64,7 @@ def find_programs( def get_program( - path: str, + path: Path, language_config: Languages, includes: 'Includes | None' = None, allow_validation_script: bool = False, @@ -98,17 +98,16 @@ def get_program( includes = Includes() - if os.path.isfile(path): + if path.is_file(): if allow_validation_script: - ext = os.path.splitext(path)[1] - if ext == '.viva': + if path.suffix == '.viva': return Viva(path) - if ext == '.ctd': + if path.suffix == '.ctd': return Checktestdata(path) files = [path] else: - build = os.path.join(path, 'build') - if os.path.isfile(build) and os.access(build, os.X_OK): + build = path / 'build' + if build.is_file() and os.access(build, os.X_OK): return BuildRun(path) files = rutil.list_files_recursive(path) diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index 9e8fff72..a9d3e675 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -15,19 +15,16 @@ class BuildRun(Program): """Class for build/run-script program.""" - def __init__(self, path: str) -> None: + def __init__(self, path: Path) -> None: """Instantiate BuildRun object. Args: path: directory containing the build script. """ - if not os.path.isdir(path): + if not path.is_dir(): raise ProgramError(f'{path} is not a directory') - if path[-1] == '/': - path = path[:-1] - name = os.path.basename(path) - super().__init__(name=name) + super().__init__(name=path.name) self._source_path = path def do_compile(self, work_dir: Path) -> CompileResult: @@ -35,16 +32,16 @@ def do_compile(self, work_dir: Path) -> CompileResult: work_dir) and run the build script.""" name = self.name run_path = work_dir / name - if os.path.exists(run_path): + if run_path.exists(): run_path = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) else: - os.makedirs(run_path) + run_path.mkdir(parents=True) self._path = run_path rutil.add_files(self._source_path, self.path) - build = os.path.join(self.path, 'build') - if not os.path.isfile(build): + build = self.path / 'build' + if not build.is_file(): raise ProgramError(f'{self._source_path} does not have a build script') if not os.access(build, os.X_OK): raise ProgramError(f'{self._source_path}/build is not executable') @@ -54,12 +51,12 @@ def do_compile(self, work_dir: Path) -> CompileResult: except subprocess.CalledProcessError as err: return CompileResult(False, err.output.decode('utf8', 'replace'), self.path) - run = os.path.join(self.path, 'run') - if not os.path.isfile(run) or not os.access(run, os.X_OK): + run = self.path / 'run' + if not run.is_file() or not os.access(run, os.X_OK): return CompileResult(False, 'build script did not produce an executable called "run"', self.path) return CompileResult(True, None, self.path) - def get_runcmd(self, cwd: str | None = None, memlim: int = 1024) -> list[str]: + def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: """Run command for the program. Must not be called until compile() has been called. @@ -68,8 +65,8 @@ def get_runcmd(self, cwd: str | None = None, memlim: int = 1024) -> list[str]: cwd: if not None, the run command is provided relative to cwd (otherwise absolute paths are given). """ - path = self.path if cwd is None else os.path.relpath(self.path, cwd) - return [os.path.join(path, 'run')] + path = self.path if cwd is None else Path(os.path.relpath(self.path, cwd)) + return [str(path / 'run')] def should_skip_memory_rlimit(self) -> bool: """Ugly hack (see program.py for details).""" diff --git a/problemtools/run/checktestdata.py b/problemtools/run/checktestdata.py index 6dcc3383..38f53856 100644 --- a/problemtools/run/checktestdata.py +++ b/problemtools/run/checktestdata.py @@ -7,19 +7,19 @@ from pathlib import Path from .executable import Executable -from .program import CompileResult +from .program import DEV_NULL, CompileResult class Checktestdata(Executable): """Wrapper class for running Checktestdata scripts.""" - def __init__(self, path: str) -> None: + def __init__(self, path: Path) -> None: """Create a Checktestdata wrapper. Args: path: path to .ctd source file """ - super().__init__(sys.executable, args=['-m', 'checktestdata', path], name=os.path.basename(path)) + super().__init__(Path(sys.executable), args=['-m', 'checktestdata', str(path)], name=path.name) def do_compile(self, work_dir: Path) -> CompileResult: """Syntax-check the Checktestdata script""" @@ -29,9 +29,9 @@ def do_compile(self, work_dir: Path) -> CompileResult: def run( self, - infile: str = '/dev/null', - outfile: str = '/dev/null', - errfile: str = '/dev/null', + infile: Path = DEV_NULL, + outfile: Path = DEV_NULL, + errfile: Path = DEV_NULL, args: list[str] | None = None, timelim: int = 1000, memlim: int = 1024, @@ -40,9 +40,9 @@ def run( """Run the Checktestdata script to validate an input file. Args: - infile: name of input file to validate - outfile: file name to save stdout of Checktestdata in - errfile: file name to save stderr of Checktestdata in + infile: input file to validate + outfile: file to save stdout of Checktestdata in + errfile: file to save stderr of Checktestdata in args: additional command-line arguments to pass to Checktestdata timelim: time limit for the Checktestdata process in seconds diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index eda047a8..338051e0 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -12,7 +12,7 @@ class Executable(Program): """Class for executable files.""" - def __init__(self, path: str, args: list[str] | None = None, name: str | None = None) -> None: + def __init__(self, path: Path, args: list[str] | None = None, name: str | None = None) -> None: """Instantiate executable object. Args: @@ -22,12 +22,12 @@ def __init__(self, path: str, args: list[str] | None = None, name: str | None = should be passed to the program every time it is executed. name: name to use for the program. Defaults to the basename of path. """ - if not os.path.isfile(path) or not os.access(path, os.X_OK): + if not path.is_file() or not os.access(path, os.X_OK): raise ProgramError(f'{path} is not an executable program') - super().__init__(name=name if name is not None else os.path.basename(path), path=Path(path)) + super().__init__(name=name if name is not None else path.name, path=path) self.args = args if args is not None else [] - def get_runcmd(self, cwd: str | None = None, memlim: int = 1024) -> list[str]: + def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: """Command to run the program.""" return [str(self.path)] + self.args diff --git a/problemtools/run/program.py b/problemtools/run/program.py index 4ac9a1cd..163b2704 100644 --- a/problemtools/run/program.py +++ b/problemtools/run/program.py @@ -14,6 +14,8 @@ log = logging.getLogger(__name__) +DEV_NULL = Path(os.devnull) + @dataclasses.dataclass(frozen=True) class CompileResult: @@ -58,14 +60,14 @@ def __str__(self) -> str: return self.name @abstractmethod - def get_runcmd(self, cwd: str | None = None, memlim: int = 1024) -> list[str]: + def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: pass def run( self, - infile: str = '/dev/null', - outfile: str = '/dev/null', - errfile: str = '/dev/null', + infile: Path = DEV_NULL, + outfile: Path = DEV_NULL, + errfile: Path = DEV_NULL, args: list[str] | None = None, timelim: int = 1000, memlim: int = 1024, @@ -74,9 +76,9 @@ def run( """Run the program. Args: - infile: name of file to pass on stdin - outfile: name of file to send stdout to - errfile: name of file to send stderr to + infile: file to pass on stdin + outfile: file to send stdout to + errfile: file to send stderr to args: additional command-line arguments to pass to the program timelim: CPU time limit in seconds memlim: memory limit in MiB @@ -136,9 +138,9 @@ def should_skip_memory_rlimit(self) -> bool: def __run_wait( self, argv: list[str], - infile: str, - outfile: str, - errfile: str, + infile: Path, + outfile: Path, + errfile: Path, timelim: int, memlim: int, work_dir: Path | None, @@ -185,7 +187,7 @@ def __run_wait( return status, rusage.ru_utime + rusage.ru_stime @staticmethod - def __setfd(fd: int, filename: str, flag: int) -> None: + def __setfd(fd: int, filename: Path, flag: int) -> None: tmpfd = os.open(filename, flag) os.dup2(tmpfd, fd) os.close(tmpfd) diff --git a/problemtools/run/rutil.py b/problemtools/run/rutil.py index 0a1631fe..2123fd3a 100644 --- a/problemtools/run/rutil.py +++ b/problemtools/run/rutil.py @@ -8,7 +8,7 @@ from .errors import ProgramError -def add_files(src: str | Path, dstdir: str | Path) -> None: +def add_files(src: Path, dstdir: Path) -> None: """Copy src to dstdir. Args: @@ -22,13 +22,12 @@ def add_files(src: str | Path, dstdir: str | Path) -> None: existing directory. """ try: - if os.path.isfile(src): + if src.is_file(): shutil.copy(src, dstdir) else: - for name in os.listdir(src): - srcfile = os.path.join(src, name) - destfile = os.path.join(dstdir, name) - if os.path.isdir(srcfile): + for srcfile in src.iterdir(): + destfile = dstdir / srcfile.name + if srcfile.is_dir(): shutil.copytree(srcfile, destfile, dirs_exist_ok=True) else: shutil.copy(srcfile, destfile) @@ -39,14 +38,14 @@ def add_files(src: str | Path, dstdir: str | Path) -> None: raise -def list_files_recursive(root: str | Path) -> list[str]: +def list_files_recursive(root: Path) -> list[Path]: """List files in a directory with subdirectories. Returns: - all file names for all files contained in a directory and its + paths of all files contained in a directory and its subdirectories. """ - ret = [] + ret: list[Path] = [] for path, _, files in os.walk(root): - ret.extend([os.path.join(root, path, filename) for filename in files]) + ret.extend(Path(path) / filename for filename in files) return ret diff --git a/problemtools/run/source.py b/problemtools/run/source.py index 284bbbce..584aac2e 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -23,7 +23,7 @@ class SourceCode(Program): """Class representing a program provided by source code.""" - def __init__(self, path: str, language: Language, includes: 'LanguageIncludes') -> None: + def __init__(self, path: Path, language: Language, includes: 'LanguageIncludes') -> None: """Instantiate SourceCode object Args: @@ -41,17 +41,14 @@ def __init__(self, path: str, language: Language, includes: 'LanguageIncludes') a mainfile, that takes precedence over the one we would otherwise have detected. """ - if path[-1] == '/': - path = path[:-1] - name = os.path.basename(path) - super().__init__(name=name) + super().__init__(name=path.name) self.language = language self._source_path = path self._includes = includes - if os.path.isfile(path): - self._code_size = os.path.getsize(path) + if path.is_file(): + self._code_size = path.stat().st_size else: - self._code_size = sum(os.path.getsize(f) for f in rutil.list_files_recursive(path)) + self._code_size = sum(f.stat().st_size for f in rutil.list_files_recursive(path)) def code_size(self) -> int: return self._code_size @@ -63,34 +60,33 @@ def do_compile(self, work_dir: Path) -> CompileResult: # Set up work-space run_path = work_dir / name - if os.path.exists(run_path): + if run_path.exists(): run_path = Path(tempfile.mkdtemp(prefix=f'{name}-', dir=work_dir)) else: - os.makedirs(run_path) + run_path.mkdir(parents=True) self._path = run_path # Copy all files rutil.add_files(self._source_path, self.path) for include_file in self._includes.files: - dest = os.path.join(self.path, include_file.path) - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, 'wb') as f: - f.write(include_file.data) + dest = self.path / include_file.path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(include_file.data) self.src = sorted(self.language.get_source_files(rutil.list_files_recursive(self.path))) if len(self.src) == 0: raise ProgramError(f'No source files found for language {self.language.lang_id} in {self.name}') if self._includes.mainfile is not None: - self.mainfile = os.path.join(self.path, self._includes.mainfile) + self.mainfile = self.path / self._includes.mainfile else: candidates = self.language.mainfile_candidates(self.src) - self.mainfile = str(candidates[0]) if candidates else self.src[0] + self.mainfile = candidates[0] if candidates else self.src[0] - self.mainclass = os.path.splitext(os.path.basename(self.mainfile))[0] + self.mainclass = self.mainfile.stem self.Mainclass = self.mainclass[0].upper() + self.mainclass[1:] - self.binary = os.path.join(self.path, 'run') + self.binary = self.path / 'run' not_installed = self.language.check_installed() if not_installed is not None: @@ -108,7 +104,7 @@ def do_compile(self, work_dir: Path) -> CompileResult: except subprocess.CalledProcessError as err: return CompileResult(False, err.output.decode('utf8', 'replace'), self.path) - def get_runcmd(self, cwd: str | None = None, memlim: int = 1024) -> list[str]: + def get_runcmd(self, cwd: Path | None = None, memlim: int = 1024) -> list[str]: """Run command for the program. Must not be called until compile() has been called. @@ -137,10 +133,10 @@ def __str__(self) -> str: def __get_substitution(self, memlim: int = 1024) -> CommandSubstitution: return CommandSubstitution( path=str(self.path), - files=' '.join(self.src), + files=' '.join(str(f) for f in self.src), memlim=memlim, - mainfile=self.mainfile, + mainfile=str(self.mainfile), mainclass=self.mainclass, Mainclass=self.Mainclass, - binary=self.binary, + binary=str(self.binary), ) diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index 94e1d7c8..be8bfc1d 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -1,9 +1,12 @@ import os +from pathlib import Path from .executable import Executable +_PACKAGE_DIR = Path(__file__).parent.parent -def get_tool_path(name: str) -> str | None: + +def get_tool_path(name: str) -> Path | None: """Find the path to one of problemtools' external tools. Args: @@ -15,8 +18,8 @@ def get_tool_path(name: str) -> str | None: """ return __locate_executable( [ - os.path.join(os.path.dirname(__file__), '..', 'support', name), - os.path.join(os.path.dirname(__file__), '..', '..', 'support', os.path.splitext(name)[0], name), + _PACKAGE_DIR / 'support' / name, + _PACKAGE_DIR.parent / 'support' / Path(name).stem / name, ] ) @@ -35,7 +38,7 @@ def get_tool(name: str) -> Executable | None: return Executable(path) if path is not None else None -def __locate_executable(candidate_paths: list[str]) -> str | None: +def __locate_executable(candidate_paths: list[Path]) -> Path | None: """Find executable among a set of paths. Args: @@ -46,4 +49,4 @@ def __locate_executable(candidate_paths: list[str]) -> str | None: first entry of candidate_paths that is an executable file, or None if no such entry. """ - return next((p for p in candidate_paths if os.path.isfile(p) and os.access(p, os.X_OK)), None) + return next((p for p in candidate_paths if p.is_file() and os.access(p, os.X_OK)), None) diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index 53036307..04d0d611 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -7,7 +7,7 @@ from .errors import ProgramError from .executable import Executable -from .program import CompileResult +from .program import DEV_NULL, CompileResult from .tools import get_tool_path @@ -16,7 +16,7 @@ class Viva(Executable): _VIVA_PATH = get_tool_path('viva.sh') - def __init__(self, path: str) -> None: + def __init__(self, path: Path) -> None: """Create a VIVA wrapper. Args: @@ -24,7 +24,7 @@ def __init__(self, path: str) -> None: """ if Viva._VIVA_PATH is None: raise ProgramError(f'Could not locate the VIVA program to run {path}') - super().__init__(Viva._VIVA_PATH, args=[path], name=os.path.basename(path)) + super().__init__(Viva._VIVA_PATH, args=[str(path)], name=path.name) def do_compile(self, work_dir: Path) -> CompileResult: """Syntax-check the VIVA script""" @@ -34,9 +34,9 @@ def do_compile(self, work_dir: Path) -> CompileResult: def run( self, - infile: str = '/dev/null', - outfile: str = '/dev/null', - errfile: str = '/dev/null', + infile: Path = DEV_NULL, + outfile: Path = DEV_NULL, + errfile: Path = DEV_NULL, args: list[str] | None = None, timelim: int = 1000, memlim: int = 1024, @@ -45,9 +45,9 @@ def run( """Run the VIVA script to validate an input file. Args: - infile: name of input file to validate - outfile: file name to save stdout of VIVA in - errfile: file name to save stderr of VIVA in + infile: input file to validate + outfile: file to save stdout of VIVA in + errfile: file to save stderr of VIVA in args: additional command-line arguments to pass to VIVA timelim: time limit for the VIVA process in seconds @@ -61,8 +61,8 @@ def run( if args is None: args = [] # VIVA takes input as argument and not on stdin - if infile != '/dev/null': - args = args + [infile] + if infile != DEV_NULL: + args = args + [str(infile)] (status, runtime) = super().run( outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir