diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index 0734c4dd73..edf0b06898 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -31,7 +31,6 @@ UncertaintyReserveBasisUnknown, ) from nexent.core.tools.parallel_executor import ParallelExecutorTool -from nexent.core.agents.sandbox import SandboxConfig from nexent.core.agents.nexent_agent import get_local_python_authorized_imports from nexent.memory import models as memory_models @@ -73,6 +72,9 @@ from utils.redis_utils import get_redis_client from consts.const import ( AGENT_WORKSPACE_ROOT, + NEXENT_SANDBOX_WORKSPACE_MODE, + NEXENT_SANDBOX_CONTAINER_WORKSPACE_ROOT, + NEXENT_SANDBOX_FAILURE_POLICY, AIDP_API_KEY, AIDP_SERVER_URL, AIDP_TENANT_ID, @@ -2914,11 +2916,16 @@ async def create_agent_run_info( # Resolve sandbox config: DB policy overrides env-var defaults. # build_sandbox_policy returns None when level=local (backward-compatible). # Import inside function body to avoid circular dependency. + from agents.sandbox_config import resolve_sandbox_config from management.services.agent.service import build_sandbox_policy, get_sandbox_minio_client sandbox_policy = build_sandbox_policy(tenant_id=tenant_id, agent_type="") agent_db_policy = getattr(agent_config, "sandbox_policy", None) - merged_policy = sandbox_policy if sandbox_policy else agent_db_policy - sandbox_config = SandboxConfig.from_dict(merged_policy) if merged_policy else None + sandbox_config = resolve_sandbox_config( + agent_db_policy, sandbox_policy, + workspace_mode=NEXENT_SANDBOX_WORKSPACE_MODE, + container_workspace_root=NEXENT_SANDBOX_CONTAINER_WORKSPACE_ROOT, + failure_policy=NEXENT_SANDBOX_FAILURE_POLICY, + ) sandbox_minio_client = ( get_sandbox_minio_client() if sandbox_config and sandbox_config.auto_sync_outputs @@ -2934,6 +2941,7 @@ async def create_agent_run_info( if ( getattr(sandbox_config.level, "value", sandbox_config.level) == "docker" and getattr(sandbox_config.scope, "value", sandbox_config.scope) == "system" + and sandbox_config.workspace_mode == "legacy" ): sandbox_config.extra_kwargs.update({ "workspace_volume_name": NEXENT_SANDBOX_WORKSPACE_VOLUME, diff --git a/backend/agents/sandbox_config.py b/backend/agents/sandbox_config.py new file mode 100644 index 0000000000..8074f06df4 --- /dev/null +++ b/backend/agents/sandbox_config.py @@ -0,0 +1,25 @@ +"""Resolve per-agent sandbox policy without reading deployment environment.""" + +from typing import Any + +from nexent.core.agents.sandbox import SandboxConfig + + +def resolve_sandbox_config( + database_policy: dict[str, Any] | None, environment_policy: dict[str, Any] | None, *, + workspace_mode: str, container_workspace_root: str, failure_policy: str, +) -> SandboxConfig | None: + """Use a complete DB policy first; omitted policy fields retain SDK defaults. + + Workspace deployment values are defaults that explicit policy fields override. + Empty DB objects represent an unconfigured policy, not an explicit local level. + """ + if database_policy is not None and not isinstance(database_policy, dict): + raise TypeError('Agent sandbox policy must be a dictionary') + policy = database_policy or environment_policy + return SandboxConfig.from_dict({ + 'workspace_mode': workspace_mode, + 'container_workspace_root': container_workspace_root, + 'failure_policy': failure_policy, + **policy, + }) if policy else None diff --git a/backend/consts/const.py b/backend/consts/const.py index 5f97fe86b6..8137f7224c 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -59,6 +59,9 @@ class VectorDatabaseType(str, Enum): MAX_CONCURRENT_UPLOADS = 5 UPLOAD_FOLDER = os.getenv('UPLOAD_FOLDER', 'uploads') AGENT_WORKSPACE_ROOT = os.getenv('AGENT_WORKSPACE_ROOT', '/mnt/nexent/workdir') +NEXENT_SANDBOX_WORKSPACE_MODE = os.getenv('NEXENT_SANDBOX_WORKSPACE_MODE', 'legacy') +NEXENT_SANDBOX_CONTAINER_WORKSPACE_ROOT = os.getenv('NEXENT_SANDBOX_CONTAINER_WORKSPACE_ROOT', '') +NEXENT_SANDBOX_FAILURE_POLICY = os.getenv('NEXENT_SANDBOX_FAILURE_POLICY', 'local') ROOT_DIR = os.getenv("ROOT_DIR") PER_WAVE_TIMEOUT = int(os.getenv("DP_SPLIT_WAIT_TIMEOUT_PER_WAVE_S", "30")) diff --git a/deploy/env/.env.example b/deploy/env/.env.example index 978759eeda..147cd65d02 100644 --- a/deploy/env/.env.example +++ b/deploy/env/.env.example @@ -321,6 +321,14 @@ NEXENT_SANDBOX_DOCKER_IMAGE=nexent/nexent-sandbox:latest # Docker named volume shared by nexent-runtime and the system-scoped sandbox. NEXENT_SANDBOX_WORKSPACE_VOLUME=nexent-agent-workspace +# Workspace deployment defaults; explicit Agent sandbox_policy fields override them. +# A non-empty DB policy replaces the env policy; its other missing fields use SDK defaults. +# For Windows native runtime with Docker Linux containers, use bind and /mnt/nexent. +NEXENT_SANDBOX_WORKSPACE_MODE=legacy +NEXENT_SANDBOX_CONTAINER_WORKSPACE_ROOT= +# local preserves fallback behavior; error reports Docker failures without local execution. +NEXENT_SANDBOX_FAILURE_POLICY=local + # Sandbox resource limits. NEXENT_SANDBOX_MEMORY_LIMIT_MB=2048 NEXENT_SANDBOX_CPU_QUOTA=1.0 diff --git a/docs/sandbox-jupyter-tls.md b/docs/sandbox-jupyter-tls.md new file mode 100644 index 0000000000..e42e7b1067 --- /dev/null +++ b/docs/sandbox-jupyter-tls.md @@ -0,0 +1,51 @@ +# Sandbox Jupyter TLS + +Docker sandbox control traffic uses HTTPS for kernel health, creation and deletion, +and WSS for execution, readiness and reconnection. Each container creates its own +ECDSA certificate before Jupyter starts. The private key stays inside the container +under `/home/sandbox/.nexent-jupyter-tls` (directory `0700`, key `0600`). +The bootstrap checks ancestor ownership and write permissions, rejects symlinks, +and uses directory-relative file descriptors and exclusive creation. Private key +permissions apply before writing. Existing identities must be complete, privately +owned, currently valid and have matching keys; unsafe files are never repaired in +place. These filesystem controls do not isolate processes sharing the same UID. + +The runtime reads only the public certificate through its existing trusted Docker +connection. Each container owner has a separate trust file and SSL context, shared +by its kernel leases and cleaned up with the owner. Certificate and hostname checks +remain enabled. HTTPS requests do not use environment proxies or follow redirects. +No global CA installation or manual certificate configuration is required. + +The certificate covers the container name, localhost and loopback. +Native runtimes continue to use dynamically allocated +loopback ports; containerized runtimes use Docker networking. Workspace mapping, +kernel isolation, execution and cancellation contracts remain unchanged. + +## Deployment and migration + +- Images must contain `cryptography >= 42` and Jupyter Kernel Gateway with + `certfile`/`keyfile` support. The SDK now declares the cryptography dependency; + both repository Dockerfiles install the SDK. A custom older image without it + must be rebuilt. The configured user must own the private TLS directory and be + able to create it under a trusted, non-publicly-writable `/home/sandbox` parent. + Both repository images already provide this layout. Startup does not download dependencies. +- Before upgrading an existing system sandbox, drain active runs and explicitly + stop its old HTTP or TLS-version-1 container. Version 2 uses the private identity + directory; it does not read or migrate files from the old public temporary path. + The runtime refuses to recover or automatically + delete a running legacy container. On the next acquisition it can remove the + stopped owned container and create a TLS container. +- Certificates are valid for 365 days and retained on container restart. Drain + and recreate a system container before expiry; merely restarting it does not + renew the certificate. There is no hot certificate rotation in this change. +- If recovery cannot load or verify a running container's certificate, it reports + an error and preserves that container. Investigate the certificate/time/Docker + connection, then drain and explicitly stop the owner before recreating it. + +Client contexts and the test HTTPS server explicitly require TLS 1.2 or newer. +The repository's Sonar configuration declares Python 3.11, matching the SDK runtime. + +This change addresses the Jupyter TLS findings in PR #3985. The separate host +tool callback bridge is outside this change. SonarCloud and Codecov results still +need to be checked on the pushed commit; no findings are suppressed or accepted +automatically. diff --git a/sdk/nexent/core/agents/nexent_agent.py b/sdk/nexent/core/agents/nexent_agent.py index 1b25731833..8aa2534657 100644 --- a/sdk/nexent/core/agents/nexent_agent.py +++ b/sdk/nexent/core/agents/nexent_agent.py @@ -10,6 +10,7 @@ import shutil import tarfile import time +from concurrent.futures import CancelledError from copy import deepcopy from dataclasses import replace from pathlib import Path @@ -26,6 +27,8 @@ from ..utils.constants import THINK_PREFIX_PATTERN, THINK_TAG_PATTERN from ..utils.observer import MessageObserver, ProcessType from .agent_model import AgentConfig, AgentHistory, ModelConfig, ToolConfig +from .core_agent import CoreAgent, convert_code_format +from .sandbox_workspace import SandboxWorkspace, probe_workspace from .clarification import choose_clarification_tool_name, clarification_policy from .core_agent import CoreAgent, convert_code_format from ...consts.mcp_errors import is_mcp_timeout_error @@ -279,6 +282,12 @@ def __init__(self, observer: MessageObserver, self.user_id = user_id self.tenant_id = tenant_id self.workspace_path = workspace_path + self.workspace_mapping = None + if sandbox_config is not None and sandbox_config.workspace_mode == "bind" and workspace_path: + from .sandbox import SandboxLevel + + if sandbox_config.level == SandboxLevel.DOCKER: + self.workspace_mapping = sandbox_config.bind_workspace().for_run(workspace_path) self.workspace_run_id = workspace_run_id self.minio_files = list(minio_files or []) self.user_context = dict(user_context or {}) @@ -672,6 +681,13 @@ def create_tool(self, tool_config: ToolConfig): tool_obj = self.create_builtin_tool(tool_config) else: raise ValueError(f"unsupported tool source: {source}") + mapping = getattr(self, "workspace_mapping", None) + if mapping is not None and class_name in { + "CreateFileTool", "ReadFileTool", "DeleteFileTool", "DownloadFromS3Tool", "UploadToS3Tool", + }: + tool_obj.workspace_mapping = mapping + if class_name in {"CreateFileTool", "ReadFileTool", "DeleteFileTool"}: + tool_obj.init_path = str(mapping.host_root / "outputs") if source in {"local", "builtin", "mcp"}: try: setattr(tool_obj, "_nexent_execute_on_host", True) @@ -724,6 +740,11 @@ def _wrap_subagent( ), ) + def _check_sandbox_cancelled(self) -> None: + """Stop initialization and retries after the owning run is cancelled.""" + if self.stop_event.is_set() is True: + raise CancelledError("Sandbox initialization cancelled") + def create_single_agent( self, agent_config: AgentConfig, @@ -748,6 +769,7 @@ def create_single_agent( _sandbox_tree_context = {} try: + self._check_sandbox_cancelled() model = self.create_model(agent_config.model_name) model.context_budget_snapshot = getattr( agent_config, @@ -912,6 +934,7 @@ def create_single_agent( timeout_seconds=skill_timeout, workspace_path=self.workspace_path, network_enabled=not self.sandbox_config.network_disabled, + workspace_mapping=getattr(self, "workspace_mapping", None), ) for tool in tool_list: bind_backend = getattr(tool, "bind_execution_backend", None) @@ -925,6 +948,7 @@ def create_single_agent( if self.sandbox_config.level != SandboxLevel.LOCAL: try: warm_start = time.time() + current_metadata = get_agent_monitoring_context() or AgentRunMetadata() warmup_metadata = replace( current_metadata, @@ -946,10 +970,23 @@ def create_single_agent( "sandbox.backend": getattr(python_executor, "_nexent_backend", "unknown"), }, ): + self._check_sandbox_cancelled() python_executor("[0, None]") + self._check_sandbox_cancelled() + warm_dur = time.time() - warm_start backend = getattr(python_executor, "_nexent_backend", "unknown") if backend == "local": + if self.sandbox_config.failure_policy == "error": + raise RuntimeError("Docker was requested but the executor is local") + self.workspace_mapping = None + for tool in tool_list: + if getattr(tool, "workspace_mapping", None) is not None: + tool.workspace_mapping = None + self.observer.add_message( + "", ProcessType.WARNING, + "Docker sandbox unavailable; execution has fallen back to LocalPythonExecutor.", + ) logger.warning( "Sandbox level '%s' unavailable; using LocalPythonExecutor instead " "(scope=%s, warm-up %.2fs)", @@ -965,7 +1002,12 @@ def create_single_agent( self.sandbox_config.level.value, self.sandbox_config.scope.value, ) + except CancelledError: + raise except Exception as warm_err: + self._check_sandbox_cancelled() + if self.sandbox_config.failure_policy == "error": + raise RuntimeError("Sandbox FAILED phase=warmup") from warm_err logger.warning( "Sandbox warm-up failed (%s): %s", self.sandbox_config.level.value, @@ -975,6 +1017,7 @@ def create_single_agent( self._sandbox_scope = self.sandbox_config.scope.value # Create the agent + self._check_sandbox_cancelled() agent = CoreAgent( observer=self.observer, tools=tool_list, @@ -1019,7 +1062,15 @@ def create_single_agent( update_step._get_user_id = agent._get_user_id return agent + except CancelledError: + self._cleanup_sandbox() + raise except Exception as e: + if getattr(self.sandbox_config, "failure_policy", None) == "error": + try: + self._cleanup_sandbox() + except Exception: + logger.exception("Failed to release sandbox resources after agent construction failed") raise ValueError(f"Error in creating agent, agent name: {agent_config.name}, Error: {e}") def add_history_to_agent(self, history: List[AgentHistory]): @@ -1343,10 +1394,12 @@ def _prepare_file_workspace(self, query: str) -> str: result = json.loads(download_tool.forward(source_url, local_filename)) downloaded.append({"name": filename, "path": result["local_path"]}) + mapping = getattr(self, "workspace_mapping", None) + display_workspace = mapping.container_root if mapping is not None else workspace file_lines = "\n".join(f"- {item['name']}: {item['path']}" for item in downloaded) workspace_note = ( - f"\n\nRun workspace: {workspace}\n" - f"Write every generated file under: {workspace / 'outputs'}\n" + f"\n\nRun workspace: {display_workspace}\n" + f"Write every generated file under: {display_workspace / 'outputs'}\n" "The code executor already runs in that outputs directory. Use bare relative " "paths such as 'report.pdf', not 'outputs/report.pdf', to avoid creating an " "outputs/outputs directory.\n" @@ -1400,6 +1453,8 @@ def _sandbox_containers(self) -> List[Any]: def _uses_shared_file_workspace(self) -> bool: """Return whether the runtime and sandbox use the same workspace volume.""" + if getattr(self, "workspace_mapping", None) is not None: + return True extra_kwargs = getattr(self.sandbox_config, "extra_kwargs", {}) or {} return bool( extra_kwargs.get("shared_workspace") @@ -1412,6 +1467,10 @@ def _push_file_workspace_to_sandbox(self) -> None: if not containers or not self.workspace_path: return workspace = Path(self.workspace_path).resolve() + mapping = getattr(self, "workspace_mapping", None) + if mapping is not None: + self._verify_bind_workspace_access(containers, workspace, mapping) + return if not workspace.exists() or workspace.drive: return shared_workspace = self._uses_shared_file_workspace() @@ -1427,11 +1486,28 @@ def _push_file_workspace_to_sandbox(self) -> None: raise RuntimeError("Failed to copy run workspace into the sandbox") self._grant_sandbox_output_access(container, workspace) + def _verify_bind_workspace_access(self, containers, workspace: Path, mapping: SandboxWorkspace) -> None: + """Probe the mounted run directory without copying or changing Windows ACLs.""" + for container in containers: + try: + if not workspace.drive: + self._grant_sandbox_output_access(container, mapping.container_root) + probe_workspace(container, mapping.container_root) + except Exception: + logger.exception( + "Sandbox FAILED phase=workspace_access run_id=%s container_id=%s", + self.workspace_run_id, getattr(container, "id", None), + ) + raise + def _initialize_sandbox_workspaces(self) -> None: """Set every Docker kernel's cwd and workspace environment for this run.""" if not self.workspace_path: return workspace = Path(self.workspace_path).resolve() + mapping = getattr(self, "workspace_mapping", None) + if mapping is not None: + workspace = mapping.container_root output_dir = workspace / "outputs" bootstrap_code = ( "import os as _nexent_os\n" @@ -1444,6 +1520,7 @@ def _initialize_sandbox_workspaces(self) -> None: ) seen_executor_ids = set() for executor in self._sandbox_executors: + self._check_sandbox_cancelled() executor_id = id(executor) if executor_id in seen_executor_ids: continue @@ -1467,7 +1544,18 @@ def _initialize_sandbox_workspaces(self) -> None: ) try: execute_bootstrap(bootstrap_code) + self._check_sandbox_cancelled() + logger.info( + "Sandbox READY run_id=%s actual_backend=docker scope=%s container_id=%s kernel_id=%s " + "host_workspace=%s container_workspace=%s", + self.workspace_run_id, getattr(getattr(self.sandbox_config, "scope", None), "value", None), + getattr(getattr(executor, "container", None), "id", None), + getattr(executor, "kernel_id", None), self.workspace_path, workspace, + ) + except CancelledError: + raise except Exception as exc: + self._check_sandbox_cancelled() # Workspace initialization is idempotent. If the kernel channel # failed and marked this lease unhealthy, retry the bootstrap in # the same run so the lease can replace its kernel immediately. @@ -1476,16 +1564,27 @@ def _initialize_sandbox_workspaces(self) -> None: if ( getattr(executor, "_nexent_kernel_recovery_supported", False) and getattr(executor, "_unhealthy", False) + # Registered bootstraps already own their bounded recovery. + and not callable(register_bootstrap) ): logger.warning( "Retrying sandbox workspace initialization with a replacement kernel: %s", exc, + exc_info=True, ) try: execute_bootstrap(bootstrap_code) + self._check_sandbox_cancelled() continue + except CancelledError: + raise except Exception as retry_exc: + self._check_sandbox_cancelled() exc = retry_exc + logger.exception( + "Sandbox FAILED phase=workspace run_id=%s: %s", self.workspace_run_id, exc, + exc_info=(type(exc), exc, exc.__traceback__), + ) raise RuntimeError( f"Failed to initialize sandbox workspace '{workspace}': {exc}" ) from exc @@ -1602,7 +1701,8 @@ def _cleanup_file_workspace(self) -> None: if workspace.name != self.workspace_run_id: return try: - for container in self._sandbox_containers(): + containers = [] if getattr(self, "workspace_mapping", None) is not None else self._sandbox_containers() + for container in containers: try: result = container.exec_run( ["rm", "-rf", "--", str(workspace)], diff --git a/sdk/nexent/core/agents/run_agent.py b/sdk/nexent/core/agents/run_agent.py index 11d8442caf..765215f784 100644 --- a/sdk/nexent/core/agents/run_agent.py +++ b/sdk/nexent/core/agents/run_agent.py @@ -3,6 +3,7 @@ import logging import math import threading +from concurrent.futures import CancelledError from contextvars import Context, copy_context from copy import deepcopy from dataclasses import replace @@ -459,7 +460,7 @@ def _agent_run_thread(agent_run_info: AgentRunInfo): _log_memory_value_assessment(agent) agent_run_info.attempt_outcome = "stopped" if agent_run_info.stop_event.is_set() else "completed" - except RunTerminated: + except (RunTerminated, CancelledError): agent_run_info.attempt_outcome = "stopped" except (ModelInvocationTerminalError, ModelOutputProtocolExhaustedError): agent_run_info.attempt_outcome = "failed" diff --git a/sdk/nexent/core/agents/sandbox.py b/sdk/nexent/core/agents/sandbox.py index ddcb1f3e92..f7042d8c6c 100644 --- a/sdk/nexent/core/agents/sandbox.py +++ b/sdk/nexent/core/agents/sandbox.py @@ -31,6 +31,7 @@ import tarfile import threading import time +from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import TimeoutError as FuturesTimeoutError from contextlib import closing, contextmanager from contextvars import Context, copy_context @@ -40,7 +41,14 @@ from pathlib import Path from types import SimpleNamespace from typing import Any, Optional +from urllib.parse import urlsplit +from nexent.core.agents.sandbox_workspace import SandboxWorkspace, validate_container_root +from nexent.core.agents.sandbox_tls import ( + TLS_BOOTSTRAP, TLS_CERTIFICATE, TLS_DIRECTORY, TLS_LABEL, TLS_VERSION, + SandboxTLSClient, SandboxTLSMigrationRequired, SandboxTLSRecoveryError, + load_container_tls, require_tls_container, +) from nexent.core.concurrency import ( ManagedTaskSpec, ManagedThreadSpec, @@ -80,11 +88,15 @@ def __init__( timeout_seconds: int = 300, workspace_path: Optional[str] = None, network_enabled: bool = False, + workspace_mapping: Optional[SandboxWorkspace] = None, ) -> None: self._executor = executor self._container = getattr(executor, "container", None) self._timeout_seconds = max(1, int(timeout_seconds)) self._workspace_path = (workspace_path or "").rstrip("/") + self._workspace_mapping = workspace_mapping + if workspace_mapping is not None: + self._workspace_path = str(workspace_mapping.container_root) self._network_enabled = bool(network_enabled) self._pnpm_store_path = "" self._pnpm_store_seeded = False @@ -290,6 +302,8 @@ def __call__( ) normalized_source = (source or "skill").strip().lower() + if self._workspace_mapping is not None and working_directory: + working_directory = str(self._workspace_mapping.to_container(working_directory)) skill_python_path = "" if normalized_source == "skill": skills_root = self._resolve_skills_root(working_directory) @@ -747,6 +761,26 @@ class SandboxConfig: output_dir: str = "/home/sandbox/workdir/output" auto_sync_outputs: bool = True extra_kwargs: dict[str, Any] = field(default_factory=dict) + workspace_mode: str = "legacy" + container_workspace_root: str = "" + failure_policy: str = "local" + + def __post_init__(self) -> None: + if self.workspace_mode not in {"legacy", "bind"}: + raise ValueError("Unknown sandbox workspace mode") + if self.failure_policy not in {"local", "error"}: + raise ValueError("Unknown sandbox failure policy") + if self.workspace_mode == "bind": + validate_container_root(self.container_workspace_root) + + def bind_workspace(self) -> Optional[SandboxWorkspace]: + """Build the explicit mount mapping; never infer a mode from the host OS.""" + if self.workspace_mode != "bind": + return None + root = self.extra_kwargs.get("workspace_root") + if not root or not Path(root).is_absolute(): + raise ValueError("bind workspace requires an absolute host workspace_root") + return SandboxWorkspace(Path(root), validate_container_root(self.container_workspace_root)) @classmethod def from_dict(cls, data: Optional[dict[str, Any]]) -> "SandboxConfig": @@ -772,6 +806,9 @@ def from_dict(cls, data: Optional[dict[str, Any]]) -> "SandboxConfig": output_dir=data.get("output_dir", "/home/sandbox/workdir/output"), auto_sync_outputs=bool(data.get("auto_sync_outputs", True)), extra_kwargs=data.get("extra_kwargs", {}), + workspace_mode=data.get("workspace_mode", "legacy"), + container_workspace_root=data.get("container_workspace_root", ""), + failure_policy=data.get("failure_policy", "local"), ) @@ -1622,11 +1659,14 @@ def _seed_pnpm_offline_store(container: Any) -> None: ) -def _kernel_gateway_command() -> list[str]: +def _kernel_gateway_command(hostname: Optional[str] = None) -> list[str]: """Return the Kernel Gateway command required by Nexent's health checks.""" return [ + "python", "-c", TLS_BOOTSTRAP, hostname or SANDBOX_CONTAINER_NAME, TLS_DIRECTORY, "jupyter", "kernelgateway", + f"--KernelGatewayApp.certfile={TLS_CERTIFICATE}", + f"--KernelGatewayApp.keyfile={TLS_DIRECTORY}/server.key", "--KernelGatewayApp.ip=0.0.0.0", f"--KernelGatewayApp.port={SANDBOX_JUPYTER_PORT}", "--KernelGatewayApp.allow_origin=*", @@ -1647,6 +1687,21 @@ def _sandbox_connection_hosts(container: Any) -> list[str]: hosts.append(network_ip) return hosts + +def _published_sandbox_port(container: Any) -> int: + """Read the effective loopback mapping, never a requested or guessed port.""" + ports = (container.attrs.get("NetworkSettings") or {}).get("Ports") or {} + for binding in ports.get(f"{SANDBOX_JUPYTER_PORT}/tcp") or []: + if binding.get("HostIp") != "127.0.0.1": + continue + try: + port = int(binding.get("HostPort", "")) + except (TypeError, ValueError): + continue + if 0 < port < 65536: + return port + raise RuntimeError("Docker sandbox has no effective loopback port mapping for Jupyter") + class _RecoveredDockerExecutor: """Minimal Docker executor facade for a container owned by another runtime.""" @@ -1657,6 +1712,7 @@ def __init__( host: str, additional_imports: Optional[list[str]] = None, port: int = SANDBOX_JUPYTER_PORT, + tls_client: Optional[SandboxTLSClient] = None, ) -> None: self.container = container self.client = container.client @@ -1664,7 +1720,8 @@ def __init__( self._logger = logger_ self.host = host self.port = port - self.base_url = f"http://{self.host}:{self.port}" + self.base_url = f"https://{self.host}:{self.port}" + self.tls_client = tls_client if tls_client is not None else load_container_tls(container) self.additional_imports = additional_imports or [] self.installed_packages = [] self._nexent_backend = "docker" @@ -1675,6 +1732,8 @@ def cleanup(self) -> None: self.container.remove(force=True) except Exception as exc: self._logger.warning("Failed to remove recovered sandbox container: %s", exc) + finally: + self.tls_client.close() class _DockerKernelLease: @@ -1688,9 +1747,6 @@ def __init__( logger_: logging.Logger, receive_timeout_seconds: float = 30, ) -> None: - import requests - from smolagents.remote_executors import _create_kernel_http - self._container_executor = container_executor self.logger = container_executor.logger self.additional_imports = getattr(container_executor, "additional_imports", []) @@ -1700,19 +1756,128 @@ def __init__( self.base_url = container_executor.base_url self.host = container_executor.host self.port = container_executor.port - self.kernel_id = _create_kernel_http(f"{self.base_url}/api/kernels", self.logger) - self._channel_session_id = secrets.token_hex(16) - self.ws_url = self._build_channels_url(self.kernel_id) self._receive_timeout_seconds = float(receive_timeout_seconds) if self._receive_timeout_seconds <= 0: raise ValueError("Sandbox WebSocket receive timeout must be positive") + self._requests = container_executor.tls_client.http + self._ssl_context = container_executor.tls_client.ssl_context + self.kernel_id = self._create_kernel() + self._channel_session_id = secrets.token_hex(16) + self.ws_url = self._build_channels_url(self.kernel_id) self._closed = False self._unhealthy = False self._nexent_kernel_recovery_supported = True - self._requests = requests self._cached_variables: Optional[dict[str, Any]] = None self._cached_tools: Optional[dict[str, Any]] = None self._kernel_bootstrap_code: list[str] = [] + self._cancellation_scope: Optional[RunCancellationScope] = None + + def _create_kernel(self) -> str: + """Create kernels using this owner's pinned trust, including on recovery.""" + response = self._requests.post( + f"{self.base_url}/api/kernels", timeout=self._receive_timeout_seconds, + ) + if response.status_code != 201: + raise RuntimeError(f"Jupyter kernel creation failed (status={response.status_code})") + return response.json()["id"] + + def _check_execution_cancelled(self) -> None: + """Cancellation must stop channel waits and framework-state replay.""" + scope = getattr(self, "_cancellation_scope", None) + if scope is not None and scope.stop_event.is_set(): + raise FuturesCancelledError("Sandbox kernel execution cancelled") + + def _wait_for_kernel_channel_ready(self, ws: Any) -> None: + """Verify shell and IOPub delivery before sending code with side effects.""" + from websocket import ABNF, WebSocketConnectionClosedException, WebSocketTimeoutException + + request_id = secrets.token_hex(16) + ws.send(json.dumps({ + "header": { + "msg_id": request_id, "session": self._channel_session_id, + "username": "nexent", "msg_type": "kernel_info_request", "version": "5.3", + }, + "parent_header": {}, "metadata": {}, "content": {}, "channel": "shell", + })) + deadline = time.monotonic() + min(3.0, self._receive_timeout_seconds) + reply_received = idle_received = False + while not (reply_received and idle_received): + self._check_execution_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise WebSocketTimeoutException("Kernel channel readiness handshake timed out") + ws.settimeout(remaining) + opcode, raw = ws.recv_data(control_frame=True) + if opcode in (ABNF.OPCODE_PING, ABNF.OPCODE_PONG): + continue + if opcode == ABNF.OPCODE_CLOSE or not raw: + raise WebSocketConnectionClosedException("Kernel channel closed during handshake") + message = json.loads(raw) + if message.get("parent_header", {}).get("msg_id") != request_id: + continue + kind = message.get("msg_type") or message.get("header", {}).get("msg_type") + reply_received |= kind == "kernel_info_reply" + idle_received |= kind == "status" and message.get("content", {}).get("execution_state") == "idle" + + @contextmanager + def _kernel_channel(self): + """Retry only the side-effect-free handshake, never a submitted execution.""" + from websocket import ( + WebSocketBadStatusException, WebSocketConnectionClosedException, + WebSocketTimeoutException, create_connection, + ) + + channel_host = urlsplit(self.ws_url).hostname + if not channel_host: + raise ValueError("Sandbox WebSocket URL must contain a hostname") + for attempt in range(3): + self._check_execution_cancelled() + ws = None + scope = getattr(self, "_cancellation_scope", None) + token = None + try: + try: + ws = create_connection( + self.ws_url, timeout=self._receive_timeout_seconds, + sslopt={"context": self._ssl_context}, + # websocket-client only retains no_proxy with an explicit proxy host. + # The matching bypass entry prevents any proxy connection to this host. + http_proxy_host=channel_host, + http_no_proxy=[channel_host], redirect_limit=0, + ) + status = ws.getstatus() + if status != 101: + raise WebSocketBadStatusException( + "Sandbox WebSocket handshake requires status 101", status, + ) + if scope is not None: + token = scope.register_closer(ws.shutdown) + self._check_execution_cancelled() + self._wait_for_kernel_channel_ready(ws) + except (TimeoutError, ConnectionError, WebSocketTimeoutException, + WebSocketConnectionClosedException, WebSocketBadStatusException) as exc: + self._check_execution_cancelled() + if isinstance(exc, WebSocketBadStatusException) and exc.status_code != 404: + raise + if attempt == 2: + self._unhealthy = True + raise RuntimeError("Sandbox kernel channel readiness failed before code submission") from exc + self._logger.warning( + "Sandbox channel handshake retry: kernel_id=%s attempt=%d error_type=%s", + self.kernel_id, attempt + 1, type(exc).__name__, + ) + continue + self._check_execution_cancelled() + yield ws + return + except Exception: + self._check_execution_cancelled() + raise + finally: + if scope is not None and token is not None: + scope.unregister_closer(token) + if ws is not None: + ws.close() def _build_channels_url(self, kernel_id: str) -> str: """Build a Kernel Gateway channel URL with a stable client session.""" @@ -1721,7 +1886,7 @@ def _build_channels_url(self, kernel_id: str) -> str: session_id = secrets.token_hex(16) self._channel_session_id = session_id return ( - f"ws://{self.host}:{self.port}/api/kernels/{kernel_id}/channels" + f"wss://{self.host}:{self.port}/api/kernels/{kernel_id}/channels" f"?session_id={session_id}" ) @@ -1745,17 +1910,15 @@ def run_code_raise_errors(self, code: str) -> Any: ABNF, WebSocketConnectionClosedException, WebSocketTimeoutException, - create_connection, ) + self._check_execution_cancelled() if self._closed: raise RuntimeError("Sandbox kernel lease is already closed") if self._unhealthy: self._replace_unhealthy_kernel() - with closing( - create_connection(self.ws_url, timeout=self._receive_timeout_seconds) - ) as ws: + with self._kernel_channel() as ws: msg_id = _websocket_send_execute_request(code, ws) outputs = [] result = None @@ -1763,6 +1926,7 @@ def run_code_raise_errors(self, code: str) -> Any: status_deadline = time.monotonic() + self._receive_timeout_seconds while True: + self._check_execution_cancelled() now = time.monotonic() if now >= status_deadline: self._check_kernel_channel_health( @@ -1813,7 +1977,7 @@ def run_code_raise_errors(self, code: str) -> Any: if parent_msg_id != msg_id: continue - msg_type = message.get("msg_type", "") + msg_type = message.get("msg_type") or message.get("header", {}).get("msg_type", "") content = message.get("content", {}) if msg_type == "stream": outputs.append(content["text"]) @@ -1843,6 +2007,7 @@ def _check_kernel_channel_health( allow_busy: bool = True, ) -> None: """Fail a lost kernel channel while allowing a genuinely busy kernel to continue.""" + self._check_execution_cancelled() state = self._get_kernel_execution_state() if allow_busy and state == "busy": self._logger.debug( @@ -1879,12 +2044,24 @@ def _get_kernel_execution_state(self) -> Optional[str]: return None def _replace_unhealthy_kernel(self) -> None: + """Bound recovery even when replaying framework state enters setup again.""" + self._check_execution_cancelled() + if getattr(self, "_kernel_replacement_in_progress", False): + raise RuntimeError("Sandbox kernel recovery failed during framework-state replay") + self._kernel_replacement_count = getattr(self, "_kernel_replacement_count", 0) + 1 + self._kernel_replacement_in_progress = True + try: + self._replace_kernel_and_restore_state() + finally: + self._kernel_replacement_in_progress = False + + def _replace_kernel_and_restore_state(self) -> None: """Replace a failed kernel and restore framework-managed execution state.""" from smolagents.remote_executors import ( RemotePythonExecutor, - _create_kernel_http, ) + self._check_execution_cancelled() previous_kernel_id = self.kernel_id try: response = self._requests.delete( @@ -1909,7 +2086,8 @@ def _replace_unhealthy_kernel(self) -> None: previous_kernel_id, ) try: - kernel_id = _create_kernel_http(f"{self.base_url}/api/kernels", self.logger) + self._check_execution_cancelled() + kernel_id = self._create_kernel() self.kernel_id = kernel_id self._channel_session_id = secrets.token_hex(16) self.ws_url = self._build_channels_url(kernel_id) @@ -1921,6 +2099,8 @@ def _replace_unhealthy_kernel(self) -> None: RemotePythonExecutor.send_tools(self, self._cached_tools) for code in self._kernel_bootstrap_code: self.run_code_raise_errors(code) + except FuturesCancelledError: + raise except Exception as exc: self._unhealthy = True self._logger.exception( @@ -1969,10 +2149,12 @@ def __call__(self, code_action: str) -> Any: def send_variables(self, variables: dict[str, Any]) -> None: from smolagents.remote_executors import RemotePythonExecutor self._cached_variables = dict(variables) + recovery_count = getattr(self, "_kernel_replacement_count", 0) try: RemotePythonExecutor.send_variables(self, variables) except Exception as exc: - if not self._unhealthy: + self._check_execution_cancelled() + if not self._can_retry_initialization(recovery_count): raise self._logger.warning( "Retrying sandbox variable registration with a replacement kernel: %s", @@ -2028,10 +2210,12 @@ def __init__(self, value): def send_tools(self, tools: dict[str, Any]) -> None: from smolagents.remote_executors import RemotePythonExecutor self._cached_tools = dict(tools) + recovery_count = getattr(self, "_kernel_replacement_count", 0) try: RemotePythonExecutor.send_tools(self, tools) except Exception as exc: - if not self._unhealthy: + self._check_execution_cancelled() + if not self._can_retry_initialization(recovery_count): raise self._logger.warning( "Retrying sandbox tool registration with a replacement kernel: %s", @@ -2043,10 +2227,12 @@ def send_tools(self, tools: dict[str, Any]) -> None: def register_kernel_bootstrap_code(self, code: str) -> Any: """Execute and retain framework bootstrap code for future kernel replacement.""" + recovery_count = getattr(self, "_kernel_replacement_count", 0) try: output = self.run_code_raise_errors(code) except Exception as exc: - if not self._unhealthy: + self._check_execution_cancelled() + if not self._can_retry_initialization(recovery_count): raise self._logger.warning( "Retrying sandbox bootstrap registration with a replacement kernel: %s", @@ -2058,6 +2244,13 @@ def register_kernel_bootstrap_code(self, code: str) -> Any: self._kernel_bootstrap_code.append(code) return output + def _can_retry_initialization(self, recovery_count: int) -> bool: + return ( + self._unhealthy + and not getattr(self, "_kernel_replacement_in_progress", False) + and getattr(self, "_kernel_replacement_count", 0) == recovery_count + ) + def cleanup(self) -> None: """Delete this kernel while leaving the shared container running.""" if self._closed: @@ -2258,6 +2451,15 @@ def acquire( ) return ex + def _check_system_startup_cancelled( + self, cancellation_scope: Optional[RunCancellationScope], + ) -> None: + """Stop startup without treating run cancellation as a shared-owner failure.""" + if self._stop_evict.is_set() or ( + cancellation_scope is not None and cancellation_scope.stop_event.is_set() + ): + raise FuturesCancelledError("System sandbox startup cancelled") + def _acquire_shared_docker_kernel( self, config: SandboxConfig, @@ -2283,14 +2485,23 @@ def discard_owner(owner: Any) -> None: self._destroy_executor(owner, logger_) def get_or_create_owner() -> Any: + self._check_system_startup_cancelled(cancellation_scope) with self._lock: owner = self._system_containers.get(pool_key) + if owner is not None: + previous = getattr(owner, "_nexent_sandbox_config", None) + if isinstance(previous, SandboxConfig): + if previous.bind_workspace() != config.bind_workspace(): + raise RuntimeError("System sandbox workspace changed; restart runtime before reuse") if owner is not None and self._is_alive(owner): return owner if owner is not None: discard_owner(owner) - with self._container_build_lock: + while not self._container_build_lock.acquire(timeout=0.1): + self._check_system_startup_cancelled(cancellation_scope) + try: + self._check_system_startup_cancelled(cancellation_scope) with self._lock: owner = self._system_containers.get(pool_key) if owner is not None and self._is_alive(owner): @@ -2305,15 +2516,26 @@ def get_or_create_owner() -> Any: ) if owner is None: self._remove_stale_docker_containers(config, logger_) - owner = self._build_executor(config, logger_, host_tools_exist) + owner = self._build_executor( + config, logger_, host_tools_exist, cancellation_scope=cancellation_scope, + ) if not hasattr(owner, "base_url") or not hasattr(owner, "container"): return owner with self._lock: - existing = self._system_containers.setdefault(pool_key, owner) + cancelled = self._stop_evict.is_set() or ( + cancellation_scope is not None and cancellation_scope.stop_event.is_set() + ) + existing = owner if cancelled else self._system_containers.setdefault(pool_key, owner) + if cancelled: + # This owner has not been published; no other run can lease it. + self._destroy_executor(owner, logger_) + raise FuturesCancelledError("System sandbox startup cancelled before publication") if existing is not owner: self._destroy_executor(owner, logger_) owner = existing return owner + finally: + self._container_build_lock.release() container_executor = None lease = None @@ -2325,6 +2547,7 @@ def get_or_create_owner() -> Any: ): return container_executor try: + self._check_system_startup_cancelled(cancellation_scope) # Revalidate immediately before creating the kernel. This closes # the restart window between owner lookup and the Kernel Gateway # request, while the one retry rebuilds stale recovered owners. @@ -2335,8 +2558,23 @@ def get_or_create_owner() -> Any: logger_, receive_timeout_seconds=config.timeout_seconds, ) + lease._cancellation_scope = cancellation_scope break + except FuturesCancelledError: + # The owner is already shared. Cancelling this run must not remove it. + raise except Exception as exc: + self._check_system_startup_cancelled(cancellation_scope) + with self._lock: + active = any( + getattr(executor, "container", None) is getattr(container_executor, "container", None) + for executor in self._executors.values() + if getattr(executor, "container", None) is not None + ) + if active: + raise RuntimeError( + "Kernel connection failed; preserving the shared container with active leases" + ) from exc discard_owner(container_executor) if attempt == 0: logger_.warning( @@ -2350,6 +2588,11 @@ def get_or_create_owner() -> Any: if lease is None: # pragma: no cover - loop either assigns or raises raise RuntimeError("Failed to create a shared sandbox kernel lease") + try: + self._check_system_startup_cancelled(cancellation_scope) + except FuturesCancelledError: + lease.cleanup() + raise if host_tools_exist: lease = _install_host_tool_bridge( lease, @@ -2361,10 +2604,17 @@ def get_or_create_owner() -> Any: lease._nexent_sandbox_config = config lease._nexent_pool_key = pool_key with self._lock: - self._in_use[id(lease)] = pool_key - self._lease_owners[id(lease)] = container_executor - self._executors[id(lease)] = lease - self._last_touch[id(lease)] = _now() + cancelled = self._stop_evict.is_set() or ( + cancellation_scope is not None and cancellation_scope.stop_event.is_set() + ) + if not cancelled: + self._in_use[id(lease)] = pool_key + self._lease_owners[id(lease)] = container_executor + self._executors[id(lease)] = lease + self._last_touch[id(lease)] = _now() + if cancelled: + lease.cleanup() + raise FuturesCancelledError("System sandbox kernel acquisition cancelled") logger_.debug( "Leased dedicated Jupyter kernel %s from shared sandbox (key=%s)", lease.kernel_id, @@ -2424,6 +2674,9 @@ def release_immediate(self, executor: Any, logger_: logging.Logger) -> None: self._destroy_executor(executor, logger_) if shared_container is not None: with self._lock: + if any(owner is shared_container for owner in self._lease_owners.values()): + logger_.info("Preserving shared sandbox with other active kernel leases") + return if self._system_containers.get(pool_key) is shared_container: self._system_containers.pop(pool_key, None) self._destroy_executor(shared_container, logger_) @@ -2508,9 +2761,11 @@ def _recover_docker_container( host_tools_exist: bool, ) -> Optional[Any]: """Recover a healthy Docker sandbox left by a previous runtime process.""" + from requests.exceptions import SSLError + + tls_client = None try: import docker - import requests client = docker.from_env() containers = [ @@ -2527,13 +2782,23 @@ def _recover_docker_container( if labels.get("com.nexent.sandbox") != "runtime": logger_.warning("Ignoring unrelated container named %s", SANDBOX_CONTAINER_NAME) return None + require_tls_container(container) if container.status != "running": logger_.warning("Persisted sandbox container is not running (status=%s)", container.status) return None workspace_volume_name = config.extra_kwargs.get("workspace_volume_name") workspace_root = config.extra_kwargs.get("workspace_root") - if workspace_volume_name and workspace_root: + mapping = config.bind_workspace() + if mapping is not None: + mounts = container.attrs.get("Mounts") or [] + if labels.get("com.nexent.workspace") != mapping.mount_id or not any( + mapping.matches_mount(mount) + for mount in mounts + ): + logger_.warning("Persisted sandbox bind workspace does not match configuration") + return None + elif workspace_volume_name and workspace_root: expected_destination = str(Path(workspace_root).resolve()) mounts = container.attrs.get("Mounts") or [] has_expected_mount = any( @@ -2552,29 +2817,37 @@ def _recover_docker_container( return None networks = (container.attrs.get("NetworkSettings") or {}).get("Networks") or {} - if SANDBOX_NETWORK_NAME not in networks: + containerized_runtime = _is_containerized_runtime() + if containerized_runtime and SANDBOX_NETWORK_NAME not in networks: logger_.warning("Persisted sandbox container is not attached to network %s", SANDBOX_NETWORK_NAME) return None - if not _is_containerized_runtime(): - ports = (container.attrs.get("NetworkSettings") or {}).get("Ports") or {} - bindings = ports.get(f"{SANDBOX_JUPYTER_PORT}/tcp") or [] - if not any(str(binding.get("HostPort")) == str(SANDBOX_JUPYTER_PORT) for binding in bindings): - logger_.warning("Persisted sandbox container does not expose host port %s", SANDBOX_JUPYTER_PORT) - return None + connection_port = SANDBOX_JUPYTER_PORT if containerized_runtime else _published_sandbox_port(container) + try: + tls_client = load_container_tls(container) + except Exception as exc: + raise SandboxTLSRecoveryError( + "Cannot load the running sandbox's TLS identity; container preserved. " + "Drain active runs and explicitly stop it before recreating." + ) from exc selected_host = None kernels = None - for candidate_host in _sandbox_connection_hosts(container): - base_url = f"http://{candidate_host}:{SANDBOX_JUPYTER_PORT}" + for candidate_host in (_sandbox_connection_hosts(container) if containerized_runtime else ["127.0.0.1"]): + base_url = f"https://{candidate_host}:{connection_port}" try: - response = requests.get(f"{base_url}/api/kernels", timeout=3) + response = tls_client.http.get(f"{base_url}/api/kernels", timeout=3) response.raise_for_status() candidate_kernels = response.json() if isinstance(candidate_kernels, list): selected_host = candidate_host kernels = candidate_kernels break + except SSLError as exc: + raise SandboxTLSRecoveryError( + "Running sandbox TLS verification failed; container preserved. " + "Drain active runs and explicitly stop it before recreating." + ) from exc except Exception: continue if selected_host is None or kernels is None: @@ -2584,7 +2857,10 @@ def _recover_docker_container( logger_, selected_host, config.extra_kwargs.get("additional_imports", []), + port=connection_port, + tls_client=tls_client, ) + tls_client = None recovered._nexent_sandbox_config = config recovered._nexent_kernel_count = len(kernels) logger_.info( @@ -2594,12 +2870,17 @@ def _recover_docker_container( len(kernels), ) return recovered + except (SandboxTLSMigrationRequired, SandboxTLSRecoveryError): + raise except Exception as exc: logger_.warning("Persisted Docker sandbox recovery failed: %s", exc) return None + finally: + if tls_client is not None: + tls_client.close() def _remove_stale_docker_containers(self, config: SandboxConfig, logger_: logging.Logger) -> None: - """Remove stale containers that would conflict with the stable sandbox name or port.""" + """Remove only Nexent-owned containers conflicting with the stable system name.""" try: import docker @@ -2607,20 +2888,19 @@ def _remove_stale_docker_containers(self, config: SandboxConfig, logger_: loggin containers = [] for container in client.containers.list(all=True): container.reload() - if container.name == SANDBOX_CONTAINER_NAME: + if container.name == SANDBOX_CONTAINER_NAME and (container.labels or {}).get( + "com.nexent.sandbox" + ) == "runtime": + require_tls_container(container) containers.append(container) - continue - if container.image.tags and config.docker_image in container.image.tags: - ports = (container.attrs.get("NetworkSettings") or {}).get("Ports") or {} - bindings = ports.get(f"{SANDBOX_JUPYTER_PORT}/tcp") or [] - if any(str(binding.get("HostPort")) == str(SANDBOX_JUPYTER_PORT) for binding in bindings): - containers.append(container) for container in containers: try: container.remove(force=True) logger_.info("Removed stale persisted sandbox container %s", container.short_id) except Exception as exc: logger_.warning("Failed to remove stale sandbox container: %s", exc) + except SandboxTLSMigrationRequired: + raise except Exception as exc: logger_.debug("Could not inspect stale sandbox containers: %s", exc) @@ -2632,7 +2912,6 @@ def _build_session_docker_executor( ) -> Any: """Create a per-session container without a fixed host-port binding.""" import docker - import requests client = docker.from_env() run_kwargs = dict(container_run_kwargs) @@ -2640,8 +2919,8 @@ def _build_session_docker_executor( container_name = f"{SANDBOX_SESSION_CONTAINER_PREFIX}-{secrets.token_hex(8)}" run_kwargs.update({ "name": container_name, - "labels": {"com.nexent.sandbox": "session"}, - "command": _kernel_gateway_command(), + "labels": {**run_kwargs.get("labels", {}), "com.nexent.sandbox": "session", TLS_LABEL: TLS_VERSION}, + "command": _kernel_gateway_command(container_name), "detach": True, # Kernel Gateway needs a network namespace for its HTTP/WebSocket # control plane. Its published endpoint remains constrained to host @@ -2680,27 +2959,25 @@ def _build_session_docker_executor( connection_port = 0 container = client.containers.run(config.docker_image, **run_kwargs) - if _is_containerized_runtime() and not config.network_disabled: - _attach_sandbox_to_control_network( - client, - container, - alias=container_name, - ) - _seed_pnpm_offline_store(container) owner = None container_group = None executor = None + tls_client = None try: + if _is_containerized_runtime() and not config.network_disabled: + _attach_sandbox_to_control_network( + client, + container, + alias=container_name, + ) + _seed_pnpm_offline_store(container) container.reload() if not _is_containerized_runtime(): - ports = (container.attrs.get("NetworkSettings") or {}).get("Ports") or {} - bindings = ports.get(f"{SANDBOX_JUPYTER_PORT}/tcp") or [] - if not bindings or not bindings[0].get("HostPort"): - raise RuntimeError("Docker did not allocate a Jupyter host port") - connection_port = int(bindings[0]["HostPort"]) + connection_port = _published_sandbox_port(container) + tls_client = load_container_tls(container, timeout=max(10, config.timeout_seconds)) deadline = time.monotonic() + max(10, config.timeout_seconds) - base_url = f"http://{connection_host}:{connection_port}" + base_url = f"https://{connection_host}:{connection_port}" while time.monotonic() < deadline: container.reload() if container.status not in (None, "created", "running"): @@ -2709,7 +2986,7 @@ def _build_session_docker_executor( f"(status={container.status})" ) try: - response = requests.get(f"{base_url}/api/kernels", timeout=1) + response = tls_client.http.get(f"{base_url}/api/kernels", timeout=1) response.raise_for_status() if isinstance(response.json(), list): break @@ -2725,7 +3002,9 @@ def _build_session_docker_executor( connection_host, config.extra_kwargs.get("additional_imports", []), port=connection_port, + tls_client=tls_client, ) + tls_client = None container_group = _SessionDockerContainerGroup(owner) executor = self._lease_session_docker_kernel( config, @@ -2733,12 +3012,14 @@ def _build_session_docker_executor( container_group, ) logger_.info( - "Created session Docker sandbox %s (url=%s)", + "Sandbox CREATED scope=session container_id=%s (url=%s)", container.short_id, executor.base_url, ) return executor except Exception: + if tls_client is not None: + tls_client.close() if executor is not None: executor.cleanup() if container_group is not None: @@ -2781,65 +3062,130 @@ def _build_system_docker_executor( config: SandboxConfig, logger_: logging.Logger, container_run_kwargs: dict[str, Any], + cancellation_scope: Optional[RunCancellationScope] = None, ) -> Any: """Create a shared Docker sandbox and connect over host or container networking.""" import docker - import requests client = docker.from_env() run_kwargs = dict(container_run_kwargs) + run_kwargs["command"] = _kernel_gateway_command() + run_kwargs["labels"] = {**run_kwargs.get("labels", {}), TLS_LABEL: TLS_VERSION} _apply_legacy_docker_seccomp_compatibility(client, run_kwargs, logger_) - if _is_containerized_runtime(): + containerized_runtime = _is_containerized_runtime() + if containerized_runtime: run_kwargs.pop("ports", None) else: + run_kwargs["network"] = "bridge" + run_kwargs["network_disabled"] = False run_kwargs["ports"] = { - f"{SANDBOX_JUPYTER_PORT}/tcp": ("127.0.0.1", SANDBOX_JUPYTER_PORT) + f"{SANDBOX_JUPYTER_PORT}/tcp": ("127.0.0.1", None) } + if config.network_disabled: + logger_.warning( + "Native system sandbox uses bridge networking for the Jupyter control endpoint; " + "the published port is loopback-only, but outbound network isolation is not enforced" + ) run_kwargs["detach"] = True + logger_.info( + "Sandbox create requested: scope=system runtime_containerized=%s image=%s " + "policy_network_disabled=%s network=%s network_mode=%s " + "docker_network_disabled=%s ports=%s", + containerized_runtime, + config.docker_image, + config.network_disabled, + run_kwargs.get("network"), + run_kwargs.get("network_mode"), + run_kwargs.get("network_disabled", False), + run_kwargs.get("ports"), + ) + self._check_system_startup_cancelled(cancellation_scope) container = client.containers.run(config.docker_image, **run_kwargs) - if _is_containerized_runtime() and not config.network_disabled: - _attach_sandbox_to_control_network( - client, - container, - alias=SANDBOX_CONTAINER_NAME, - ) - _seed_pnpm_offline_store(container) + tls_client = None try: + self._check_system_startup_cancelled(cancellation_scope) + if containerized_runtime and not config.network_disabled: + _attach_sandbox_to_control_network( + client, + container, + alias=SANDBOX_CONTAINER_NAME, + ) container.reload() + host_config = container.attrs.get("HostConfig", {}) + network_settings = container.attrs.get("NetworkSettings", {}) + logger_.info( + "Sandbox container inspected: scope=system container_id=%s name=%s " + "network_mode=%s configured_bindings=%s effective_ports=%s networks=%s", + container.short_id, + container.name, + host_config.get("NetworkMode"), + host_config.get("PortBindings"), + network_settings.get("Ports"), + sorted(network_settings.get("Networks", {})), + ) + _seed_pnpm_offline_store(container) + connection_port = SANDBOX_JUPYTER_PORT if containerized_runtime else _published_sandbox_port(container) + tls_client = load_container_tls( + container, timeout=max(10, config.timeout_seconds), + check_cancelled=lambda: self._check_system_startup_cancelled(cancellation_scope), + ) deadline = time.monotonic() + max(10, config.timeout_seconds) selected_host = None + last_probe_errors: dict[str, str] = {} while time.monotonic() < deadline: + self._check_system_startup_cancelled(cancellation_scope) container.reload() - for candidate_host in _sandbox_connection_hosts(container): - base_url = f"http://{candidate_host}:{SANDBOX_JUPYTER_PORT}" + for candidate_host in (_sandbox_connection_hosts(container) if containerized_runtime else ["127.0.0.1"]): + base_url = f"https://{candidate_host}:{connection_port}" try: - response = requests.get(f"{base_url}/api/kernels", timeout=1) + response = tls_client.http.get(f"{base_url}/api/kernels", timeout=1) response.raise_for_status() if isinstance(response.json(), list): selected_host = candidate_host break - except Exception: + last_probe_errors[base_url] = "Kernel API response is not a list" + except Exception as exc: + last_probe_errors[base_url] = f"{type(exc).__name__}: {exc}" continue if selected_host is not None: break - time.sleep(0.5) + self._stop_evict.wait(0.5) + self._check_system_startup_cancelled(cancellation_scope) if selected_host is None: + logger_.error( + "Sandbox gateway failed: scope=system container_id=%s " + "effective_ports=%s last_probe_errors=%s", + container.short_id, + container.attrs.get("NetworkSettings", {}).get("Ports"), + last_probe_errors, + ) raise RuntimeError("Jupyter kernel API did not become ready") + logger_.info( + "Sandbox gateway ready: scope=system container_id=%s endpoint=https://%s:%s/api/kernels", + container.short_id, + selected_host, + connection_port, + ) executor = _RecoveredDockerExecutor( container, logger_, selected_host, config.extra_kwargs.get("additional_imports", []), + port=connection_port, + tls_client=tls_client, ) + tls_client = None executor._nexent_sandbox_config = config logger_.info( - "Created shared Docker sandbox %s (url=%s, network=%s)", + "Sandbox CREATED scope=system container_id=%s (url=%s, network=%s)", container.short_id, executor.base_url, - SANDBOX_NETWORK_NAME, + sorted(network_settings.get("Networks", {})), ) return executor except Exception: + if tls_client is not None: + tls_client.close() try: container.remove(force=True) except Exception: @@ -2861,6 +3207,8 @@ def _build_docker_executor( if DockerExecutor is None: raise ImportError("DockerExecutor is unavailable") except ImportError: + if config.failure_policy == "error": + raise RuntimeError("Docker sandbox FAILED phase=dependency; local fallback is disabled") logger_.error( "DockerExecutor requires smolagents[docker]. " "Install it with: pip install 'smolagents[docker]'. " @@ -2883,7 +3231,10 @@ def _build_docker_executor( else False ), } - if host_tools_exist and not _is_containerized_runtime(): + if ( + host_tools_exist and not _is_containerized_runtime() + and not (config.workspace_mode == "bind" and Path().resolve().drive) + ): import docker docker_client = docker.from_env() @@ -2895,7 +3246,16 @@ def _build_docker_executor( container_environment.update(_ONLINE_PACKAGE_ENV) container_run_kwargs["environment"] = container_environment workspace_root = config.extra_kwargs.get("workspace_root") - if workspace_root: + mapping = config.bind_workspace() + if mapping is not None: + from docker.types import Mount + + mapping.host_root.mkdir(parents=True, exist_ok=True) + container_run_kwargs["mounts"] = [Mount( + source=str(mapping.host_root), target=str(mapping.container_root), type="bind", + )] + container_run_kwargs["labels"] = {"com.nexent.workspace": mapping.mount_id} + elif workspace_root: resolved_workspace_root = str(Path(workspace_root).resolve()) workspace_volume_name = config.extra_kwargs.get("workspace_volume_name") if not workspace_volume_name: @@ -2911,21 +3271,23 @@ def _build_docker_executor( import docker docker_client = docker.from_env() - _ensure_sandbox_control_network(docker_client) + if _is_containerized_runtime(): + _ensure_sandbox_control_network(docker_client) container_run_kwargs.update({ "name": SANDBOX_CONTAINER_NAME, - "labels": {"com.nexent.sandbox": "runtime"}, + "labels": {**container_run_kwargs.get("labels", {}), "com.nexent.sandbox": "runtime"}, "command": _kernel_gateway_command(), }) - if config.network_disabled: + if _is_containerized_runtime() and config.network_disabled: container_run_kwargs["network"] = SANDBOX_NETWORK_NAME else: container_run_kwargs.pop("network", None) - logger_.debug("Using Docker network %s for system sandbox", SANDBOX_NETWORK_NAME) except Exception as exc: + if config.failure_policy == "error": + raise RuntimeError("Docker sandbox FAILED phase=network_setup") from exc logger_.warning("Could not prepare Docker network %s: %s", SANDBOX_NETWORK_NAME, exc) - if host_tools_exist and config.network_disabled: + if host_tools_exist and config.network_disabled and config.scope != SandboxScope.SYSTEM: logger_.warning( "Docker network isolation is relaxed to bridge mode so sandbox code can call " "token-authenticated Nexent host tools" @@ -2937,6 +3299,7 @@ def _build_docker_executor( config, logger_, container_run_kwargs, + cancellation_scope=cancellation_scope, ) else: if session_container_group is None: @@ -2952,6 +3315,8 @@ def _build_docker_executor( session_container_group, ) executor._nexent_sandbox_config = config # store for pool bookkeeping + if config.scope == SandboxScope.SESSION: + executor._cancellation_scope = cancellation_scope executor._nexent_backend = "docker" logger_.debug( "DockerExecutor created (image=%s, mem=%dm, network=%s)", @@ -2959,7 +3324,12 @@ def _build_docker_executor( config.memory_limit_mb, network_mode, ) + except FuturesCancelledError: + raise except Exception as exc: + if config.failure_policy == "error": + logger_.exception("Docker sandbox FAILED phase=create_or_connect: %s", exc) + raise RuntimeError("Docker sandbox unavailable; local fallback is disabled") from exc logger_.error( "DockerExecutor construction failed: %s. " "Falling back to LocalPythonExecutor.", @@ -3153,9 +3523,13 @@ def build_python_executor( the existing container instead of starting another container. Returns: - A wrapped python_executor. Never raises — always returns a usable - executor (falls back to LocalPythonExecutor on any error). + A wrapped python_executor. Strict Docker policy propagates startup + failures; the compatibility policy permits local construction fallback. """ + logger_.info( + "Sandbox requested_backend=%s scope=%s workspace_mode=%s failure_policy=%s image=%s", + config.level.value, config.scope.value, config.workspace_mode, config.failure_policy, config.docker_image, + ) pool = SandboxPoolManager.get_instance() if config.scope == SandboxScope.SESSION: diff --git a/sdk/nexent/core/agents/sandbox_tls.py b/sdk/nexent/core/agents/sandbox_tls.py new file mode 100644 index 0000000000..8604db25c3 --- /dev/null +++ b/sdk/nexent/core/agents/sandbox_tls.py @@ -0,0 +1,95 @@ +"""Per-container Jupyter TLS, with trust obtained through the Docker API.""" + +import io +import ssl +import tarfile +import tempfile +import time +from pathlib import Path + + +TLS_LABEL = 'com.nexent.jupyter.tls' +TLS_VERSION = '2' +TLS_DIRECTORY = '/home/sandbox/.nexent-jupyter-tls' +TLS_CERTIFICATE = TLS_DIRECTORY + '/server.crt' + + +class SandboxTLSMigrationRequired(RuntimeError): + """A running plaintext container must be drained before upgrading.""" + + +class SandboxTLSRecoveryError(RuntimeError): + """Preserve a running owner when its TLS identity cannot be verified.""" + + +# Executed by the image's configured user, before any kernel can run. The key +# never leaves the container, and neither the command nor Docker labels contain it. +TLS_BOOTSTRAP = Path(__file__).with_name('sandbox_tls_bootstrap.py').read_text(encoding='utf-8') + + +def require_tls_container(container) -> None: + """Never delete a running owner with an older transport or identity layout.""" + if container.status == 'running' and (container.labels or {}).get(TLS_LABEL) != TLS_VERSION: + raise SandboxTLSMigrationRequired( + 'Running sandbox uses a legacy transport or TLS identity layout. Drain its active runs ' + 'and explicitly stop the container before upgrading; the container was preserved.' + ) + + +class SandboxTLSClient: + """One owner's verified HTTPS session and WSS trust; leases borrow it.""" + + def __init__(self, certificate: bytes): + import requests + + class HTTPSOnlySession(requests.Session): + def request(self, method, url, **kwargs): + if not url.startswith('https://'): + raise ValueError('Sandbox control requests require HTTPS') + kwargs['allow_redirects'] = False + return super().request(method, url, **kwargs) + + self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 + self.ssl_context.load_verify_locations(cadata=certificate.decode('ascii')) + self._directory = tempfile.TemporaryDirectory(prefix='nexent-tls-trust-') + self.ca_file = Path(self._directory.name) / 'server.crt' + try: + self.ca_file.write_bytes(certificate) + self.http = HTTPSOnlySession() + self.http.verify = str(self.ca_file) + # Internal Docker control traffic must not follow external proxy settings. + self.http.trust_env = False + except Exception: + self._directory.cleanup() + raise + + def close(self) -> None: + self.http.close() + self._directory.cleanup() + + +def load_container_tls(container, *, timeout: float = 10, check_cancelled=lambda: None) -> SandboxTLSClient: + """Read only the public certificate over the already trusted Docker channel.""" + from docker.errors import NotFound + + deadline = time.monotonic() + timeout + while True: + check_cancelled() + try: + stream, _ = container.get_archive(TLS_CERTIFICATE) + data = bytearray() + for chunk in stream: + data.extend(chunk) + if len(data) > 65536: + raise ValueError('Sandbox TLS certificate archive is too large') + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + member = archive.getmember('server.crt') + if not member.isfile() or member.size > 16384: + raise ValueError('Invalid sandbox TLS certificate member') + certificate = archive.extractfile(member).read() + return SandboxTLSClient(certificate) + except NotFound: + if time.monotonic() >= deadline: + raise RuntimeError('Sandbox TLS certificate was not generated before the startup deadline') from None + time.sleep(0.1) diff --git a/sdk/nexent/core/agents/sandbox_tls_bootstrap.py b/sdk/nexent/core/agents/sandbox_tls_bootstrap.py new file mode 100644 index 0000000000..1c65c19753 --- /dev/null +++ b/sdk/nexent/core/agents/sandbox_tls_bootstrap.py @@ -0,0 +1,119 @@ +"""Standalone Linux container bootstrap, sent as source to existing sandbox images.""" + +import ipaddress +import os +import stat +import sys +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import PurePosixPath + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + + +@contextmanager +def private_directory(path): + """Walk without following links; pin every lookup to the verified parent.""" + path = PurePosixPath(path) + if not path.is_absolute() or '..' in path.parts or len(path.parts) < 3: + raise ValueError('TLS directory must be an absolute private path') + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + fd = os.open('/', flags) + try: + for part in path.parts[1:-1]: + child = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = child + info = os.fstat(fd) + if info.st_uid not in (0, os.geteuid()) or info.st_mode & 0o022: + raise ValueError('Unsafe TLS parent directory') + try: + os.mkdir(path.name, mode=0o700, dir_fd=fd) + except FileExistsError: + pass + child = os.open(path.name, flags, dir_fd=fd) + os.close(fd) + fd = child + info = os.fstat(fd) + if info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError('TLS directory must be owned by the runtime user with mode 0700') + yield fd + finally: + os.close(fd) + + +def read_identity_file(directory_fd, name): + try: + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd) + except FileNotFoundError: + return None + with os.fdopen(fd, 'rb') as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_size > 16384): + raise ValueError('Unsafe TLS identity file') + return stream.read(16385) + + +def write_identity_file(directory_fd, name, contents): + """Exclusive creation with private permissions before the first byte is written.""" + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) + with os.fdopen(fd, 'wb') as stream: + stream.write(contents) + stream.flush() + os.fsync(stream.fileno()) + + +def generate_identity(hostname): + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)]) + now = datetime.now(timezone.utc) + cert = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=5)).not_valid_after(now + timedelta(days=365)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension(x509.SubjectAlternativeName([ + x509.DNSName(hostname), x509.DNSName('localhost'), + x509.IPAddress(ipaddress.ip_address('127.0.0.1')), + ]), critical=False).sign(key, hashes.SHA256())) + return ( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()), + cert.public_bytes(serialization.Encoding.PEM), + ) + + +def validate_identity(key_data, cert_data): + key = serialization.load_pem_private_key(key_data, password=None) + cert = x509.load_pem_x509_certificate(cert_data) + encoding = serialization.Encoding.DER + public_format = serialization.PublicFormat.SubjectPublicKeyInfo + if key.public_key().public_bytes(encoding, public_format) != cert.public_key().public_bytes(encoding, public_format): + raise ValueError('TLS certificate and private key do not match') + now = datetime.now(timezone.utc) + if not cert.not_valid_before_utc <= now < cert.not_valid_after_utc: + raise ValueError('TLS certificate is not currently valid; drain and recreate the container') + + +def prepare_identity(hostname, directory): + with private_directory(directory) as fd: + key_data = read_identity_file(fd, 'server.key') + cert_data = read_identity_file(fd, 'server.crt') + if key_data is None and cert_data is None: + key_data, cert_data = generate_identity(hostname) + validate_identity(key_data, cert_data) + write_identity_file(fd, 'server.key', key_data) + write_identity_file(fd, 'server.crt.pending', cert_data) + os.replace('server.crt.pending', 'server.crt', src_dir_fd=fd, dst_dir_fd=fd) + elif key_data is None or cert_data is None: + raise ValueError('Incomplete TLS identity; refusing to replace it') + else: + validate_identity(key_data, cert_data) + + +if __name__ == '__main__': + prepare_identity(sys.argv[1], sys.argv[2]) + os.execvp(sys.argv[3], sys.argv[3:]) diff --git a/sdk/nexent/core/agents/sandbox_workspace.py b/sdk/nexent/core/agents/sandbox_workspace.py new file mode 100644 index 0000000000..0bdd62bfb4 --- /dev/null +++ b/sdk/nexent/core/agents/sandbox_workspace.py @@ -0,0 +1,96 @@ +"""Explicit host/container workspace mapping for bind-mounted executors.""" + +import hashlib +import json +import os +from dataclasses import dataclass +from pathlib import Path, PurePosixPath, PureWindowsPath + + +def validate_container_root(value: str) -> PurePosixPath: + """Reject host paths and traversal before constructing a container path.""" + path = PurePosixPath(value) + if not path.is_absolute() or path == PurePosixPath('/') or '\\' in value or ':' in value or '..' in path.parts: + raise ValueError('container workspace must be a non-root POSIX absolute path') + return path + + +@dataclass(frozen=True) +class SandboxWorkspace: + host_root: Path + container_root: PurePosixPath + + def __post_init__(self): + object.__setattr__(self, 'host_root', Path(self.host_root).resolve()) + object.__setattr__(self, 'container_root', validate_container_root(str(self.container_root))) + + def to_container(self, path: str | Path) -> PurePosixPath: + relative = Path(path).resolve().relative_to(self.host_root) + return self.container_root.joinpath(*relative.parts) + + def to_host(self, value: str) -> Path: + """Translate an absolute kernel path and enforce the resolved host boundary.""" + path = validate_container_root(value) + relative = path.relative_to(self.container_root) + # Reject Windows-special components, including alternate data streams. + if any(PureWindowsPath(part).is_reserved() for part in relative.parts): + raise ValueError('Reserved host filename') + target = self.host_root.joinpath(*relative.parts).resolve() + target.relative_to(self.host_root) + return target + + def resolve_file(self, value: str, base: str | Path) -> Path: + """Resolve a tool path without changing that tool's relative-path convention.""" + base_path = Path(base).resolve() + base_path.relative_to(self.host_root) + if value.startswith('/'): + target = self.to_host(value) + else: + if '..' in PureWindowsPath(value).parts: + raise ValueError('Workspace traversal is not allowed') + target = (base_path / value).resolve() + target.relative_to(base_path) + return target + + def for_run(self, host_run: str | Path) -> 'SandboxWorkspace': + return SandboxWorkspace(Path(host_run), self.to_container(host_run)) + + @property + def mount_id(self) -> str: + payload = json.dumps([os.path.normcase(str(self.host_root)), str(self.container_root)]) + return hashlib.sha256(payload.encode()).hexdigest() + + def matches_mount(self, mount: dict) -> bool: + """Compare Desktop's Linux view of drive paths with the configured source.""" + source = str(mount.get('Source', '')).replace('\\', '/') + expected = str(self.host_root).replace('\\', '/') + if self.host_root.drive: + for prefix in ('/run/desktop/mnt/host/', '/host_mnt/'): + if source.startswith(prefix): + tail = source[len(prefix):] + if len(tail) >= 2 and tail[1] == '/': + source = tail[0] + ':' + tail[1:] + break + source, expected = source.casefold(), expected.casefold() + return ( + mount.get('Type') == 'bind' and mount.get('RW') is True + and mount.get('Destination') == str(self.container_root) + and source.rstrip('/') == expected.rstrip('/') + ) + + +def probe_workspace(container, workspace: PurePosixPath) -> None: + """Verify traversal and real read/write access as the container's configured user.""" + code = ( + 'import os,tempfile; from pathlib import Path; ' + f'p=Path({str(workspace)!r}); ' + 'os.chdir(p.parent); os.chdir(p.name); ' + '[(f.open("rb").read(1)) for f in (p/"inputs").rglob("*") if f.is_file()]; ' + 't=tempfile.TemporaryDirectory(prefix=".nexent-probe-",dir=p/"outputs"); ' + 'f=Path(t.name)/"nested"/"file.txt"; f.parent.mkdir(); ' + 'f.write_text("probe",encoding="utf-8"); ' + 'assert f.read_text(encoding="utf-8")=="probe"; t.cleanup()' + ) + result = container.exec_run(['python', '-c', code]) + if result.exit_code != 0: + raise RuntimeError(f'Sandbox workspace access failed: {result.output!r}') diff --git a/sdk/nexent/core/tools/create_file_tool.py b/sdk/nexent/core/tools/create_file_tool.py index b8846af660..9e0847af81 100644 --- a/sdk/nexent/core/tools/create_file_tool.py +++ b/sdk/nexent/core/tools/create_file_tool.py @@ -71,6 +71,7 @@ def __init__(self, raise ValueError("init_path cannot be empty. Use a non-empty path or omit to use the default '/mnt/nexent'.") self.init_path = os.path.abspath(init_path if init_path else "/mnt/nexent") self.observer = observer + self.workspace_mapping = None def _validate_path(self, file_path: str) -> str: """Validate and resolve file path within the workspace. @@ -84,6 +85,8 @@ def _validate_path(self, file_path: str) -> str: Raises: Exception: If path is outside workspace or invalid """ + if self.workspace_mapping is not None: + return str(self.workspace_mapping.resolve_file(file_path, self.init_path)) # Check for absolute path if os.path.isabs(file_path): abs_path = os.path.abspath(file_path) @@ -144,7 +147,9 @@ def forward(self, file_path: str, content: str = "", encoding: str = "utf-8") -> success_msg = { "status": "success", "file_path": relative_path, - "absolute_path": abs_path, + "absolute_path": ( + str(self.workspace_mapping.to_container(abs_path)) if self.workspace_mapping else abs_path + ), "content_length": len(content) if content else 0, "file_size_bytes": file_size, "encoding": encoding, diff --git a/sdk/nexent/core/tools/delete_file_tool.py b/sdk/nexent/core/tools/delete_file_tool.py index 9c9f154551..bff63a9dfe 100644 --- a/sdk/nexent/core/tools/delete_file_tool.py +++ b/sdk/nexent/core/tools/delete_file_tool.py @@ -57,6 +57,7 @@ def __init__(self, raise ValueError("init_path cannot be empty. Use a non-empty path or omit to use the default '/mnt/nexent'.") self.init_path = os.path.abspath(init_path if init_path else "/mnt/nexent") self.observer = observer + self.workspace_mapping = None def _validate_path(self, file_path: str) -> str: """Validate and resolve file path within the workspace. @@ -70,6 +71,8 @@ def _validate_path(self, file_path: str) -> str: Raises: Exception: If path is outside workspace or invalid """ + if self.workspace_mapping is not None: + return str(self.workspace_mapping.resolve_file(file_path, self.init_path)) # Check for absolute path if os.path.isabs(file_path): abs_path = os.path.abspath(file_path) @@ -135,7 +138,9 @@ def forward(self, file_path: str) -> str: success_msg = { "status": "success", "file_path": relative_path, - "absolute_path": abs_path, + "absolute_path": ( + str(self.workspace_mapping.to_container(abs_path)) if self.workspace_mapping else abs_path + ), "file_name": file_name, "file_size_bytes": file_size, "message": f"File deleted successfully: {relative_path}" diff --git a/sdk/nexent/core/tools/download_from_s3_tool.py b/sdk/nexent/core/tools/download_from_s3_tool.py index 7092550b19..c0b08bc7fe 100644 --- a/sdk/nexent/core/tools/download_from_s3_tool.py +++ b/sdk/nexent/core/tools/download_from_s3_tool.py @@ -71,6 +71,7 @@ def __init__( on_download: object = Field(description="Download synchronization callback", default=None, exclude=True), ): super().__init__() + self.workspace_mapping = None # Guard against FieldInfo objects when called without arguments _default_ws = "/mnt/nexent" if not isinstance(workspace_path, str): @@ -180,6 +181,8 @@ def _resolve_local_path(self, object_key: str, local_filename: Optional[str]) -> if not filename: raise ValueError(f"Cannot determine filename from S3 path: {object_key}") + if self.workspace_mapping is not None: + return self.workspace_mapping.resolve_file(filename, self.workspace_path) workspace = Path(self.workspace_path).resolve() local_path = (workspace / filename).resolve() try: @@ -254,6 +257,9 @@ def forward(self, s3_path: str, local_filename: str = None) -> str: } if self.on_download is not None: self.on_download(dict(result)) + if self.workspace_mapping is not None: + result["local_path"] = str(self.workspace_mapping.to_container(local_path)) + result["relative_path"] = Path(relative_path).as_posix() return json.dumps(result, ensure_ascii=False) except ValueError as e: diff --git a/sdk/nexent/core/tools/read_file_tool.py b/sdk/nexent/core/tools/read_file_tool.py index 857653faf9..f216158748 100644 --- a/sdk/nexent/core/tools/read_file_tool.py +++ b/sdk/nexent/core/tools/read_file_tool.py @@ -64,6 +64,7 @@ def __init__(self, raise ValueError("init_path cannot be empty. Use a non-empty path or omit to use the default '/mnt/nexent'.") self.init_path = os.path.abspath(init_path if init_path else "/mnt/nexent") self.observer = observer + self.workspace_mapping = None def _validate_path(self, file_path: str) -> str: """Validate and resolve file path within the workspace. @@ -77,6 +78,8 @@ def _validate_path(self, file_path: str) -> str: Raises: Exception: If path is outside workspace or invalid """ + if self.workspace_mapping is not None: + return str(self.workspace_mapping.resolve_file(file_path, self.init_path)) # Check for absolute path if os.path.isabs(file_path): abs_path = os.path.abspath(file_path) @@ -142,7 +145,9 @@ def forward(self, file_path: str, encoding: str = "utf-8") -> str: success_msg = { "status": "success", "file_path": relative_path, - "absolute_path": abs_path, + "absolute_path": ( + str(self.workspace_mapping.to_container(abs_path)) if self.workspace_mapping else abs_path + ), "content": content, "content_length": len(content), "file_size_bytes": file_size, diff --git a/sdk/nexent/core/tools/upload_to_s3_tool.py b/sdk/nexent/core/tools/upload_to_s3_tool.py index 0146ca6927..b780cdfb2b 100644 --- a/sdk/nexent/core/tools/upload_to_s3_tool.py +++ b/sdk/nexent/core/tools/upload_to_s3_tool.py @@ -89,6 +89,7 @@ def __init__( self.observer = observer if hasattr(observer, 'add_message') else None self.run_id = run_id if isinstance(run_id, str) else "" self.on_upload = on_upload if callable(on_upload) else None + self.workspace_mapping = None self.ensure_local_file = ensure_local_file if callable(ensure_local_file) else None self.uploaded_paths: set[str] = uploaded_paths if isinstance(uploaded_paths, set) else set() @@ -101,6 +102,8 @@ def _validate_path(self, file_path: str) -> str: Raises: Exception: If path is outside workspace or invalid. """ + if self.workspace_mapping is not None: + return str(self.workspace_mapping.resolve_file(file_path, self.workspace_path)) workspace = Path(self.workspace_path).resolve() if os.path.isabs(file_path): abs_path = Path(file_path).resolve() diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 39f7088dd9..77658d2a4f 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ + "cryptography>=42.0.0", "aiofiles>=24.1.0", "elasticsearch==8.17.2", "exa_py==1.14.0", diff --git a/sonar-project.properties b/sonar-project.properties index 3a465a32a9..ef76be6cbd 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,5 +1,6 @@ sonar.projectKey=ModelEngine-Group_nexent sonar.organization=modelengine-group +sonar.python.version=3.11 sonar.exclusions=**/init.sql # Fresh-install, upgrade, and preflight SQL deliberately share schema/backfill diff --git a/test/backend/agents/test_sandbox_config.py b/test/backend/agents/test_sandbox_config.py new file mode 100644 index 0000000000..dcf31cb556 --- /dev/null +++ b/test/backend/agents/test_sandbox_config.py @@ -0,0 +1,63 @@ +"""Policy precedence and SDK validation at the backend configuration boundary.""" + +import pytest +from agents.sandbox_config import resolve_sandbox_config +from nexent.core.agents.sandbox import SandboxLevel, SandboxScope + + +def resolve(db_policy, env_policy): + return resolve_sandbox_config( + db_policy, env_policy, workspace_mode='legacy', + container_workspace_root='', failure_policy='local', + ) + + +@pytest.mark.parametrize('db_policy', [None, {}]) +def test_absent_database_policy_uses_environment(db_policy): + config = resolve(db_policy, {'level': 'docker', 'scope': 'system'}) + assert config.level == SandboxLevel.DOCKER + assert config.scope == SandboxScope.SYSTEM + + +def test_explicit_local_database_policy_overrides_environment(): + config = resolve({'level': 'local'}, {'level': 'docker', 'scope': 'system'}) + assert config.level == SandboxLevel.LOCAL + assert config.scope == SandboxScope.SESSION + + +def test_partial_database_policy_uses_sdk_defaults_without_mixing_environment(): + config = resolve({'level': 'docker'}, {'level': 'docker', 'memory_limit_mb': 8192}) + assert config.memory_limit_mb == 2048 + + +def test_database_only_policy(): + assert resolve({'level': 'docker'}, None).level == SandboxLevel.DOCKER + + +def test_no_policy_preserves_disabled_sandbox(): + assert resolve(None, None) is None + + +@pytest.mark.parametrize('policy', [{'level': 'invalid'}, {'failure_policy': 'invalid'}, [], 'docker']) +def test_invalid_database_policy_does_not_fall_back(policy): + with pytest.raises((TypeError, ValueError)): + resolve(policy, {'level': 'docker'}) + + +def test_agent_workspace_fields_override_deployment_defaults(): + policy = {'level': 'docker', 'workspace_mode': 'legacy', 'failure_policy': 'local'} + config = resolve_sandbox_config( + policy, None, workspace_mode='bind', container_workspace_root='/mnt/deployment', failure_policy='error', + ) + assert config.workspace_mode == 'legacy' + assert config.failure_policy == 'local' + assert policy == {'level': 'docker', 'workspace_mode': 'legacy', 'failure_policy': 'local'} + + +def test_missing_workspace_fields_use_deployment_defaults(): + config = resolve_sandbox_config( + {'level': 'docker'}, None, workspace_mode='bind', container_workspace_root='/mnt/work', failure_policy='error', + ) + assert config.workspace_mode == 'bind' + assert config.container_workspace_root == '/mnt/work' + assert config.failure_policy == 'error' diff --git a/test/sdk/core/agents/test_nexent_agent.py b/test/sdk/core/agents/test_nexent_agent.py index 3a77b7e903..a8598c4bb7 100644 --- a/test/sdk/core/agents/test_nexent_agent.py +++ b/test/sdk/core/agents/test_nexent_agent.py @@ -4769,7 +4769,7 @@ def __call__(self, code): assert len(unmarked_kernel_lease.calls) == 1 assert "NEXENT_OUTPUT_DIR" in unmarked_kernel_lease.calls[0] - def test_initialize_sandbox_workspaces_retries_unhealthy_kernel_lease( + def test_initialize_sandbox_workspaces_retries_unhealthy_legacy_executor( self, nexent_agent_instance, tmp_path ): class RecoveringKernelLease: @@ -4780,7 +4780,6 @@ def __init__(self): self.container = object() self._unhealthy = False self.calls = [] - self.registered_bootstrap = [] def __call__(self, code): self.calls.append(code) @@ -4790,11 +4789,6 @@ def __call__(self, code): self._unhealthy = False return ["workspace", "outputs"] - def register_kernel_bootstrap_code(self, code): - result = self(code) - self.registered_bootstrap.append(code) - return result - workspace = tmp_path / "user" / "run" (workspace / "outputs").mkdir(parents=True) executor = RecoveringKernelLease() @@ -4806,9 +4800,8 @@ def register_kernel_bootstrap_code(self, code): assert len(executor.calls) == 2 assert executor.calls[0] == executor.calls[1] assert executor._unhealthy is False - assert executor.registered_bootstrap == [executor.calls[0]] - def test_initialize_sandbox_workspaces_reports_retry_failure( + def test_initialize_sandbox_workspaces_reports_registered_bootstrap_failure_without_retry( self, nexent_agent_instance, tmp_path ): class FailingKernelLease: @@ -4817,17 +4810,24 @@ class FailingKernelLease: _unhealthy = True container = object() + def __init__(self): + self.calls = [] + def register_kernel_bootstrap_code(self, code): + self.calls.append(code) raise RuntimeError("replacement bootstrap failed") workspace = tmp_path / "user" / "run" (workspace / "outputs").mkdir(parents=True) nexent_agent_instance.workspace_path = str(workspace) - nexent_agent_instance._sandbox_executors = [FailingKernelLease()] + executor = FailingKernelLease() + nexent_agent_instance._sandbox_executors = [executor] with pytest.raises(RuntimeError, match="replacement bootstrap failed"): nexent_agent_instance._initialize_sandbox_workspaces() + assert len(executor.calls) == 1 + def test_initialize_sandbox_workspaces_does_not_retry_healthy_failure( self, nexent_agent_instance, tmp_path ): @@ -6801,6 +6801,7 @@ def test_docker_sandbox_binds_skill_script_execution_backend( timeout_seconds=300, workspace_path=nexent_agent_instance.workspace_path, network_enabled=False, + workspace_mapping=None, ) tool.bind_execution_backend.assert_called_once_with( runner, diff --git a/test/sdk/core/agents/test_nexent_agent_bind_workspace.py b/test/sdk/core/agents/test_nexent_agent_bind_workspace.py new file mode 100644 index 0000000000..78e4388d42 --- /dev/null +++ b/test/sdk/core/agents/test_nexent_agent_bind_workspace.py @@ -0,0 +1,216 @@ +import json +import logging +from concurrent.futures import CancelledError +from pathlib import Path, PurePosixPath +from threading import Event +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from nexent.core.agents.nexent_agent import NexentAgent +from nexent.core.agents.sandbox import SandboxConfig, SandboxLevel +from nexent.core.agents.sandbox_workspace import SandboxWorkspace +from nexent.core.tools.download_from_s3_tool import DownloadFromS3Tool + + +@pytest.fixture +def bind_agent(tmp_path): + workspace = tmp_path / 'user' / 'run' + (workspace / 'inputs').mkdir(parents=True) + (workspace / 'outputs').mkdir() + agent = object.__new__(NexentAgent) + agent.stop_event = Event() + agent.workspace_path = str(workspace) + agent.workspace_run_id = 'run' + agent.workspace_mapping = SandboxWorkspace(workspace, '/mnt/work/user/run') + agent.sandbox_config = SandboxConfig( + level=SandboxLevel.DOCKER, workspace_mode='bind', + container_workspace_root='/mnt/work', failure_policy='error', + ) + executor = Mock() + executor._nexent_backend = 'docker' + executor._nexent_kernel_recovery_supported = False + executor._unhealthy = False + executor.container.id = 'test-container' + executor.container.exec_run.return_value = SimpleNamespace(exit_code=0, output=b'1000') + agent._sandbox_executors = [executor] + agent.agent = SimpleNamespace(python_executor=executor) + return agent + + +def test_kernel_bootstrap_uses_container_paths(bind_agent, caplog): + with caplog.at_level(logging.INFO): + bind_agent._initialize_sandbox_workspaces() + code = bind_agent._sandbox_executors[0].call_args.args[0] + assert '/mnt/work/user/run/outputs' in code + assert str(bind_agent.workspace_path) not in code + assert 'Sandbox READY' in caplog.text + + +def test_cwd_failure_does_not_report_ready(bind_agent, caplog): + bind_agent._sandbox_executors[0].side_effect = PermissionError('cwd denied') + with caplog.at_level(logging.INFO), pytest.raises(RuntimeError, match='initialize sandbox workspace'): + bind_agent._initialize_sandbox_workspaces() + assert 'Sandbox READY' not in caplog.text + assert 'phase=workspace' in caplog.text + + +def test_cancelled_bootstrap_does_not_log_ready_or_initialize_next_kernel(bind_agent, caplog): + first = bind_agent._sandbox_executors[0] + second = Mock() + bind_agent._sandbox_executors.append(second) + first.side_effect = lambda code: bind_agent.stop_event.set() + with caplog.at_level(logging.INFO), pytest.raises(CancelledError): + bind_agent._initialize_sandbox_workspaces() + first.assert_called_once() + second.assert_not_called() + assert 'Sandbox READY' not in caplog.text + + +def test_bind_push_verifies_access_without_copying(bind_agent): + bind_agent._push_file_workspace_to_sandbox() + container = bind_agent._sandbox_executors[0].container + container.put_archive.assert_not_called() + commands = [call.args[0] for call in container.exec_run.call_args_list] + assert any(command[:2] == ['python', '-c'] for command in commands) + if Path(bind_agent.workspace_path).drive: + assert not any(command[0] in {'chmod', 'chgrp'} for command in commands) + + +def test_permission_probe_error_propagates(bind_agent, caplog): + bind_agent._sandbox_executors[0].container.exec_run.return_value = SimpleNamespace( + exit_code=1, output=b'permission denied', + ) + with pytest.raises(RuntimeError): + bind_agent._push_file_workspace_to_sandbox() + assert 'phase=workspace_access' in caplog.text + + +def test_download_returns_kernel_path_but_writes_host_path(tmp_path): + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + client = Mock() + client.get_file_size.return_value = 5 + + def download(key, path, bucket): + Path(path).write_text('hello', encoding='utf-8') + return True, '' + + client.download_file.side_effect = download + tool = DownloadFromS3Tool( + workspace_path=str(tmp_path), minio_client=client, + validate_url_access=lambda _: None, + ) + tool.workspace_mapping = mapping + result = json.loads(tool.forward('s3://bucket/input.txt', 'inputs/input.txt')) + assert result['local_path'] == '/mnt/run/inputs/input.txt' + assert (tmp_path / 'inputs' / 'input.txt').read_text(encoding='utf-8') == 'hello' + + +def test_bootstrap_retries_only_failed_initialization(bind_agent): + executor = bind_agent._sandbox_executors[0] + executor._nexent_kernel_recovery_supported = True + executor._unhealthy = True + executor.side_effect = [RuntimeError('lost channel'), None] + bind_agent._initialize_sandbox_workspaces() + assert executor.call_count == 2 + assert executor.call_args_list[0] == executor.call_args_list[1] + + +def test_bootstrap_retry_logs_and_chains_actual_retry_failure(bind_agent, caplog): + executor = bind_agent._sandbox_executors[0] + executor._nexent_kernel_recovery_supported = True + executor._unhealthy = True + retry_error = PermissionError('replacement cwd denied') + executor.side_effect = [RuntimeError('lost channel'), retry_error] + with pytest.raises(RuntimeError, match='replacement cwd denied') as caught: + bind_agent._initialize_sandbox_workspaces() + assert caught.value.__cause__ is retry_error + failure = next(record for record in caplog.records if 'phase=workspace' in record.message) + assert failure.exc_info[1] is retry_error + assert executor.call_count == 2 + + +def test_registered_bootstrap_does_not_receive_outer_recovery_retry(bind_agent): + class Executor: + _nexent_backend = 'docker' + _nexent_kernel_recovery_supported = True + _unhealthy = True + calls = 0 + + def register_kernel_bootstrap_code(self, code): + self.calls += 1 + raise RuntimeError('replacement already failed') + + executor = Executor() + bind_agent._sandbox_executors = [executor] + with pytest.raises(RuntimeError, match='replacement already failed'): + bind_agent._initialize_sandbox_workspaces() + assert executor.calls == 1 + + +@pytest.mark.parametrize('cancel_during_retry', [False, True]) +def test_bootstrap_retry_preserves_cancellation(bind_agent, cancel_during_retry): + executor = bind_agent._sandbox_executors[0] + executor._nexent_kernel_recovery_supported = True + executor._unhealthy = True + def bootstrap(code): + if cancel_during_retry and executor.call_count == 1: + raise RuntimeError('lost channel') + bind_agent.stop_event.set() + raise RuntimeError('closed during cancellation') + executor.side_effect = bootstrap + with pytest.raises(CancelledError): + bind_agent._initialize_sandbox_workspaces() + assert executor.call_count == (2 if cancel_during_retry else 1) + + +def test_bootstrap_deduplicates_executors_and_skips_local(bind_agent): + executor = bind_agent._sandbox_executors[0] + local = Mock(_nexent_backend='local') + other = Mock(_nexent_backend='wasm', container=None) + bind_agent._sandbox_executors = [executor, executor, local, other] + bind_agent._initialize_sandbox_workspaces() + executor.assert_called_once() + local.assert_not_called() + other.assert_not_called() + + +def test_bind_probe_checks_each_distinct_container(bind_agent, mocker): + second = Mock() + second.container.id = 'second-container' + bind_agent._sandbox_executors.append(second) + probe = mocker.patch('nexent.core.agents.nexent_agent.probe_workspace') + grant = mocker.patch.object(bind_agent, '_grant_sandbox_output_access') + bind_agent._push_file_workspace_to_sandbox() + assert probe.call_count == 2 + assert [call.args[0] for call in probe.call_args_list] == [ + bind_agent._sandbox_executors[0].container, second.container, + ] + assert grant.call_count == (0 if Path(bind_agent.workspace_path).drive else 2) + + +def test_no_workspace_does_not_initialize_or_copy(bind_agent): + bind_agent.workspace_path = None + bind_agent._initialize_sandbox_workspaces() + bind_agent._push_file_workspace_to_sandbox() + bind_agent._sandbox_executors[0].assert_not_called() + bind_agent._sandbox_executors[0].container.exec_run.assert_not_called() + + +def test_direct_retry_cancellation_is_not_wrapped(bind_agent): + executor = bind_agent._sandbox_executors[0] + executor._nexent_kernel_recovery_supported = True + executor._unhealthy = True + executor.side_effect = [RuntimeError('lost channel'), CancelledError('stopped')] + with pytest.raises(CancelledError, match='stopped'): + bind_agent._initialize_sandbox_workspaces() + assert executor.call_count == 2 + + +def test_posix_bind_access_grants_permissions_before_probe(bind_agent, mocker): + container = bind_agent._sandbox_executors[0].container + grant = mocker.patch.object(bind_agent, '_grant_sandbox_output_access') + probe = mocker.patch('nexent.core.agents.nexent_agent.probe_workspace') + bind_agent._verify_bind_workspace_access([container], PurePosixPath('/host/run'), bind_agent.workspace_mapping) + grant.assert_called_once_with(container, bind_agent.workspace_mapping.container_root) + probe.assert_called_once_with(container, bind_agent.workspace_mapping.container_root) diff --git a/test/sdk/core/agents/test_nexent_agent_sandbox_construction.py b/test/sdk/core/agents/test_nexent_agent_sandbox_construction.py new file mode 100644 index 0000000000..f6b03b9f45 --- /dev/null +++ b/test/sdk/core/agents/test_nexent_agent_sandbox_construction.py @@ -0,0 +1,112 @@ +"""Construction and fallback preserve the run's mapping and resource ownership.""" + +from concurrent.futures import CancelledError +from threading import Event +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from nexent.core.agents import nexent_agent as agent_module +from nexent.core.agents import sandbox +from nexent.core.agents.agent_model import AgentConfig, ToolConfig +from nexent.core.agents.nexent_agent import NexentAgent +from nexent.core.utils.observer import MessageObserver + + +@pytest.fixture +def factory(tmp_path): + config = sandbox.SandboxConfig( + level=sandbox.SandboxLevel.DOCKER, workspace_mode='bind', + container_workspace_root='/mnt/work', failure_policy='error', + extra_kwargs={'workspace_root': str(tmp_path)}, + ) + return NexentAgent( + observer=Mock(spec=MessageObserver), model_config_list=[], stop_event=Event(), + sandbox_config=config, workspace_path=str(tmp_path / 'run'), workspace_run_id='run', + ) + + +def test_constructor_creates_run_scoped_mapping(factory, tmp_path): + assert factory.workspace_mapping.host_root == tmp_path / 'run' + assert str(factory.workspace_mapping.container_root) == '/mnt/work/run' + + +def test_explicit_local_bind_config_keeps_mapping_disabled(tmp_path): + config = sandbox.SandboxConfig(workspace_mode='bind', container_workspace_root='/mnt/work') + agent = NexentAgent(Mock(spec=MessageObserver), [], Event(), sandbox_config=config, workspace_path=str(tmp_path)) + assert agent.workspace_mapping is None + + +@pytest.mark.parametrize('class_name', [ + 'CreateFileTool', 'ReadFileTool', 'DeleteFileTool', 'DownloadFromS3Tool', 'UploadToS3Tool', 'OtherTool', +]) +def test_tool_binding_preserves_each_tools_directory(factory, mocker, class_name): + tool = SimpleNamespace(init_path='original') + mocker.patch.object(factory, 'create_local_tool', return_value=tool) + config = ToolConfig(name=class_name, class_name=class_name, source='local', params={}) + assert factory.create_tool(config) is tool + if class_name == 'OtherTool': + assert not hasattr(tool, 'workspace_mapping') + else: + assert tool.workspace_mapping is factory.workspace_mapping + expected = str(factory.workspace_mapping.host_root / 'outputs') if class_name in { + 'CreateFileTool', 'ReadFileTool', 'DeleteFileTool', + } else 'original' + assert tool.init_path == expected + + +@pytest.fixture +def construction(factory, mocker): + mocker.patch.object(factory, 'create_model', return_value=Mock()) + executor = Mock(_nexent_backend='docker', _nexent_session_container_group=None) + mocker.patch.object(sandbox, 'build_python_executor', return_value=executor) + core = mocker.patch.object(agent_module, 'CoreAgent', return_value=SimpleNamespace(enable_planning=False)) + cleanup = mocker.patch.object(factory, '_cleanup_sandbox') + config = AgentConfig(name='agent', description='test', model_name='model', tools=[], managed_agents=[]) + return SimpleNamespace(factory=factory, executor=executor, core=core, cleanup=cleanup, config=config) + + +@pytest.mark.parametrize('error', [RuntimeError('warm failed'), CancelledError('cancelled')]) +def test_strict_warmup_failure_releases_acquired_executor(construction, error): + c = construction + c.executor.side_effect = error + expected = CancelledError if isinstance(error, CancelledError) else ValueError + with pytest.raises(expected): + c.factory.create_single_agent(c.config) + c.cleanup.assert_called_once() + c.core.assert_not_called() + + +def test_strict_mode_rejects_unexpected_local_executor(construction): + c = construction + c.executor._nexent_backend = 'local' + with pytest.raises(ValueError, match='phase=warmup'): + c.factory.create_single_agent(c.config) + c.cleanup.assert_called_once() + + +def test_cleanup_failure_does_not_hide_construction_failure(construction, caplog): + c = construction + c.executor.side_effect = RuntimeError('warm failed') + c.cleanup.side_effect = RuntimeError('cleanup failed') + with pytest.raises(ValueError, match='phase=warmup'): + c.factory.create_single_agent(c.config) + assert 'Failed to release sandbox resources' in caplog.text + + +def test_allowed_local_fallback_clears_mappings_and_warns(construction, mocker): + c = construction + c.factory.sandbox_config.failure_policy = 'local' + c.executor._nexent_backend = 'local' + tool = Mock(name='file-tool') + tool.workspace_mapping = c.factory.workspace_mapping + plain_tool = Mock(name='plain-tool', workspace_mapping=None) + c.config.tools = [ToolConfig(name='read', class_name='ReadFileTool', source='local', params={})] * 2 + mocker.patch.object(c.factory, 'create_tool', side_effect=[tool, plain_tool]) + mocker.patch.object(agent_module, '_wrap_tool_with_monitoring', side_effect=lambda tool, _: tool) + result = c.factory.create_single_agent(c.config) + assert result is c.core.return_value + assert c.factory.workspace_mapping is None + assert tool.workspace_mapping is None + c.factory.observer.add_message.assert_called_once() + c.cleanup.assert_not_called() diff --git a/test/sdk/core/agents/test_run_agent.py b/test/sdk/core/agents/test_run_agent.py index bfb5171c80..221f7d7582 100644 --- a/test/sdk/core/agents/test_run_agent.py +++ b/test/sdk/core/agents/test_run_agent.py @@ -940,6 +940,17 @@ def test_normalize_mcp_config_keeps_proxy_by_default(): } +def test_agent_run_thread_marks_kernel_cancellation_stopped(basic_agent_run_info, monkeypatch): + from concurrent.futures import CancelledError + + instance = MagicMock() + instance.create_single_agent.side_effect = CancelledError("cancelled during bootstrap") + monkeypatch.setattr(run_agent, "NexentAgent", MagicMock(return_value=instance)) + run_agent.agent_run_thread(basic_agent_run_info) + assert basic_agent_run_info.attempt_outcome == "stopped" + assert not any("Run Agent Error" in str(call) for call in basic_agent_run_info.observer.add_message.call_args_list) + + def test_agent_run_thread_handles_internal_exception( basic_agent_run_info, mock_memory_context, monkeypatch ): diff --git a/test/sdk/core/agents/test_sandbox.py b/test/sdk/core/agents/test_sandbox.py index e41b00de04..457b83ed60 100644 --- a/test/sdk/core/agents/test_sandbox.py +++ b/test/sdk/core/agents/test_sandbox.py @@ -61,6 +61,21 @@ def _load_sandbox_module(): seed_pnpm_offline_store = sandbox_module._seed_pnpm_offline_store +@pytest.fixture(autouse=True) +def mock_container_tls_transport(monkeypatch, request): + """Keep lifecycle tests independent of Docker certificate I/O. + + Certificate loading and real TLS verification are tested in test_sandbox_tls. + Existing per-test HTTP mocks remain the transport collaborators. + """ + if request.node.cls is not None and request.node.cls.__name__ == 'TestDockerIntegration': + return + def transport(*args, **kwargs): + import requests + return SimpleNamespace(http=requests, ssl_context=object(), close=MagicMock()) + monkeypatch.setattr(sandbox_module, 'load_container_tls', transport) + + def test_docker_bridge_gateway_returns_concrete_ipv4_address(): network = MagicMock( attrs={ @@ -1429,12 +1444,12 @@ def test_recover_running_named_container(self, monkeypatch): container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.short_id = "abc123" container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.client = MagicMock() container.attrs = { "NetworkSettings": { "Networks": {sandbox_module.SANDBOX_NETWORK_NAME: {}}, - "Ports": {"8888/tcp": [{"HostPort": "8888"}]}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, } } @@ -1454,7 +1469,7 @@ def test_recover_running_named_container(self, monkeypatch): assert recovered is not None assert recovered.container is container - assert recovered.base_url == "http://127.0.0.1:8888" + assert recovered.base_url == "https://127.0.0.1:49173" assert recovered._nexent_backend == "docker" container.reload.assert_called_once() @@ -1465,7 +1480,10 @@ def test_system_creation_uses_localhost_on_host_runtime(self, monkeypatch): container = MagicMock() container.short_id = "host123" container.client = MagicMock() - container.attrs = {"NetworkSettings": {"Networks": {}}} + container.attrs = {"NetworkSettings": { + "Networks": {}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, + }} run = MagicMock(return_value=container) docker_module = SimpleNamespace( from_env=lambda: SimpleNamespace(containers=SimpleNamespace(run=run)) @@ -1482,8 +1500,8 @@ def test_system_creation_uses_localhost_on_host_runtime(self, monkeypatch): executor = pm._build_system_docker_executor(cfg, logger, {"name": "sandbox"}) - assert executor.base_url == "http://127.0.0.1:8888" - assert run.call_args.kwargs["ports"] == {"8888/tcp": ("127.0.0.1", 8888)} + assert executor.base_url == "https://127.0.0.1:49173" + assert run.call_args.kwargs["ports"] == {"8888/tcp": ("127.0.0.1", None)} def test_system_creation_uses_container_dns_without_host_port(self, monkeypatch): pm = SandboxPoolManager.get_instance() @@ -1513,10 +1531,10 @@ def get(url, **kwargs): {"name": sandbox_module.SANDBOX_CONTAINER_NAME, "ports": {"old": "mapping"}}, ) - assert executor.base_url == "http://nexent-runtime-sandbox:8888" + assert executor.base_url == "https://nexent-runtime-sandbox:8888" assert "ports" not in run.call_args.kwargs assert requests_made == [ - ("http://nexent-runtime-sandbox:8888/api/kernels", {"timeout": 1}) + ("https://nexent-runtime-sandbox:8888/api/kernels", {"timeout": 1}) ] def test_recovery_rejects_container_without_nexent_network(self, monkeypatch): @@ -1526,7 +1544,7 @@ def test_recovery_rejects_container_without_nexent_network(self, monkeypatch): container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = { "NetworkSettings": { "Networks": {}, @@ -1555,7 +1573,7 @@ def test_recovery_rejects_container_with_wrong_workspace_mount(self, monkeypatch container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = { "Mounts": [{ "Type": "bind", @@ -1616,7 +1634,7 @@ def test_system_owner_does_not_install_host_tool_bridge(self, monkeypatch): bridge_installer = MagicMock(side_effect=AssertionError("owner bridge installation")) monkeypatch.setitem(sys.modules, "docker", docker_module) monkeypatch.setattr(sandbox_module, "_is_containerized_runtime", lambda: True) - monkeypatch.setattr(pm, "_build_system_docker_executor", lambda *args: owner) + monkeypatch.setattr(pm, "_build_system_docker_executor", lambda *args, **kwargs: owner) monkeypatch.setattr(sandbox_module, "_install_host_tool_bridge", bridge_installer) executor = pm._build_docker_executor(cfg, logger, host_tools_exist=True) @@ -1643,7 +1661,7 @@ def test_system_docker_mounts_named_workspace_volume(self, monkeypatch): monkeypatch.setattr( pm, "_build_system_docker_executor", - lambda config, logger_, kwargs: captured_kwargs.update(kwargs) or owner, + lambda config, logger_, kwargs, **options: captured_kwargs.update(kwargs) or owner, ) cfg = SandboxConfig( level=SandboxLevel.DOCKER, @@ -1699,7 +1717,7 @@ def install_bridge( bridge_timeouts.append(request_timeout_seconds) return executor - monkeypatch.setattr(pm, "_build_executor", lambda *args: owner) + monkeypatch.setattr(pm, "_build_executor", lambda *args, **kwargs: owner) monkeypatch.setattr(pm, "_recover_docker_container", lambda *args: None) lease_timeouts = [] @@ -2544,22 +2562,25 @@ def _lease(): lease = object.__new__(sandbox_module._DockerKernelLease) lease.logger = MagicMock() lease._logger = MagicMock() - lease.base_url = "http://sandbox:8888" + lease.base_url = "https://sandbox:8888" lease.host = "sandbox" lease.port = 8888 lease.kernel_id = "kernel-1" lease._channel_session_id = "session-1" lease.ws_url = ( - "ws://sandbox:8888/api/kernels/kernel-1/channels?session_id=session-1" + "wss://sandbox:8888/api/kernels/kernel-1/channels?session_id=session-1" ) lease._receive_timeout_seconds = 0.25 lease._closed = False lease._unhealthy = False lease._nexent_kernel_recovery_supported = True lease._requests = MagicMock() + lease._ssl_context = object() lease._cached_variables = None lease._cached_tools = None lease._kernel_bootstrap_code = [] + # These tests isolate execution/recovery; readiness has dedicated protocol tests. + lease._wait_for_kernel_channel_ready = lambda ws: None return lease def test_busy_kernel_continues_after_receive_timeout(self, monkeypatch): @@ -2568,6 +2589,7 @@ def test_busy_kernel_continues_after_receive_timeout(self, monkeypatch): lease = self._lease() lease._get_kernel_execution_state = MagicMock(return_value="busy") websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.side_effect = [ WebSocketTimeoutException("poll timeout"), ( @@ -2603,21 +2625,25 @@ def test_busy_kernel_continues_after_receive_timeout(self, monkeypatch): assert result.logs == "done\n" assert lease._unhealthy is False lease._get_kernel_execution_state.assert_called_once_with() - create_connection.assert_called_once_with(lease.ws_url, timeout=0.25) + create_connection.assert_called_once_with( + lease.ws_url, timeout=0.25, sslopt={"context": lease._ssl_context}, + http_proxy_host="sandbox", http_no_proxy=["sandbox"], redirect_limit=0, + ) websocket.close.assert_called_once_with() def test_kernel_lease_uses_stable_gateway_session_id(self, monkeypatch): container_executor = SimpleNamespace( + tls_client=SimpleNamespace(http=MagicMock(), ssl_context=object()), logger=MagicMock(), additional_imports=[], installed_packages=[], _nexent_backend="docker", - base_url="http://sandbox:8888", + base_url="https://sandbox:8888", host="sandbox", port=8888, ) monkeypatch.setattr( - "smolagents.remote_executors._create_kernel_http", + sandbox_module._DockerKernelLease, "_create_kernel", MagicMock(return_value="kernel-1"), ) monkeypatch.setattr(sandbox_module.secrets, "token_hex", lambda _size: "stable-session") @@ -2625,7 +2651,7 @@ def test_kernel_lease_uses_stable_gateway_session_id(self, monkeypatch): lease = sandbox_module._DockerKernelLease(container_executor, MagicMock()) assert lease.ws_url == ( - "ws://sandbox:8888/api/kernels/kernel-1/channels" + "wss://sandbox:8888/api/kernels/kernel-1/channels" "?session_id=stable-session" ) assert lease._build_channels_url("kernel-1") == lease.ws_url @@ -2639,6 +2665,7 @@ def test_idle_kernel_without_terminal_message_fails_and_marks_lease_unhealthy( lease = self._lease() lease._get_kernel_execution_state = MagicMock(return_value="idle") websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.side_effect = WebSocketTimeoutException("terminal message lost") monkeypatch.setattr("websocket.create_connection", MagicMock(return_value=websocket)) monkeypatch.setattr( @@ -2660,6 +2687,7 @@ def test_unhealthy_lease_replaces_kernel_before_next_execution(self, monkeypatch side_effect=lambda: setattr(lease, "_unhealthy", False) ) websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = ( ABNF.OPCODE_TEXT, json.dumps( @@ -2698,7 +2726,7 @@ def test_kernel_replacement_replays_framework_state(self, monkeypatch): monkeypatch.setattr(RemotePythonExecutor, "send_variables", send_variables) monkeypatch.setattr(RemotePythonExecutor, "send_tools", send_tools) monkeypatch.setattr( - "smolagents.remote_executors._create_kernel_http", + sandbox_module._DockerKernelLease, "_create_kernel", MagicMock(return_value="kernel-2"), ) monkeypatch.setattr(lease, "run_code_raise_errors", run_code) @@ -2706,12 +2734,12 @@ def test_kernel_replacement_replays_framework_state(self, monkeypatch): lease._replace_unhealthy_kernel() lease._requests.delete.assert_called_once_with( - "http://sandbox:8888/api/kernels/kernel-1", + "https://sandbox:8888/api/kernels/kernel-1", timeout=5, ) assert lease.kernel_id == "kernel-2" assert lease.ws_url.startswith( - "ws://sandbox:8888/api/kernels/kernel-2/channels?session_id=" + "wss://sandbox:8888/api/kernels/kernel-2/channels?session_id=" ) assert lease.ws_url.endswith(lease._channel_session_id) assert lease._unhealthy is False @@ -2784,6 +2812,7 @@ def test_unrelated_messages_do_not_postpone_watchdog(self, monkeypatch): lease = self._lease() lease._get_kernel_execution_state = MagicMock(return_value="idle") websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = ( ABNF.OPCODE_TEXT, json.dumps( @@ -2815,6 +2844,7 @@ def test_control_frames_do_not_postpone_watchdog(self, monkeypatch): lease = self._lease() lease._get_kernel_execution_state = MagicMock(return_value="idle") websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = (ABNF.OPCODE_PING, b"heartbeat") monkeypatch.setattr( sandbox_module.time, @@ -2839,6 +2869,7 @@ def test_closed_websocket_marks_busy_kernel_lease_unhealthy(self, monkeypatch): lease = self._lease() lease._get_kernel_execution_state = MagicMock(return_value="busy") websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.side_effect = WebSocketConnectionClosedException("channel closed") monkeypatch.setattr("websocket.create_connection", MagicMock(return_value=websocket)) monkeypatch.setattr( @@ -2858,7 +2889,7 @@ def test_kernel_state_query_is_bounded_and_returns_none_on_error(self): assert lease._get_kernel_execution_state() is None lease._requests.get.assert_called_once_with( - "http://sandbox:8888/api/kernels/kernel-1", + "https://sandbox:8888/api/kernels/kernel-1", timeout=0.25, ) lease._logger.warning.assert_called_once() @@ -2867,12 +2898,13 @@ def test_inherits_docker_backend_marker(self, monkeypatch): """System kernel leases must remain identifiable as Docker executors.""" from smolagents import remote_executors - monkeypatch.setattr(remote_executors, "_create_kernel_http", lambda *_args: "kernel-1") + monkeypatch.setattr(sandbox_module._DockerKernelLease, "_create_kernel", lambda *_args: "kernel-1") owner = SimpleNamespace( + tls_client=SimpleNamespace(http=MagicMock(), ssl_context=object()), logger=MagicMock(), additional_imports=[], installed_packages=[], - base_url="http://sandbox:8888", + base_url="https://sandbox:8888", host="sandbox", port=8888, _nexent_backend="docker", @@ -3327,7 +3359,7 @@ def test_recovery_skips_non_running_container(self, monkeypatch): container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "exited" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = { "NetworkSettings": { "Networks": {sandbox_module.SANDBOX_NETWORK_NAME: {}}, @@ -3392,7 +3424,7 @@ def test_recovery_skips_when_no_port_mapping(self, monkeypatch): container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = { "NetworkSettings": { "Networks": {sandbox_module.SANDBOX_NETWORK_NAME: {}}, @@ -3421,6 +3453,7 @@ def test_removes_named_stale_containers(self, monkeypatch): cfg = SandboxConfig(level=SandboxLevel.DOCKER, scope=SandboxScope.SYSTEM) stale_container = MagicMock() + stale_container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} stale_container.name = sandbox_module.SANDBOX_CONTAINER_NAME stale_container.short_id = "stale123" stale_container.attrs = {"NetworkSettings": {"Ports": {}}} @@ -3444,6 +3477,7 @@ def test_handles_removal_exception_gracefully(self, monkeypatch): cfg = SandboxConfig(level=SandboxLevel.DOCKER, scope=SandboxScope.SYSTEM) stale_container = MagicMock() + stale_container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} stale_container.name = sandbox_module.SANDBOX_CONTAINER_NAME stale_container.short_id = "stale456" stale_container.attrs = {"NetworkSettings": {"Ports": {}}} @@ -3579,7 +3613,10 @@ def test_waits_for_kernel_ready(self, monkeypatch): container = MagicMock() container.short_id = "ready123" - container.attrs = {"NetworkSettings": {"Networks": {}}} + container.attrs = {"NetworkSettings": { + "Networks": {}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, + }} container.reload = MagicMock() call_count = [0] @@ -3607,7 +3644,7 @@ def mock_get(url, **kwargs): executor = pm._build_system_docker_executor(cfg, logger, {"name": "test-sandbox"}) - assert executor.base_url == "http://127.0.0.1:8888" + assert executor.base_url == "https://127.0.0.1:49173" assert call_count[0] >= 2 assert run.call_args.kwargs["security_opt"] == ["seccomp=unconfined"] @@ -3623,7 +3660,10 @@ def test_removes_container_on_failure(self, monkeypatch): container = MagicMock() container.short_id = "fail123" - container.attrs = {"NetworkSettings": {"Networks": {}}} + container.attrs = {"NetworkSettings": { + "Networks": {}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, + }} container.reload = MagicMock() container.remove = MagicMock() @@ -3684,7 +3724,7 @@ def test_host_runtime_uses_docker_allocated_port(self, monkeypatch): client=MagicMock(), attrs={ "NetworkSettings": { - "Ports": {"8888/tcp": [{"HostPort": "49152"}]} + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49152"}]} } }, ) @@ -3711,7 +3751,7 @@ def test_host_runtime_uses_docker_allocated_port(self, monkeypatch): assert run.call_args.kwargs["ports"] == {"8888/tcp": ("127.0.0.1", None)} assert run.call_args.kwargs["network_disabled"] is False assert run.call_args.kwargs["security_opt"] == ["seccomp=unconfined"] - assert captured["owner"].base_url == "http://127.0.0.1:49152" + assert captured["owner"].base_url == "https://127.0.0.1:49152" assert captured["installed"] == ["numpy"] assert executor.installed_packages == ["numpy"] @@ -3774,7 +3814,7 @@ class NotFound(Exception): aliases=["nexent-runtime-sandbox-session-unique"], ) assert captured["owner"].base_url == ( - "http://nexent-runtime-sandbox-session-unique:8888" + "https://nexent-runtime-sandbox-session-unique:8888" ) def test_cleanup_removes_shared_container_after_last_kernel(self, monkeypatch): @@ -3855,10 +3895,12 @@ def test_missing_dynamic_port_removes_container(self, monkeypatch): monkeypatch.setitem(sys.modules, "requests", SimpleNamespace()) monkeypatch.setattr(sandbox_module, "_is_containerized_runtime", lambda: False) - with pytest.raises(RuntimeError, match="did not allocate"): + config = SandboxConfig(level=SandboxLevel.DOCKER, scope=SandboxScope.SESSION) + logger_ = MagicMock() + with pytest.raises(RuntimeError, match="no effective loopback port mapping"): pm._build_session_docker_executor( - SandboxConfig(level=SandboxLevel.DOCKER, scope=SandboxScope.SESSION), - MagicMock(), + config, + logger_, {}, ) @@ -3869,7 +3911,7 @@ def test_stopped_container_is_removed_before_jupyter_ready(self, monkeypatch): container = MagicMock( attrs={ "NetworkSettings": { - "Ports": {"8888/tcp": [{"HostPort": "49153"}]} + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49153"}]} } }, status="exited", @@ -3900,7 +3942,7 @@ def test_jupyter_timeout_preserves_error_when_container_remove_fails(self, monke container = MagicMock( attrs={ "NetworkSettings": { - "Ports": {"8888/tcp": [{"HostPort": "49154"}]} + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49154"}]} } }, status="running", @@ -3945,7 +3987,7 @@ def test_owner_cleanup_runs_when_group_construction_fails(self, monkeypatch): client=MagicMock(), attrs={ "NetworkSettings": { - "Ports": {"8888/tcp": [{"HostPort": "49155"}]} + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49155"}]} } }, status="running", @@ -3984,7 +4026,7 @@ def test_post_lease_failure_cleans_executor_and_unused_group(self, monkeypatch): short_id="session3", attrs={ "NetworkSettings": { - "Ports": {"8888/tcp": [{"HostPort": "49156"}]} + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49156"}]} } }, status="running", @@ -4316,7 +4358,7 @@ def test_acquire_with_different_images_creates_separate_containers(self, monkeyp executor1 = _FakeExecutor(image="image1:latest", alive=True) executor2 = _FakeExecutor(image="image2:latest", alive=True) - def mock_build_executor(config, logger_, host_tools=False): + def mock_build_executor(config, logger_, host_tools=False, **kwargs): if config.docker_image == "image1:latest": return executor1 return executor2 @@ -4767,7 +4809,7 @@ def test_kernel_lease_rejects_non_positive_receive_timeout(self, monkeypatch): monkeypatch.setitem(sys.modules, "smolagents.remote_executors", remote_module) owner = SimpleNamespace( logger=MagicMock(), - base_url="http://sandbox:8888", + base_url="https://sandbox:8888", host="sandbox", port=8888, ) @@ -4810,6 +4852,7 @@ def test_kernel_watchdog_refreshes_deadline_after_busy_health_check( from websocket import ABNF websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = ( ABNF.OPCODE_TEXT, json.dumps( @@ -4863,10 +4906,12 @@ def test_kernel_replacement_logs_delete_failures( send_variables=MagicMock(), send_tools=MagicMock(), ), - _create_kernel_http=MagicMock(return_value="kernel-2"), ) monkeypatch.setitem(sys.modules, "smolagents.remote_executors", remote_module) + lease._requests.post.return_value = SimpleNamespace( + status_code=201, json=lambda: {"id": "kernel-2"} + ) lease._replace_unhealthy_kernel() assert lease.kernel_id == "kernel-2" @@ -4879,10 +4924,10 @@ def test_kernel_replacement_failure_keeps_lease_unhealthy(self, monkeypatch): lease._requests.delete.return_value = SimpleNamespace(status_code=204) remote_module = SimpleNamespace( RemotePythonExecutor=SimpleNamespace(), - _create_kernel_http=MagicMock(side_effect=RuntimeError("create failed")), ) monkeypatch.setitem(sys.modules, "smolagents.remote_executors", remote_module) + lease._requests.post.side_effect = RuntimeError("create failed") with pytest.raises(RuntimeError, match="Failed to replace unhealthy sandbox kernel"): lease._replace_unhealthy_kernel() @@ -4941,6 +4986,7 @@ def test_kernel_lease_handles_terminal_message_types( "content": {"execution_state": "idle"}, } websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.side_effect = [ (ABNF.OPCODE_TEXT, json.dumps(message)), (ABNF.OPCODE_TEXT, json.dumps(idle)), @@ -4974,6 +5020,7 @@ def test_kernel_lease_rejects_closed_or_empty_frames(self, monkeypatch, frame): from websocket import ABNF websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = ( ABNF.OPCODE_CLOSE if frame == "close" else ABNF.OPCODE_TEXT, b"closed" if frame == "close" else b"", @@ -5098,6 +5145,8 @@ def test_cleanup_ignores_container_kill_failure(self): container.kill.assert_called_once() def test_kernel_lease_execution_and_cleanup_paths(self, monkeypatch): + from websocket import WebSocketBadStatusException + class FakeABNF: OPCODE_TEXT = 1 OPCODE_CLOSE = 8 @@ -5105,6 +5154,7 @@ class FakeABNF: OPCODE_PONG = 10 websocket = MagicMock() + websocket.getstatus.return_value = 101 websocket.recv_data.return_value = ( FakeABNF.OPCODE_TEXT, json.dumps( @@ -5117,6 +5167,7 @@ class FakeABNF: ) websocket_module = SimpleNamespace( ABNF=FakeABNF, + WebSocketBadStatusException=WebSocketBadStatusException, create_connection=MagicMock(return_value=websocket), WebSocketConnectionClosedException=ConnectionError, WebSocketTimeoutException=TimeoutError, @@ -5133,18 +5184,23 @@ class FakeABNF: lease = object.__new__(sandbox_module._DockerKernelLease) lease._closed = False + lease._wait_for_kernel_channel_ready = lambda ws: None lease._unhealthy = False lease._receive_timeout_seconds = 5 - lease.ws_url = "ws://kernel" + lease.ws_url = "wss://kernel" + lease._ssl_context = object() lease.logger = MagicMock() - lease.base_url = "http://kernel" + lease.base_url = "https://kernel" lease.kernel_id = "kernel-id" lease._logger = MagicMock() lease._requests = SimpleNamespace(delete=MagicMock(return_value=SimpleNamespace(status_code=500))) assert lease.run_code_raise_errors("1 + 1") == "result" remote_module._websocket_send_execute_request.assert_called_once_with("1 + 1", websocket) - websocket_module.create_connection.assert_called_once_with("ws://kernel", timeout=5) + websocket_module.create_connection.assert_called_once_with( + "wss://kernel", timeout=5, sslopt={"context": lease._ssl_context}, + http_proxy_host="kernel", http_no_proxy=["kernel"], redirect_limit=0, + ) lease.cleanup() lease._logger.warning.assert_called_once() lease._requests.delete.assert_called_once() @@ -5349,7 +5405,7 @@ def test_recovery_tries_next_connection_host(self, monkeypatch): container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = {"NetworkSettings": {"Networks": {sandbox_module.SANDBOX_NETWORK_NAME: {}}}} container.client = MagicMock() docker_module = SimpleNamespace(from_env=lambda: SimpleNamespace( @@ -5374,7 +5430,7 @@ def test_recovery_returns_none_when_all_connection_hosts_fail(self, monkeypatch) container = MagicMock() container.name = sandbox_module.SANDBOX_CONTAINER_NAME container.status = "running" - container.labels = {"com.nexent.sandbox": "runtime"} + container.labels = {"com.nexent.sandbox": "runtime", sandbox_module.TLS_LABEL: sandbox_module.TLS_VERSION} container.attrs = {"NetworkSettings": {"Networks": {sandbox_module.SANDBOX_NETWORK_NAME: {}}}} docker_module = SimpleNamespace(from_env=lambda: SimpleNamespace( containers=SimpleNamespace(list=lambda **kwargs: [container]) @@ -5388,7 +5444,7 @@ def test_recovery_returns_none_when_all_connection_hosts_fail(self, monkeypatch) SandboxConfig(level=SandboxLevel.DOCKER, scope=SandboxScope.SYSTEM), MagicMock(), False ) is None - def test_remove_stale_container_using_image_and_port(self, monkeypatch): + def test_preserves_unrelated_container_using_image_and_port(self, monkeypatch): pool = SandboxPoolManager.get_instance() container = MagicMock( name="old-name", @@ -5402,11 +5458,14 @@ def test_remove_stale_container_using_image_and_port(self, monkeypatch): pool._remove_stale_docker_containers(SandboxConfig(docker_image="custom:image"), MagicMock()) - container.remove.assert_called_once_with(force=True) + container.remove.assert_not_called() def test_system_docker_cleanup_preserves_original_error_when_remove_fails(self, monkeypatch): pool = SandboxPoolManager.get_instance() - container = MagicMock(attrs={"NetworkSettings": {"Networks": {}}}) + container = MagicMock(attrs={"NetworkSettings": { + "Networks": {}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, + }}) container.remove.side_effect = RuntimeError("remove failed") docker_module = SimpleNamespace(from_env=lambda: SimpleNamespace( containers=SimpleNamespace(run=MagicMock(return_value=container)) @@ -5478,6 +5537,7 @@ def test_build_docker_executor_leases_from_existing_session_group(self, monkeypa def test_system_docker_creates_missing_network(self, monkeypatch): pool = SandboxPoolManager.get_instance() + monkeypatch.setattr(sandbox_module, "_is_containerized_runtime", lambda: True) executor = SimpleNamespace(__call__=MagicMock(return_value="ok")) networks = SimpleNamespace( get=MagicMock(side_effect=KeyError("missing")), diff --git a/test/sdk/core/agents/test_sandbox_channel_readiness.py b/test/sdk/core/agents/test_sandbox_channel_readiness.py new file mode 100644 index 0000000000..7621e49b61 --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_channel_readiness.py @@ -0,0 +1,323 @@ +"""Channel readiness must precede execution; cancelled requests must not retry.""" + +import json +import ssl +from concurrent.futures import CancelledError +from unittest.mock import MagicMock + +import pytest +from nexent.core.agents import sandbox as sb +from websocket import ( + ABNF, + WebSocketBadStatusException, + WebSocketConnectionClosedException, + WebSocketTimeoutException, +) + + +def message(request_id, kind, content=None): + return (ABNF.OPCODE_TEXT, json.dumps({ + "parent_header": {"msg_id": request_id}, + "header": {"msg_type": kind}, "content": content or {}, + })) + + +@pytest.fixture +def lease(): + item = object.__new__(sb._DockerKernelLease) + item._closed = item._unhealthy = False + item._channel_session_id = "probe-session" + item.kernel_id = "owned-kernel" + item.ws_url = "ws://127.0.0.1:58023/api/kernels/owned-kernel/channels" + item._receive_timeout_seconds = 0.05 + item._ssl_context = object() + item._cancellation_scope = sb.RunCancellationScope() + item._logger = item.logger = MagicMock() + item._get_kernel_execution_state = MagicMock(return_value="idle") + return item + + +def socket_for_execution(*, handshake_lost=False, execution_lost=False): + ws = MagicMock() + ws.getstatus.return_value = 101 + frames = [] + sent = [] + + def send(payload): + request = json.loads(payload) + sent.append(request) + request_id = request["header"]["msg_id"] + if request["header"]["msg_type"] == "kernel_info_request": + if handshake_lost: + return + frames.extend([ + message("stale", "status", {"execution_state": "idle"}), + message(request_id, "kernel_info_reply"), + message(request_id, "status", {"execution_state": "idle"}), + ]) + elif not execution_lost: + frames.extend([ + message(request_id, "stream", {"text": "OK\n"}), + message(request_id, "execute_reply", {"status": "ok"}), + message(request_id, "status", {"execution_state": "idle"}), + ]) + + def receive(**kwargs): + if frames: + return frames.pop(0) + raise WebSocketTimeoutException("No matching frames") + + ws.send.side_effect = send + ws.recv_data.side_effect = receive + return ws, sent + + +def test_reconnects_only_handshake_then_submits_code_once(lease, monkeypatch): + first, first_sent = socket_for_execution(handshake_lost=True) + second, second_sent = socket_for_execution() + connect = MagicMock(side_effect=[first, second]) + monkeypatch.setattr("websocket.create_connection", connect) + result = lease.run_code_raise_errors("print('OK')") + assert result.logs == "OK\n" + assert [m["header"]["msg_type"] for m in first_sent] == ["kernel_info_request"] + assert [m["header"]["msg_type"] for m in second_sent] == ["kernel_info_request", "execute_request"] + first.close.assert_called_once() + second.close.assert_called_once() + assert not lease._cancellation_scope._closers + + +def test_handshake_retry_is_bounded_and_never_sends_code(lease, monkeypatch): + ws, sent = socket_for_execution(handshake_lost=True) + connect = MagicMock(return_value=ws) + monkeypatch.setattr("websocket.create_connection", connect) + with pytest.raises(RuntimeError, match="before code submission"): + lease.run_code_raise_errors("side_effect()") + assert connect.call_count == 3 + assert all(m["header"]["msg_type"] == "kernel_info_request" for m in sent) + assert lease._unhealthy + + +def test_connection_timeout_is_retried_before_submission(lease, monkeypatch): + ws, sent = socket_for_execution() + connect = MagicMock(side_effect=[TimeoutError('connect timed out'), ws]) + monkeypatch.setattr('websocket.create_connection', connect) + assert lease.run_code_raise_errors('pass').logs == 'OK\n' + assert connect.call_count == 2 + assert sum(m['header']['msg_type'] == 'execute_request' for m in sent) == 1 + + +def test_initialization_replaces_kernel_after_channel_failure(lease, monkeypatch): + lease._cached_variables = lease._cached_tools = None + lease._kernel_bootstrap_code = [] + lease.base_url = 'https://sandbox:8888' + lease.host, lease.port = 'sandbox', 8888 + lease._requests = MagicMock() + lease._requests.delete.return_value.status_code = 204 + lease._create_kernel = MagicMock(return_value='replacement') + ws, sent = socket_for_execution() + connect = MagicMock(side_effect=[TimeoutError('failed')] * 3 + [ws]) + monkeypatch.setattr('websocket.create_connection', connect) + lease.register_kernel_bootstrap_code('initialize_workspace()') + assert lease.kernel_id == 'replacement' + lease._create_kernel.assert_called_once() + assert lease._kernel_bootstrap_code == ['initialize_workspace()'] + assert sum(m['header']['msg_type'] == 'execute_request' for m in sent) == 1 + + +@pytest.fixture +def recoverable_lease(lease): + lease._cached_variables = lease._cached_tools = None + lease._kernel_bootstrap_code = [] + lease.base_url = 'https://sandbox:8888' + lease.host, lease.port = 'sandbox', 8888 + lease._requests = MagicMock() + lease._requests.delete.return_value.status_code = 204 + lease._create_kernel = MagicMock(return_value='replacement') + return lease + + +@pytest.mark.parametrize('operation', ['variables', 'tools', 'bootstrap']) +def test_failed_replacement_does_not_recurse(recoverable_lease, monkeypatch, operation): + lease = recoverable_lease + from smolagents.remote_executors import RemotePythonExecutor + + monkeypatch.setattr(RemotePythonExecutor, 'send_variables', lambda self, _: self.run_code_raise_errors('variables')) + monkeypatch.setattr(RemotePythonExecutor, 'send_tools', lambda self, _: self.run_code_raise_errors('tools')) + connect = MagicMock(side_effect=TimeoutError('unavailable')) + monkeypatch.setattr('websocket.create_connection', connect) + call = { + 'variables': lambda: lease.send_variables({'x': 1}), + 'tools': lambda: lease.send_tools({}), + 'bootstrap': lambda: lease.register_kernel_bootstrap_code('workspace'), + }[operation] + with pytest.raises(RuntimeError): + call() + lease._create_kernel.assert_called_once() + assert connect.call_count == 6 + assert lease._unhealthy + assert not lease._kernel_replacement_in_progress + + +def test_initially_unhealthy_setup_has_only_one_replacement(recoverable_lease, monkeypatch): + lease = recoverable_lease + lease._unhealthy = True + monkeypatch.setattr('websocket.create_connection', MagicMock(side_effect=TimeoutError('unavailable'))) + with pytest.raises(RuntimeError): + lease.register_kernel_bootstrap_code('workspace') + lease._create_kernel.assert_called_once() + + +@pytest.mark.parametrize('failure', [ + ssl.SSLCertVerificationError('wrong certificate'), + WebSocketBadStatusException('unauthorized', status_code=401), + WebSocketBadStatusException('forbidden', status_code=403), +]) +def test_security_failure_does_not_retry_or_replace(recoverable_lease, monkeypatch, failure): + lease = recoverable_lease + connect = MagicMock(side_effect=failure) + monkeypatch.setattr('websocket.create_connection', connect) + exception_type = type(failure) + with pytest.raises(exception_type): + lease.register_kernel_bootstrap_code('workspace') + connect.assert_called_once() + lease._create_kernel.assert_not_called() + + +def test_missing_kernel_can_be_replaced(recoverable_lease, monkeypatch): + lease = recoverable_lease + ws, _ = socket_for_execution() + failure = WebSocketBadStatusException('missing kernel', status_code=404) + connect = MagicMock(side_effect=[failure, failure, failure, ws]) + monkeypatch.setattr('websocket.create_connection', connect) + lease.register_kernel_bootstrap_code('workspace') + lease._create_kernel.assert_called_once() + + +def test_cancel_during_connection_failure_stops_recovery(recoverable_lease, monkeypatch): + lease = recoverable_lease + def connect(*args, **kwargs): + lease._cancellation_scope.cancel() + raise TimeoutError('cancelled connection') + connect_mock = MagicMock(side_effect=connect) + monkeypatch.setattr('websocket.create_connection', connect_mock) + with pytest.raises(CancelledError): + lease.register_kernel_bootstrap_code('workspace') + connect_mock.assert_called_once() + lease._create_kernel.assert_not_called() + + +def test_lost_execution_is_not_replayed_or_accepted_from_http_idle(lease, monkeypatch): + ws, sent = socket_for_execution(execution_lost=True) + connect = MagicMock(return_value=ws) + monkeypatch.setattr("websocket.create_connection", connect) + with pytest.raises(RuntimeError, match="kernel channel failed"): + lease.run_code_raise_errors("side_effect()") + assert connect.call_count == 1 + assert sum(m["header"]["msg_type"] == "execute_request" for m in sent) == 1 + assert lease._unhealthy + + +@pytest.mark.parametrize("phase", ["handshake", "execution"]) +def test_cancellation_closes_only_current_channel_and_never_retries(lease, monkeypatch, phase): + ws, _ = socket_for_execution() + original_send = ws.send.side_effect + + def send(payload): + original_send(payload) + kind = json.loads(payload)["header"]["msg_type"] + if kind == ("kernel_info_request" if phase == "handshake" else "execute_request"): + lease._cancellation_scope.cancel() + + ws.send.side_effect = send + connect = MagicMock(return_value=ws) + monkeypatch.setattr("websocket.create_connection", connect) + replace = MagicMock() + monkeypatch.setattr(lease, "_replace_unhealthy_kernel", replace) + with pytest.raises(CancelledError): + lease.run_code_raise_errors("side_effect()") + connect.assert_called_once() + ws.shutdown.assert_called_once() + ws.close.assert_called_once() + replace.assert_not_called() + assert not lease._cancellation_scope._closers + + +def test_close_induced_oserror_is_cancellation(lease, monkeypatch): + ws, _ = socket_for_execution() + def receive(**kwargs): + lease._cancellation_scope.cancel() + raise OSError("socket closed") + ws.recv_data.side_effect = receive + monkeypatch.setattr("websocket.create_connection", MagicMock(return_value=ws)) + with pytest.raises(CancelledError): + lease.run_code_raise_errors("side_effect()") + + +def test_cancelled_unhealthy_lease_is_not_replaced(lease): + lease._unhealthy = True + lease._cancellation_scope.cancel() + lease._replace_unhealthy_kernel = MagicMock() + with pytest.raises(CancelledError): + lease.register_kernel_bootstrap_code("pass") + lease._replace_unhealthy_kernel.assert_not_called() + + +def test_handshake_requires_both_matching_shell_reply_and_iopub_idle(lease): + ws = MagicMock() + ws.getstatus.return_value = 101 + frames = [] + def send(payload): + request_id = json.loads(payload)["header"]["msg_id"] + frames.extend([ + message(request_id, "kernel_info_reply"), + message("other-request", "status", {"execution_state": "idle"}), + ]) + ws.send.side_effect = send + def receive(**kwargs): + if frames: + return frames.pop(0) + raise WebSocketTimeoutException("No matching idle") + ws.recv_data.side_effect = receive + with pytest.raises(WebSocketTimeoutException): + lease._wait_for_kernel_channel_ready(ws) + + +def test_handshake_tolerates_control_frames(lease): + ws = MagicMock() + ws.getstatus.return_value = 101 + frames = [(ABNF.OPCODE_PING, b'ping'), (ABNF.OPCODE_PONG, b'pong')] + def send(payload): + request_id = json.loads(payload)['header']['msg_id'] + frames.extend([message(request_id, 'kernel_info_reply'), + message(request_id, 'status', {'execution_state': 'idle'})]) + ws.send.side_effect = send + ws.recv_data.side_effect = lambda **_: frames.pop(0) + lease._wait_for_kernel_channel_ready(ws) + assert not frames + + +@pytest.mark.parametrize('frame', [(ABNF.OPCODE_CLOSE, b'closed'), (ABNF.OPCODE_TEXT, b'')]) +def test_handshake_rejects_closed_channels(lease, frame): + ws = MagicMock() + ws.getstatus.return_value = 101 + ws.recv_data.return_value = frame + with pytest.raises(WebSocketConnectionClosedException): + lease._wait_for_kernel_channel_ready(ws) + + +def test_handshake_obeys_total_deadline(lease, monkeypatch): + times = iter([0, 1]) + monkeypatch.setattr(sb.time, 'monotonic', lambda: next(times)) + ws = MagicMock() + ws.getstatus.return_value = 101 + with pytest.raises(WebSocketTimeoutException, match='handshake timed out'): + lease._wait_for_kernel_channel_ready(ws) + ws.recv_data.assert_not_called() + + +def test_execution_without_cancellation_scope_still_closes_channel(lease, monkeypatch): + lease._cancellation_scope = None + ws, _ = socket_for_execution() + monkeypatch.setattr('websocket.create_connection', MagicMock(return_value=ws)) + assert lease.run_code_raise_errors("print('OK')").logs == 'OK\n' + ws.close.assert_called_once() diff --git a/test/sdk/core/agents/test_sandbox_proxy.py b/test/sdk/core/agents/test_sandbox_proxy.py new file mode 100644 index 0000000000..e20b865ea0 --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_proxy.py @@ -0,0 +1,129 @@ +"""Exercise real websocket-client proxy selection without external network I/O.""" + +import base64 +import hashlib +import socket +import ssl +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from nexent.core.agents import sandbox as sb +from nexent.core.agents.sandbox_tls import SandboxTLSClient +from nexent.core.agents.sandbox_tls_bootstrap import generate_identity +from websocket import WebSocketException +from websocket import _core, _http +from websocket._url import get_proxy_info + + +@pytest.fixture +def transport(monkeypatch): + resolve = MagicMock(return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('127.0.0.1', 8888))]) + monkeypatch.setattr(_http.socket, 'getaddrinfo', resolve) + sock = MagicMock() + sock.recv.return_value = b'' + sock.send.side_effect = len + monkeypatch.setattr(_http, '_open_socket', MagicMock(return_value=sock)) + monkeypatch.setattr(_http, '_tunnel', MagicMock(side_effect=AssertionError('Sandbox used the proxy'))) + monkeypatch.setattr(_http, '_ssl_socket', lambda sock, options, hostname: sock) + handshake = MagicMock(return_value=SimpleNamespace(status=101, headers={}, subprotocol=None)) + monkeypatch.setattr(_core, 'handshake', handshake) + return resolve, sock, handshake + + +def make_lease(host): + lease = object.__new__(sb._DockerKernelLease) + lease.host, lease.port = host, 8888 + lease._closed = lease._unhealthy = False + lease._receive_timeout_seconds = 1 + lease._ssl_context = object() + lease._logger = MagicMock() + lease._cancellation_scope = sb.RunCancellationScope() + lease.ws_url = lease._build_channels_url('original') + lease._wait_for_kernel_channel_ready = MagicMock() + return lease + + +@pytest.mark.parametrize('host', ['127.0.0.1', 'nexent-runtime-sandbox', 'nexent-runtime-sandbox-session-abc123']) +@pytest.mark.parametrize('proxy_key', ['HTTPS_PROXY', 'https_proxy']) +@pytest.mark.parametrize('excluded', [None, 'unrelated.internal']) +def test_environment_proxy_is_bypassed_only_for_sandbox(monkeypatch, transport, host, proxy_key, excluded): + for name in ('HTTPS_PROXY', 'https_proxy', 'NO_PROXY', 'no_proxy'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv(proxy_key, 'http://proxy.example:3128') + if excluded is not None: + monkeypatch.setenv('NO_PROXY', excluded) + lease = make_lease(host) + resolve, sock, handshake = transport + for kernel in ('original', 'replacement'): + lease.ws_url = lease._build_channels_url(kernel) + with lease._kernel_channel() as ws: + assert ws.sock_opt.sslopt['context'] is lease._ssl_context + assert [call.args[0] for call in resolve.call_args_list] == [host, host] + for call in handshake.call_args_list: + assert call.kwargs['http_no_proxy'] == [host] + assert get_proxy_info('external.example', True)[:2] == ('proxy.example', 3128) + assert not lease._cancellation_scope._closers + assert sock.close.call_count == 2 + + +@pytest.mark.parametrize('status', [301, 302, 303, 307, 308, 200]) +def test_non_upgrade_response_is_closed_without_redirect_or_execution(transport, status): + lease = make_lease('nexent-runtime-sandbox') + resolve, sock, handshake = transport + handshake.return_value.status = status + handshake.return_value.headers = {'location': 'ws://other.example/channels'} + with pytest.raises(WebSocketException, match='Redirect limit exhausted|handshake requires status 101'): + with lease._kernel_channel(): + pytest.fail('An invalid handshake must not expose an execution channel') + assert resolve.call_count == 1 + lease._wait_for_kernel_channel_ready.assert_not_called() + assert sock.close.called + assert not lease._cancellation_scope._closers + + +def test_real_wss_handshake_bypasses_unreachable_environment_proxy(monkeypatch, tmp_path): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + challenge = self.headers['Sec-WebSocket-Key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' + # SHA-1 is mandated by the WebSocket handshake, not used for TLS security. + accept = base64.b64encode(hashlib.sha1(challenge.encode(), usedforsecurity=False).digest()).decode() + self.send_response(101) + self.send_header('Upgrade', 'websocket') + self.send_header('Connection', 'Upgrade') + self.send_header('Sec-WebSocket-Accept', accept) + self.end_headers() + + def log_message(self, *args): + pass + + key, cert = generate_identity('sandbox-test') + key_file, cert_file = tmp_path / 'key.pem', tmp_path / 'cert.pem' + key_file.write_bytes(key) + cert_file.write_bytes(cert) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(cert_file, key_file) + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = Thread(target=server.serve_forever, daemon=True) + client = SandboxTLSClient(cert) + thread.start() + try: + for name in ('HTTPS_PROXY', 'https_proxy'): + monkeypatch.setenv(name, 'http://127.0.0.1:1') + for name in ('NO_PROXY', 'no_proxy'): + monkeypatch.delenv(name, raising=False) + lease = make_lease('127.0.0.1') + lease.port = server.server_port + lease.ws_url = lease._build_channels_url('test') + lease._ssl_context = client.ssl_context + with lease._kernel_channel() as ws: + assert ws.getstatus() == 101 + finally: + client.close() + server.shutdown() + server.server_close() + thread.join(timeout=3) diff --git a/test/sdk/core/agents/test_sandbox_system_network.py b/test/sdk/core/agents/test_sandbox_system_network.py new file mode 100644 index 0000000000..8d6ec4c22d --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_system_network.py @@ -0,0 +1,276 @@ +"""Native system endpoints and cancellation ownership regressions.""" + +import logging +import sys +from concurrent.futures import CancelledError +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from nexent.core.agents import sandbox as sb + + +@pytest.fixture +def system(monkeypatch): + pool = sb.SandboxPoolManager() + config = sb.SandboxConfig(level=sb.SandboxLevel.DOCKER, scope=sb.SandboxScope.SYSTEM) + container = MagicMock() + container.attrs = {"NetworkSettings": { + "Networks": {"bridge": {}}, + "Ports": {"8888/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]}, + }} + container.labels = {"com.nexent.sandbox": "runtime", sb.TLS_LABEL: sb.TLS_VERSION} + container.name = sb.SANDBOX_CONTAINER_NAME + container.status = "running" + run = MagicMock(return_value=container) + client = SimpleNamespace( + version=lambda: {"Version": "20.10.24"}, + containers=SimpleNamespace(run=run, list=lambda **kwargs: [container]), + ) + get = MagicMock(return_value=SimpleNamespace(raise_for_status=lambda: None, json=list)) + tls = SimpleNamespace(http=SimpleNamespace(get=get), ssl_context=object(), close=MagicMock()) + monkeypatch.setattr(sb, 'load_container_tls', lambda *args, **kwargs: tls) + monkeypatch.setitem(sys.modules, "docker", SimpleNamespace(from_env=lambda: client)) + monkeypatch.setitem(sys.modules, "requests", SimpleNamespace(get=get)) + monkeypatch.setattr(sb, "_is_containerized_runtime", lambda: False) + monkeypatch.setattr(sb, "_seed_pnpm_offline_store", lambda container: None) + return SimpleNamespace( + pool=pool, config=config, container=container, run=run, get=get, + logger=logging.getLogger(__name__), scope=sb.RunCancellationScope(), + ) + + +def test_native_system_uses_actual_dynamic_port(system): + s = system + owner = s.pool._build_system_docker_executor(s.config, s.logger, {"network": sb.SANDBOX_NETWORK_NAME}) + assert s.run.call_args.kwargs["network"] == "bridge" + assert s.run.call_args.kwargs["ports"] == {"8888/tcp": ("127.0.0.1", None)} + assert owner.base_url == "https://127.0.0.1:49173" + assert owner.port == 49173 + s.get.assert_called_once_with("https://127.0.0.1:49173/api/kernels", timeout=1) + + +def test_recovery_reads_dynamic_mapping_on_bridge(system): + s = system + owner = s.pool._recover_docker_container(s.config, s.logger, False) + assert owner.base_url == "https://127.0.0.1:49173" + s.run.assert_not_called() + + +@pytest.mark.parametrize("bindings", [[], [{"HostIp": "0.0.0.0", "HostPort": "49173"}]]) +def test_missing_or_public_mapping_fails_without_probing_other_service(system, bindings): + s = system + s.container.attrs["NetworkSettings"]["Ports"]["8888/tcp"] = bindings + with pytest.raises(RuntimeError, match="no effective loopback"): + s.pool._build_system_docker_executor(s.config, s.logger, {}) + s.get.assert_not_called() + s.container.remove.assert_called_once_with(force=True) + + +@pytest.mark.parametrize("phase", ["before_create", "after_create", "probe", "shutdown"]) +def test_cancel_unpublished_startup_cleans_only_created_container(system, phase): + s = system + if phase == "before_create": + s.scope.cancel() + elif phase == "after_create": + def create(*args, **kwargs): + s.scope.cancel() + return s.container + s.run.side_effect = create + else: + def probe(*args, **kwargs): + if phase == "shutdown": + s.pool._stop_evict.set() + else: + s.scope.cancel() + raise OSError("connection closed") + s.get.side_effect = probe + with pytest.raises(CancelledError): + s.pool._build_system_docker_executor(s.config, s.logger, {}, s.scope) + if phase == "before_create": + s.run.assert_not_called() + s.container.remove.assert_not_called() + else: + s.container.remove.assert_called_once_with(force=True) + assert not s.pool._system_containers + + +def test_cancel_after_kernel_creation_preserves_shared_owner_and_b(system, monkeypatch): + s = system + owner = SimpleNamespace(container=s.container, base_url="http://127.0.0.1:49173") + s.pool._system_containers[s.config.docker_image] = owner + b = MagicMock(container=s.container) + s.pool._executors[id(b)] = b + s.pool._lease_owners[id(b)] = owner + lease = MagicMock(container=s.container) + def create_lease(*args, **kwargs): + s.scope.cancel() + return lease + monkeypatch.setattr(s.pool, "_is_alive", lambda owner: True) + monkeypatch.setattr(sb, "_DockerKernelLease", create_lease) + with pytest.raises(CancelledError): + s.pool._acquire_shared_docker_kernel(s.config, s.logger, False, s.scope) + lease.cleanup.assert_called_once() + b.cleanup.assert_not_called() + s.container.remove.assert_not_called() + assert s.pool._system_containers[s.config.docker_image] is owner + + +def test_immediate_release_of_a_preserves_b_and_shared_container(system, monkeypatch): + s = system + owner = MagicMock() + a, b = MagicMock(), MagicMock() + s.pool._system_containers[s.config.docker_image] = owner + for lease in (a, b): + s.pool._lease_owners[id(lease)] = owner + s.pool._executors[id(lease)] = lease + s.pool._in_use[id(lease)] = s.config.docker_image + destroy = MagicMock() + monkeypatch.setattr(s.pool, "_destroy_executor", destroy) + s.pool.release_immediate(a, s.logger) + destroy.assert_called_once_with(a, s.logger) + assert s.pool._system_containers[s.config.docker_image] is owner + assert s.pool._executors[id(b)] is b + + +def test_unrelated_same_name_container_is_not_removed(system): + s = system + s.container.labels = {} + s.pool._remove_stale_docker_containers(s.config, s.logger) + s.container.remove.assert_not_called() + + +def test_cancel_before_owner_publication_cleans_unpublished_owner(system, monkeypatch): + s = system + owner = SimpleNamespace(container=s.container, base_url="http://127.0.0.1:49173") + monkeypatch.setattr(s.pool, "_recover_docker_container", lambda *args: None) + monkeypatch.setattr(s.pool, "_remove_stale_docker_containers", lambda *args: None) + def build(*args, cancellation_scope=None): + assert cancellation_scope is s.scope + s.scope.cancel() + return owner + monkeypatch.setattr(s.pool, "_build_executor", build) + destroy = MagicMock() + monkeypatch.setattr(s.pool, "_destroy_executor", destroy) + with pytest.raises(CancelledError): + s.pool._acquire_shared_docker_kernel(s.config, s.logger, False, s.scope) + destroy.assert_called_once_with(owner, s.logger) + assert not s.pool._system_containers + assert s.pool._container_build_lock.acquire(blocking=False) + s.pool._container_build_lock.release() + + +def test_system_cancellation_never_falls_back_to_local(system, monkeypatch): + s = system + monkeypatch.setitem(sys.modules, "smolagents.remote_executors", SimpleNamespace(DockerExecutor=object())) + builder = MagicMock(side_effect=CancelledError("cancelled")) + local = MagicMock() + monkeypatch.setattr(s.pool, "_build_system_docker_executor", builder) + monkeypatch.setattr(sb, "_make_local_executor", local) + with pytest.raises(CancelledError): + s.pool._build_docker_executor(s.config, s.logger, cancellation_scope=s.scope) + assert builder.call_args.kwargs["cancellation_scope"] is s.scope + local.assert_not_called() + + +@pytest.mark.parametrize('port', ['', None, 'bad', '0', '-1', '65536']) +def test_invalid_published_ports_are_not_used(system, port): + system.container.attrs['NetworkSettings']['Ports']['8888/tcp'] = [ + {'HostIp': '127.0.0.1', 'HostPort': port}, + ] + with pytest.raises(RuntimeError, match='no effective loopback'): + sb._published_sandbox_port(system.container) + assert system.pool._recover_docker_container(system.config, system.logger, False) is None + system.get.assert_not_called() + + +def test_port_parser_skips_invalid_binding_and_uses_valid_loopback(system): + system.container.attrs['NetworkSettings']['Ports']['8888/tcp'] = [ + {'HostIp': '0.0.0.0', 'HostPort': '8888'}, + {'HostIp': '127.0.0.1', 'HostPort': 'bad'}, + {'HostIp': '127.0.0.1', 'HostPort': '65535'}, + ] + assert sb._published_sandbox_port(system.container) == 65535 + + +@pytest.mark.parametrize('matching_mount', [False, True]) +def test_recovery_requires_matching_bind_mount(system, tmp_path, matching_mount): + system.config.workspace_mode = 'bind' + system.config.container_workspace_root = '/mnt/work' + system.config.extra_kwargs = {'workspace_root': str(tmp_path)} + mapping = system.config.bind_workspace() + system.container.labels['com.nexent.workspace'] = mapping.mount_id + system.container.attrs['Mounts'] = [{ + 'Type': 'bind', 'Source': str(tmp_path if matching_mount else tmp_path / 'wrong'), + 'Destination': '/mnt/work', 'RW': True, + }] + result = system.pool._recover_docker_container(system.config, system.logger, False) + if matching_mount: + assert result.container is system.container + system.get.assert_called_once() + else: + assert result is None + system.get.assert_not_called() + system.container.remove.assert_not_called() + + +@pytest.mark.parametrize('response', ['invalid', 'exception']) +def test_system_startup_timeout_cleans_created_container(system, monkeypatch, response): + times = iter([0, 0, 1000]) + monkeypatch.setattr(sb.time, 'monotonic', lambda: next(times)) + monkeypatch.setattr(system.pool._stop_evict, 'wait', lambda _: None) + if response == 'invalid': + system.get.return_value = SimpleNamespace(raise_for_status=lambda: None, json=dict) + else: + system.get.side_effect = OSError('unavailable') + with pytest.raises(RuntimeError, match='did not become ready'): + system.pool._build_system_docker_executor(system.config, system.logger, {}) + system.container.remove.assert_called_once_with(force=True) + assert not system.pool._system_containers + + +def test_changed_system_workspace_does_not_destroy_active_owner(system, tmp_path, monkeypatch): + original = sb.SandboxConfig( + level=sb.SandboxLevel.DOCKER, workspace_mode='bind', container_workspace_root='/mnt/old', + extra_kwargs={'workspace_root': str(tmp_path)}, + ) + owner = SimpleNamespace(_nexent_sandbox_config=original, container=system.container) + system.pool._system_containers[system.config.docker_image] = owner + destroy = MagicMock() + monkeypatch.setattr(system.pool, '_destroy_executor', destroy) + with pytest.raises(RuntimeError, match='workspace changed'): + system.pool._acquire_shared_docker_kernel(system.config, system.logger, False) + destroy.assert_not_called() + assert system.pool._system_containers[system.config.docker_image] is owner + + +def test_failed_new_lease_preserves_owner_with_active_leases(system, monkeypatch): + owner = SimpleNamespace(_nexent_sandbox_config=system.config, container=system.container, base_url='http://127.0.0.1') + system.pool._system_containers[system.config.docker_image] = owner + active = SimpleNamespace(container=system.container) + system.pool._executors[id(active)] = active + monkeypatch.setattr(system.pool, '_is_alive', lambda _: True) + monkeypatch.setattr(sb, '_DockerKernelLease', MagicMock(side_effect=OSError('connect failed'))) + destroy = MagicMock() + monkeypatch.setattr(system.pool, '_destroy_executor', destroy) + with pytest.raises(RuntimeError, match='preserving the shared container'): + system.pool._acquire_shared_docker_kernel(system.config, system.logger, False) + destroy.assert_not_called() + assert system.pool._executors[id(active)] is active + + +def test_cancel_before_lease_registration_preserves_shared_owner(system, monkeypatch): + owner = SimpleNamespace(_nexent_sandbox_config=system.config, container=system.container, base_url='http://127.0.0.1') + system.pool._system_containers[system.config.docker_image] = owner + lease = MagicMock() + monkeypatch.setattr(system.pool, '_is_alive', lambda _: True) + monkeypatch.setattr(sb, '_DockerKernelLease', lambda *args, **kwargs: lease) + def wrap(executor, *args): + system.scope.cancel() + return executor + monkeypatch.setattr(sb, '_wrap_executor', wrap) + with pytest.raises(CancelledError, match='acquisition cancelled'): + system.pool._acquire_shared_docker_kernel(system.config, system.logger, False, system.scope) + lease.cleanup.assert_called_once() + assert not system.pool._executors + assert system.pool._system_containers[system.config.docker_image] is owner diff --git a/test/sdk/core/agents/test_sandbox_tls.py b/test/sdk/core/agents/test_sandbox_tls.py new file mode 100644 index 0000000000..c0576dd071 --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_tls.py @@ -0,0 +1,308 @@ +"""Verified TLS transport and non-destructive migration of legacy containers.""" + +import io +import logging +import socket +import ssl +import tarfile +from concurrent.futures import CancelledError +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from unittest.mock import Mock + +import pytest +from cryptography import x509 +from nexent.core.agents import sandbox +from nexent.core.agents import sandbox_tls as tls +from nexent.core.agents import sandbox_tls_bootstrap as bootstrap +from nexent.core.agents.sandbox_tls_bootstrap import generate_identity +from requests.exceptions import SSLError + + +@pytest.fixture +def certificate(tmp_path): + key, cert = generate_identity('sandbox-test') + (tmp_path / 'server.key').write_bytes(key) + (tmp_path / 'server.crt').write_bytes(cert) + return cert + + +@pytest.fixture +def expired_certificate(monkeypatch, tmp_path): + class PastClock: + @staticmethod + def now(tz): + return datetime.now(timezone.utc) - timedelta(days=366) + + monkeypatch.setattr(bootstrap, 'datetime', PastClock) + key, cert = generate_identity('sandbox-test') + (tmp_path / 'server.key').write_bytes(key) + (tmp_path / 'server.crt').write_bytes(cert) + monkeypatch.setattr(bootstrap, 'datetime', datetime) + return key, cert + + +def test_generated_certificate_covers_native_and_docker_hosts(certificate): + cert = x509.load_pem_x509_certificate(certificate) + names = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + assert 'sandbox-test' in names.get_values_for_type(x509.DNSName) + assert '127.0.0.1' in [str(value) for value in names.get_values_for_type(x509.IPAddress)] + + +def test_verified_transport_keeps_certificate_checks_enabled(certificate): + client = tls.SandboxTLSClient(certificate) + try: + assert client.ssl_context.verify_mode == ssl.CERT_REQUIRED + assert client.ssl_context.check_hostname + assert client.ssl_context.minimum_version == ssl.TLSVersion.TLSv1_2 + assert client.http.verify == str(client.ca_file) + assert client.http.trust_env is False + assert client.ca_file.read_bytes() == certificate + finally: + client.close() + assert not client.ca_file.exists() + + +def test_recovered_executor_uses_https(certificate): + client = tls.SandboxTLSClient(certificate) + owner = sandbox._RecoveredDockerExecutor(Mock(), logging.getLogger('tls-test'), '127.0.0.1', + port=49000, tls_client=client) + try: + assert owner.base_url == 'https://127.0.0.1:49000' + finally: + owner.cleanup() + + +@pytest.mark.parametrize('status', [200, 302, 500]) +def test_kernel_creation_rejects_non_created_status(status): + lease = object.__new__(sandbox._DockerKernelLease) + lease.base_url = 'https://sandbox-test:8888' + lease._receive_timeout_seconds = 3 + lease._requests = Mock() + lease._requests.post.return_value.status_code = status + with pytest.raises(RuntimeError, match=f'status={status}'): + lease._create_kernel() + lease._requests.post.assert_called_once_with('https://sandbox-test:8888/api/kernels', timeout=3) + + +def test_trust_file_creation_failure_cleans_temporary_directory(certificate, monkeypatch, tmp_path): + directory = Mock(name=str(tmp_path)) + directory.name = str(tmp_path) + monkeypatch.setattr(tls.tempfile, 'TemporaryDirectory', lambda **kwargs: directory) + monkeypatch.setattr(tls.Path, 'write_bytes', Mock(side_effect=OSError('disk full'))) + with pytest.raises(OSError, match='disk full'): + tls.SandboxTLSClient(certificate) + directory.cleanup.assert_called_once() + + +def test_tls_gateway_command_includes_certificate_and_key(): + command = sandbox._kernel_gateway_command('session-test') + assert '--KernelGatewayApp.certfile=' + tls.TLS_CERTIFICATE in command + assert '--KernelGatewayApp.keyfile=' + tls.TLS_DIRECTORY + '/server.key' in command + assert command[:2] == ['python', '-c'] + assert 'session-test' in command + + +def test_running_legacy_container_requires_explicit_migration(): + container = Mock(status='running', labels={'com.nexent.sandbox': 'runtime'}) + with pytest.raises(tls.SandboxTLSMigrationRequired, match='Drain'): + tls.require_tls_container(container) + container.remove.assert_not_called() + + +def test_certificate_loaded_via_docker_api(certificate): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode='w') as archive: + member = tarfile.TarInfo('server.crt') + member.size = len(certificate) + archive.addfile(member, io.BytesIO(certificate)) + container = Mock() + container.get_archive.return_value = ([buffer.getvalue()], {}) + client = tls.load_container_tls(container) + try: + assert client.ca_file.read_bytes() == certificate + container.get_archive.assert_called_once_with(tls.TLS_CERTIFICATE) + finally: + client.close() + + +@pytest.fixture +def https_server(certificate, tmp_path, request): + if getattr(request, 'param', None) == 'expired': + key, cert = request.getfixturevalue('expired_certificate') + (tmp_path / 'server.key').write_bytes(key) + (tmp_path / 'server.crt').write_bytes(cert) + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(302 if self.path == '/redirect' else 200) + self.send_header('Location', 'http://127.0.0.1:1/plaintext') + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(tmp_path / 'server.crt', tmp_path / 'server.key') + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + +def test_real_https_verification_and_no_plaintext_redirect(certificate, https_server): + host, port = https_server + client = tls.SandboxTLSClient(certificate) + try: + assert client.http.get(f'https://{host}:{port}', timeout=3).status_code == 200 + response = client.http.get(f'https://{host}:{port}/redirect', timeout=3) + assert response.status_code == 302 + assert not response.history + with pytest.raises(ValueError, match='require HTTPS'): + client.http.get(f'http://{host}:{port}', timeout=3) + finally: + client.close() + + +def test_real_handshake_rejects_wrong_host_and_untrusted_ca(certificate, https_server): + client = tls.SandboxTLSClient(certificate) + try: + for context, hostname in [ + (client.ssl_context, 'wrong-sandbox'), + (ssl.create_default_context(), 'localhost'), + ]: + with socket.create_connection(https_server, timeout=3) as raw, pytest.raises(ssl.SSLCertVerificationError): + context.wrap_socket(raw, server_hostname=hostname) + # The same context used by WSS also validates the Docker DNS SAN. + with ( + socket.create_connection(https_server, timeout=3) as raw, + client.ssl_context.wrap_socket(raw, server_hostname='sandbox-test') as secured, + ): + assert secured.version() is not None + finally: + client.close() + + +@pytest.mark.parametrize('failure', ['wrong_ca', 'wrong_hostname']) +def test_https_and_wss_reject_invalid_peer(certificate, https_server, failure, monkeypatch): + from websocket import create_connection + + host, port = https_server + trust = generate_identity('other-container')[1] if failure == 'wrong_ca' else certificate + client = tls.SandboxTLSClient(trust) + sslopt = {'context': client.ssl_context} + if failure == 'wrong_hostname': + # Connect to the local test listener while validating an incorrect DNS name. + original_getaddrinfo = socket.getaddrinfo + + def resolve_test_host(name, *args, **kwargs): + return original_getaddrinfo('127.0.0.1' if name == 'wrong-sandbox' else name, *args, **kwargs) + + monkeypatch.setattr(socket, 'getaddrinfo', resolve_test_host) + host = 'wrong-sandbox' + try: + with pytest.raises(SSLError): + client.http.get(f'https://{host}:{port}', timeout=3) + with pytest.raises(ssl.SSLCertVerificationError): + create_connection(f'wss://{host}:{port}', sslopt=sslopt, timeout=3, http_no_proxy=[host]) + finally: + client.close() + + +def test_expired_identity_is_rejected_before_launch(expired_certificate): + key, cert = expired_certificate + with pytest.raises(ValueError, match='not currently valid'): + bootstrap.validate_identity(key, cert) + + +@pytest.mark.parametrize('https_server', ['expired'], indirect=True) +def test_https_and_wss_reject_expired_certificate(expired_certificate, https_server): + from websocket import create_connection + + host, port = https_server + client = tls.SandboxTLSClient(expired_certificate[1]) + sslopt = {'context': client.ssl_context} + try: + with pytest.raises(SSLError, match='expired'): + client.http.get(f'https://{host}:{port}', timeout=3) + with pytest.raises(ssl.SSLCertVerificationError, match='expired'): + create_connection(f'wss://{host}:{port}', sslopt=sslopt, timeout=3, http_no_proxy=[host]) + finally: + client.close() + + +def test_certificate_wait_handles_retry_timeout_and_cancellation(certificate, monkeypatch): + from docker.errors import NotFound + + container = Mock() + container.get_archive.side_effect = NotFound('not ready') + with pytest.raises(RuntimeError, match='startup deadline'): + tls.load_container_tls(container, timeout=0) + monkeypatch.setattr(tls.time, 'sleep', Mock(side_effect=CancelledError)) + with pytest.raises(CancelledError): + tls.load_container_tls(container) + container.reset_mock() + check_cancelled = Mock(side_effect=CancelledError) + with pytest.raises(CancelledError): + tls.load_container_tls(container, check_cancelled=check_cancelled) + container.get_archive.assert_not_called() + + +@pytest.mark.parametrize('kind', ['archive_size', 'member_size', 'symlink', 'invalid_cert']) +def test_rejects_invalid_certificate_archives(kind): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode='w') as archive: + member = tarfile.TarInfo('server.crt') + data = b'x' * (16385 if kind == 'member_size' else 4) + member.size = len(data) + if kind == 'symlink': + member.type = tarfile.SYMTYPE + member.linkname = '/etc/shadow' + archive.addfile(member, io.BytesIO(data)) + container = Mock() + container.get_archive.return_value = ([b'x' * 65537 if kind == 'archive_size' else buffer.getvalue()], {}) + with pytest.raises((ValueError, ssl.SSLError)): + tls.load_container_tls(container) + + +@pytest.mark.parametrize('failure', ['legacy', 'old_tls', 'certificate', 'handshake']) +def test_recovery_security_failure_preserves_running_owner(monkeypatch, failure): + container = Mock(status='running', labels={'com.nexent.sandbox': 'runtime'}) + container.name = sandbox.SANDBOX_CONTAINER_NAME + container.attrs = {'NetworkSettings': {'Ports': {'8888/tcp': [ + {'HostIp': '127.0.0.1', 'HostPort': '49000'}, + ]}}} + if failure != 'legacy': + container.labels[tls.TLS_LABEL] = tls.TLS_VERSION + if failure == 'old_tls': + container.labels[tls.TLS_LABEL] = '1' + client = Mock() + client.containers.list.return_value = [container] + monkeypatch.setattr('docker.from_env', lambda: client) + monkeypatch.setattr(sandbox, '_is_containerized_runtime', lambda: False) + transport = Mock() + transport.http.get.side_effect = SSLError('expired or wrong identity') + loader = Mock(return_value=transport) + if failure == 'certificate': + loader.side_effect = ssl.SSLError('invalid certificate') + monkeypatch.setattr(sandbox, 'load_container_tls', loader) + config = sandbox.SandboxConfig(level=sandbox.SandboxLevel.DOCKER, scope=sandbox.SandboxScope.SYSTEM) + pool = sandbox.SandboxPoolManager() + logger_ = logging.getLogger(__name__) + with pytest.raises((tls.SandboxTLSMigrationRequired, tls.SandboxTLSRecoveryError)): + pool._acquire_shared_docker_kernel(config, logger_, False) + container.remove.assert_not_called() + client.containers.run.assert_not_called() + if failure == 'handshake': + transport.close.assert_called_once() + if failure in ('legacy', 'old_tls'): + with pytest.raises(tls.SandboxTLSMigrationRequired): + pool._remove_stale_docker_containers(config, logger_) + container.remove.assert_not_called() diff --git a/test/sdk/core/agents/test_sandbox_tls_files.py b/test/sdk/core/agents/test_sandbox_tls_files.py new file mode 100644 index 0000000000..e7793d579e --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_tls_files.py @@ -0,0 +1,149 @@ +"""Real POSIX filesystem boundaries of the container's TLS bootstrap.""" + +import importlib.util +import os +import runpy +import stat +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import serialization + +pytestmark = pytest.mark.skipif(sys.platform != 'linux', reason='Container POSIX permissions require Linux') + + +@pytest.fixture +def bootstrap(monkeypatch): + source = Path(__file__).resolve().parents[4] / 'sdk/nexent/core/agents/sandbox_tls.py' + spec = importlib.util.spec_from_file_location('standalone_tls', source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory(prefix='tls-tests-', dir=Path.home()) as root: + directory = Path(root) / 'identity' + + def run(path=directory): + launched = [] + monkeypatch.setattr(os, 'execvp', lambda command, argv: launched.append(command)) + monkeypatch.setattr(sys, 'argv', ['bootstrap', 'sandbox-test', str(path), 'jupyter']) + try: + runpy.run_path(str(source.with_name('sandbox_tls_bootstrap.py')), run_name='__main__') + except (ValueError, OSError) as exc: + return SimpleNamespace(returncode=1, stdout='', stderr=str(exc)) + return SimpleNamespace(returncode=0, stdout='gateway launched' if launched else '', stderr='') + + yield directory, run + + +def test_private_identity_and_restart(bootstrap): + directory, run = bootstrap + result = run() + assert result.returncode == 0, result.stderr + key_path = directory / 'server.key' + cert_path = directory / 'server.crt' + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 + assert stat.S_IMODE(key_path.stat().st_mode) == 0o600 + original = key_path.read_bytes(), cert_path.read_bytes() + key = serialization.load_pem_private_key(original[0], password=None) + cert = x509.load_pem_x509_certificate(original[1]) + assert key.public_key().public_numbers() == cert.public_key().public_numbers() + assert run().returncode == 0 + assert (key_path.read_bytes(), cert_path.read_bytes()) == original + + +@pytest.mark.parametrize('target', ['directory', 'server.key', 'server.crt']) +def test_rejects_symlinks_without_modifying_target(bootstrap, tmp_path, target): + directory, run = bootstrap + outside = tmp_path / 'outside' + if target == 'directory': + outside.mkdir(mode=0o700) + directory.symlink_to(outside, target_is_directory=True) + else: + directory.mkdir(mode=0o700) + outside.write_bytes(b'untouched') + (directory / target).symlink_to(outside) + result = run() + assert result.returncode != 0 + assert 'gateway launched' not in result.stdout + if target == 'directory': + assert list(outside.iterdir()) == [] + else: + assert outside.read_bytes() == b'untouched' + + +def test_rejects_unsafe_directory_without_chmod(bootstrap): + directory, run = bootstrap + directory.mkdir(mode=0o700) + directory.chmod(0o777) + result = run() + assert result.returncode != 0 + assert stat.S_IMODE(directory.stat().st_mode) == 0o777 + assert list(directory.iterdir()) == [] + + +@pytest.mark.parametrize('damage', ['mismatched_key', 'public_key_file', 'missing_certificate', 'hardlink']) +def test_rejects_damaged_existing_identity(bootstrap, tmp_path, damage): + directory, run = bootstrap + assert run().returncode == 0 + key = directory / 'server.key' + if damage == 'mismatched_key': + other = directory.parent / 'other' + assert run(other).returncode == 0 + key.write_bytes((other / 'server.key').read_bytes()) + elif damage == 'public_key_file': + key.chmod(0o644) + elif damage == 'missing_certificate': + (directory / 'server.crt').unlink() + else: + os.link(key, tmp_path / 'key-link') + original = key.read_bytes() + result = run() + assert result.returncode != 0 + assert 'gateway launched' not in result.stdout + assert key.read_bytes() == original + + +def test_rejects_untrusted_parent(bootstrap): + directory, run = bootstrap + directory.parent.chmod(0o777) + result = run() + assert result.returncode != 0 + assert not directory.exists() + + +def test_private_mode_is_applied_at_creation(bootstrap, monkeypatch): + _, run = bootstrap + original_open = os.open + created_modes = [] + + def observe_open(path, flags, mode=0o777, **kwargs): + fd = original_open(path, flags, mode, **kwargs) + if path == 'server.key' and flags & os.O_CREAT: + created_modes.append(stat.S_IMODE(os.fstat(fd).st_mode)) + return fd + + monkeypatch.setattr(os, 'open', observe_open) + result = run() + assert result.returncode == 0, result.stderr + assert created_modes == [0o600] + + +def test_rejects_wrong_directory_owner(bootstrap, monkeypatch): + directory, run = bootstrap + directory.mkdir(mode=0o700) + inode = directory.stat().st_ino + original_fstat = os.fstat + + def wrong_owner(fd): + info = original_fstat(fd) + if info.st_ino == inode: + return SimpleNamespace(st_uid=os.geteuid() + 1, st_mode=info.st_mode) + return info + + with monkeypatch.context() as patch: + patch.setattr(os, 'fstat', wrong_owner) + assert run().returncode != 0 + assert list(directory.iterdir()) == [] diff --git a/test/sdk/core/agents/test_sandbox_workspace.py b/test/sdk/core/agents/test_sandbox_workspace.py new file mode 100644 index 0000000000..18a9313534 --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_workspace.py @@ -0,0 +1,217 @@ +import json +import logging +from pathlib import Path, PurePosixPath, PureWindowsPath +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from nexent.core.agents import sandbox +from nexent.core.agents.sandbox_workspace import SandboxWorkspace, probe_workspace +from nexent.core.tools.create_file_tool import CreateFileTool +from nexent.core.tools.read_file_tool import ReadFileTool +from nexent.core.tools.upload_to_s3_tool import UploadToS3Tool + + +def test_mapping_roundtrip_and_run_boundary(tmp_path): + mapping = SandboxWorkspace(tmp_path, '/mnt/work') + run = mapping.for_run(tmp_path / 'user' / 'run') + path = tmp_path / 'user' / 'run' / 'outputs' / '中文 folder' / 'result.txt' + assert str(run.to_container(path)) == '/mnt/work/user/run/outputs/中文 folder/result.txt' + assert run.to_host(str(run.to_container(path))) == path + with pytest.raises(ValueError): + run.to_host('/mnt/work/user/other/file.txt') + with pytest.raises(ValueError): + run.to_container(tmp_path / 'user' / 'run-other' / 'file.txt') + + +@pytest.mark.parametrize('value', ['/mnt/work/../secret', r'/mnt/work/a\b', '/mnt/work/a:b', 'D:/work']) +def test_reject_unsafe_container_paths(tmp_path, value): + mapping = SandboxWorkspace(tmp_path, '/mnt/work') + with pytest.raises(ValueError): + mapping.to_host(value) + + +def test_symlink_escape(tmp_path): + root = tmp_path / 'run' + root.mkdir() + outside = tmp_path / 'outside' + outside.mkdir() + try: + (root / 'link').symlink_to(outside, target_is_directory=True) + except OSError: + pytest.skip('Host does not permit symlink creation') + mapping = SandboxWorkspace(root, '/mnt/run') + with pytest.raises(ValueError): + mapping.to_host('/mnt/run/link/secret') + + +def test_permission_probe_failure_is_not_ignored(mocker): + container = Mock() + container.exec_run.return_value = SimpleNamespace(exit_code=1, output=b'Permission denied') + with pytest.raises(RuntimeError, match='workspace access failed'): + probe_workspace(container, Path('/mnt/run')) + + +@pytest.mark.parametrize('scope', list(sandbox.SandboxScope)) +def test_bind_mount_has_distinct_source_and_target(tmp_path, mocker, scope): + manager = object.__new__(sandbox.SandboxPoolManager) + config = sandbox.SandboxConfig( + level=sandbox.SandboxLevel.DOCKER, scope=scope, workspace_mode='bind', + container_workspace_root='/mnt/work', failure_policy='error', + extra_kwargs={'workspace_root': str(tmp_path)}, + ) + mocker.patch('smolagents.remote_executors.DockerExecutor', Mock()) + mocker.patch.object(sandbox, '_wrap_executor', side_effect=lambda executor, *_: executor) + mocker.patch.object(sandbox, '_ensure_sandbox_control_network') + mocker.patch('docker.from_env') + method = '_build_system_docker_executor' if scope == sandbox.SandboxScope.SYSTEM else '_build_session_docker_executor' + build = mocker.patch.object(manager, method, return_value=SimpleNamespace()) + manager._build_docker_executor(config, logging.getLogger('test')) + kwargs = build.call_args.args[2] + mount = kwargs['mounts'][0] + assert mount['Source'] == str(tmp_path) + assert mount['Target'] == '/mnt/work' + assert mount['Type'] == 'bind' + assert 'volumes' not in kwargs + + +@pytest.mark.parametrize('scope', list(sandbox.SandboxScope)) +@pytest.mark.parametrize('error', [RuntimeError('create failed'), TimeoutError('connect failed')]) +def test_strict_creation_failure_never_constructs_local(mocker, scope, error): + manager = object.__new__(sandbox.SandboxPoolManager) + config = sandbox.SandboxConfig(level=sandbox.SandboxLevel.DOCKER, scope=scope, failure_policy='error') + mocker.patch('smolagents.remote_executors.DockerExecutor', Mock()) + mocker.patch.object(sandbox, '_ensure_sandbox_control_network') + mocker.patch('docker.from_env') + local = mocker.patch.object(sandbox, '_make_local_executor') + method = '_build_system_docker_executor' if scope == sandbox.SandboxScope.SYSTEM else '_build_session_docker_executor' + mocker.patch.object(manager, method, side_effect=error) + logger = logging.getLogger('test') + with pytest.raises(RuntimeError, match='fallback is disabled'): + manager._build_docker_executor(config, logger) + local.assert_not_called() + + +def test_tool_paths_roundtrip(tmp_path): + outputs = tmp_path / 'outputs' + outputs.mkdir() + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + create = CreateFileTool(init_path=str(outputs), observer=None) + read = ReadFileTool(init_path=str(outputs), observer=None) + upload = UploadToS3Tool(workspace_path=str(tmp_path)) + for tool in (create, read, upload): + tool.workspace_mapping = mapping + result = json.loads(create.forward('/mnt/run/outputs/file.txt', 'hello')) + assert result['absolute_path'] == '/mnt/run/outputs/file.txt' + assert 'hello' in read.forward(result['absolute_path']) + assert upload._upload_path_candidates(result['absolute_path']) == [str(outputs / 'file.txt')] + assert upload._upload_path_candidates('file.txt')[0] == str(outputs / 'file.txt') + + +def test_mount_comparison_rejects_wrong_source(tmp_path): + mapping = SandboxWorkspace(tmp_path, '/mnt/work') + mount = {'Type': 'bind', 'Source': str(tmp_path), 'Destination': '/mnt/work', 'RW': True} + assert mapping.matches_mount(mount) + assert not mapping.matches_mount({**mount, 'Source': str(tmp_path / 'other')}) + assert not mapping.matches_mount({**mount, 'RW': False}) + + +def test_skill_runner_translates_host_working_directory(tmp_path, mocker): + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + executor = SimpleNamespace(_nexent_backend='docker', container=Mock()) + runner = sandbox.SandboxSkillScriptRunner(executor, workspace_path=str(tmp_path), workspace_mapping=mapping) + resolve = mocker.patch.object(runner, '_resolve_workspace_script', side_effect=ValueError('stop after mapping')) + with pytest.raises(ValueError, match='stop after mapping'): + runner( + manager=None, skill_name='', script_path='outputs/probe.py', params=None, + tenant_id=None, working_directory=str(tmp_path), source='workspace', + ) + resolve.assert_called_once_with('outputs/probe.py', '/mnt/run') + + +@pytest.mark.parametrize('value', ['/mnt/run/inputs/secret.txt', '/mnt/run/outputs-other/secret.txt', '../inputs/secret.txt']) +def test_resolve_file_enforces_tool_base(tmp_path, value): + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + base = tmp_path / 'outputs' + with pytest.raises(ValueError): + mapping.resolve_file(value, base) + + +def test_resolve_file_rejects_base_outside_workspace(tmp_path): + mapping = SandboxWorkspace(tmp_path / 'run', '/mnt/run') + base = tmp_path / 'outside' + with pytest.raises(ValueError): + mapping.resolve_file('/mnt/run/outputs/file.txt', base) + + +def test_resolve_file_accepts_new_nested_targets(tmp_path): + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + base = tmp_path / 'outputs' + target = base / 'new' / 'file.txt' + assert mapping.resolve_file('new/file.txt', base) == target + assert mapping.resolve_file('/mnt/run/outputs/new/file.txt', base) == target + assert not target.exists() + + +@pytest.mark.parametrize('value', ['link/new.txt', '/mnt/run/outputs/link/new.txt']) +def test_tool_base_rejects_symlink_into_other_workspace_directory(tmp_path, value): + outputs = tmp_path / 'outputs' + inputs = tmp_path / 'inputs' + outputs.mkdir() + inputs.mkdir() + try: + (outputs / 'link').symlink_to(inputs, target_is_directory=True) + except OSError: + pytest.skip('Host does not permit symlink creation') + mapping = SandboxWorkspace(tmp_path, '/mnt/run') + with pytest.raises(ValueError): + mapping.resolve_file(value, outputs) + + +@pytest.mark.parametrize('source', ['/run/desktop/mnt/host/c/work', '/host_mnt/c/work', 'C:/WORK']) +def test_windows_mount_comparison_is_portable(source): + mapping = SimpleNamespace(host_root=PureWindowsPath('C:/work'), container_root=PurePosixPath('/mnt/work')) + mount = {'Type': 'bind', 'Source': source, 'Destination': '/mnt/work', 'RW': True} + assert SandboxWorkspace.matches_mount(mapping, mount) + assert not SandboxWorkspace.matches_mount(mapping, {**mount, 'Destination': '/mnt/other'}) + assert not SandboxWorkspace.matches_mount(mapping, {**mount, 'Type': 'volume'}) + + +@pytest.mark.parametrize('name', ['NUL', 'CON.txt', 'COM1']) +def test_reserved_windows_names_rejected_on_all_hosts(tmp_path, name): + mapping = SandboxWorkspace(tmp_path, '/mnt/work') + with pytest.raises(ValueError, match='Reserved host filename'): + mapping.to_host('/mnt/work/' + name) + + +@pytest.mark.parametrize('scope', list(sandbox.SandboxScope)) +def test_allowed_creation_failure_keeps_local_fallback(mocker, scope): + manager = object.__new__(sandbox.SandboxPoolManager) + config = sandbox.SandboxConfig(level=sandbox.SandboxLevel.DOCKER, scope=scope, failure_policy='local') + mocker.patch('smolagents.remote_executors.DockerExecutor', Mock()) + mocker.patch.object(sandbox, '_ensure_sandbox_control_network') + mocker.patch('docker.from_env') + local = mocker.patch.object(sandbox, '_make_local_executor', return_value=SimpleNamespace()) + wrap = mocker.patch.object(sandbox, '_wrap_executor', side_effect=lambda executor, *_: executor) + method = '_build_system_docker_executor' if scope == sandbox.SandboxScope.SYSTEM else '_build_session_docker_executor' + mocker.patch.object(manager, method, side_effect=RuntimeError('create failed')) + result = manager._build_docker_executor(config, logging.getLogger('test')) + assert result is local.return_value + local.assert_called_once() + wrap.assert_called_once() + + +@pytest.mark.parametrize('strict', [False, True]) +def test_missing_docker_dependency_respects_failure_policy(mocker, strict): + manager = object.__new__(sandbox.SandboxPoolManager) + config = sandbox.SandboxConfig(level=sandbox.SandboxLevel.DOCKER, failure_policy='error' if strict else 'local') + mocker.patch('smolagents.remote_executors.DockerExecutor', None) + local = mocker.patch.object(sandbox, '_make_local_executor', return_value=SimpleNamespace()) + mocker.patch.object(sandbox, '_wrap_executor', side_effect=lambda executor, *_: executor) + logger = logging.getLogger('test') + if strict: + with pytest.raises(RuntimeError, match='phase=dependency'): + manager._build_docker_executor(config, logger) + local.assert_not_called() + else: + assert manager._build_docker_executor(config, logger) is local.return_value diff --git a/test/sdk/core/agents/test_sandbox_workspace_config.py b/test/sdk/core/agents/test_sandbox_workspace_config.py new file mode 100644 index 0000000000..e1dd79ea8b --- /dev/null +++ b/test/sdk/core/agents/test_sandbox_workspace_config.py @@ -0,0 +1,19 @@ +import pytest +from nexent.core.agents.sandbox import SandboxConfig + + +def test_bind_requires_posix_container_root(): + with pytest.raises(ValueError, match="container"): + SandboxConfig.from_dict({"workspace_mode": "bind"}) + + +@pytest.mark.parametrize("field,value", [("workspace_mode", "typo"), ("failure_policy", "ignore")]) +def test_invalid_workspace_policy_is_rejected(field, value): + with pytest.raises(ValueError): + SandboxConfig.from_dict({field: value}) + + +def test_defaults_preserve_existing_policy(): + config = SandboxConfig.from_dict({}) + assert config.workspace_mode == "legacy" + assert config.failure_policy == "local" diff --git a/test/sdk/core/tools/test_file_tools_bind_boundary.py b/test/sdk/core/tools/test_file_tools_bind_boundary.py new file mode 100644 index 0000000000..ce51d8b8b0 --- /dev/null +++ b/test/sdk/core/tools/test_file_tools_bind_boundary.py @@ -0,0 +1,91 @@ +"""Bind-mode tools preserve successful operations and reject cross-base effects.""" + +import json +from pathlib import Path +from unittest.mock import Mock + +import pytest +from nexent.core.agents.sandbox_workspace import SandboxWorkspace +from nexent.core.tools.create_file_tool import CreateFileTool +from nexent.core.tools.delete_file_tool import DeleteFileTool +from nexent.core.tools.download_from_s3_tool import DownloadFromS3Tool +from nexent.core.tools.read_file_tool import ReadFileTool +from nexent.core.tools.upload_to_s3_tool import UploadToS3Tool + + +@pytest.fixture +def workspace(tmp_path): + (tmp_path / 'outputs').mkdir() + (tmp_path / 'inputs').mkdir() + (tmp_path / 'inputs' / 'secret.txt').write_text('keep', encoding='utf-8') + return SandboxWorkspace(tmp_path, '/mnt/run') + + +@pytest.mark.parametrize('tool_type', [CreateFileTool, ReadFileTool, DeleteFileTool]) +def test_cross_base_absolute_path_has_no_side_effect(workspace, tool_type): + tool = tool_type(init_path=str(workspace.host_root / 'outputs'), observer=None) + tool.workspace_mapping = workspace + arguments = ['/mnt/run/inputs/secret.txt'] + if tool_type is CreateFileTool: + arguments.append('overwrite') + with pytest.raises(Exception, match='outputs'): + tool.forward(*arguments) + assert (workspace.host_root / 'inputs' / 'secret.txt').read_text(encoding='utf-8') == 'keep' + + +@pytest.mark.parametrize('absolute', [False, True]) +def test_create_read_delete_roundtrip(workspace, absolute): + tools = [tool_type(init_path=str(workspace.host_root / 'outputs'), observer=None) + for tool_type in (CreateFileTool, ReadFileTool, DeleteFileTool)] + for tool in tools: + tool.workspace_mapping = workspace + path = '/mnt/run/outputs/new/file.txt' if absolute else 'new/file.txt' + created = json.loads(tools[0].forward(path, 'hello')) + assert created['absolute_path'] == '/mnt/run/outputs/new/file.txt' + assert 'hello' in tools[1].forward(created['absolute_path']) + deleted = json.loads(tools[2].forward(created['absolute_path'])) + assert deleted['absolute_path'] == created['absolute_path'] + assert not (workspace.host_root / 'outputs' / 'new' / 'file.txt').exists() + + +def test_delete_missing_file_reports_failure(workspace): + tool = DeleteFileTool(init_path=str(workspace.host_root / 'outputs'), observer=None) + tool.workspace_mapping = workspace + with pytest.raises(Exception, match='does not exist'): + tool.forward('missing.txt') + + +def test_delete_permission_failure_preserves_file(workspace, mocker): + target = workspace.host_root / 'outputs' / 'file.txt' + target.write_text('keep', encoding='utf-8') + tool = DeleteFileTool(init_path=str(target.parent), observer=None) + tool.workspace_mapping = workspace + remove = mocker.patch('nexent.core.tools.delete_file_tool.os.remove', side_effect=PermissionError('denied')) + with pytest.raises(Exception, match='Permission denied'): + tool.forward('file.txt') + remove.assert_called_once_with(str(target)) + assert target.read_text(encoding='utf-8') == 'keep' + + +def test_upload_keeps_access_to_inputs(workspace): + tool = UploadToS3Tool(workspace_path=str(workspace.host_root)) + tool.workspace_mapping = workspace + assert tool._upload_path_candidates('/mnt/run/inputs/secret.txt') == [ + str(workspace.host_root / 'inputs' / 'secret.txt'), + ] + + +def test_download_creates_new_nested_target(workspace): + client = Mock() + client.get_file_size.return_value = 5 + def download(key, path, bucket): + Path(path).write_text('hello', encoding='utf-8') + return True, '' + client.download_file.side_effect = download + tool = DownloadFromS3Tool( + workspace_path=str(workspace.host_root), minio_client=client, validate_url_access=lambda _: None, + ) + tool.workspace_mapping = workspace + result = json.loads(tool.forward('s3://bucket/input.txt', '/mnt/run/inputs/new/input.txt')) + assert result['local_path'] == '/mnt/run/inputs/new/input.txt' + assert (workspace.host_root / 'inputs' / 'new' / 'input.txt').read_text(encoding='utf-8') == 'hello'