D: gitlink mounts, pinned gitlink parents and a pre-launch tree check (#81, #99)

agents/adapters/types.ts

@@ -1,5 +1,6 @@
 import type { InvocationInput } from '../contract.ts';
 import type { TaskFilesystems } from '../container/storage.ts';
+import type { TaskTreeCheck } from '../container/changes.ts';
 import { assertBuiltAgentImage } from '../container/image.ts';
 import { assertResourceOwner } from '../labels.ts';
 import type { CaptureLimits } from './supervisor.ts';
@@ -15,6 +16,8 @@ export interface AgentAdapterRequest {
    * call so recovery can match them to the attempt (#51 item 3).
    */
   readonly networkAllocationId: string;
+  /** Execute and fix only, and required there: `checkTaskTree`'s result for `filesystems`, made just before this start. */
+  readonly treeCheck?: TaskTreeCheck;
 }
 /** Check a start call's synchronous input, so invalid input throws before a handle exists. */
 export function assertAdapterRequest(request: AgentAdapterRequest): void {

agents/container/changes.ts

@@ -1,5 +1,6 @@
 import { constants } from 'node:buffer';
 import { createHash } from 'node:crypto';
+import { DockerError } from '../docker.ts';
 import { resolveMetadataBaseline, runStorageScript, type RecoveredTaskStorage, type StorageScriptOptions,
   type TaskFilesystems } from './storage.ts';
 import { MAXIMUM_CHANGES, MAXIMUM_DECLARED_LINKS, MAXIMUM_NAME_BYTES, MAXIMUM_TARGET_ENTRIES, MAXIMUM_TREE_OUTPUT,
@@ -350,6 +351,192 @@ export async function inspectTaskChanges(storage: TaskFilesystems | RecoveredTas
   return manifestOf(options.base, input, JSON.parse(stdout));
 }
 
+/** One file operation a plan item declares, as the plan names it (`renamedFrom` only for a rename). */
+export interface DeclaredOperation {
+  readonly kind: 'add' | 'edit' | 'delete' | 'rename';
+  readonly path: string;
+  readonly renamedFrom?: string | null;
+}
+export interface TreeCheckOptions extends StorageScriptOptions {
+  /** The recorded head the storage was seeded from: the clone's head. */
+  readonly base: string;
+  /** The item's file operations, checked against the tree of `base`. */
+  readonly operations: readonly DeclaredOperation[];
+  /** As for `inspectTaskChanges`. */
+  readonly metadataBaseline?: string;
+}
+/**
+ * What `checkTaskTree` found, for the execute or fix profile of the same storage. Each check is one-shot: one profile
+ * uses it, and the next invocation needs a new check.
+ */
+export interface TaskTreeCheck {
+  readonly base: string;
+  /**
+   * Every gitlink of `base`, each an empty directory now; each gets an empty read-only mount, and every directory above
+   * one (as `gitlinkParents(gitlinks)` lists them) is pinned with a mount of its own.
+   */
+  readonly gitlinks: readonly string[];
+}
+/**
+ * The item must not launch on this storage: the tree is not a clean copy of the head, an operation does not fit the
+ * head, or a gitlink cannot be mounted. Each entry of `differences` says why. A person must look; retrying the same
+ * head and plan gives the same answer.
+ */
+export class TaskTreeRefused extends Error {
+  readonly differences: readonly string[];
+  constructor(differences: readonly string[]) {
+    const shown = differences.slice(0, 10).join('; ');
+    super(`The pre-launch tree check refused: ${shown}${differences.length > 10
+      ? `; and ${differences.length - 10} more` : ''}.`);
+    this.name = 'TaskTreeRefused';
+    this.differences = Object.freeze([...differences]);
+  }
+}
+/** The most gitlinks one profile mounts, and the most parent directories it pins; a head with more refuses the launch. */
+export const MAXIMUM_GITLINK_MOUNTS = 256;
+/**
+ * The most bytes of path the gitlink mounts add to the `docker create` arguments (each pin names its directory twice,
+ * each gitlink once), well inside what one exec can carry; a head that needs more refuses the launch.
+ */
+export const MAXIMUM_GITLINK_MOUNT_BYTES = 128 * 1024;
+/**
+ * Every directory above a gitlink below the top level, parents before children (#99). Each is mounted again at its own
+ * path from the same work volume, so it is a mountpoint: renaming or removing it fails (EBUSY), and the gitlink's own
+ * mount cannot be moved away with it. What is inside stays writable.
+ */
+export function gitlinkParents(gitlinks: readonly string[]): readonly string[] {
+  const parents = new Set<string>();
+  for (const path of gitlinks) {
+    const parts = path.split('/');
+    for (let index = 1; index < parts.length; index++) parents.add(parts.slice(0, index).join('/'));
+  }
+  return Object.freeze([...parents].sort((a, b) => a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0)));
+}
+// The storage each check was made over, and whether a profile has used it.
+const treeChecks = new WeakMap<TaskTreeCheck, { filesystems: TaskFilesystems; used: boolean }>();
+// Docker reads `--mount` as comma-separated fields that a double quote can open: a name with either cannot be mounted
+// exactly where it is.
+const unmountable = (path: string) => /[,"]/.test(path);
+const KINDS = new Set(['add', 'edit', 'delete', 'rename']);
+interface DeclaredFact { readonly path: string; readonly type: EntryType | 'absent'; readonly blockedBy?: string;
+  readonly blockedByType?: EntryType }
+
+// Why the storage is not a fresh copy of `base`: every change, nested content, commit or metadata difference.
+function treeDifferences(manifest: TaskChangeManifest): string[] {
+  if (manifest.metadataChanged) return ['the Git metadata differs from what the seeder recorded'];
+  const q = JSON.stringify, found: string[] = [];
+  if (manifest.agentCommits.length) found.push('HEAD is not the recorded head');
+  for (const change of manifest.changes) {
+    if (change.kind === 'add') found.push(`${q(change.path)} is a ${change.newType} the head does not have`);
+    else if (change.kind === 'delete') found.push(`${q(change.path)} is missing`);
+    else if (change.kind === 'rename') found.push(`${q(change.path)} is ${q(change.oldPath)} moved`);
+    else if (change.oldType !== change.newType) found.push(`${q(change.path)} is a ${change.newType} where the head has a ${change.oldType}`);
+    else found.push(`${q(change.path)} differs from the head`);
+  }
+  for (const path of manifest.nestedGitlinkContent) found.push(`gitlink ${q(path)} has content or cannot be read`);
+  return found;
+}
+
+/**
+ * Check task storage immediately before an execute or fix invocation (plan-format.md, "Clean invocation state" and
+ * "Submodules in version 1"): read without following links, the work tree must be exactly the tree of `base` (no
+ * untracked or ignored entry, no changed type or content, nothing in a gitlink directory, HEAD at `base`, metadata as
+ * seeded), and each declared operation must fit that tree: an add or rename destination is free, an edit, delete or
+ * rename source is a file or symlink, and no declared path lies beneath a file, symlink or gitlink. It refuses with
+ * `TaskTreeRefused` otherwise, and when a gitlink cannot be given its empty read-only mount (more than
+ * `MAXIMUM_GITLINK_MOUNTS`, or a name Docker cannot take). Runs the inspection's read-only, no-network container.
+ * The result is the capability `createContainerProfile` needs for an execute or fix profile over this storage.
+ */
+export async function checkTaskTree(storage: TaskFilesystems, options: TreeCheckOptions): Promise<TaskTreeCheck> {
+  if (typeof options.base !== 'string' || !COMMIT_ID.test(options.base)) throw new Error('base must be a full commit ID.');
+  const baseline = resolveMetadataBaseline(storage, options.metadataBaseline);
+  if (!Array.isArray(options.operations)) throw new Error('Declared operations must be a list.');
+  const operations = options.operations.map(operation => {
+    const { kind, path, renamedFrom } = operation ?? {} as DeclaredOperation;
+    if (!KINDS.has(kind)) throw new Error(`Declared operation kind ${JSON.stringify(kind)} is not add, edit, delete or rename.`);
+    assertDeclaredPath(path);
+    if (kind === 'rename') assertDeclaredPath(renamedFrom);
+    else if (renamedFrom !== undefined && renamedFrom !== null) throw new Error('Only a rename has renamedFrom.');
+    return { kind, path, renamedFrom: kind === 'rename' ? renamedFrom! : undefined };
+  });
+  const paths = [...new Set(operations.flatMap(operation => [operation.path, ...operation.renamedFrom ? [operation.renamedFrom] : []]))];
+  if (paths.length > MAXIMUM_DECLARED_LINKS) throw new Error(`At most ${MAXIMUM_DECLARED_LINKS} declared paths are checked.`);
+  assertArguments(paths);
+  let stdout: string;
+  try {
+    stdout = await runStorageScript(storage, { kind: 'inspect', operation: 'Pre-launch tree check',
+      consequence: 'the task tree cannot be checked', entrypoint: 'perl', maxOutputBytes: MAXIMUM_TREE_OUTPUT + OUTPUT_SLACK,
+      memory: INSPECTION_MEMORY, tmpBytes: INSPECTION_TMP,
+      args: ['-e', TREE_SCRIPT, 'prelaunch', baseline, options.base, ...paths] },
+    { ...options, timeoutMs: options.timeoutMs ?? 120_000 });
+  } catch (error) {
+    // The script found the tree itself unfit: something it cannot read (6), a name the manifest cannot carry (8), or
+    // too many changes or too much output (9). A fresh copy of the head has none of these, so each is a refusal like
+    // any other difference, not a failure to run. Its message escapes every name it shows.
+    if (error instanceof DockerError && [6, 8, 9].includes(error.status ?? -1)) {
+      const reason = error.stderr.trim().split('\n').at(-1)!.slice(0, 512);
+      throw new TaskTreeRefused([`the tree cannot be checked whole (${JSON.stringify(reason)})`]);
+    }
+    throw error;
+  }
+  const output = JSON.parse(stdout) as InspectOutput & { metadataOnly?: boolean; gitlinks?: unknown; declared?: unknown };
+  const manifest = manifestOf(options.base, { baseline, links: [], recorded: {}, linkArgs: [] }, output);
+  const differences = treeDifferences(manifest);
+  if (manifest.metadataChanged) throw new TaskTreeRefused(differences);
+  const gitlinks = output.gitlinks, declared = output.declared as DeclaredFact[] | undefined;
+  if (!Array.isArray(gitlinks) || gitlinks.some(path => { try { assertDeclaredPath(path); return false; } catch { return true; } })
+    || !Array.isArray(declared)
+    || declared.length !== paths.length || declared.some((fact, index) => fact?.path !== paths[index]))
+    throw new Error('The pre-launch tree check returned an unexpected result.');
+  const facts = new Map(declared.map(fact => [fact.path, fact]));
+  const q = JSON.stringify;
+  for (const operation of operations) {
+    const sources = operation.kind === 'add' ? [] : [operation.renamedFrom ?? operation.path];
+    const destinations = operation.kind === 'add' || operation.kind === 'rename' ? [operation.path] : [];
+    for (const path of [...sources, ...destinations]) {
+      const fact = facts.get(path)!;
+      if (fact.blockedBy !== undefined) differences.push(`${q(path)} lies beneath the ${fact.blockedByType} ${q(fact.blockedBy)}`);
+    }
+    for (const path of sources) {
+      const type = facts.get(path)!.type;
+      if (type === 'gitlink') differences.push(`${q(path)} is a gitlink, which a plan item cannot change`);
+      else if (type !== 'file' && type !== 'symlink') differences.push(`${operation.kind} source ${q(path)} is ${type === 'absent' ? 'missing' : `a ${type}`}`);
+    }
+    for (const path of destinations) {
+      const type = facts.get(path)!.type;
+      if (type !== 'absent') differences.push(`${operation.kind} destination ${q(path)} is occupied by a ${type}`);
+    }
+  }
+  if (gitlinks.length > MAXIMUM_GITLINK_MOUNTS)
+    differences.push(`the head has ${gitlinks.length} gitlinks, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be mounted`);
+  const parents = gitlinkParents(gitlinks as string[]).length;
+  if (parents > MAXIMUM_GITLINK_MOUNTS)
+    differences.push(`the head's gitlinks are below ${parents} directories, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be pinned`);
+  const mountBytes = (gitlinks as string[]).reduce((total, path) => total + Buffer.byteLength(path), 0)
+    + gitlinkParents(gitlinks as string[]).reduce((total, path) => total + 2 * Buffer.byteLength(path), 0);
+  if (mountBytes > MAXIMUM_GITLINK_MOUNT_BYTES)
+    differences.push(`the gitlink mounts would name ${mountBytes} bytes of paths, more than the ${MAXIMUM_GITLINK_MOUNT_BYTES} a launch passes`);
+  for (const path of gitlinks as string[]) if (unmountable(path)) differences.push(`gitlink ${q(path)} has a name Docker cannot mount`);
+  if (differences.length) throw new TaskTreeRefused([...new Set(differences)]);
+  const check = Object.freeze({ base: options.base, gitlinks: Object.freeze([...gitlinks as string[]]) });
+  treeChecks.set(check, { filesystems: storage, used: false });
+  return check;
+}
+
+/**
+ * For the profile builder: the gitlinks of a check `checkTaskTree` made over these filesystems at `base`, marking it
+ * used. Refuses a copied, reused or mismatched check.
+ */
+export function useTaskTreeCheck(check: TaskTreeCheck | undefined, filesystems: TaskFilesystems, base: string): readonly string[] {
+  const record = check ? treeChecks.get(check) : undefined;
+  if (!record) throw new Error('An execute or fix profile requires a pre-launch tree check from checkTaskTree.');
+  if (record.filesystems !== filesystems || check!.base !== base)
+    throw new Error('The pre-launch tree check was made over other task storage or another head.');
+  if (record.used) throw new Error('The pre-launch tree check was already used; check the tree again before each invocation.');
+  record.used = true;
+  return check!.gitlinks;
+}
+
 /** The most bundle bytes a runner commit returns; a larger one fails the commit. */
 export const MAXIMUM_BUNDLE_BYTES = 64 * 1024 * 1024;
 const MAXIMUM_MESSAGE_BYTES = 64 * 1024;

agents/container/profile.ts

@@ -6,6 +6,7 @@ import { join } from 'node:path';
 import { assertCapturedInvocation, type InvocationInput, type Phase, type UnreleasedResource } from '../contract.ts';
 import { assertBuiltAgentImage } from './image.ts';
 import { assertTaskFilesystems, taskFilesystemOwner, type TaskFilesystems } from './storage.ts';
+import { gitlinkParents, useTaskTreeCheck, type TaskTreeCheck } from './changes.ts';
 import { ownerLabelArgs, type ResourceOwner } from '../labels.ts';
 import { assertVendorNetwork, removeVendorNetwork, vendorNetworkResources,
   type VendorNetwork } from '../network/network.ts';
@@ -24,6 +25,10 @@ export interface ContainerProfile {
   readonly network: VendorNetwork;
   readonly policy: PhasePolicy;
   readonly deferredOutput: boolean;
+  /** Execute and fix: every gitlink path, each covered by an empty read-only tmpfs. Empty in read-only phases. */
+  readonly gitlinks: readonly string[];
+  /** Execute and fix: every directory above a gitlink below the top level, each remounted from the work volume (#99). */
+  readonly gitlinkParents: readonly string[];
 }
 export interface ProfileOptions {
   readonly invocation: InvocationInput;
@@ -36,6 +41,11 @@ export interface ProfileOptions {
   readonly network: VendorNetwork;
   readonly policy: PhasePolicy;
   readonly deferredOutput?: boolean;
+  /**
+   * Execute and fix only, and required there: what `checkTaskTree` returned for these filesystems at the clone's head,
+   * just before this launch. Each check serves one profile.
+   */
+  readonly treeCheck?: TaskTreeCheck;
   /** Remaining invocation budget for Docker-backed profile validation. */
   readonly timeoutMs?: number;
   /** Cancels creation; whatever was staged is removed before the promise rejects. */
@@ -214,6 +224,9 @@ const safeName = (value: string) => {
   const prefix = value.replace(/[^a-zA-Z0-9_.-]/g, '-').slice(0, 24);
   return `${prefix}-${createHash('sha256').update(value).digest('hex').slice(0, 16)}`;
 };
+/** The size and mode of the empty read-only tmpfs at each gitlink path. */
+export const GITLINK_MOUNT_BYTES = 4096;
+export const GITLINK_MOUNT_MODE = '0555';
 const mount = (parts: Record<string, string | boolean>) => Object.entries(parts)
   .map(([key, value]) => value === true ? key : `${key}=${value}`).join(',');
 const mountSource = (path: string, kind: string) => {
@@ -277,6 +290,11 @@ export async function createContainerProfile(options: ProfileOptions): Promise<C
     }
     const name = `codeboost-agent-${safeName(invocation.attemptId)}`, ownershipId = randomUUID();
     const readOnlyWork = ['planning', 'questions', 'review'].includes(invocation.phase);
+    // In a read-only phase nothing can be written beneath a gitlink, and no check is made. Execute and fix use theirs
+    // up: the next profile needs a new check, made just before its own launch.
+    if (readOnlyWork && options.treeCheck) throw new Error('Only an execute or fix profile takes a pre-launch tree check.');
+    const gitlinks = readOnlyWork ? [] : useTaskTreeCheck(options.treeCheck, filesystems, invocation.clone.head);
+    const parents = gitlinkParents(gitlinks);
     const args = ['create', '--name', name, '--read-only', '--user', '10001:10001', '--cap-drop=ALL',
       '--security-opt=no-new-privileges', '--security-opt=seccomp=builtin', '--runtime=runc', '--pids-limit=128', '--memory=512m', '--memory-swap=512m',
       '--cpus=1', '--shm-size=16m', '--ipc=private', '--cgroupns=private',
@@ -293,7 +311,16 @@ export async function createContainerProfile(options: ProfileOptions): Promise<C
       '--tmpfs', '/home/codeboost:rw,nosuid,nodev,size=1048576,nr_inodes=128,uid=10001,gid=10001,mode=0700',
       '--mount', mount({ type: 'volume', source: filesystems.workVolume, target: '/work', readonly: readOnlyWork }),
       '--mount', mount({ type: 'volume', source: filesystems.metadataVolume, target: '/work/.git', readonly: true }),
-      '--mount', mount({ type: 'bind', source: inputIdentity.inputDirectory, target: '/run/codeboost-input', readonly: true })];
+      '--mount', mount({ type: 'bind', source: inputIdentity.inputDirectory, target: '/run/codeboost-input', readonly: true }),
+      // Each directory above a nested gitlink is a real directory: the check proved it, and nothing writes the work
+      // volume between the check and this container's start (Docker would follow a link it found there). Mounting it
+      // again from the work volume makes it a mountpoint, which cannot be renamed, so the gitlink's mount cannot be moved
+      // aside and its path refilled during the run (#99). Docker mounts parents before children whatever the order here.
+      ...parents.flatMap(path => ['--mount', mount({ type: 'volume', source: filesystems.workVolume, target: `/work/${path}`,
+        'volume-subpath': path })]),
+      // Each gitlink is an empty directory (the check proved it): an empty read-only tmpfs over it keeps it empty.
+      ...gitlinks.flatMap(path => ['--mount', mount({ type: 'tmpfs', target: `/work/${path}`, readonly: true,
+        'tmpfs-mode': GITLINK_MOUNT_MODE, 'tmpfs-size': String(GITLINK_MOUNT_BYTES) })])];
     if (options.deferredOutput) {
       if (invocation.vendor !== 'codex') throw new Error('Deferred output is available only for Codex.');
       args.push('--env', 'CODEBOOST_DEFERRED_OUTPUT=1',
@@ -311,7 +338,7 @@ export async function createContainerProfile(options: ProfileOptions): Promise<C
       phase: invocation.phase, vendor: invocation.vendor,
       filesystems: capturedFilesystems, inputDirectory: inputIdentity.inputDirectory, codexAuthFile,
       command: Object.freeze([...command]), ownershipId, network: options.network, policy: options.policy,
-      deferredOutput: options.deferredOutput === true });
+      deferredOutput: options.deferredOutput === true, gitlinks: Object.freeze([...gitlinks]), gitlinkParents: parents });
     identities.set(profile, Object.freeze({ inputDirectory: inputIdentity.inputDirectory, schema: inputIdentity.schema,
       auth: authIdentity,
       cleanupDirectories: Object.freeze([...cleanupDirectories]), filesystems, clone: invocation.clone,