diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index c3df1eac..3e391095 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -26,6 +26,7 @@ export interface Node { tlsVerify: boolean | null; imageStorage: string; volumeStorage: string; + sharedVolumeStorage: string | null; networkBridge: string; nvidiaAvailable: boolean; hasSecret: boolean; diff --git a/create-a-container/client/src/pages/nodes/NodeFormPage.tsx b/create-a-container/client/src/pages/nodes/NodeFormPage.tsx index 74b139df..225ee126 100644 --- a/create-a-container/client/src/pages/nodes/NodeFormPage.tsx +++ b/create-a-container/client/src/pages/nodes/NodeFormPage.tsx @@ -29,6 +29,7 @@ const schema = z.object({ tlsVerify: z.boolean().optional(), imageStorage: z.string().min(1, 'Required'), volumeStorage: z.string().min(1, 'Required'), + sharedVolumeStorage: z.string().optional(), networkBridge: z.string().min(1, 'Required'), nvidiaAvailable: z.boolean().optional(), }); @@ -77,6 +78,7 @@ export function NodeFormPage() { tlsVerify: node.tlsVerify ?? true, imageStorage: node.imageStorage, volumeStorage: node.volumeStorage, + sharedVolumeStorage: node.sharedVolumeStorage ?? '', networkBridge: node.networkBridge, nvidiaAvailable: node.nvidiaAvailable, }); @@ -199,9 +201,20 @@ export function NodeFormPage() { )}
- +
+ { allowNull: false, defaultValue: 'local-lvm' }, + // Path-backed shared storage (cephfs/nfs) for persistent volumes. Null + // falls back to volumeStorage. See Node#volumesStorageName. + sharedVolumeStorage: { + type: DataTypes.STRING(255), + allowNull: true, + defaultValue: null + }, networkBridge: { type: DataTypes.STRING(255), allowNull: false, diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index 81f6e935..aa90491d 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -332,6 +332,7 @@ components: tlsVerify: { type: boolean, nullable: true } imageStorage: { type: string } volumeStorage: { type: string } + sharedVolumeStorage: { type: string, nullable: true } networkBridge: { type: string } nvidiaAvailable: { type: boolean } hasSecret: { type: boolean } @@ -360,7 +361,17 @@ components: description: Proxmox API token secret. Never returned (see `hasSecret`). On update, blank/omitted keeps the existing secret. tlsVerify: { type: boolean, nullable: true } imageStorage: { type: string, default: local } - volumeStorage: { type: string, default: local-lvm } + volumeStorage: + type: string + default: local-lvm + description: Proxmox storage for container root filesystems (`rootdir`); may be block storage (lvmthin/rbd/zfspool). + sharedVolumeStorage: + type: string + nullable: true + description: >- + Path-backed shared storage (cephfs/nfs/dir) hosting persistent + volumes under `/volumes`. Null/blank falls back to + `volumeStorage`. networkBridge: { type: string, default: vmbr0 } nvidiaAvailable: { type: boolean, default: false } Job: diff --git a/create-a-container/routers/api/v1/nodes.js b/create-a-container/routers/api/v1/nodes.js index 816e9086..7f169a8b 100644 --- a/create-a-container/routers/api/v1/nodes.js +++ b/create-a-container/routers/api/v1/nodes.js @@ -6,6 +6,7 @@ const express = require('express'); const https = require('https'); const { Node, Site, Container } = require('../../../models'); const { isValidDockerHost } = require('../../../utils/docker-api'); +const { volumesStorageName } = require('../../../utils/volumes'); const { apiAuth, apiAdmin, asyncHandler, ok, created, noContent, ApiError } = require('../../../middlewares/api'); @@ -25,6 +26,7 @@ function serialize(n) { tlsVerify: n.tlsVerify, imageStorage: n.imageStorage, volumeStorage: n.volumeStorage, + sharedVolumeStorage: n.sharedVolumeStorage, networkBridge: n.networkBridge, nvidiaAvailable: n.nvidiaAvailable, hasSecret: !!n.secret, @@ -56,7 +58,7 @@ function normalizeNodeType(nodeType) { */ async function volumeStorageWarnings(node) { if (node.nodeType !== 'proxmox' || !node.hasApiAccess()) return []; - const storageName = node.volumeStorage || node.imageStorage || 'local'; + const storageName = volumesStorageName(node); const warnings = []; try { const client = await node.api(); @@ -75,12 +77,12 @@ async function volumeStorageWarnings(node) { const shared = cfg.shared === 1 || cfg.shared === '1' || cfg.shared === true; if (!cfg.path) { warnings.push( - `Volume storage "${storageName}" (type ${cfg.type || 'unknown'}) has no host path; ` + - 'persistent volumes require a path-backed storage (dir/nfs/cephfs). Move the volumes root to shared storage.', + `Shared volume storage "${storageName}" (type ${cfg.type || 'unknown'}) has no host path; ` + + 'persistent volumes require a path-backed storage (dir/nfs/cephfs). Set "Shared volume storage" to a shared filesystem.', ); } else if (!shared) { warnings.push( - `Volume storage "${storageName}" is not marked shared across the cluster; ` + + `Shared volume storage "${storageName}" is not marked shared across the cluster; ` + 'persistent volume data is not guaranteed to follow containers across nodes. ' + 'Move the volumes root to a path-backed shared filesystem (CephFS or NFS) available on every node.', ); @@ -104,7 +106,7 @@ async function volumeStorageWarnings(node) { const missing = [...nodeNames].filter((n) => !presentOn.has(n)); if (missing.length > 0) { warnings.push( - `Volume storage "${storageName}" is not present/active on every node ` + + `Shared volume storage "${storageName}" is not present/active on every node ` + `(missing on: ${missing.join(', ')}). Persistent volumes will not be creatable/durable ` + 'where a container lands on those nodes.', ); @@ -252,7 +254,7 @@ router.post( apiAdmin, asyncHandler(async (req, res) => { const site = await loadSite(req); - const { name, nodeType, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage, volumeStorage, networkBridge, nvidiaAvailable } = + const { name, nodeType, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage, volumeStorage, sharedVolumeStorage, networkBridge, nvidiaAvailable } = req.body || {}; const type = validateNodeInput({ nodeType, apiUrl }); const node = await Node.create({ @@ -266,6 +268,7 @@ router.post( tlsVerify === '' || tlsVerify === null || tlsVerify === undefined ? null : tlsVerify === true || tlsVerify === 'true', imageStorage: imageStorage || 'local', volumeStorage: volumeStorage || 'local-lvm', + sharedVolumeStorage: sharedVolumeStorage ? String(sharedVolumeStorage).trim() || null : null, networkBridge: networkBridge || 'vmbr0', nvidiaAvailable: nvidiaAvailable === true || nvidiaAvailable === 'true', siteId: site.id, @@ -284,7 +287,7 @@ router.put( where: { id: parseInt(req.params.id, 10), siteId: site.id }, }); if (!node) throw new ApiError(404, 'not_found', 'Node not found'); - const { name, nodeType, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage, volumeStorage, networkBridge, nvidiaAvailable } = + const { name, nodeType, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage, volumeStorage, sharedVolumeStorage, networkBridge, nvidiaAvailable } = req.body || {}; const type = validateNodeInput({ nodeType: nodeType || node.nodeType, @@ -300,6 +303,7 @@ router.put( tlsVerify === '' || tlsVerify === null || tlsVerify === undefined ? null : tlsVerify === true || tlsVerify === 'true', imageStorage: imageStorage || 'local', volumeStorage: volumeStorage || 'local-lvm', + sharedVolumeStorage: sharedVolumeStorage ? String(sharedVolumeStorage).trim() || null : null, networkBridge: networkBridge || 'vmbr0', nvidiaAvailable: nvidiaAvailable === true || nvidiaAvailable === 'true', }; diff --git a/create-a-container/utils/__tests__/volumes.test.js b/create-a-container/utils/__tests__/volumes.test.js index 24ba3a4a..6e9a43d2 100644 --- a/create-a-container/utils/__tests__/volumes.test.js +++ b/create-a-container/utils/__tests__/volumes.test.js @@ -9,6 +9,7 @@ const { Volume } = require('../../models'); const volumeModule = require('../../models/volume'); const { resolveVolumesRoot, + volumesStorageName, containerVolumeHostPath, } = require('../volumes'); @@ -174,6 +175,30 @@ describe('resolveVolumesRoot', () => { const res = await resolveVolumesRoot(client, { ...node, volumeStorage: 'local' }); expect(res.shared).toBe(false); }); + + test('prefers sharedVolumeStorage over block rootfs volumeStorage', async () => { + const client = { + async storageConfig(storage) { + expect(storage).toBe('cephfs'); + return { storage, type: 'cephfs', path: '/mnt/pve/cephfs', shared: 1 }; + }, + }; + const res = await resolveVolumesRoot(client, { + ...node, + volumeStorage: 'local-lvm', + sharedVolumeStorage: 'cephfs', + }); + expect(res).toEqual({ root: '/mnt/pve/cephfs/volumes', storage: 'cephfs', shared: true }); + }); +}); + +describe('volumesStorageName', () => { + test('falls back sharedVolumeStorage -> volumeStorage -> imageStorage -> local', () => { + expect(volumesStorageName({ sharedVolumeStorage: 'nfs', volumeStorage: 'lvm' })).toBe('nfs'); + expect(volumesStorageName({ sharedVolumeStorage: null, volumeStorage: 'lvm' })).toBe('lvm'); + expect(volumesStorageName({ imageStorage: 'img' })).toBe('img'); + expect(volumesStorageName({})).toBe('local'); + }); }); describe('Volume DB validation', () => { diff --git a/create-a-container/utils/volumes.js b/create-a-container/utils/volumes.js index 8622de00..57494ad8 100644 --- a/create-a-container/utils/volumes.js +++ b/create-a-container/utils/volumes.js @@ -12,9 +12,22 @@ // reject a user volume that would collide with a pre-#421 backfilled row. const { QUICK_AND_DIRTY_NAME, QUICK_AND_DIRTY_MOUNT } = require('../models/volume'); +/** + * Name of the Proxmox storage that hosts persistent volumes on a node. + * `sharedVolumeStorage` (path-backed shared FS) takes precedence; when unset we + * fall back to `volumeStorage` (rootfs storage) for backward compatibility. + * + * @param {object} node - Node instance or plain object + * @returns {string} + */ +function volumesStorageName(node) { + return node.sharedVolumeStorage || node.volumeStorage || node.imageStorage || 'local'; +} + /** * Resolve the on-disk root directory for user volumes on a node, derived from - * the node's configured `volumeStorage` (falling back to `imageStorage`). The + * the storage chosen by `volumesStorageName` (sharedVolumeStorage → + * volumeStorage → imageStorage). The * path is taken from Proxmox's storage config (`GET /storage/{storage}` → * `path`), never assumed. Volumes live under `/volumes`. * @@ -30,7 +43,7 @@ async function resolveVolumesRoot(client, node) { return { root: '/var/lib/opensource-server/volumes', storage: 'docker', shared: false }; } - const storageName = node.volumeStorage || node.imageStorage || 'local'; + const storageName = volumesStorageName(node); if (typeof client.storageConfig !== 'function') { // Dummy nodes and any client without storage-config support: fall back to a // conventional path so the simulated create path still runs end to end. @@ -158,6 +171,7 @@ async function deriveVolumeHostPaths(volumes, { volumesRoot, siteId, owner, host } module.exports = { + volumesStorageName, resolveVolumesRoot, containerVolumeHostPath, deriveVolumeHostPaths, diff --git a/mie-opensource-landing/docs/admins/core-concepts/nodes.md b/mie-opensource-landing/docs/admins/core-concepts/nodes.md index 49a95c1a..0ab79cb7 100644 --- a/mie-opensource-landing/docs/admins/core-concepts/nodes.md +++ b/mie-opensource-landing/docs/admins/core-concepts/nodes.md @@ -11,13 +11,14 @@ Nodes are Proxmox VE servers within a site that host containers. - **Authentication**: Username/password or API token - **TLS Verification**: Enable/disable certificate validation - **Template Storage**: Proxmox storage for CT template images (`vztmpl` content) -- **Volume Storage**: Proxmox storage for container root filesystems (`rootdir` content) and persistent [volumes](volumes.md) +- **Root Disk Storage** (`volumeStorage`): Proxmox storage for container root filesystems (`rootdir` content); thin-provisioned block storage (LVM-thin, Ceph RBD, ZFS) is fine +- **Shared Volume Storage** (`sharedVolumeStorage`, optional): path-backed shared filesystem (CephFS, NFS) hosting persistent [volumes](volumes.md). When blank, volumes fall back to the root disk storage -!!! warning "Volume storage should be shared across the cluster" +!!! warning "Shared volume storage should be shared across the cluster" Persistent [volumes](volumes.md) require their host directories to exist on - whichever node a container lands on. Place the **volume storage** on storage + whichever node a container lands on. Set **shared volume storage** to storage that is shared across every node (a path-backed shared filesystem such as CephFS or NFS). On save, the manager - warns (but does not block) if the chosen volume storage is not shared or is + warns (but does not block) if the chosen shared volume storage is not shared or is not active on every node. Single-node sites are unaffected. ## Adding Nodes @@ -63,12 +64,13 @@ Proxmox uses self-signed certificates by default. Either disable TLS verificatio ## Storage Configuration -Nodes have two storage settings because most Proxmox storage types only support one content type. +Nodes have separate storage settings because most Proxmox storage types only support one content type, and because root disks and persistent volumes need different semantics (block vs. filesystem). | Setting | Content Type | Default | Purpose | |---|---|---|---| | **Template Storage** | `vztmpl` | `local` | Stores pulled Docker/OCI images as CT templates | -| **Volume Storage** | `rootdir` | `local-lvm` | Stores container root filesystems | +| **Root Disk Storage** | `rootdir` | `local-lvm` | Stores container root filesystems | +| **Shared Volume Storage** | path-backed FS | *(root disk storage)* | Hosts persistent volumes under `/volumes` | If a configured storage does not support the required content type, the system falls back to the largest enabled storage on the node that does. If no storage supports the required content type, container creation fails. diff --git a/mie-opensource-landing/docs/admins/core-concepts/volumes.md b/mie-opensource-landing/docs/admins/core-concepts/volumes.md index 942a8686..4e1588c4 100644 --- a/mie-opensource-landing/docs/admins/core-concepts/volumes.md +++ b/mie-opensource-landing/docs/admins/core-concepts/volumes.md @@ -19,7 +19,8 @@ it is not re-applied and new containers never receive it.) (e.g. `/mnt/data`), and a **mode** (`ro` read-only or `rw` read-write). 2. The host directory lives under `/site-///`, where `` - is derived from the node's **volume storage** actual configured path (not + is derived from the node's **shared volume storage** (falling back to the root + disk storage when unset) actual configured path (not assumed) plus `/volumes`. The `site-` segment isolates containers that share a hostname across sites on the same shared storage; the `` segment (the container owner's username) ensures retained data is only ever @@ -42,16 +43,17 @@ it is not re-applied and new containers never receive it.) !!! warning "Place the volumes root on shared storage" A single per-site agent creates volume directories on the shared volumes root that is bind-mounted into it. For a directory to exist wherever a - container is placed or migrated, the **volume storage must be a path-backed + container is placed or migrated, the node's **shared volume storage must be a path-backed shared filesystem** — CephFS or NFS (`shared=1`) — that every node mounts. Block storages (Ceph RBD, LVM, ZFS) expose no host directory path and - therefore cannot host volumes. + therefore cannot host volumes — keep those for root disks and set + **Shared volume storage** separately. On node-local storage (`dir`/`lvm`/`zfspool` with `shared=0`), the directory will not exist where a container on another node lands and the data will not follow it. Single-node sites are unaffected. - When you save a node's configuration, the manager checks the chosen volume + When you save a node's configuration, the manager checks the chosen shared volume storage against the cluster topology and **warns** (it does not block) if the storage is not shared or is not active on every node. Move the volumes root to shared storage to clear the warning. diff --git a/mie-opensource-landing/docs/admins/deploying-agents.md b/mie-opensource-landing/docs/admins/deploying-agents.md index 847d9e54..133c93f4 100644 --- a/mie-opensource-landing/docs/admins/deploying-agents.md +++ b/mie-opensource-landing/docs/admins/deploying-agents.md @@ -80,7 +80,8 @@ must be on storage shared across every node so a directory the agent creates exists wherever a container lands. The volumes root is `/volumes` — where `` -is the configured **path** of the node's volume storage (e.g. a CephFS/NFS mount +is the configured **path** of the node's **shared volume storage** (or the root +disk storage, if shared volume storage is unset) (e.g. a CephFS/NFS mount like `/mnt/pve/cephfs`). Do the following once per site, on the Proxmox host that runs the agent: diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index 799d4932..a498ae50 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -48,6 +48,7 @@ erDiagram boolean disableTlsVerification string imageStorage "default: local" string volumeStorage "default: local-lvm" + string sharedVolumeStorage "nullable" string networkBridge "default: vmbr0" boolean nvidiaAvailable "default: false" int siteId FK @@ -202,7 +203,7 @@ erDiagram Top-level organizational unit. Has many Nodes. Has many ExternalDomains (as default site). `externalIp` is the public IP used as the target for Cloudflare DNS A records when cross-site HTTP services are created. ### Node -Proxmox VE server within a site. `name` must match Proxmox hostname (unique). `imageStorage` defaults to `'local'` (CT templates). `volumeStorage` defaults to `'local-lvm'` (container rootfs). `networkBridge` defaults to `'vmbr0'` (Proxmox bridge used in container net0 config). `nvidiaAvailable` indicates the node has NVIDIA drivers and nvidia-container-toolkit configured for GPU passthrough. Belongs to Site, has many Containers. +Proxmox VE server within a site. `name` must match Proxmox hostname (unique). `imageStorage` defaults to `'local'` (CT templates). `volumeStorage` defaults to `'local-lvm'` (container rootfs). `sharedVolumeStorage` (nullable) names the path-backed shared storage hosting persistent volumes; null falls back to `volumeStorage`. `networkBridge` defaults to `'vmbr0'` (Proxmox bridge used in container net0 config). `nvidiaAvailable` indicates the node has NVIDIA drivers and nvidia-container-toolkit configured for GPU passthrough. Belongs to Site, has many Containers. ### Agent Site agent registered by its check-in (`POST /api/v1/agents`, every 30s). Unique composite index on `(siteId, hostname)`. `services` stores the per-service status reported by the agent (`{ nginx: { state, lastApply }, ... }`); `lastCheckinAt` drives the online/offline health shown on the web client's Agents page. Belongs to Site. See [agent](agent.md).