diff --git a/.agents/skills/deploy-vllm-omni/SKILL.md b/.agents/skills/deploy-vllm-omni/SKILL.md new file mode 100644 index 000000000..c546b6c97 --- /dev/null +++ b/.agents/skills/deploy-vllm-omni/SKILL.md @@ -0,0 +1,82 @@ +--- +name: deploy-vllm-omni +description: Inspect a Linux GPU host, select a compatible vLLM-Omni model recipe, deploy an isolated service, connect OpenDirector and verify real generated assets. Use for self-hosted model setup or deployment repair; ordinary application installation uses install-opendirector. +--- + +# Deploy vLLM-Omni for OpenDirector + +Deliver a reproducible working model connection and actual output. Reply in the +user's language. Follow the user's chosen host, assigned GPU, directory and model. +A discussion or plan alone does not authorize installation or inference; an +authorized deployment-and-validation request does not require confirmation at +each routine step. + +Use the installed `opendirector vllm-omni` CLI on the target host. Discover its +options with `opendirector vllm-omni deploy --help` and +`opendirector vllm-omni verify --help`. The application package includes the +profiles and deployment tools; model dependencies belong to an isolated environment. +Read the matching [English guide](../../../docs/guides/vllm-omni.md) or +[Chinese guide](../../../docs/guides/vllm-omni.zh-CN.md) when available. + +## Inspect before changing anything + +Run `opendirector vllm-omni deploy --root DATA_DIRECTORY doctor` with a Python 3.12 model interpreter on the target host. +The helper is read-only. Resolve SSH aliases using actual SSH configuration, and +record the detected hardware rather than inferring it from an alias. Respect +assigned resources, inherited device visibility and other workloads. + +Run `plan` for an explicit physical index or UUID and model profile. Inspect OS, +GPU architecture, driver, free VRAM, available host memory and the relevant data +disk. Multiple cards do not form a larger single GPU. Unknown measurements do not +establish readiness. In particular, the repository's FastH3 NVFP4/SM120 recipe +does not work on an Ampere RTX A6000. Eligibility and measured qualification are +different facts; do not claim support for an untested combination. + +## Deploy and observe the original operation + +Use `install`, `download`, `start`, `status` and `stop` from the guide. Reuse the +fixed profile and mature uv/Hugging Face mechanisms. Keep runtime data, caches, +logs and outputs under the selected data directory. Use the profile's isolated +environment, preserve existing application configuration and do not change +global drivers or package registries. Choose a reachable trusted mirror only +when needed; keep it scoped to the deployment. +If observed long-transfer failures warrant it, use the guide's range backend, +which reuses the repository's existing pinned downloader. Preserve original +operations and partial files; do not invent another downloader or run duplicate +transports for the same model. Reuse existing weights only after manifest checks. + +Observe exit status, readiness and the reported log. A started PID is not ready +inference. `startup_pending` refers to the original possibly loading process; +query it before another action. Preserve conflicting listeners and shared +workloads. After a transient failure retry the same operation once only when +evidence supports it; continued failure requires diagnosis or changed conditions. + +## Prove delivery through OpenDirector + +Use `opendirector vllm-omni verify` for read-only health/model identity and `opendirector vllm-omni verify --infer` for +one authorized real request. Inspect the PNG or listen to the audio, and report +technical validity separately from creative review. Preserve verification +receipts, exact versions, parameters, artifacts and resource sampling scope. +Inference timeouts are unknown submissions; do not automatically resubmit them. + +Connect through Settings → Model access or existing model configuration APIs. +Choose the URL reachable from the OpenDirector backend, which may be on a +different machine from the browser. Keep remote model services on loopback and +use an SSH tunnel for the selected topology. + +Finish in an isolated real OpenDirector project using existing business +commands: create generation intent, submit, wait by returned job identity, inspect +the registered asset and adoption state, reopen and continue. Use CLI discovery +and returned next commands; never edit project files or credential storage to +bypass business rules. The `qwen3-asr-06b` companion uses native vLLM in the same +environment; use `opendirector vllm-omni verify --infer --audio-file` for transcription. Reference +text is only a local comparison, never a model hint. OpenDirector has no automatic +ASR tool: register transcripts through existing text-asset commands and describe +this handoff explicitly. Do not treat music-to-score as ASR or invent capabilities +from the provider name. Record unavailable integrations explicitly. Follow the +guide's first-use protocol for a real participant; keep unobserved human feedback +pending and distinguish cached setup from clean installation. + +Hand off copyable restart/status/stop commands for the actual host and +data directory, the model connection, measured results and remaining limitations. +Stop test processes you created unless the user requested a running service. diff --git a/.agents/skills/deploy-vllm-omni/agents/openai.yaml b/.agents/skills/deploy-vllm-omni/agents/openai.yaml new file mode 100644 index 000000000..a8061e42a --- /dev/null +++ b/.agents/skills/deploy-vllm-omni/agents/openai.yaml @@ -0,0 +1,3 @@ +interface: + display_name: "Deploy vLLM-Omni" + short_description: "Deploy GPU models and verify OpenDirector integration" diff --git a/MANIFEST.in b/MANIFEST.in index f29dfedc8..1957e751f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,8 @@ recursive-include src * recursive-include packages/core/src *.py recursive-include apps/api/src *.py recursive-include packages/agent/skills * +recursive-include tools/vllm_omni *.py *.json +include tools/fasth3/__init__.py tools/fasth3/download_models.py prune packages/core/src/opendirector/_bundled prune node_modules prune dist diff --git a/docs/README.md b/docs/README.md index cc14c4725..285309c79 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ This directory is the entry point for the repository's shared documentation. The | [Business CLI](guides/cli.md) | Querying, editing, and generating through a terminal or Agent Shell | Commands, authorization, or receipt semantics change | | [DeepSeek Harness plugin](../plugins/deepseek-harness/README.md) | Installing the DSH bundle and editing an OpenDirector project from DSH | Plugin setup, tools, or creation workflows change | | [Video model capabilities](guides/video-model-capabilities.md) | Declaring output specifications supported by a service | Configuration fields or capability integration changes | +| [Local vLLM-Omni deployment](guides/vllm-omni.md) ([中文](guides/vllm-omni.zh-CN.md)) | Hardware inspection, fixed model recipes, service lifecycle and real inference verification | Deployment tooling, recipes or operational guidance change | | [OpenAI Images](guides/openai-images.md) | Wiring gpt-image as an optional image service, request shape and failures | Integration fields, capability scope, or the OpenAI interface change | | [Kling video](guides/kling-video.md) ([中文](guides/kling-video.zh-CN.md)) | Optional Kling 4.0 and Flash setup, model-specific inputs, and recovery limits | Kling inputs, model declarations, or recovery behavior change | | [BigModel video](guides/bigmodel-video.md) ([中文](guides/bigmodel-video.zh-CN.md)) | CogVideoX-3 text, single-image and first/last-frame generation, task recovery and adoption | BigModel protocol, inputs, output sizes or recovery behavior change | diff --git a/docs/development/packaging.md b/docs/development/packaging.md index 19e8b05a0..b3ff65dfe 100644 --- a/docs/development/packaging.md +++ b/docs/development/packaging.md @@ -42,4 +42,4 @@ python -m pip check python scripts/verify_install.py --python /absolute/path/to/venv/bin/python ``` -Windows 对应传入 `venv/Scripts/python.exe`。验证脚本在临时工作目录运行,移除 `PYTHONPATH` 并收窄 PATH,检查内置 OpenCode、技能、正式 serve 入口、健康接口、首页与静态资源、CLI 创建和重读工程,最后清理自己启动的进程。它不调用真实模型,也不覆盖媒体处理或所有平台验收。 +Windows 对应传入 `venv/Scripts/python.exe`。验证脚本在临时工作目录运行,移除 `PYTHONPATH` 并收窄 PATH,检查部署命令帮助、全部配方、分段下载器计划、内置 OpenCode、技能、正式 serve 入口、健康接口、首页与静态资源、CLI 创建和重读工程,最后清理自己启动的进程。它不调用真实模型,也不覆盖媒体处理或所有平台验收。 diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 92939cd1d..15653ab3f 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -14,6 +14,8 @@ Installing dependencies and calling remote models require a network connection. Prepare self-hosted model services separately; starting this application does not install or deploy models. +For a Linux NVIDIA model host, see [local vLLM-Omni deployment](vllm-omni.md). + You can install and open the application, upload local media, and edit projects before connecting a model. AI chat and generation require a configured service; running the application does not require a local GPU when using a remote service. #### macOS prerequisites diff --git a/docs/guides/installation.zh-CN.md b/docs/guides/installation.zh-CN.md index 121183388..3403ef9e4 100644 --- a/docs/guides/installation.zh-CN.md +++ b/docs/guides/installation.zh-CN.md @@ -14,6 +14,8 @@ 安装依赖和调用远程模型需要网络。自部署模型服务需另外准备,启动本应用不会自动安装或部署模型。 +Linux NVIDIA 模型主机的配置与验证参见 [vLLM-Omni 本地部署](vllm-omni.zh-CN.md)。 + 尚未接入模型也可以安装、打开应用、上传本地素材和编辑工程。AI 对话和生成需要配置模型服务;使用远程服务时,运行本应用的 Mac 无需本地 GPU 推理环境。 #### macOS 环境准备 diff --git a/docs/guides/vllm-omni.md b/docs/guides/vllm-omni.md new file mode 100644 index 000000000..d9ba2fddf --- /dev/null +++ b/docs/guides/vllm-omni.md @@ -0,0 +1,224 @@ +# Deploy local models with vLLM-Omni + +[中文](vllm-omni.zh-CN.md) + +Use these optional tools on a Linux x86_64 GPU host with Python 3.12. OpenDirector +and the model server have separate environments. Install OpenDirector using the +[installation guide](installation.md); installing the application does not deploy +models. Eligibility is not a claim that an untested GPU is supported. +The host needs an existing C++ compiler for CUDA JIT compilation, and `ffmpeg` / +`ffprobe` for media checks. The helpers do not install system packages. + +For Agent assistance, load the repository's +[deployment skill](../../.agents/skills/deploy-vllm-omni/SKILL.md) in an external +coding Agent with authorized shell/SSH access. The deployment tools and profiles +are included in the OpenDirector package; model dependencies remain isolated. + +## Inspect and select + +After installing OpenDirector, use `opendirector vllm-omni --help`, +`opendirector vllm-omni deploy --help` and `opendirector vllm-omni verify --help` +to discover the commands. Choose a data disk with room for weights, the CUDA environment, +caches and outputs. `doctor`, `plan` and `status` do not create directories or submit +inference. An index refers to a physical device in `nvidia-smi`; use the returned +UUID for subsequent operations. Respect the server's resource allocation and inherited +`CUDA_VISIBLE_DEVICES`. Free memory alone does not allocate a shared GPU to you. +The model environment defaults to the interpreter running OpenDirector. If it is +not Python 3.12, pass `--python /path/to/python3.12` to deployment commands. + +```sh +OMNI_ROOT=/path/to/data/opendirector-models +opendirector vllm-omni deploy --root "$OMNI_ROOT" doctor +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID plan +``` + +The bundled `zimage-bf16` profile fixes a Z-Image-Turbo snapshot and the matching +vLLM / vLLM-Omni 0.32.0 combination. It downloads about 32.85 GB. `qwen3-tts-06b` +uses a 0.6B CustomVoice snapshot (about 2.50 GB). Resource budgets are conservative +preflight reservations, not measured minimums. An existing matching environment +and cached weights reduce the additional storage budget. + +The separate [FastH3 recipe](../../tools/fasth3/README.txt) requires Blackwell SM120 +and NVFP4 kernels. Its validation does not cover Ampere RTX A6000 cards. Do not +infer compatibility from a similar GPU name or add multiple cards' memory together. + +## Install, download and start + +Use the same root and GPU for each command. Installation creates `omni-env` under +the chosen root, verifies dependencies and runs a small CUDA operation and a real +FlashInfer sampling kernel. Compiler, CRT and NVVM versions are fixed at 13.0.88 +to match the cu130 runtime. The helper discovers the environment's compiler and +creates a private linker alias for the runtime library; it preserves an explicitly +selected `CUDA_HOME`/`CUDA_PATH`. Downloads +use Hugging Face's cache/locks/resume support, a fixed revision and file hashes. +No model service starts during installation or downloading. + +```sh +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID install +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID download +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID start +opendirector vllm-omni deploy --root "$OMNI_ROOT" status +``` + +To select a mirror, add `--hf-endpoint https://your-trusted-mirror` before the +command. It is scoped to this deployment. Keep the cache on the data disk; avoid +multiple downloads of the same snapshot. A failed download preserves SDK resume +files. Investigate its status and network condition before retrying the same +command. The tool does not change a global package index, driver or CUDA install. +The SDK chooses its transfer backend. If your selected network cannot access Xet, +set `HF_HUB_DISABLE_XET=1` for that process; the tool preserves this setting. +For a network with persistently slow or failing long transfers, add +`--download-backend ranges` to `download`. This reuses +the packaged FastH3 range downloader with an explicit complete manifest, preserving +its original FastH3 defaults. `DOWNLOAD_WORKERS=16` bounds concurrent ranges; +proxy settings and the selected endpoint remain scoped to this process. Completed +files and range journals are preserved. Do not run two transports concurrently +for the same model. Existing weights may be copied into the selected profile's +range directory and are verified before reuse; never rename a different checkpoint +to make it appear compatible. + +Startup binds `127.0.0.1:8091`, records the owned process, and waits for health +and the exact Model ID. `startup_pending` means the original process may still be +loading: query `status` and inspect the reported log before taking another action. +Repeated `start` does not launch another live instance; it requires the same +profile, port and explicitly selected GPU within the current device visibility. +Use `--port` before `start` to select a free port. `stop` only signals the recorded process group after checking +its owner and process start identity. +`status` describes only the model service; `stopped` does not mean that a separate +installation or download process has finished. +Compilation caches for vLLM, TorchInductor, Triton and FlashInfer also stay under +the selected data directory; no shared home-directory cache is required. + +```sh +opendirector vllm-omni deploy --root "$OMNI_ROOT" stop +``` + +For TTS add `--profile qwen3-tts-06b` to the commands and use `--port 8092` for +startup. Stop the image model first when validating resource use in isolation. +The TTS configuration comes from the installed upstream pipeline, with per-stage +memory utilization 0.15 and maximum sequences 4. The 0.6B CustomVoice checkpoint +uses built-in speakers such as `vivian`; cloning requires a different Base model. +Initial compilation and graph capture can exceed the upstream 300-second stage +deadline. This profile allows 1200 seconds; use `--timeout 1500 start` to extend +the helper's observation window. An observation timeout requires querying the +original process before another action. + +## Verify actual output and connect OpenDirector + +```sh +# Health and model identity only; no inference or output files. +opendirector vllm-omni verify --root "$OMNI_ROOT" +# One real request, saved output, decoding and a verification receipt. +opendirector vllm-omni verify --root "$OMNI_ROOT" --infer +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-tts-06b --infer +``` + +Inspect the PNG and listen to the MP3. Technical validation leaves creative review +pending. Receipts include fixed request parameters, model revision, file hash, +elapsed time and sampled whole-device memory. Those samples include other workloads +and are not process allocator peaks. A timeout preserves an unknown-submission +receipt and never automatically resubmits inference. + +In Settings → Model access select the image or speech vLLM-Omni preset, self-hosted, +no authentication, the exact Model ID in the profile, and the reachable API base +ending in `/v1`. Use `proxyMode: direct` for a loopback tunnel. Configure through +the existing UI or `/api/model-configs` API, not by editing `models.json`. + +If both services run on the GPU host, the model URL is `http://127.0.0.1:8091/v1` +and forward OpenDirector's port to the browser machine. If OpenDirector runs on +your computer and Omni runs remotely, forward the model port: + +```sh +ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -N -L 18091:127.0.0.1:8091 YOUR_GPU_HOST +``` + +Then the model URL is `http://127.0.0.1:18091/v1`. `localhost` always refers to +the machine running the OpenDirector backend, not necessarily the browser. + +Finish with an OpenDirector generation: create a project and generation item, +submit through existing commands, wait on the returned job identity, inspect the +registered asset and adoption state, and reopen the project. Use CLI `--help` and +the next command in the receipt instead of inventing image/TTS convenience commands. +Keep model-list readiness, direct inference, project delivery and creative approval +as separate results. ASR is a separate integration; the music-to-score preset is +not speech transcription. Z-Image sampling alignment is tracked in [#230](https://github.com/ThinkFlowLab/OpenDirector/issues/230) +and [#231](https://github.com/ThinkFlowLab/OpenDirector/pull/231); compare performance +only with matching actual request parameters. + +## ASR companion service + +The `qwen3-asr-06b` profile pins Qwen3-ASR-0.6B (about 1.88 GB) in the same +isolated environment. It uses **native vLLM 0.32.0**, which implements this model's +OpenAI transcription endpoint. It does not add an Omni pipeline. Run the normal +`plan`, `download`, `start`, `status` and `stop` commands with +`--profile qwen3-asr-06b`; use `--port 8093` at startup. The 16 GiB GPU budget is a +preflight reservation, not a measured hardware minimum. The profile limits engine +memory utilization to 0.10, sequence length to 4096 and concurrent sequences to 4. +The pinned image and TTS recipes continue using `--omni`. + +```sh +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-asr-06b +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-asr-06b \ + --infer --audio-file /path/to/authorized-recording.mp3 \ + --expected-text 'The known recording transcript' +``` + +Read-only verification needs no audio. Inference accepts WAV, MP3, FLAC, M4A, +OGG, WebM or MP4 up to 25 MiB and saves the exact uploaded bytes, their SHA256, +media facts, raw JSON output and recognized text. `--expected-text` is optional +and **never sent to the model**. Comparison reports character edit distance and +error rate after NFKC normalization, case folding and removal of punctuation and +spaces; both normalized strings are limited to 4096 characters. This is a +content check, not a pronunciation, listening or general ASR benchmark score. +Keep authorized recordings and transcripts private; receipts contain their text. + +OpenDirector currently has no speech-to-text model preset or automatic ASR tool. +Do not configure ASR as the music-to-score model or a speech synthesizer. An +external Agent can call this verifier and use the existing `asset write-text` +and `canvas add-material` commands to register its transcript in the project. +Users can inspect that text and continue editing through the existing UI. This +manual handoff does not claim an in-application ASR integration. The selected +Qwen3-TTS CustomVoice checkpoint also rejects uploaded voice cloning; a transcript +does not change its supported capabilities. + +## First-use participant trial + +Use a real participant unfamiliar with this deployment. Before starting, record +their experience, OS, host, assigned GPU, checkout revision, network route, and +whether weights, environment or compilation caches already exist. A trial using +cached weights measures an assisted cached setup, not a clean installation. + +Give the participant the deployment skill and this guide. Ask them to inspect the +host, select an eligible recipe, start the service, connect OpenDirector, produce +one asset and reopen the project. Let them make ordinary decisions; record every +manual action and help intervention. If exercising recovery, stop only their own +SSH forward, observe the readable failure and preserved asset, restore it, and +query the original operation before any new attempt. Never interrupt shared jobs. + +Record timestamps for start, ready service, direct inference and first usable +project asset; count manual actions, help requests, retries and recovery time. +Ask the participant to inspect the image or actually listen to the audio and +report omissions, pronunciation, noise, naturalness and suitability. Record +technical result, participant observations and creative approval separately. +Leave unanswered fields pending; an Agent run, successful playback or ASR match +cannot supply human observations. + +## Failure codes + +Commands emit JSON and exit 1 on a blocked plan or failure; argument errors exit 2. + +| Code | Next step | +| --- | --- | +| `gpu_probe_failed`, `gpu_probe_incomplete` | Check NVIDIA driver visibility and `nvidia-smi`; missing data is not zero usage. | +| `gpu_selection_required`, `gpu_visibility_unresolved` | Select an assigned physical index or UUID within the existing visibility. | +| `unsupported_gpu_architecture`, `driver_too_old` | Select a compatible recipe/environment; retain existing system drivers. | +| `insufficient_free_vram`, `insufficient_host_memory`, `insufficient_disk_space` | Use allocated resources with sufficient headroom; budgets are not minimum-hardware claims. | +| `model_download_failed`, `model_file_invalid` | Check the fixed snapshot, mirror and available space; preserve partial files and inspect invalid files. | +| `port_in_use`, `running_configuration_conflict` | Preserve the listener; select a different port or stop the owned deployment before changing it. | +| `server_exited`, `startup_pending` | Inspect the reported log and original process; do not start duplicate model loads. | +| `cuda_linker_conflict` | Inspect the reported private runtime-library alias; preserve the conflicting file before correcting its target. | +| `served_model_mismatch` | Correct the endpoint or served model identity; health alone does not establish compatibility. | +| `service_not_started` | Run `status`/`start`, or supply `opendirector vllm-omni verify --url` for an existing service. | +| `audio_input_required`, `audio_input_unreadable`, `unsupported_audio_format`, `invalid_audio_input_size` | Supply a readable ASR input in a supported format, nonempty and at most 25 MiB. | +| `invalid_comparison_text`, `comparison_requires_transcription` | Use a nonempty ASR reference with at most 4096 normalized characters; the option is only for transcription. | +| `inference_result_unknown` | Inspect the saved receipt and server before deciding whether to start a new attempt. | diff --git a/docs/guides/vllm-omni.zh-CN.md b/docs/guides/vllm-omni.zh-CN.md new file mode 100644 index 000000000..b1d7799a8 --- /dev/null +++ b/docs/guides/vllm-omni.zh-CN.md @@ -0,0 +1,171 @@ +# 使用 vLLM-Omni 部署本地模型 + +[English](vllm-omni.md) + +本流程面向 Linux x86_64、Python 3.12 的 GPU 主机。OpenDirector 与模型服务使用独立环境。 +应用安装参见[安装指南](installation.zh-CN.md);安装应用本身不会部署模型。 +通过资源预检不代表未测试硬件已获支持。 +主机需已有用于 CUDA JIT 编译的 C++ 编译器,以及媒体检查所需的 `ffmpeg` / `ffprobe`;工具不安装系统软件包。 + +需要 Agent 协助时,在具备已授权 shell/SSH 访问的外部编程 Agent 中加载仓库的 +[部署技能](../../.agents/skills/deploy-vllm-omni/SKILL.md)。部署工具与配方已包含在 OpenDirector 安装包中, +无需源码目录;模型依赖仍安装在独立环境。 + +## 探测与选择 + +安装 OpenDirector 后,在目标主机执行 `opendirector vllm-omni --help` 查看命令, +通过 `opendirector vllm-omni deploy --help` 和 `opendirector vllm-omni verify --help` 查看参数。数据目录应能容纳权重、CUDA 环境、下载缓存、临时文件和输出。 +`doctor`、`plan`、`status` 不创建目录,也不调用生成模型。索引对应 `nvidia-smi` 的物理设备, +后续操作建议使用返回的 UUID。遵守共享服务器的资源分配和已有 `CUDA_VISIBLE_DEVICES`,空闲显存不等于已分配资源。 +模型虚拟环境默认使用运行 OpenDirector 的 Python;若不是 3.12,使用 `--python /path/to/python3.12` 指定模型环境解释器。 + +```sh +OMNI_ROOT=/path/to/data/opendirector-models +opendirector vllm-omni deploy --root "$OMNI_ROOT" doctor +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID plan +``` + +默认 `zimage-bf16` 配方固定 Z-Image-Turbo 快照和配套的 vLLM / vLLM-Omni 0.32.0, +模型文件约 32.85 GB。`qwen3-tts-06b` 使用 0.6B CustomVoice 快照,约 2.50 GB。 +资源预算是保守的预检预留量,不是测出的最低配置。已存在的匹配环境和缓存权重会减少新增磁盘预算。 + +独立的 [FastH3 配方](../../tools/fasth3/README.txt)要求 Blackwell SM120 和 NVFP4 内核, +其验收不覆盖 Ampere RTX A6000。不能仅凭名称相近判断兼容,也不能将多卡显存直接相加。 + +## 安装、下载与启动 + +每条命令沿用同一数据目录及分配的 GPU。安装在数据目录下创建 `omni-env`,检查依赖完整性并执行小型 CUDA 运算。 +安装还会编译并运行真实的 FlashInfer 采样内核。编译器、CRT 和 NVVM 固定为与 cu130 运行时匹配的 13.0.88; +工具发现独立环境内的编译器,并在数据目录下创建运行库的私有链接别名,保留显式指定的 `CUDA_HOME`/`CUDA_PATH`。 +下载复用 Hugging Face 的缓存、锁和续传机制,固定 revision 并核对文件哈希。安装和下载阶段不启动模型服务。 + +```sh +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID install +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID download +opendirector vllm-omni deploy --root "$OMNI_ROOT" --gpu GPU-UUID start +opendirector vllm-omni deploy --root "$OMNI_ROOT" status +``` + +如果需要镜像,在命令前添加 `--hf-endpoint https://your-trusted-mirror`,只对本次部署生效。 +缓存应放在数据盘上,同一快照避免并发重复下载。失败后保留 SDK 续传文件,先检查错误状态与网络条件,再重试原命令。 +工具不修改全局包索引、驱动或系统 CUDA。 +传输方式由 SDK 选择;如果当前网络无法访问 Xet,可为该进程设置 `HF_HUB_DISABLE_XET=1`,工具会保留这个设置。 +长连接持续过慢或失败时,可为 `download` 增加 `--download-backend ranges`,复用 +安装包内的 FastH3 分段下载器和完整清单下载模式,保留它原来的 FastH3 默认行为。 +通过 `DOWNLOAD_WORKERS=16` 限定并行分段数, +代理和端点仍只对当前进程生效。保留已完成文件和分段日志,同一模型不要并发运行两种传输。 +已有权重可复制到该配方的分段下载目录,通过校验后复用;不能把其他检查点改名冒充当前模型。 + +服务默认绑定 `127.0.0.1:8091`,记录本次启动的进程,并等待健康检查与目标 Model ID。 +`startup_pending` 表示原进程可能仍在加载,应查询 `status` 和回执中的日志,不要重复启动。 +再次执行 `start` 时必须指定相同配方、端口和当前可见范围内的同一张 GPU。 +通过 `--port` 指定其他空闲端口。停止前会核对进程所有者和启动身份,只向记录的进程组发送停止信号。 +`status` 只描述模型服务;返回 `stopped` 不代表另一个安装或下载进程已经结束。 +vLLM、TorchInductor、Triton 和 FlashInfer 的编译缓存也保存在所选数据目录下。 + +```sh +opendirector vllm-omni deploy --root "$OMNI_ROOT" stop +``` + +部署 TTS 时,各命令增加 `--profile qwen3-tts-06b`,启动使用 `--port 8092`。 +需要独立测量资源时,先停止图片模型。TTS 复用已安装上游流水线,逐阶段显存比例设为 0.15、最大序列数设为 4。 +CustomVoice 使用 `vivian` 等内置音色;声音克隆需要不同的 Base 模型。 +首次编译和图捕获可能超过上游默认的 300 秒。配方将阶段初始化期限设为 1200 秒; +可通过 `--timeout 1500 start` 延长工具的观察窗口。观察超时后先查询原进程,不重复启动。 + +## 验证真实产物并连接 OpenDirector + +```sh +# 只检查服务健康与模型身份,不推理、不写输出。 +opendirector vllm-omni verify --root "$OMNI_ROOT" +# 提交一次真实生成,保存产物、解码并记录验证回执。 +opendirector vllm-omni verify --root "$OMNI_ROOT" --infer +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-tts-06b --infer +``` + +请查看 PNG、试听 MP3。技术校验通过后,创作品质仍标为待审。 +回执保留实际请求参数、模型 revision、文件哈希、耗时和整卡显存采样。 +整卡数据包含其他负载,不能当作本服务的分配器峰值。推理超时会保留提交结果未知的回执,不自动重发。 + +在设置 → 模型接入中选择 vLLM-Omni 图片或语音接口、自部署服务、无需鉴权,填写配方里的准确 Model ID 和可访问的 `/v1` 地址。 +对回环隧道使用直连代理模式。通过现有界面或 `/api/model-configs` 配置,不直接修改 `models.json`。 + +两个服务均在 GPU 主机时,模型地址为 `http://127.0.0.1:8091/v1`,将 OpenDirector 端口转发给浏览器所在电脑即可。 +OpenDirector 在个人电脑、Omni 在服务器时,转发模型端口: + +```sh +ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -N -L 18091:127.0.0.1:8091 YOUR_GPU_HOST +``` + +此时模型地址为 `http://127.0.0.1:18091/v1`。`localhost` 指向 OpenDirector 后端所在机器,不一定是浏览器所在机器。 + +最终还应在 OpenDirector 新建工程和生成节点,通过现有命令提交、按返回身份等待任务,核对素材登记与采用状态,并重开工程。 +CLI 参数以 `--help` 和业务回执里的下一步命令为准,不自行猜测图片或 TTS 便捷命令。 +分别记录服务就绪、直接推理、工程交付与创作品质验收。音乐“扒谱”不是 ASR。 +Z-Image 采样参数对齐见 [#230](https://github.com/ThinkFlowLab/OpenDirector/issues/230)、[#231](https://github.com/ThinkFlowLab/OpenDirector/pull/231), +性能比较必须使用一致的实际请求参数。 + +## ASR 配套服务 + +`qwen3-asr-06b` 固定 Qwen3-ASR-0.6B 快照,约 1.88 GB,复用同一独立环境。 +该模型由 **vLLM 0.32.0 原生**提供 OpenAI 转录接口,没有新增 Omni 流水线。 +在常规 `plan`、`download`、`start`、`status`、`stop` 命令中加 +`--profile qwen3-asr-06b`,启动时使用 `--port 8093`。16 GiB 显存预算是预检 +保留量,不是实测最低要求。配方将引擎显存利用率设为 0.10,序列长度限制为 4096, +并发序列数为 4。图像和 TTS 配方仍使用 `--omni`。 + +```sh +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-asr-06b +opendirector vllm-omni verify --root "$OMNI_ROOT" --profile qwen3-asr-06b \ + --infer --audio-file /path/to/authorized-recording.mp3 \ + --expected-text '录音的已知逐字稿' +``` + +只读探测不需要音频。真实转录接受不超过 25 MiB 的 WAV、MP3、FLAC、M4A、OGG、 +WebM 或 MP4,保存实际上传字节、SHA256、媒体信息、原始 JSON 和识别文本。 +`--expected-text` 可选,**不会发给模型**;仅在本地按 NFKC、大小写折叠、移除标点 +和空白后的字符序列计算编辑距离与字符错误率,规范化后的两段文本均不超过 4096 字符。 +这是内容核对,不是发音、听感或通用 ASR 基准。请使用有授权的录音;回执含原文, +录音、转录与回执应按实际隐私要求保存。 + +OpenDirector 当前没有语音转文字模型预设或自动 ASR 工具。不要将 ASR 配成扒谱或 +语音合成模型。外部 Agent 可以调用验证器,再通过已有 `asset write-text` 和 +`canvas add-material` 命令把转录登记到工程,用户随后在现有 UI 中审看、继续编辑。 +这一手动交接不代表应用内 ASR 接入。已选 Qwen3-TTS CustomVoice 权重不支持上传 +录音克隆音色,转录文本不会改变该模型的能力。 + +## 真人首次试用 + +由不熟悉本部署的真实参与者操作。开始前记录其经验、系统、主机、分配的 GPU、源码版本、 +网络路径,以及环境、权重和编译缓存是否已存在。复用缓存的试用应写为“有辅助的缓存部署”, +不能当成全新安装耗时。 + +向参与者提供部署技能和本指南,任务为检查主机、选择可用配方、启动服务、连接应用、 +生成一份素材并重开工程。记录每次手动操作和帮助介入。若检验故障恢复,仅停止自己 +的 SSH 转发,观察可读错误与既有素材保留,再恢复转发;新尝试之前先查询原操作。 +不得中断共享作业。 + +记录开始、服务就绪、直接推理、工程内首次可用素材的时间,统计手动操作、求助、重试次数 +和恢复耗时。请参与者实际审图或听完整段音频,反馈漏字、错读、杂音、自然度和用途适合程度。 +技术结果、参与者观察、创意认可分别记录。未回答的字段保持待验,不能用 Agent 操作、 +播放成功或 ASR 匹配代替真人观察。 + +## 失败代码 + +工具输出 JSON,预检阻断或执行失败退出 1,参数错误退出 2。 + +| 代码 | 下一步 | +| --- | --- | +| `gpu_probe_failed`、`gpu_probe_incomplete` | 检查 NVIDIA 驱动及 `nvidia-smi`;缺失数据不能按零占用处理。 | +| `gpu_selection_required`、`gpu_visibility_unresolved` | 在已有可见设备和分配范围内选择物理索引或 UUID。 | +| `unsupported_gpu_architecture`、`driver_too_old` | 选择兼容配方或环境,保留现有系统驱动。 | +| `insufficient_free_vram`、`insufficient_host_memory`、`insufficient_disk_space` | 使用有足够余量的已分配资源;预算不是最低硬件承诺。 | +| `model_download_failed`、`model_file_invalid` | 检查固定快照、镜像与空间,保留断点并核查无效文件。 | +| `port_in_use`、`running_configuration_conflict` | 保留已有监听,换端口,或先停止本工具管理的部署再改配置。 | +| `server_exited`、`startup_pending` | 检查回执中的日志与原进程,不重复加载模型。 | +| `cuda_linker_conflict` | 检查回执中的私有运行库链接,保留冲突文件,再纠正链接目标。 | +| `served_model_mismatch` | 核对地址与模型身份;健康检查不能证明模型匹配。 | +| `service_not_started` | 先运行 `status`/`start`,或为现有服务指定 `opendirector vllm-omni verify --url`。 | +| `audio_input_required`、`audio_input_unreadable`、`unsupported_audio_format`、`invalid_audio_input_size` | 提供可读、格式受支持、非空且不超过 25 MiB 的 ASR 输入。 | +| `invalid_comparison_text`、`comparison_requires_transcription` | 对照文本仅用于转录,应非空,规范化后不超过 4096 字符。 | +| `inference_result_unknown` | 查阅保存的回执和服务端情况,再决定是否发起新尝试。 | diff --git a/packages/core/src/opendirector/cli.py b/packages/core/src/opendirector/cli.py index 72771a219..640fa3d1e 100644 --- a/packages/core/src/opendirector/cli.py +++ b/packages/core/src/opendirector/cli.py @@ -23,10 +23,12 @@ from opendirector.domain.creation.commands import COMMAND_MODELS, command_fingerprint, parse_command from opendirector.domain.projects import ProjectError from opendirector.domain.timeline_errors import timeline_error, timeline_error_message +from opendirector.vllm_cli import app as vllm_omni_app app = typer.Typer(no_args_is_help=True, help="Start OpenDirector and create or edit projects.", epilog="Run opendirector COMMAND --help for subcommands and usage. " "Place --project, --url and other global options before the command.") +app.add_typer(vllm_omni_app, name='vllm-omni') GROUP_HELP = { "project": "List, inspect, create, rename, trash or restore projects.", "model": "Inspect available model configurations and input capabilities.", @@ -429,7 +431,7 @@ def add_material(ctx: typer.Context, asset: Annotated[UUID, typer.Option(help="R reference["assetId"] = source["id"] emit(client.execute("canvas.show", {"item": { "id": str(item_id or client.operation_id or uuid4()), "projectId": snapshot["project"]["id"], - "kind": source["kind"], "position": {"x": x, "y": y}, + "kind": source["kind"], "title": source["name"], "position": {"x": x, "y": y}, "binding": {"role": "material", "objectRef": reference}, }})) diff --git a/packages/core/src/opendirector/vllm_cli.py b/packages/core/src/opendirector/vllm_cli.py new file mode 100644 index 000000000..af4e7d8c5 --- /dev/null +++ b/packages/core/src/opendirector/vllm_cli.py @@ -0,0 +1,64 @@ +"""Installed CLI entry points for the isolated local model tools.""" +from enum import Enum +from pathlib import Path +import sys +from typing import Annotated + +import typer + +app = typer.Typer(no_args_is_help=True, help="Deploy and verify isolated local vLLM-Omni model services.") + + +class Operation(str, Enum): + doctor = 'doctor' + plan = 'plan' + install = 'install' + download = 'download' + start = 'start' + status = 'status' + stop = 'stop' + + +class DownloadBackend(str, Enum): + sdk = 'sdk' + ranges = 'ranges' + + +@app.command('deploy', help='Inspect, install, download, start, inspect status or stop a model service. Returns JSON.') +def deploy( + command: Annotated[Operation, typer.Argument(help='Deployment operation to run.')], + root: Annotated[Path, typer.Option(help='Directory for model environments, weights and service state.')], + profile: Annotated[str, typer.Option(help='Profile: zimage-bf16, qwen3-tts-06b or qwen3-asr-06b.')] = 'zimage-bf16', + gpu: Annotated[str, typer.Option(help='Assigned physical GPU index or full UUID.')] = '', + python: Annotated[str, typer.Option(help='Python 3.12 interpreter for the isolated model environment.')] = sys.executable, + hf_endpoint: Annotated[str | None, typer.Option(help='Trusted Hugging Face endpoint for this deployment.')] = None, + download_backend: Annotated[DownloadBackend, typer.Option(help='Hugging Face SDK or pinned range downloader.')] = DownloadBackend.sdk, + port: Annotated[int, typer.Option(help='Loopback service port.')] = 8091, + timeout: Annotated[int, typer.Option(help='Startup observation window in seconds.')] = 600, +): + from opendirector.vllm_omni import deploy as implementation + args = [f'--root={root}', f'--profile={profile}', f'--gpu={gpu}', f'--python={python}', + f'--download-backend={download_backend.value}', f'--port={port}', f'--timeout={timeout}'] + if hf_endpoint is not None: + args.append(f'--hf-endpoint={hf_endpoint}') + raise typer.Exit(implementation.main([*args, command.value])) + + +@app.command('verify', help='Check service readiness or submit one real verification request. Returns JSON.') +def verify( + root: Annotated[Path, typer.Option(help='Deployment data directory.')], + profile: Annotated[str, typer.Option(help='Profile: zimage-bf16, qwen3-tts-06b or qwen3-asr-06b.')] = 'zimage-bf16', + url: Annotated[str | None, typer.Option(help='Existing service API base URL ending in /v1.')] = None, + infer: Annotated[bool, typer.Option(help='Submit one real request; save and inspect the result.')] = False, + timeout: Annotated[int, typer.Option(help='Request timeout in seconds.')] = 240, + audio_file: Annotated[Path | None, typer.Option(help='ASR input file, at most 25 MiB.')] = None, + expected_text: Annotated[str | None, typer.Option(help='Optional local ASR comparison text, never sent to the model.')] = None, +): + from opendirector.vllm_omni import verify as implementation + args = [f'--root={root}', f'--profile={profile}', f'--timeout={timeout}'] + for option, value in [('--url', url), ('--audio-file', audio_file), ('--expected-text', expected_text)]: + if value is not None: + args.append(f'{option}={value}') + if infer: + args.append('--infer') + raise typer.Exit(implementation.main(args)) diff --git a/scripts/verify_install.py b/scripts/verify_install.py index a458e7596..a0940bdee 100644 --- a/scripts/verify_install.py +++ b/scripts/verify_install.py @@ -32,6 +32,28 @@ "import subprocess; assert any(builtin_skills().rglob('SKILL.md')); " "subprocess.run([executable('default'), '--version'], check=True)"], cwd=temporary, env=env, check=True) + for command, expected in [(['vllm-omni'], 'verify'), (['vllm-omni', 'deploy'], '--download-backend'), + (['vllm-omni', 'verify'], '--audio-file')]: + result = subprocess.run([str(cli), *command, '--help'], cwd=temporary, env=env, + capture_output=True, text=True, check=True) + assert expected in result.stdout + model_root = Path(temporary) / 'model-data' + for profile in ('zimage-bf16', 'qwen3-tts-06b', 'qwen3-asr-06b'): + result = subprocess.run([str(cli), 'vllm-omni', 'deploy', '--root', str(model_root), + '--profile', profile, 'status'], cwd=temporary, env=env, + capture_output=True, text=True, check=True) + assert json.loads(result.stdout)['status'] == 'stopped' + assert not model_root.exists() + # Exercise the packaged downloader in its standalone, model-environment form. + subprocess.run([str(python), '-c', + "import json, subprocess, sys; from pathlib import Path; " + "from opendirector.vllm_omni.deploy import load_profile; " + "from opendirector._fasth3 import download_models; " + "p=load_profile('qwen3-tts-06b'); " + "f=Path(sys.argv[1])/'manifest.json'; " + "f.write_text(json.dumps([dict(repo=p['modelId'],sha=p['revision'],files=p['files'])])); " + "subprocess.run([sys.executable,download_models.__file__,'--manifest',str(f),'--all-files','--plan'],check=True)", temporary], + cwd=temporary, env=env, check=True, capture_output=True) with socket.socket() as listener: listener.bind(('127.0.0.1', 0)) port = listener.getsockname()[1] @@ -63,7 +85,7 @@ def request(path): subprocess.run([*command, 'project', 'create', '--name', 'Installed smoke'], cwd=temporary, env=env, check=True, capture_output=True) result = subprocess.run([*command, 'project', 'show'], cwd=temporary, env=env, check=True, capture_output=True, text=True) assert json.loads(result.stdout)['project']['name'] == 'Installed smoke' - print('PASS installed CLI, OpenCode, skills, health, web assets and persisted project (no Node on PATH)') + print('PASS installed CLI, deployment profiles and downloader, OpenCode, skills, health, web assets and persisted project (no Node on PATH)') except BaseException: log.seek(0) print(log.read()) diff --git a/setup.py b/setup.py index ec0e4eb53..0bbe3dda4 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,13 @@ def get_tag(self): setup( - packages=find_packages('packages/core/src') + find_packages('apps/api/src'), + packages=find_packages('packages/core/src') + find_packages('apps/api/src') + + ['opendirector.vllm_omni', 'opendirector._fasth3'], package_dir={'opendirector': 'packages/core/src/opendirector', - 'opendirector_api': 'apps/api/src/opendirector_api'}, - package_data={'opendirector': ['_bundled/**/*']}, + 'opendirector_api': 'apps/api/src/opendirector_api', + 'opendirector.vllm_omni': 'tools/vllm_omni', + 'opendirector._fasth3': 'tools/fasth3'}, + package_data={'opendirector': ['_bundled/**/*'], 'opendirector.vllm_omni': ['profiles/*.json']}, include_package_data=True, distclass=BundledDistribution, cmdclass={'bdist_wheel': PlatformWheel}, diff --git a/tests/test_cli.py b/tests/test_cli.py index 3c6689eb2..1d4facb2e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -154,6 +154,7 @@ def invoke(arguments): assert shown["canvasItems"][0]["width"] == 352 item = invoke(["project", "show"])["canvasItems"][0] assert item["binding"]["objectRef"]["assetId"] == asset["id"] + assert item["title"] == asset["name"] # UI text revisions need a valid file name. file = tmp_path / "upload.txt" file.write_text("Imported text") diff --git a/tests/test_vllm_omni_cli.py b/tests/test_vllm_omni_cli.py new file mode 100644 index 000000000..dd0b84b58 --- /dev/null +++ b/tests/test_vllm_omni_cli.py @@ -0,0 +1,55 @@ +"""Exercise the installed CLI adapter against the shared deployment implementation.""" +import json +import sys + +import pytest +from typer.testing import CliRunner + +from opendirector import cli +from tools import vllm_omni +from tools.vllm_omni import verify + + +@pytest.fixture(autouse=True) +def source_mapping(monkeypatch): + # Editable source tests use the same files that are mapped into the wheel. + monkeypatch.setitem(sys.modules, 'opendirector.vllm_omni', vllm_omni) + + +def test_cli_help_discovers_operations_and_options(): + runner = CliRunner() + for command, words in [([], ['vllm-omni']), (['vllm-omni'], ['deploy', 'verify']), + (['vllm-omni', 'deploy'], ['doctor', 'plan', 'download', '--root', '--gpu', 'qwen3-tts-06b']), + (['vllm-omni', 'verify'], ['--infer', '--audio-file', '--expected-text'])]: + result = runner.invoke(cli.app, [*command, '--help'], color=False) + assert result.exit_code == 0, result.output + for word in words: + assert word in result.output + + +@pytest.mark.parametrize('profile', ['zimage-bf16', 'qwen3-tts-06b', 'qwen3-asr-06b']) +def test_deploy_status_loads_each_profile_without_creating_root(tmp_path, profile): + root = tmp_path / 'absent' + result = CliRunner().invoke(cli.app, ['vllm-omni', 'deploy', '--root', str(root), + '--profile', profile, 'status']) + assert result.exit_code == 0, result.output + assert json.loads(result.output)['status'] == 'stopped' + assert not root.exists() + + +def test_cli_preserves_structured_failure_and_verification_options(tmp_path, monkeypatch): + runner = CliRunner() + result = runner.invoke(cli.app, ['vllm-omni', 'deploy', '--root', str(tmp_path), + '--profile', 'missing', 'status']) + assert result.exit_code == 1 + assert json.loads(result.output)['code'] == 'unknown_profile' + calls = [] + monkeypatch.setattr(verify, 'verify', lambda *args, **kwargs: calls.append((args, kwargs)) or {'status': 'ok'}) + result = runner.invoke(cli.app, ['vllm-omni', 'verify', '--root', str(tmp_path), + '--profile', 'qwen3-asr-06b', '--url', 'http://localhost:8093/v1', '--infer', + '--audio-file', str(tmp_path / 'speech.wav'), '--expected-text=-Hello world', '--timeout', '30']) + assert result.exit_code == 0, result.output + args, kwargs = calls[0] + assert args[1]['id'] == 'qwen3-asr-06b' + assert kwargs == dict(infer=True, url='http://localhost:8093/v1', timeout=30, + audio_file=tmp_path / 'speech.wav', expected_text='-Hello world') diff --git a/tests/test_vllm_omni_deployment.py b/tests/test_vllm_omni_deployment.py new file mode 100644 index 000000000..f7c8b1d08 --- /dev/null +++ b/tests/test_vllm_omni_deployment.py @@ -0,0 +1,393 @@ +"""Deployment decisions and failure boundaries need neither a GPU nor model downloads.""" +from types import SimpleNamespace + +from PIL import Image +import pytest + +from tools.vllm_omni import deploy, verify + + +@pytest.fixture +def profile(): + return deploy.load_profile('zimage-bf16') + + +def host(profile): + gpu = dict(index=1, uuid='GPU-assigned', name='test GPU', totalMiB=100000, + freeMiB=profile['gpuBudgetMiB'], driver=profile['minDriver'], computeCapability='12.0') + return dict(system='Linux', architecture='x86_64', python='3.12.7', gpus=[gpu], + visibleGpuUuids=[gpu['uuid']], memory={'MemAvailable': 128 * deploy.GIB}, + diskFreeBytes=profile['downloadBytes'] + profile['environmentBudgetGiB'] * deploy.GIB) + + +def test_doctor_is_read_only_before_root_exists(tmp_path, monkeypatch): + root = tmp_path / 'not-created' + monkeypatch.setattr(deploy, 'gpu_facts', lambda: host(deploy.load_profile('zimage-bf16'))['gpus']) + monkeypatch.setattr(deploy, 'capture', lambda *a, **kw: '3.12.7') + monkeypatch.delenv('CUDA_VISIBLE_DEVICES', raising=False) + assert deploy.inspect(root, 'python')['diskFreeBytes'] > 0 + assert not root.exists() + + +def test_download_transport_uses_sdk_defaults_and_operator_settings(tmp_path, monkeypatch): + monkeypatch.delenv('HF_HUB_DISABLE_XET', raising=False) + assert 'HF_HUB_DISABLE_XET' not in deploy.environment(tmp_path) + monkeypatch.setenv('HF_HUB_DISABLE_XET', '1') + assert deploy.environment(tmp_path)['HF_HUB_DISABLE_XET'] == '1' + + +def test_private_cuda_linker_alias_preserves_conflicting_files(tmp_path): + runtime = tmp_path / 'omni-env/lib/python3.12/site-packages/nvidia/cu13/lib/libcudart.so.13' + runtime.parent.mkdir(parents=True) + runtime.touch() + deploy.prepare(tmp_path) + alias = tmp_path / 'cache/cuda-linker/libcudart.so' + assert alias.resolve() == runtime.resolve() + alias.unlink() + alias.write_bytes(b'operator-owned file') + with pytest.raises(deploy.DeploymentError, match='cuda_linker_conflict'): + deploy.prepare(tmp_path) + assert alias.read_bytes() == b'operator-owned file' + + +def test_environment_receipt_digest_matches_the_written_requirements(tmp_path, profile, monkeypatch): + import hashlib + monkeypatch.setattr(deploy, 'require_plan', lambda *a: {'gpu': {'uuid': 'GPU-assigned'}}) + monkeypatch.setattr(deploy, 'checked', lambda *a, **k: None) + monkeypatch.setattr(deploy, 'capture', lambda *a, **k: 'example==1.0') + deploy.install(SimpleNamespace(root=tmp_path, python='python', hf_endpoint=None), profile) + receipt = deploy.read_json(tmp_path / 'environment.json') + requirements = (tmp_path / 'installed-requirements.txt').read_bytes() + assert receipt['requirementsSha256'] == hashlib.sha256(requirements).hexdigest() + + +def test_selection_respects_assigned_devices(profile): + facts = host(profile) + facts['visibleGpuUuids'] = [] + assert deploy.plan(profile, facts, '1')['blockers'] == ['gpu_selection_required'] + with pytest.raises(deploy.DeploymentError, match='gpu_visibility_unresolved'): + deploy.visible_gpus(facts['gpus'], 'GPU-unknown') + assert deploy.visible_gpus(facts['gpus'], '-1') == [] + assert deploy.visible_gpus(facts['gpus'], 'GPU-assign') == facts['gpus'] + + +def test_vram_is_not_summed_across_cards(profile): + facts = host(profile) + facts['gpus'][0]['freeMiB'] = profile['gpuBudgetMiB'] - 1 + facts['gpus'].append(dict(facts['gpus'][0], index=2, uuid='GPU-other', freeMiB=100000)) + assert deploy.plan(profile, facts, '1')['blockers'] == ['insufficient_free_vram'] + + +def test_architecture_is_checked_before_install(profile, tmp_path, monkeypatch): + # The existing FastH3 NVFP4 recipe requires SM120, independent of VRAM. + profile = dict(profile, computeCapabilities=['12.0']) + facts = host(profile) + facts['gpus'][0]['computeCapability'] = '8.6' + monkeypatch.setattr(deploy, 'inspect', lambda *a: facts) + args = SimpleNamespace(root=tmp_path / 'unused', gpu='1', python='python') + with pytest.raises(deploy.DeploymentError) as error: + deploy.install(args, profile) + assert error.value.facts['plan']['blockers'] == ['unsupported_gpu_architecture'] + assert not args.root.exists() + + +@pytest.mark.parametrize('change,expected', [ + (lambda f, p: f.update(diskFreeBytes=0), 'insufficient_disk_space'), + (lambda f, p: f['memory'].update(cgroupAvailable=0), 'insufficient_host_memory'), + (lambda f, p: f['gpus'][0].update(driver='570.0'), 'driver_too_old'), + (lambda f, p: f.update(python='3.10.12'), 'python_312_required'), +]) +def test_missing_resources_produce_actionable_codes(profile, change, expected): + facts = host(profile) + change(facts, profile) + assert deploy.plan(profile, facts, '1')['blockers'] == [expected] + + +def test_eligible_is_distinct_from_qualification(profile): + planned = deploy.plan(profile, host(profile), 'GPU-assigned') + assert planned['status'] == 'eligible' + assert planned['qualification'] == profile['qualification'] + + +def test_image_smoke_request_is_an_output_declared_by_the_application(profile): + from opendirector.domain.input_rules import input_rules + from opendirector.domain.image_output import resolve_image_output + config = dict(kind='image', modelId=profile['modelId'], protocol='openai', connectionType='self_hosted', url='http://localhost:8091/v1') + rules = input_rules(config) + size = profile['verification']['request']['size'] + assert resolve_image_output(config, rules, {'size': size}) == size + + +def test_storage_budget_reuses_cached_weights_and_installed_environment(tmp_path, profile): + profile = dict(profile, downloadBytes=123, files=[dict(path='config.json', size=123)]) + snapshot = tmp_path / 'cache/huggingface/hub' / ('models--' + profile['modelId'].replace('/', '--')) / 'snapshots' / profile['revision'] + snapshot.mkdir(parents=True) + (snapshot / 'config.json').write_bytes(b'x' * 123) + python = tmp_path / 'omni-env/bin/python' + python.parent.mkdir(parents=True) + python.touch() + deploy.write_json(tmp_path / 'environment.json', dict(packages=profile['packages'])) + assert deploy.remaining_disk(profile, tmp_path) == 5 * deploy.GIB + + +def test_range_manifest_opt_in_preserves_fasth3_selection(): + from tools.fasth3.download_models import selected_files + manifest = [dict(repo='FastVideo/FastH3-NVFP4-VAE-Decoder', sha='pinned', files=[ + dict(path='fast-8L/config.json', size=1), dict(path='model.safetensors', size=2)])] + assert [item['path'] for _, item in selected_files(manifest)] == ['fast-8L/config.json'] + assert [item['path'] for _, item in selected_files(manifest, all_files=True)] == ['fast-8L/config.json', 'model.safetensors'] + manifest[0]['files'] = [dict(path='../outside', size=1)] + with pytest.raises(ValueError, match='Unsafe manifest path'): + list(selected_files(manifest, all_files=True)) + + +def test_wrong_model_is_not_readiness(monkeypatch): + calls = [] + def respond(url): + calls.append(url) + return b'{}' if url.endswith('/health') else b'{"data":[{"id":"another-model"}]}' + monkeypatch.setattr(deploy, 'request_json', respond) + with pytest.raises(deploy.DeploymentError, match='served_model_mismatch'): + deploy.readiness('http://127.0.0.1:8091/v1', 'expected') + assert calls == ['http://127.0.0.1:8091/health', 'http://127.0.0.1:8091/v1/models'] + + +def test_recycled_pid_never_stops_another_process(tmp_path, profile, monkeypatch): + state = tmp_path / 'services' / profile['id'] / 'state.json' + deploy.write_json(state, dict(pid=123, processStart='original')) + monkeypatch.setattr(deploy, 'process_identity', lambda pid: 'recycled') + monkeypatch.setattr(deploy.os, 'killpg', lambda *a: pytest.fail('unrelated process was signalled')) + assert deploy.stop(SimpleNamespace(root=tmp_path), profile)['status'] == 'stopped' + + +def test_repeat_start_does_not_spawn_while_original_is_starting(tmp_path, profile, monkeypatch): + record = dict(pid=123, processStart='same', profileSha256=deploy.profile_digest(profile), + url='http://127.0.0.1:8091/v1', modelId=profile['modelId'], gpuUuid='GPU-assigned') + deploy.write_json(tmp_path / 'services' / profile['id'] / 'state.json', record) + monkeypatch.setattr(deploy, 'owned', lambda _: True) + monkeypatch.setattr(deploy, 'gpu_facts', lambda: host(profile)['gpus']) + monkeypatch.delenv('CUDA_VISIBLE_DEVICES', raising=False) + monkeypatch.setattr(deploy, 'readiness', lambda *a: (_ for _ in ()).throw(deploy.DeploymentError('service_unreachable'))) + monkeypatch.setattr(deploy.subprocess, 'Popen', lambda *a, **k: pytest.fail('duplicate server')) + with pytest.raises(deploy.DeploymentError, match='startup_pending'): + deploy.start(SimpleNamespace(root=tmp_path, port=8091, gpu='GPU-assigned'), profile) + + +@pytest.mark.parametrize('selector,mask,expected', [ + ('1', None, 'ready'), + ('GPU-assigned', None, 'ready'), + ('2', None, 'running_configuration_conflict'), + ('GPU-other', None, 'running_configuration_conflict'), + ('GPU-assigned', '2', 'running_configuration_conflict'), + ('', None, 'running_configuration_conflict'), +]) +def test_repeat_start_respects_selected_and_visible_gpu(tmp_path, profile, monkeypatch, selector, mask, expected): + record = dict(pid=123, processStart='same', profileSha256=deploy.profile_digest(profile), + url='http://127.0.0.1:8091/v1', modelId=profile['modelId'], gpuUuid='GPU-assigned') + deploy.write_json(tmp_path / 'services' / profile['id'] / 'state.json', record) + assigned = host(profile)['gpus'][0] + monkeypatch.setattr(deploy, 'gpu_facts', lambda: [assigned, dict(assigned, index=2, uuid='GPU-other')]) + monkeypatch.setattr(deploy, 'owned', lambda _: True) + monkeypatch.setattr(deploy, 'status', lambda *args: {'status': 'ready', **record}) + monkeypatch.setattr(deploy.subprocess, 'Popen', lambda *a, **k: pytest.fail('duplicate server')) + if mask is None: + monkeypatch.delenv('CUDA_VISIBLE_DEVICES', raising=False) + else: + monkeypatch.setenv('CUDA_VISIBLE_DEVICES', mask) + args = SimpleNamespace(root=tmp_path, port=8091, gpu=selector) + if expected == 'ready': + assert deploy.start(args, profile)['gpuUuid'] == record['gpuUuid'] + else: + with pytest.raises(deploy.DeploymentError, match=expected): + deploy.start(args, profile) + + +def test_unknown_generation_is_recorded_without_retry(tmp_path, profile, monkeypatch): + calls = [] + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + def timeout(*a): + calls.append(a) + raise deploy.DeploymentError('service_unreachable') + monkeypatch.setattr(verify, 'request_json', timeout) + with pytest.raises(deploy.DeploymentError, match='inference_result_unknown') as error: + verify.verify(tmp_path, profile, infer=True, url='http://127.0.0.1:8091/v1') + assert len(calls) == 1 + record = deploy.read_json(error.value.facts['receipt']) + assert record['status'] == 'submission_unknown' + assert record['request']['model'] == profile['modelId'] + + +def test_health_probe_does_not_generate_or_write(tmp_path, profile, monkeypatch): + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + monkeypatch.setattr(verify, 'request_json', lambda *a: pytest.fail('generation during discovery')) + assert verify.verify(tmp_path, profile, url='http://127.0.0.1:8091/v1')['status'] == 'service_ready' + assert not list(tmp_path.iterdir()) + + +def test_verification_before_start_is_a_structured_failure(tmp_path, profile): + with pytest.raises(deploy.DeploymentError, match='service_not_started') as error: + verify.verify(tmp_path, profile) + assert error.value.facts['profile'] == profile['id'] + assert not list(tmp_path.iterdir()) + + +def test_real_png_decoding_rejects_corruption(tmp_path): + import sys + path = tmp_path / 'image.png' + Image.new('RGB', (32, 24), 'red').save(path) + assert verify.media_facts(path, 'image', sys.executable)['width'] == 32 + path.write_bytes(path.read_bytes()[:35]) + with pytest.raises(deploy.DeploymentError, match='invalid_image_output'): + verify.media_facts(path, 'image', sys.executable) + + +def test_png_with_valid_checksums_still_requires_decodable_pixels(tmp_path): + import struct + import sys + import zlib + path = tmp_path / 'image.png' + Image.new('RGB', (32, 24), 'red').save(path) + source = path.read_bytes() + chunks, offset = [source[:8]], 8 + while offset < len(source): + size = struct.unpack('>I', source[offset:offset + 4])[0] + tag = source[offset + 4:offset + 8] + body = source[offset + 8:offset + 8 + size] + if tag == b'IDAT': + body = b'not a zlib stream' + chunks.append(struct.pack('>I', len(body)) + tag + body + struct.pack('>I', zlib.crc32(tag + body))) + offset += size + 12 + path.write_bytes(b''.join(chunks)) + with Image.open(path) as image: + image.verify() # Header/chunk validation alone accepts this invalid PNG. + with pytest.raises(deploy.DeploymentError, match='invalid_image_output'): + verify.media_facts(path, 'image', sys.executable) + + +@pytest.mark.parametrize('name,omni', [('qwen3-asr-06b', False), ('zimage-bf16', True)]) +def test_start_uses_the_models_upstream_engine(tmp_path, name, omni, monkeypatch): + profile = deploy.load_profile(name) + deploy.write_json(tmp_path / 'models' / (name + '.json'), + dict(profileSha256=deploy.profile_digest(profile), path='/verified-snapshot')) + monkeypatch.setattr(deploy, 'require_plan', lambda *a: {'gpu': {'uuid': 'GPU-assigned'}}) + monkeypatch.setattr(deploy, 'process_identity', lambda *a: 'owned') + commands = [] + def spawn(command, **kwargs): + commands.append(command) + return SimpleNamespace(pid=123) + monkeypatch.setattr(deploy.subprocess, 'Popen', spawn) + with pytest.raises(deploy.DeploymentError, match='startup_pending'): + deploy.start(SimpleNamespace(root=tmp_path, port=0, hf_endpoint=None, timeout=0), profile) + assert ('--omni' in commands[0]) is omni + assert commands[0][1:3] == ['serve', '/verified-snapshot'] + + +def test_asr_health_does_not_require_audio_or_write(tmp_path, monkeypatch): + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + profile = deploy.load_profile('qwen3-asr-06b') + assert verify.verify(tmp_path, profile, url='http://localhost/v1')['status'] == 'service_ready' + with pytest.raises(deploy.DeploymentError, match='audio_input_required'): + verify.verify(tmp_path, profile, url='http://localhost/v1', infer=True) + assert not list(tmp_path.iterdir()) + + +@pytest.mark.parametrize('output', [b'{}', b'{"text":" "}', b'{"text":[]}', b'not json']) +def test_asr_rejects_invalid_transcripts(tmp_path, output, monkeypatch): + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + monkeypatch.setattr(verify, 'media_facts', lambda *a: {'durationSeconds': 1}) + monkeypatch.setattr(verify, 'request_transcription', lambda *a: output) + source = tmp_path / 'source.mp3' + source.write_bytes(b'audio') + with pytest.raises(deploy.DeploymentError, match='invalid_provider_output') as error: + verify.verify(tmp_path, deploy.load_profile('qwen3-asr-06b'), url='http://localhost/v1', + infer=True, audio_file=source) + assert deploy.read_json(error.value.facts['receipt'])['status'] == 'failed' + + +def test_asr_uploads_exact_snapshot_and_keeps_reference_out_of_request(tmp_path, monkeypatch): + import hashlib + import json + from email.parser import BytesParser + from email.policy import default + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + monkeypatch.setattr(verify, 'media_facts', lambda *a: {'durationSeconds': 1}) + source = tmp_path / 'source.MP3' + source.write_bytes(b'exact audio\x00\xff') + calls = [] + def request(url, body, content_type, timeout): + message = BytesParser(policy=default).parsebytes(('Content-Type: ' + content_type + '\r\n\r\n').encode() + body) + fields = {part.get_param('name', header='content-disposition'): part.get_payload(decode=True) + for part in message.iter_parts()} + calls.append(fields) + return json.dumps({'text': '你好,Hello!'}).encode() + monkeypatch.setattr(verify, 'request_bytes', request) + result = verify.verify(tmp_path, deploy.load_profile('qwen3-asr-06b'), url='http://localhost/v1', + infer=True, audio_file=source, expected_text='你好 hello') + assert len(calls) == 1 + assert calls[0]['file'] == source.read_bytes() + assert calls[0]['model'] == b'Qwen/Qwen3-ASR-0.6B' + assert set(calls[0]) == {'file', 'model', 'response_format', 'temperature'} + assert result['input']['artifactSha256'] == hashlib.sha256(source.read_bytes()).hexdigest() + assert result['textComparison']['characterErrorRate'] == 0 + assert result['creativeReview'] == 'pending' + + +def test_asr_unknown_submission_is_not_retried(tmp_path, monkeypatch): + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + monkeypatch.setattr(verify, 'media_facts', lambda *a: {'durationSeconds': 1}) + source = tmp_path / 'source.wav' + source.write_bytes(b'audio') + calls = [] + def timeout(*args): + calls.append(args) + raise deploy.DeploymentError('service_unreachable') + monkeypatch.setattr(verify, 'request_transcription', timeout) + with pytest.raises(deploy.DeploymentError, match='inference_result_unknown') as error: + verify.verify(tmp_path, deploy.load_profile('qwen3-asr-06b'), url='http://localhost/v1', + infer=True, audio_file=source) + assert len(calls) == 1 + assert deploy.read_json(error.value.facts['receipt'])['status'] == 'submission_unknown' + + +def test_invalid_asr_audio_fails_before_sampling_or_submission(tmp_path, monkeypatch): + profile = deploy.load_profile('qwen3-asr-06b') + deploy.write_json(tmp_path / 'services' / profile['id'] / 'state.json', dict(url='http://localhost/v1', gpuUuid='GPU-assigned')) + source = tmp_path / 'source.wav' + source.write_bytes(b'corrupt') + monkeypatch.setattr(verify, 'readiness', lambda *a: None) + def reject(*args): + raise deploy.DeploymentError('invalid_audio_output') + monkeypatch.setattr(verify, 'media_facts', reject) + monkeypatch.setattr(verify, 'request_transcription', lambda *a: pytest.fail('submitted invalid input')) + with pytest.raises(deploy.DeploymentError, match='invalid_audio_output') as error: + verify.verify(tmp_path, profile, infer=True, audio_file=source) + assert deploy.read_json(error.value.facts['receipt'])['status'] == 'failed' + + +@pytest.mark.parametrize('actual,expected,distance', [('abc', 'ab', 1), ('ab', 'abc', 1), ('axc', 'abc', 1), ('你好 HELLO!', '你好,hello', 0)]) +def test_transcript_comparison_reports_insertions_deletions_and_substitutions(actual, expected, distance): + assert verify.compare_text(actual, expected)['editDistance'] == distance + + +@pytest.mark.parametrize('live_listener', [False, True]) +def test_port_preflight_allows_time_wait_but_preserves_live_listener(tmp_path, profile, monkeypatch, live_listener): + import socket + with socket.socket() as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen() + if not live_listener: + with socket.create_connection(('127.0.0.1', port)) as client: + accepted, _ = listener.accept() + accepted.close() # Active close puts the server address in TIME_WAIT. + assert client.recv(1) == b'' + listener.close() + deploy.write_json(tmp_path / 'models' / (profile['id'] + '.json'), + dict(profileSha256=deploy.profile_digest(profile), path='/verified-snapshot')) + monkeypatch.setattr(deploy, 'require_plan', lambda *a: {'gpu': {'uuid': 'GPU-assigned'}}) + monkeypatch.setattr(deploy, 'process_identity', lambda *a: 'owned') + monkeypatch.setattr(deploy.subprocess, 'Popen', lambda *a, **k: SimpleNamespace(pid=123)) + expected = 'port_in_use' if live_listener else 'startup_pending' + with pytest.raises(deploy.DeploymentError, match=expected): + deploy.start(SimpleNamespace(root=tmp_path, port=port, hf_endpoint=None, timeout=0), profile) diff --git a/tools/fasth3/__init__.py b/tools/fasth3/__init__.py new file mode 100644 index 000000000..c1a5396fc --- /dev/null +++ b/tools/fasth3/__init__.py @@ -0,0 +1 @@ +"""Shared pinned range downloader for local model deployment.""" diff --git a/tools/fasth3/download_models.py b/tools/fasth3/download_models.py index 3ffc9e372..00b236469 100644 --- a/tools/fasth3/download_models.py +++ b/tools/fasth3/download_models.py @@ -14,12 +14,14 @@ CHUNK = 4 * 1024 * 1024 -def selected_files(manifest): +def selected_files(manifest, *, all_files=False): native=any(info['repo']=='MiniMaxAI/MiniMax-H3' for info in manifest) for info in manifest: for item in info['files']: name = item['path'] - if info['repo']=='MiniMaxAI/MiniMax-H3': + if all_files: + selected=True + elif info['repo']=='MiniMaxAI/MiniMax-H3': selected=name.startswith(('Ref2VA/video_vae/','Ref2VA/audio_vae/')) elif 'QAD' in info['repo']: selected=name.startswith(PREFIXES) or '/' not in name and name.endswith('.json') @@ -50,12 +52,13 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--root', type=Path, default=Path('/root/autodl-tmp/fasth3/models')) parser.add_argument('--manifest', type=Path, default=Path(__file__).with_name('model-manifest.json')) + parser.add_argument('--all-files', action='store_true', help='Select every file in an explicitly supplied pinned manifest instead of the FastH3 component subset') parser.add_argument('--plan', action='store_true') parser.add_argument('--progress', action='store_true', help='Show completed bytes from local resume records, without downloading or hashing') parser.add_argument('--verify-only', action='store_true') parser.add_argument('--exclude-path', action='append', default=[], help='Download only: exclude a path relative to model root') args = parser.parse_args() - entries = list(selected_files(json.loads(args.manifest.read_text(encoding='utf-8-sig')))) + entries = list(selected_files(json.loads(args.manifest.read_text(encoding='utf-8-sig')), all_files=args.all_files)) if args.exclude_path: if args.verify_only: parser.error('Verification must include the full pipeline') diff --git a/tools/vllm_omni/__init__.py b/tools/vllm_omni/__init__.py new file mode 100644 index 000000000..f3a07928f --- /dev/null +++ b/tools/vllm_omni/__init__.py @@ -0,0 +1 @@ +"""Optional host deployment tooling; independent of application runtime dependencies.""" diff --git a/tools/vllm_omni/deploy.py b/tools/vllm_omni/deploy.py new file mode 100644 index 000000000..e967b3e20 --- /dev/null +++ b/tools/vllm_omni/deploy.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""Hardware-aware, local vLLM-Omni deployment. JSON codes are localized by the calling Agent. + +Run with Python 3.12. doctor/plan/status do not create directories or call a model. +Installation uses uv; downloads reuse huggingface_hub or the existing pinned range +downloader. Project changes belong to OpenDirector's existing API/CLI. +""" +import argparse +from contextlib import contextmanager +import csv +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import shutil +import signal +import socket +import subprocess +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.request import ProxyHandler, Request, build_opener +from uuid import uuid4 + + +HERE = Path(__file__).resolve().parent +GIB = 1024 ** 3 +# Keep the compiler/CRT/NVVM aligned with the cu130 runtime selected below. +CUDA_COMPILER_PACKAGES = ['nvidia-cuda-nvcc==13.0.88', 'nvidia-cuda-crt==13.0.88', 'nvidia-nvvm==13.0.88'] + + +class DeploymentError(Exception): + def __init__(self, code, **facts): + self.code, self.facts = code, facts + super().__init__(code) + + +def read_json(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + '.' + str(uuid4()) + '.tmp') + try: + with temporary.open('x', encoding='utf-8') as out: + json.dump(value, out, ensure_ascii=False, indent=2) + out.flush() + os.fsync(out.fileno()) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def capture(command, *, env=None, timeout=20): + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, env=env) + return result.stdout.strip() if result.returncode == 0 else None + except (OSError, subprocess.TimeoutExpired): + return None + + +def version(value): + return tuple(int(part) for part in value.split('.')) + + +def gpu_facts(): + fields = 'index,name,uuid,memory.total,memory.free,driver_version,compute_cap' + output = capture(['nvidia-smi', '--query-gpu=' + fields, '--format=csv,noheader,nounits']) + if output is None: + raise DeploymentError('gpu_probe_failed') + result = [] + try: + for row in csv.reader(io.StringIO(output), skipinitialspace=True): + index, name, identity, total, free, driver, capability = (field.strip() for field in row) + result.append(dict(index=int(index), name=name, uuid=identity, totalMiB=int(total), + freeMiB=int(free), driver=driver, computeCapability=capability)) + except (ValueError, TypeError): + raise DeploymentError('gpu_probe_incomplete') from None + if not result: + raise DeploymentError('gpu_probe_failed') + return result + + +def visible_gpus(gpus, mask): + if mask is None: + return gpus + tokens = mask.split(',') if mask.strip() and mask != '-1' else [] + selected = [] + for token in tokens: + matches = [gpu for gpu in gpus if token.strip() == str(gpu['index']) or + (token.strip().startswith('GPU-') and gpu['uuid'].startswith(token.strip()))] + if len(matches) != 1: + raise DeploymentError('gpu_visibility_unresolved') + if matches[0] not in selected: + selected.append(matches[0]) + return selected + + +def disk_free(path): + path = Path(path).resolve() + while not path.exists(): + path = path.parent + return shutil.disk_usage(path).free + + +def memory_facts(): + values = {} + try: + for line in Path('/proc/meminfo').read_text().splitlines(): + key, value = line.split(':', 1) + if key in {'MemTotal', 'MemAvailable'}: + values[key] = int(value.split()[0]) * 1024 + limit = Path('/sys/fs/cgroup/memory.max') + current = Path('/sys/fs/cgroup/memory.current') + if limit.is_file() and limit.read_text().strip() != 'max': + values['cgroupAvailable'] = max(0, int(limit.read_text()) - int(current.read_text())) + except (OSError, ValueError): + pass + return values + + +def inspect(root, python): + gpus = gpu_facts() + return dict(schemaVersion=1, root=str(Path(root).resolve()), system=platform.system(), architecture=platform.machine(), + python=capture([python, '-c', 'import platform; print(platform.python_version())']), + gpus=gpus, visibleGpuUuids=[gpu['uuid'] for gpu in visible_gpus(gpus, os.environ.get('CUDA_VISIBLE_DEVICES'))], + memory=memory_facts(), diskFreeBytes=disk_free(root), + sharedMemoryFreeBytes=disk_free('/dev/shm'), + tools={name: shutil.which(name) for name in ('ffmpeg', 'ffprobe', 'nvcc')}) + + +def load_profile(name): + if name not in {path.stem for path in (HERE / 'profiles').glob('*.json')}: + raise DeploymentError('unknown_profile', profile=name) + return read_json(HERE / 'profiles' / (name + '.json')) + + +def profile_digest(profile): + return hashlib.sha256(json.dumps(profile, sort_keys=True).encode()).hexdigest() + + +def remaining_disk(profile, root=None): + remaining = profile['downloadBytes'] + environment_budget = profile['environmentBudgetGiB'] * GIB + if root is not None: + root = Path(root) + snapshot = root / 'cache/huggingface/hub' / ('models--' + profile['modelId'].replace('/', '--')) / 'snapshots' / profile['revision'] + model = root / 'models' / (profile['id'] + '.json') + if model.is_file(): + record = read_json(model) + if record.get('profileSha256') == profile_digest(profile): + snapshot = Path(record['path']) + for item in profile['files']: + path = snapshot / item['path'] + if path.is_file() and path.stat().st_size == item['size']: + remaining -= item['size'] + record = root / 'environment.json' + if record.is_file() and (root / 'omni-env/bin/python').is_file(): + installed = read_json(record) + if installed.get('packages') == profile['packages']: + # Keep room for outputs and temporary files even when the exact + # recorded package combination and cached model already exist. + environment_budget = 5 * GIB + return remaining + environment_budget + + +def plan(profile, facts, selector): + checks = [] + def require(ok, code): + if not ok: + checks.append(code) + require(facts['system'] == 'Linux' and facts['architecture'] == 'x86_64', 'unsupported_platform') + require(bool(facts['python']) and facts['python'].startswith('3.12.'), 'python_312_required') + candidates = [gpu for gpu in facts['gpus'] if gpu['uuid'] in facts['visibleGpuUuids'] and + selector in {str(gpu['index']), gpu['uuid']}] + gpu = candidates[0] if len(candidates) == 1 else None + require(gpu is not None, 'gpu_selection_required') + if gpu: + try: + require(version(gpu['computeCapability']) >= version(profile['minComputeCapability']), 'unsupported_gpu_architecture') + require(version(gpu['driver']) >= version(profile['minDriver']), 'driver_too_old') + except (ValueError, TypeError): + checks.append('gpu_probe_incomplete') + if profile.get('computeCapabilities'): + require(gpu['computeCapability'] in profile['computeCapabilities'], 'unsupported_gpu_architecture') + require(gpu['freeMiB'] >= profile['gpuBudgetMiB'], 'insufficient_free_vram') + required_disk = remaining_disk(profile, facts.get('root')) + require(facts['diskFreeBytes'] >= required_disk, 'insufficient_disk_space') + available_memory = facts['memory'].get('MemAvailable') + if 'cgroupAvailable' in facts['memory']: + available_memory = min(available_memory, facts['memory']['cgroupAvailable']) if available_memory is not None else facts['memory']['cgroupAvailable'] + require(available_memory is not None and available_memory >= profile['hostMemoryBudgetGiB'] * GIB, 'insufficient_host_memory') + return dict(schemaVersion=1, status='blocked' if checks else 'eligible', blockers=checks, + profile=profile['id'], profileSha256=profile_digest(profile), gpu=gpu, + requiredDiskBytes=required_disk, qualification=profile['qualification'], facts=facts) + + +def environment(root, gpu=None, hf_endpoint=None): + result = dict(os.environ) + result.pop('PYTHONPATH', None) + result.update(HF_HOME=str(root / 'cache/huggingface'), + PIP_CACHE_DIR=str(root / 'cache/pip'), + UV_CACHE_DIR=str(root / 'cache/uv'), TMPDIR=str(root / 'cache/tmp'), + VLLM_CACHE_ROOT=str(root / 'cache/vllm'), + TRITON_CACHE_DIR=str(root / 'cache/triton'), + TORCHINDUCTOR_CACHE_DIR=str(root / 'cache/torchinductor'), + FLASHINFER_WORKSPACE_BASE=str(root / 'cache/flashinfer'), + PYTHONNOUSERSITE='1', VLLM_WORKER_MULTIPROC_METHOD='spawn', + VIRTUAL_ENV=str(root / 'omni-env'), + PATH=str(root / 'omni-env/bin') + os.pathsep + result.get('PATH', '')) + toolkit = root / 'omni-env/lib/python3.12/site-packages/nvidia/cu13' + if not result.get('CUDA_HOME') and not result.get('CUDA_PATH') and (toolkit / 'bin/nvcc').is_file(): + result['CUDA_HOME'] = str(toolkit) + result['PATH'] = str(toolkit / 'bin') + os.pathsep + result['PATH'] + result['LIBRARY_PATH'] = str(root / 'cache/cuda-linker') + os.pathsep + result.get('LIBRARY_PATH', '') + if hf_endpoint: + result['HF_ENDPOINT'] = hf_endpoint + if gpu: + result['CUDA_VISIBLE_DEVICES'] = gpu['uuid'] + return result + + +def checked(command, *, env): + result = subprocess.run(command, env=env, stdout=sys.stderr, stderr=sys.stderr) + if result.returncode: + raise DeploymentError('command_failed', executable=Path(command[0]).name, exitCode=result.returncode) + + +@contextmanager +def exclusive(path): + import fcntl + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise DeploymentError('deployment_busy') from None + yield + + +def prepare(root): + (root / 'cache/tmp').mkdir(parents=True, exist_ok=True) + # The runtime wheel provides the versioned library, while JIT linkers ask + # for libcudart.so. Keep this development alias outside installed packages. + runtime = root / 'omni-env/lib/python3.12/site-packages/nvidia/cu13/lib/libcudart.so.13' + if runtime.is_file(): + alias = root / 'cache/cuda-linker/libcudart.so' + alias.parent.mkdir(parents=True, exist_ok=True) + if alias.exists() or alias.is_symlink(): + if alias.resolve() != runtime.resolve(): + raise DeploymentError('cuda_linker_conflict', path=str(alias)) + else: + alias.symlink_to(runtime) + + +def require_plan(args, profile): + value = plan(profile, inspect(args.root, args.python), args.gpu) + if value['blockers']: + raise DeploymentError('preflight_failed', plan=value) + return value + + +def install(args, profile): + require_plan(args, profile) + prepare(args.root) + env = environment(args.root, hf_endpoint=args.hf_endpoint) + python = str(args.root / 'omni-env/bin/python') + with exclusive(args.root / 'environment.lock'): + if not Path(python).exists(): + checked([args.python, '-m', 'venv', str(args.root / 'omni-env')], env=env) + else: + existing_version = capture([python, '-c', 'import sys; print(sys.version.split()[0])']) + if not existing_version or not existing_version.startswith('3.12.'): + raise DeploymentError('existing_environment_invalid') + checked([python, '-m', 'pip', 'install', 'uv==0.13.7'], env=env) + uv = str(args.root / 'omni-env/bin/uv') + checked([uv, 'pip', 'install', '--python', python, '--torch-backend', 'cu130', + *profile['packages'], *CUDA_COMPILER_PACKAGES], env=env) + prepare(args.root) + checked([uv, 'pip', 'check', '--python', python], env=env) + cuda_env = environment(args.root, require_plan(args, profile)['gpu'], args.hf_endpoint) + checked([python, '-c', 'import torch; assert torch.cuda.is_available(); ' + 'assert torch.cuda.device_count() == 1; ' + 'assert (torch.ones(1, device="cuda") + 1).item() == 2; ' + 'import flashinfer; x=torch.arange(1024,device="cuda",dtype=torch.float32).repeat(2,1); ' + 'assert torch.isfinite(flashinfer.top_k_mask_logits(x,50)).sum().item()==100'], env=cuda_env) + frozen = capture([uv, 'pip', 'freeze', '--python', python], env=env) + if frozen is None: + raise DeploymentError('environment_inspection_failed') + frozen += '\n' + (args.root / 'installed-requirements.txt').write_text(frozen, encoding='utf-8') + write_json(args.root / 'environment.json', dict(packages=profile['packages'], + requirementsSha256=hashlib.sha256(frozen.encode()).hexdigest())) + return dict(status='installed', environment=str(args.root / 'omni-env')) + + +def download(args, profile): + require_plan(args, profile) + prepare(args.root) + env = environment(args.root, hf_endpoint=args.hf_endpoint) + if args.download_backend == 'ranges': + if __package__ and __package__.startswith('opendirector.'): + from opendirector._fasth3 import download_models + downloader = Path(download_models.__file__).resolve() + else: + downloader = HERE.parent / 'fasth3/download_models.py' + if not downloader.is_file(): + raise DeploymentError('ranged_downloader_unavailable') + directory = args.root / 'range-models' / profile['id'] + manifest = [dict(repo=profile['modelId'], sha=profile['revision'], files=[ + dict(path=item['path'], size=item['size'], + **({'lfs': {'oid': item['sha256']}} if item.get('sha256') else {})) + for item in profile['files']])] + manifest_path = args.root / 'manifests' / (profile['id'] + '.json') + write_json(manifest_path, manifest) + env.update(DOWNLOAD_ENDPOINT=args.hf_endpoint or env.get('HF_ENDPOINT', 'https://huggingface.co'), + DOWNLOAD_USE_PROXY='1') + checked([str(args.root / 'omni-env/bin/python'), str(downloader), '--all-files', + '--manifest', str(manifest_path), '--root', str(directory)], env=env) + snapshot = directory / profile['modelId'].split('/')[-1] + return register_snapshot(args.root, profile, snapshot) + # The SDK owns caching, locking, partial downloads and revision resolution. + script = '''import json,sys +from huggingface_hub import snapshot_download +p=json.loads(sys.argv[1]) +try: + path=snapshot_download(p['modelId'],revision=p['revision'],allow_patterns=p['allowPatterns'],max_workers=4) + print(json.dumps({'path':path})) +except Exception as error: + print(json.dumps({'errorType':type(error).__name__,'httpStatus':getattr(getattr(error,'response',None),'status_code',None)})) + sys.exit(1) +''' + output = subprocess.run([str(args.root / 'omni-env/bin/python'), '-c', script, json.dumps(profile)], + env=env, capture_output=True, text=True) + if output.returncode: + # Do not put signed download URLs or raw provider errors in receipts. + try: + facts = json.loads(output.stdout) + except ValueError: + facts = {} + raise DeploymentError('model_download_failed', **facts) + snapshot = Path(json.loads(output.stdout)['path']) + return register_snapshot(args.root, profile, snapshot) + + +def register_snapshot(root, profile, snapshot): + for item in profile['files']: + path = snapshot / item['path'] + if not path.is_file() or path.stat().st_size != item['size']: + raise DeploymentError('model_file_invalid', file=item['path']) + if item.get('sha256'): + with path.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if digest != item['sha256']: + raise DeploymentError('model_file_invalid', file=item['path']) + record = dict(profileSha256=profile_digest(profile), revision=profile['revision'], path=str(snapshot)) + write_json(root / 'models' / (profile['id'] + '.json'), record) + return dict(status='download_verified', **record) + + +def process_identity(pid): + try: + path = Path('/proc') / str(pid) + fields = (path / 'stat').read_text().rsplit(') ', 1)[1].split() + if path.stat().st_uid != os.getuid() or fields[0] == 'Z': + return None + return fields[19] + except (OSError, IndexError): + return None + + +def owned(record): + return bool(record and process_identity(record['pid']) == record['processStart'] and + record.get('processStart') is not None) + + +def request_json(url, payload=None, timeout=10): + return request_bytes(url, json.dumps(payload).encode() if payload is not None else None, + 'application/json', timeout) + + +def request_bytes(url, data=None, content_type='application/json', timeout=10): + # These are explicit local service probes. Never inherit a proxy for loopback. + request = Request(url, data=data, headers={'Content-Type': content_type}) + try: + with build_opener(ProxyHandler({})).open(request, timeout=timeout) as response: + data = response.read(128 * 1024 ** 2 + 1) + if len(data) > 128 * 1024 ** 2: + raise DeploymentError('response_too_large') + return data + except HTTPError as error: + raise DeploymentError('service_http_error', httpStatus=error.code) from None + except (URLError, TimeoutError, OSError): + raise DeploymentError('service_unreachable') from None + + +def readiness(url, model): + request_json(url.removesuffix('/v1') + '/health') + try: + data = json.loads(request_json(url + '/models')) + models = data['data'] + if not isinstance(models, list) or any(not isinstance(item, dict) for item in models): + raise ValueError() + except (ValueError, TypeError, KeyError): + raise DeploymentError('service_invalid_response') from None + if model not in [item.get('id') for item in models]: + raise DeploymentError('served_model_mismatch', modelId=model) + + +def deployment_config(root, profile): + """Copy the installed upstream pipeline, changing only declared resource budgets.""" + if not profile.get('deployConfig'): + return [] + target = root / 'services' / profile['id'] / 'deploy.yaml' + script = '''import importlib.resources,json,pathlib,sys,yaml +p=json.loads(sys.argv[1]) +source=importlib.resources.files('vllm_omni')/p['deployConfig'] +config=yaml.safe_load(source.read_text()) +for stage in config['stages']: + stage.update(p.get('stageOverrides',{})) +pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump(config,sort_keys=False)) +''' + checked([str(root / 'omni-env/bin/python'), '-c', script, json.dumps(profile), str(target)], + env=environment(root)) + return ['--deploy-config', str(target)] + + +def start(args, profile): + service = args.root / 'services' / profile['id'] + with exclusive(service / 'operation.lock'): + state_path = service / 'state.json' + record = read_json(state_path) if state_path.exists() else None + if owned(record): + if (record['profileSha256'] != profile_digest(profile) or + record['url'] != f'http://127.0.0.1:{args.port}/v1'): + raise DeploymentError('running_configuration_conflict') + visible = visible_gpus(gpu_facts(), os.environ.get('CUDA_VISIBLE_DEVICES')) + if not any(gpu['uuid'] == record['gpuUuid'] and args.gpu in {str(gpu['index']), gpu['uuid']} + for gpu in visible): + raise DeploymentError('running_configuration_conflict') + existing = status(args, profile) + if existing['status'] != 'ready': + raise DeploymentError('startup_pending', **record) + return existing + preflight = require_plan(args, profile) + model_path = args.root / 'models' / (profile['id'] + '.json') + if not model_path.is_file(): + raise DeploymentError('model_download_required') + model = read_json(model_path) + if model['profileSha256'] != profile_digest(profile): + raise DeploymentError('profile_changed_verify_download') + # Binding is also attempted by the child; a race cannot turn somebody else's + # listener into successful startup because the owned child must remain alive. + with socket.socket() as probe: + # Match the HTTP server: TIME_WAIT after an owned shutdown is not a listener. + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(('127.0.0.1', args.port)) + except OSError: + raise DeploymentError('port_in_use', port=args.port) from None + env = environment(args.root, preflight['gpu'], args.hf_endpoint) + env.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1') + engine_args = [] if profile.get('engine') == 'vllm' else ['--omni'] + command = [str(args.root / 'omni-env/bin/vllm'), 'serve', model['path'], *engine_args, + '--served-model-name', profile['modelId'], '--host', '127.0.0.1', '--port', str(args.port), + *deployment_config(args.root, profile), *profile.get('serveArgs', [])] + prepare(args.root) + with (service / 'server.log').open('ab') as log: + child = subprocess.Popen(command, env=env, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + record = dict(pid=child.pid, processStart=process_identity(child.pid), gpuUuid=preflight['gpu']['uuid'], + profileSha256=profile_digest(profile), url=f'http://127.0.0.1:{args.port}/v1', + modelId=profile['modelId'], log=str(service / 'server.log')) + write_json(state_path, record) + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + if child.poll() is not None: + raise DeploymentError('server_exited', exitCode=child.returncode, log=record['log']) + try: + readiness(record['url'], profile['modelId']) + if child.poll() is None: + return dict(status='ready', **record) + except DeploymentError: + pass + time.sleep(1) + raise DeploymentError('startup_pending', **record) + + +def status(args, profile): + path = args.root / 'services' / profile['id'] / 'state.json' + record = read_json(path) if path.is_file() else None + if not owned(record): + return dict(status='stopped', scope='model_service') + try: + readiness(record['url'], record['modelId']) + return dict(status='ready', scope='model_service', **record) + except DeploymentError as error: + return dict(status='starting_or_unhealthy', scope='model_service', reason=error.code, **record) + + +def stop(args, profile): + service = args.root / 'services' / profile['id'] + if not service.exists(): + return dict(status='stopped') + with exclusive(service / 'operation.lock'): + path = service / 'state.json' + record = read_json(path) if path.is_file() else None + if not owned(record): + return dict(status='stopped') + if os.getpgid(record['pid']) != record['pid']: + raise DeploymentError('process_ownership_unresolved') + os.killpg(record['pid'], signal.SIGTERM) + deadline = time.monotonic() + 30 + while owned(record) and time.monotonic() < deadline: + time.sleep(0.2) + if owned(record): + raise DeploymentError('shutdown_pending', pid=record['pid']) + return dict(status='stopped') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--profile', default='zimage-bf16') + parser.add_argument('--gpu', default='', help='Explicit physical index or full GPU UUID within the assigned CUDA visibility.') + parser.add_argument('--python', default=sys.executable, help='Python 3.12 interpreter for the isolated model environment.') + parser.add_argument('--hf-endpoint', help='Optional user-selected Hugging Face mirror, scoped to this deployment.') + parser.add_argument('--download-backend', choices=['sdk', 'ranges'], default='sdk', + help='Use the Hugging Face SDK or the existing resumable pinned range downloader.') + parser.add_argument('--port', type=int, default=8091) + parser.add_argument('--timeout', type=int, default=600) + parser.add_argument('command', choices=['doctor', 'plan', 'install', 'download', 'start', 'status', 'stop']) + args = parser.parse_args(argv) + args.root = args.root.resolve() + try: + if args.command == 'doctor': + result = inspect(args.root, args.python) + else: + profile = load_profile(args.profile) + if args.command == 'plan': + result = plan(profile, inspect(args.root, args.python), args.gpu) + else: + result = globals()[args.command](args, profile) + print(json.dumps(result, ensure_ascii=False)) + return 1 if result.get('status') == 'blocked' else 0 + except DeploymentError as error: + print(json.dumps(dict(status='failed', code=error.code, **error.facts), ensure_ascii=False)) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/vllm_omni/profiles/qwen3-asr-06b.json b/tools/vllm_omni/profiles/qwen3-asr-06b.json new file mode 100644 index 000000000..6204f8a09 --- /dev/null +++ b/tools/vllm_omni/profiles/qwen3-asr-06b.json @@ -0,0 +1,84 @@ +{ + "id": "qwen3-asr-06b", + "modelId": "Qwen/Qwen3-ASR-0.6B", + "revision": "5eb144179a02acc5e5ba31e748d22b0cf3e303b0", + "allowPatterns": [ + "*.json", + "*.txt", + "*.safetensors" + ], + "gpuBudgetMiB": 16384, + "hostMemoryBudgetGiB": 16, + "serveArgs": [ + "--gpu-memory-utilization", + "0.10", + "--max-model-len", + "4096", + "--max-num-seqs", + "4" + ], + "verification": { + "kind": "transcription", + "path": "/audio/transcriptions", + "request": { + "response_format": "json", + "temperature": 0 + } + }, + "schemaVersion": 1, + "minComputeCapability": "8.0", + "minDriver": "580.65.06", + "packages": [ + "vllm==0.32.0", + "vllm-omni==0.32.0" + ], + "environmentBudgetGiB": 30, + "qualification": { + "status": "unvalidated", + "note": "Budgets are conservative preflight reservations, not measured minimum requirements." + }, + "files": [ + { + "path": "chat_template.json", + "size": 1161, + "sha256": null + }, + { + "path": "config.json", + "size": 6193, + "sha256": null + }, + { + "path": "generation_config.json", + "size": 142, + "sha256": null + }, + { + "path": "merges.txt", + "size": 1671853, + "sha256": null + }, + { + "path": "model.safetensors", + "size": 1876091704, + "sha256": "79d6cbd4c98c7bbffe9db2edac07f56cd6637d0d5944b27f6c2b8353840323ea" + }, + { + "path": "preprocessor_config.json", + "size": 330, + "sha256": null + }, + { + "path": "tokenizer_config.json", + "size": 12487, + "sha256": null + }, + { + "path": "vocab.json", + "size": 2776833, + "sha256": null + } + ], + "downloadBytes": 1880560703, + "engine": "vllm" +} diff --git a/tools/vllm_omni/profiles/qwen3-tts-06b.json b/tools/vllm_omni/profiles/qwen3-tts-06b.json new file mode 100644 index 000000000..87810a776 --- /dev/null +++ b/tools/vllm_omni/profiles/qwen3-tts-06b.json @@ -0,0 +1,98 @@ +{ + "id": "qwen3-tts-06b", + "modelId": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", + "revision": "85e237c12c027371202489a0ec509ded67b5e4b5", + "allowPatterns": [ + "*.json", + "*.txt", + "*.safetensors", + "speech_tokenizer/*" + ], + "gpuBudgetMiB": 32768, + "hostMemoryBudgetGiB": 16, + "serveArgs": ["--stage-init-timeout", "1200"], + "verification": { + "kind": "audio", + "path": "/audio/speech", + "request": { + "input": "你好,这是一段在本地模型上生成的旁白。Hello, this is a locally generated narration.", + "voice": "vivian", + "response_format": "mp3" + } + }, + "schemaVersion": 1, + "minComputeCapability": "8.0", + "minDriver": "580.65.06", + "packages": [ + "vllm==0.32.0", + "vllm-omni==0.32.0" + ], + "environmentBudgetGiB": 30, + "qualification": { + "status": "unvalidated", + "note": "Budgets are conservative preflight reservations, not measured minimum requirements." + }, + "files": [ + { + "path": "config.json", + "size": 4908, + "sha256": null + }, + { + "path": "generation_config.json", + "size": 245, + "sha256": null + }, + { + "path": "merges.txt", + "size": 1671839, + "sha256": null + }, + { + "path": "model.safetensors", + "size": 1811626576, + "sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb" + }, + { + "path": "preprocessor_config.json", + "size": 127, + "sha256": null + }, + { + "path": "speech_tokenizer/config.json", + "size": 2336, + "sha256": null + }, + { + "path": "speech_tokenizer/configuration.json", + "size": 76, + "sha256": null + }, + { + "path": "speech_tokenizer/model.safetensors", + "size": 682293092, + "sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258" + }, + { + "path": "speech_tokenizer/preprocessor_config.json", + "size": 234, + "sha256": null + }, + { + "path": "tokenizer_config.json", + "size": 7344, + "sha256": null + }, + { + "path": "vocab.json", + "size": 2776833, + "sha256": null + } + ], + "downloadBytes": 2498383610, + "deployConfig": "deploy/qwen3_tts.yaml", + "stageOverrides": { + "gpu_memory_utilization": 0.15, + "max_num_seqs": 4 + } +} diff --git a/tools/vllm_omni/profiles/zimage-bf16.json b/tools/vllm_omni/profiles/zimage-bf16.json new file mode 100644 index 000000000..d26e19f8d --- /dev/null +++ b/tools/vllm_omni/profiles/zimage-bf16.json @@ -0,0 +1,139 @@ +{ + "id": "zimage-bf16", + "modelId": "Tongyi-MAI/Z-Image-Turbo", + "revision": "f332072aa78be7aecdf3ee76d5c247082da564a6", + "allowPatterns": [ + "model_index.json", + "scheduler/*", + "text_encoder/*", + "tokenizer/*", + "transformer/*", + "vae/*" + ], + "gpuBudgetMiB": 45056, + "hostMemoryBudgetGiB": 48, + "serveArgs": [], + "verification": { + "kind": "image", + "path": "/images/generations", + "request": { + "prompt": "A small red fishing boat leaving a quiet harbor at sunrise, lighthouse in the background, cinematic", + "size": "1344x768", + "num_inference_steps": 9, + "guidance_scale": 0, + "seed": 7, + "response_format": "b64_json", + "output_format": "png" + } + }, + "schemaVersion": 1, + "minComputeCapability": "8.0", + "minDriver": "580.65.06", + "packages": [ + "vllm==0.32.0", + "vllm-omni==0.32.0" + ], + "environmentBudgetGiB": 30, + "qualification": { + "status": "unvalidated", + "note": "Budgets are conservative preflight reservations, not measured minimum requirements." + }, + "files": [ + { + "path": "model_index.json", + "size": 467, + "sha256": null + }, + { + "path": "scheduler/scheduler_config.json", + "size": 173, + "sha256": null + }, + { + "path": "text_encoder/config.json", + "size": 726, + "sha256": null + }, + { + "path": "text_encoder/generation_config.json", + "size": 239, + "sha256": null + }, + { + "path": "text_encoder/model-00001-of-00003.safetensors", + "size": 3957900840, + "sha256": "328a91d3122359d5547f9d79521205bc0a46e1f79a792dfe650e99fc2d651223" + }, + { + "path": "text_encoder/model-00002-of-00003.safetensors", + "size": 3987450520, + "sha256": "6cd087b316306a68c562436b5492edbcf6e16c6dba3a1308279caa5a58e21ca5" + }, + { + "path": "text_encoder/model-00003-of-00003.safetensors", + "size": 99630640, + "sha256": "7ca841ee75b9c61267c0c6148fd8d096d3d21b6d3e161256a9b878154f91fc52" + }, + { + "path": "text_encoder/model.safetensors.index.json", + "size": 32819, + "sha256": null + }, + { + "path": "tokenizer/merges.txt", + "size": 1671853, + "sha256": null + }, + { + "path": "tokenizer/tokenizer.json", + "size": 11422654, + "sha256": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4" + }, + { + "path": "tokenizer/tokenizer_config.json", + "size": 9732, + "sha256": null + }, + { + "path": "tokenizer/vocab.json", + "size": 2776833, + "sha256": null + }, + { + "path": "transformer/config.json", + "size": 473, + "sha256": null + }, + { + "path": "transformer/diffusion_pytorch_model-00001-of-00003.safetensors", + "size": 9973693184, + "sha256": "95facd593e2549e8252acb571c653d57f7ddb7f1060d4e81712f152555a88804" + }, + { + "path": "transformer/diffusion_pytorch_model-00002-of-00003.safetensors", + "size": 9973714824, + "sha256": "a4bbe43ee184a1fb5af4b412d27555f532893bdc3165b1149e304ed82b5d7015" + }, + { + "path": "transformer/diffusion_pytorch_model-00003-of-00003.safetensors", + "size": 4672282880, + "sha256": "aba4e37a590e63210878160a718d916d80398f4e1f78ab6c9b2b2a00d92769fa" + }, + { + "path": "transformer/diffusion_pytorch_model.safetensors.index.json", + "size": 48969, + "sha256": null + }, + { + "path": "vae/config.json", + "size": 805, + "sha256": null + }, + { + "path": "vae/diffusion_pytorch_model.safetensors", + "size": 167666902, + "sha256": "f5b59a26851551b67ae1fe58d32e76486e1e812def4696a4bea97f16604d40a3" + } + ], + "downloadBytes": 32848305533 +} diff --git a/tools/vllm_omni/verify.py b/tools/vllm_omni/verify.py new file mode 100644 index 000000000..788d5185a --- /dev/null +++ b/tools/vllm_omni/verify.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Verify a deployed service; --infer explicitly submits one real generation. + +Uses Pillow or ffprobe for actual output validation. Reports distinguish technical +success from creative review, and never retry a possibly accepted generation. +""" +import argparse +import base64 +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import subprocess +import threading +import time +import unicodedata +from uuid import uuid4 + +if __package__: + from .deploy import (DeploymentError, capture, load_profile, profile_digest, read_json, + readiness, request_bytes, request_json, write_json) +else: + from deploy import (DeploymentError, capture, load_profile, profile_digest, read_json, + readiness, request_bytes, request_json, write_json) + + +MAX_AUDIO_BYTES = 25 * 1024 ** 2 +AUDIO_TYPES = {'.wav': 'audio/wav', '.mp3': 'audio/mpeg', '.flac': 'audio/flac', + '.m4a': 'audio/mp4', '.ogg': 'audio/ogg', '.webm': 'audio/webm', '.mp4': 'audio/mp4'} + + +def read_audio(path): + if path is None: + raise DeploymentError('audio_input_required') + path = Path(path) + if path.suffix.lower() not in AUDIO_TYPES: + raise DeploymentError('unsupported_audio_format') + try: + with path.open('rb') as stream: + raw = stream.read(MAX_AUDIO_BYTES + 1) + except OSError: + raise DeploymentError('audio_input_unreadable') from None + if not raw or len(raw) > MAX_AUDIO_BYTES: + raise DeploymentError('invalid_audio_input_size', limitBytes=MAX_AUDIO_BYTES) + return raw, path.suffix.lower() + + +def request_transcription(url, payload, audio, suffix, timeout): + """One standard multipart request to the upstream ASR endpoint; no retry.""" + boundary = 'opendirector-' + uuid4().hex + parts = [] + for key, value in payload.items(): + parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{key}"\r\n\r\n{value}\r\n'.encode()) + parts.extend([f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="input{suffix}"\r\n' + f'Content-Type: {AUDIO_TYPES[suffix]}\r\n\r\n'.encode(), audio, + f'\r\n--{boundary}--\r\n'.encode()]) + return request_bytes(url, b''.join(parts), 'multipart/form-data; boundary=' + boundary, timeout) + + +def compare_text(actual, expected): + """Character error rate after NFKC, case folding and punctuation/space removal. + + This reproducible content check is not a listening or pronunciation score. + """ + def normalize(value): + return ''.join(c for c in unicodedata.normalize('NFKC', value).casefold() if c.isalnum()) + reference, hypothesis = normalize(expected), normalize(actual) + if not reference or max(len(reference), len(hypothesis)) > 4096: + raise DeploymentError('invalid_comparison_text', limitCharacters=4096) + row = list(range(len(reference) + 1)) + for i, char in enumerate(hypothesis, 1): + current = [i] + for j, other in enumerate(reference, 1): + current.append(min(current[-1] + 1, row[j] + 1, row[j - 1] + (char != other))) + row = current + return dict(normalization='NFKC_casefold_alphanumeric', referenceCharacters=len(reference), + editDistance=row[-1], characterErrorRate=row[-1] / len(reference), exactMatch=reference == hypothesis) + + +def media_facts(path, kind, python): + if kind == 'image': + script = '''import json,sys +from PIL import Image +with Image.open(sys.argv[1]) as image: + facts={'format':image.format,'width':image.width,'height':image.height} + image.verify() +with Image.open(sys.argv[1]) as image: + image.load() +print(json.dumps(facts)) +''' + value = capture([python, '-c', script, str(path)]) + if value is None: + raise DeploymentError('invalid_image_output') + facts = json.loads(value) + if facts['format'] != 'PNG': + raise DeploymentError('invalid_image_output') + return facts + value = capture(['ffprobe', '-v', 'error', '-show_streams', '-show_format', '-of', 'json', str(path)]) + if value is None: + raise DeploymentError('invalid_audio_output') + probe = json.loads(value) + streams = [item for item in probe.get('streams', []) if item['codec_type'] == 'audio'] + if not streams or float(probe.get('format', {}).get('duration', 0)) <= 0: + raise DeploymentError('invalid_audio_output') + return dict(codec=streams[0]['codec_name'], sampleRate=int(streams[0]['sample_rate']), + channels=streams[0]['channels'], durationSeconds=float(probe['format']['duration'])) + + +class GpuSampler: + """Whole-device samples include other workloads; they are not allocator peaks.""" + def __init__(self, identity): + self.identity = identity + self.samples = [] + self.done = threading.Event() + self.thread = threading.Thread(target=self.sample, daemon=True) + + def sample(self): + while not self.done.is_set(): + value = capture(['nvidia-smi', '--id=' + self.identity, + '--query-gpu=memory.used', '--format=csv,noheader,nounits'], timeout=3) + try: + self.samples.append(int(value)) + except (TypeError, ValueError): + pass + self.done.wait(0.5) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *_): + self.done.set() + self.thread.join(timeout=4) + + def facts(self): + return dict(scope='whole_device_including_other_workloads', intervalSeconds=0.5, + sampleCount=len(self.samples), sampledPeakMiB=max(self.samples) if self.samples else None) + + +def verify(root, profile, *, infer=False, url=None, timeout=240, audio_file=None, expected_text=None): + state_path = root / 'services' / profile['id'] / 'state.json' + if url is None and not state_path.is_file(): + raise DeploymentError('service_not_started', profile=profile['id']) + state = read_json(state_path) if url is None else {} + url = (url or state['url']).rstrip('/') + readiness(url, profile['modelId']) + report = dict(schemaVersion=1, runId=str(uuid4()), status='service_ready', modelId=profile['modelId'], + revision=profile['revision'], profileSha256=profile_digest(profile), url=url, + startedAt=datetime.now(timezone.utc).isoformat(), creativeReview='pending') + if not infer: + return report + kind = profile['verification']['kind'] + audio, suffix = read_audio(audio_file) if kind == 'transcription' else (None, None) + if expected_text is not None: + if kind != 'transcription': + raise DeploymentError('comparison_requires_transcription') + compare_text('', expected_text) + directory = root / 'outputs' / report['runId'] + directory.mkdir(parents=True) + receipt = directory / 'verification.json' + payload = dict(profile['verification']['request'], model=profile['modelId']) + report.update(status='preparing_input', request=payload) + write_json(receipt, report) + sampler = GpuSampler(state['gpuUuid']) if state else None + sampled = False + started = time.monotonic() + try: + if kind == 'transcription': + source = directory / ('input' + suffix) + source.write_bytes(audio) + report['input'] = dict(artifact=str(source), artifactSha256=hashlib.sha256(audio).hexdigest(), + bytes=len(audio), media=media_facts(source, 'audio', None)) + report['status'] = 'submission_started' + write_json(receipt, report) + if sampler: + sampler.__enter__() + sampled = True + raw = (request_transcription(url + profile['verification']['path'], payload, audio, suffix, timeout) + if kind == 'transcription' else request_json(url + profile['verification']['path'], payload, timeout)) + elapsed = time.monotonic() - started + if kind == 'image': + raw = base64.b64decode(json.loads(raw)['data'][0]['b64_json'], validate=True) + path = directory / {'image': 'result.png', 'audio': 'result.mp3', 'transcription': 'result.json'}[kind] + with path.open('xb') as out: + out.write(raw) + if kind == 'transcription': + text = json.loads(raw)['text'] + if not isinstance(text, str) or not text.strip(): + raise DeploymentError('invalid_provider_output') + report['transcript'] = text + facts = dict(characters=len(text)) + if expected_text is not None: + report['textComparison'] = dict(referenceText=expected_text, **compare_text(text, expected_text)) + else: + facts = media_facts(path, kind, str(root / 'omni-env/bin/python')) + if kind == 'image': + expected = profile['verification']['request']['size'] + if f"{facts['width']}x{facts['height']}" != expected: + raise DeploymentError('output_dimensions_mismatch', actual=facts, expected=expected) + report.update(status='inference_verified', elapsedSeconds=round(elapsed, 3), artifact=str(path), + artifactSha256=hashlib.sha256(raw).hexdigest(), media=facts) + except DeploymentError as error: + report.update(status='submission_unknown' if error.code == 'service_unreachable' else 'failed', code=error.code, + elapsedSeconds=round(time.monotonic() - started, 3)) + write_json(receipt, report) + raise DeploymentError('inference_result_unknown' if report['status'] == 'submission_unknown' else error.code, + receipt=str(receipt)) from None + except (ValueError, KeyError, TypeError, IndexError): + report.update(status='failed', code='invalid_provider_output') + write_json(receipt, report) + raise DeploymentError('invalid_provider_output', receipt=str(receipt)) from None + finally: + if sampled: + sampler.__exit__() + report['gpuMemory'] = sampler.facts() + write_json(receipt, report) + return report + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--profile', default='zimage-bf16') + parser.add_argument('--url', help='Explicit API base for an already deployed service, ending in /v1.') + parser.add_argument('--infer', action='store_true', help='Submit one real request, save and probe its result. No automatic retry.') + parser.add_argument('--timeout', type=int, default=240) + parser.add_argument('--audio-file', type=Path, help='ASR input, at most 25 MiB. Required only for transcription --infer.') + parser.add_argument('--expected-text', help='Optional reference for a character error measurement; never sent to the model.') + args = parser.parse_args(argv) + try: + result = verify(args.root.resolve(), load_profile(args.profile), infer=args.infer, url=args.url, timeout=args.timeout, + audio_file=args.audio_file, expected_text=args.expected_text) + print(json.dumps(result, ensure_ascii=False)) + return 0 + except DeploymentError as error: + print(json.dumps(dict(status='failed', code=error.code, **error.facts), ensure_ascii=False)) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main())