diff --git a/docs/architecture/unified-project.md b/docs/architecture/unified-project.md index 700080e2f..aeb927527 100644 --- a/docs/architecture/unified-project.md +++ b/docs/architecture/unified-project.md @@ -148,4 +148,6 @@ UI、Agent 和后台完成写入共享可恢复变更通道。通知提示对象 模型接入使用能力契约和协议适配器,参数依次贯穿表单、生成配置、实际执行和历史依据。无法支持的参数明确限制,不静默忽略或降级。 +生成结果保留服务端返回的原始文件,不自动裁剪、丢帧、重编码或替换音轨。请求时长与帧数属于执行输入,输出媒体事实依据实际文件记录;模型对齐产生的差异如实保留。片段范围通过时间线显式编辑,转码和音轨调整产生独立派生素材,保留原始结果及来源关系。首次下载、失败恢复及各操作入口遵循同一规则。 + 浏览器与后台的执行职责必须明确。Agent 可以修改时间线,不代表后台已经能够导出视频;只有存在可调用渲染服务并验证真实输出,才可声称支持关闭浏览器后的自主导出。浏览器合成也可以作为明确需要客户端参与的能力保留。 diff --git a/docs/guides/user-guide.md b/docs/guides/user-guide.md index dedc3237c..96645333c 100644 --- a/docs/guides/user-guide.md +++ b/docs/guides/user-guide.md @@ -35,6 +35,8 @@ For a self-hosted chat service, allow at least 8192 tokens of context: the application sends `max_tokens: 8192`, so a smaller `--max-model-len` (for example vLLM's 4096 default) rejects every request with HTTP 400. +For vLLM-Omni video connections, enter fractional seconds directly, for example `6.583333`. The CLI accepts `opendirector video create --duration 6.583333` together with the other required options. Read `opendirector video create --help` and `opendirector model list` first; `inputRules.fractionalDuration` identifies support, and each output's duration list defines the allowed range. H3 can align output to its supported frame count, so inspect the actual result duration before assembling segments. + For detailed video settings, see [Video model capability configuration (Chinese)](video-model-capabilities.md). When no explicit output specification is provided, use **Auto** or **Service default duration**. Automatic region selection in images and video matting require a model with segmentation capabilities. Image generation and editing still require the corresponding generation or editing model. diff --git a/docs/guides/user-guide.zh-CN.md b/docs/guides/user-guide.zh-CN.md index 377ba8cf0..084955699 100644 --- a/docs/guides/user-guide.zh-CN.md +++ b/docs/guides/user-guide.zh-CN.md @@ -47,6 +47,8 @@ HTTP 400。 在设置的“通用”页中可切换主题、调整去除水印选项。水印设置仅影响后续生成的内容,实际效果取决于模型服务是否支持。 +vLLM-Omni 视频连接支持直接输入小数秒,例如 `6.583333`。CLI 可使用 `opendirector video create --duration 6.583333` 并填写其余必需参数。先运行 `opendirector video create --help` 和 `opendirector model list`;`inputRules.fractionalDuration` 表示支持小数秒,各输出规格的时长列表定义允许范围。H3 可能按模型支持的帧数对齐,拼接分段前应核对结果的实际时长。 + ## 制作第一支短片 建议先完成一个镜头的生成与导出,再扩展到多段素材。 diff --git a/packages/core/src/opendirector/application/creation_generation.py b/packages/core/src/opendirector/application/creation_generation.py index fb5fb207e..41af9c224 100644 --- a/packages/core/src/opendirector/application/creation_generation.py +++ b/packages/core/src/opendirector/application/creation_generation.py @@ -29,10 +29,12 @@ from opendirector.domain.input_rules import input_rules from opendirector.domain.minimax_speech import is_minimax_speech from opendirector.domain.projects import ProjectError +from opendirector.domain.rule_facts import duration_matches from opendirector.domain.video_output import apply_video_resolution from opendirector.ports.gateway import gateway from opendirector.integrations.http_settings import MODEL_JOB_CONTEXT from opendirector.integrations.fasth3 import video_parameters +from opendirector.integrations.vllm_omni import FPS, video_frame_parameters from opendirector.integrations.model_adapters import capabilities_of from opendirector.integrations.providers import GenerationProviders from opendirector.media.filenames import with_extension @@ -195,9 +197,6 @@ def _parameters(self, spec, config, node, records): seconds = intent.get('durationSeconds') if rules['profile'] == 'fasth3-omni': params = video_parameters(node, records, duration_seconds=seconds) - # FastH3's wire contract sends duration; its derived frame - # estimate is not an explicitly submitted execution parameter. - params.pop('num_frames', None) params['seed'] = new_seed() elif rules['profile'] == 'standard-video': params = {} @@ -212,7 +211,7 @@ def _parameters(self, spec, config, node, records): if node.get('videoMode') == 'continuation': raise ProjectError('接续沿用上一段视频的画幅,请将比例设为自动。', 422) caps = rules.get('videoCapabilities') or {} - choices = [item for item in caps.get('outputs', []) if seconds is None or seconds in item['durations']] + choices = [item for item in caps.get('outputs', []) if seconds is None or duration_matches(seconds, item['durations'], rules.get('fractionalDuration', False))] if not choices: raise ProjectError('指定视频比例前,请在模型接入中配置输出规格。', 422) preferred = next((item for item in choices if item['resolution'] == caps.get('defaultResolution')), choices[0]) @@ -226,9 +225,8 @@ def _parameters(self, spec, config, node, records): params['ratio'] = '16:9' else: apply_video_resolution(params, node, records, config, output) - if rules['profile'] == 'vllm-omni-h3': - from opendirector.integrations.vllm_omni import FPS, h3_frames - params.update(fps=FPS, num_frames=h3_frames(params['duration'])) + if rules['profile'] in {'vllm-omni-h3', 'fasth3-omni'} and 'duration' in params: + params.update(video_frame_parameters(params['duration'], params.get('fps', FPS))) if spec.kind in {'image', 'video'} and config['protocol'] == 'volcengine': params['watermark'] = not self.settings.get()['removeWatermark'] if spec.kind == 'image' and input_rules(config)['profile'] == 'moss-image': diff --git a/packages/core/src/opendirector/application/generation_runtime.py b/packages/core/src/opendirector/application/generation_runtime.py index c0da6625c..e104391d0 100644 --- a/packages/core/src/opendirector/application/generation_runtime.py +++ b/packages/core/src/opendirector/application/generation_runtime.py @@ -9,7 +9,6 @@ from opendirector.domain.generation import GenerationError, GenerationFailure from opendirector.ports.gateway import gateway from opendirector.integrations.http_settings import MODEL_JOB_CONTEXT -from opendirector.integrations.fasth3 import finalize_video_edit from opendirector.integrations.generation_errors import transport_failure def _self_hosted_video(task): @@ -152,7 +151,6 @@ async def result_ready(suffix, url): suffix, data = await self.provider.generate(config, task, submitted, result_ready=result_ready) else: suffix, data = await self.provider.generate(config, task, submitted) - suffix, data = await asyncio.to_thread(finalize_video_edit, self.store, task, suffix, data) elif cached: suffix, data = cached # Preserve complete bytes before catalog/link writes. Cancellation diff --git a/packages/core/src/opendirector/cli.py b/packages/core/src/opendirector/cli.py index 2b66ec8ad..ef7dac3ec 100644 --- a/packages/core/src/opendirector/cli.py +++ b/packages/core/src/opendirector/cli.py @@ -434,7 +434,7 @@ def add_material(ctx: typer.Context, asset: Annotated[UUID, typer.Option(help="R def video_create(ctx: typer.Context, model: Annotated[str, typer.Option(help="Video model configuration UUID or unique exact model name from opendirector model list.")], prompt: Annotated[str, typer.Option(help="Video generation prompt.")], - duration: Annotated[float, typer.Option(min=0, help="Video duration in seconds.")], + duration: Annotated[float, typer.Option(min=0, help="Video duration in seconds, e.g. 6.583333. Fractional values require model inputRules.fractionalDuration; inspect model list for supported ranges. Native H3 accepts 4-15 seconds and aligns frames upward to 17k+5 at 24 fps (15 seconds becomes 362 frames).")], x: Annotated[float, typer.Option(help="Canvas X coordinate. Choose using the current item positions and sizes.")], y: Annotated[float, typer.Option(help="Canvas Y coordinate. Choose using the current item positions and sizes.")], first_frame: Annotated[UUID | None, typer.Option(help="Source image canvas item for first-frame mode.")] = None, diff --git a/packages/core/src/opendirector/domain/creation/generation.py b/packages/core/src/opendirector/domain/creation/generation.py index 91b6f0b40..9b4222ed1 100644 --- a/packages/core/src/opendirector/domain/creation/generation.py +++ b/packages/core/src/opendirector/domain/creation/generation.py @@ -13,7 +13,7 @@ FASTH3_MEDIA_REFERENCE_LIMIT, FRAME_IMAGE_LIMIT, LYRICS_MAX_LENGTH, MAX_REFERENCES, MUSIC_SECONDS_INITIAL, MUSIC_SECONDS_MAX, MUSIC_SECONDS_MIN, PROMPT_MAX_LENGTH, REFERENCE_MEDIA_SECONDS, STORED_VOICE_PATTERN, TEXT_DEFAULT_MAX_TOKENS, TEXT_MAX_TOKENS, - effective_video_mode, generate_audio_options, + duration_matches, effective_video_mode, generate_audio_options, ) from .assets import AssetRef, MediaMetadata from .base import CanvasItemVersion, Contract, Digest, GenerationKind, NonNegative, ObjectVersion, PositiveInt, ProjectEntity, Revision, digest, unique @@ -36,7 +36,7 @@ class ImageParameters(Contract): class VideoParameters(Contract): generateAudio: bool = Field(default=True, description='Include audio; value must be allowed by model inputRules.generateAudioOptions.') videoMode: VideoMode = Field(default='first_frame', description='Video generation input mode.') - durationSeconds: Annotated[float, Field(strict=True, ge=1, le=60)] | None = Field(default=None, description='Duration in seconds.') + durationSeconds: Annotated[float, Field(strict=True, ge=1, le=60)] | None = Field(default=None, description='Duration in seconds; fractional values require model inputRules.fractionalDuration. Generated frames may be aligned by the model.') aspectRatio: str | None = Field(default=None, pattern=r'^[1-9]\d?:[1-9]\d?$', description='Output width-to-height ratio.') resolution: str = Field(default='auto', pattern=r'^(?:auto|[A-Z0-9][A-Z0-9_.-]{0,23})$', description='Requested output resolution.') @@ -490,5 +490,5 @@ def validate_generation(spec: GenerationSpec, model: ModelConfigPublic, referenc caps = rules.videoCapabilities if caps and params.durationSeconds is not None: choices = [output for output in caps.outputs if params.resolution == 'auto' or output.resolution == params.resolution] - if not any(params.durationSeconds in output.durations for output in choices): + if not any(duration_matches(params.durationSeconds, output.durations, rules.fractionalDuration) for output in choices): raise ValueError('当前时长与分辨率组合不受支持。') diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 87484feae..65cdb9601 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -46,6 +46,7 @@ class VoiceLibraryCapabilities(Contract): class InputRules(Contract): + fractionalDuration: bool = Field(default=False, exclude_if=lambda value: not value, description="Accept fractional video seconds within each output duration range.") profile: str = Field(min_length=1, max_length=100, description='Model capability profile.') voiceLibrary: VoiceLibraryCapabilities | None = Field(default=None, exclude_if=lambda value: value is None, description='Supported voice registry and adapter requirements; omitted for unsupported models.') diff --git a/packages/core/src/opendirector/domain/rule_facts.py b/packages/core/src/opendirector/domain/rule_facts.py index 756fd44aa..91f96d755 100644 --- a/packages/core/src/opendirector/domain/rule_facts.py +++ b/packages/core/src/opendirector/domain/rule_facts.py @@ -102,6 +102,12 @@ def aspect_ratio_choices(spec): return ['auto'] +def duration_matches(seconds: float, durations: list[int] | tuple[int, ...], fractional: bool = False) -> bool: + if fractional: + return bool(durations) and min(durations) <= seconds <= max(durations) + return seconds in durations + + def resolution_choices(spec): """Sizes offered before submit. Callers report whether an image reference is connected.""" kind = spec['kind'] @@ -122,7 +128,7 @@ def resolution_choices(spec): duration = spec.get('durationSeconds') if outputs is not None and duration is not None: choices = [label for label in choices - if any(item['resolution'] == label and duration in item['durations'] for item in outputs)] + if any(item['resolution'] == label and duration_matches(duration, item['durations'], spec.get('fractionalDuration', False)) for item in outputs)] return ['auto', *choices] @@ -336,7 +342,14 @@ def derivation_vectors(): return ["auto"]; } +export function durationMatches(seconds: number, durations: readonly number[], fractional = false): boolean { + return fractional + ? durations.length > 0 && seconds >= Math.min(...durations) && seconds <= Math.max(...durations) + : durations.includes(seconds); +} + export function resolutionChoices(input: { + readonly fractionalDuration?: boolean; readonly kind: string; readonly videoMode?: string | null; readonly aspectRatio?: string | null; @@ -380,7 +393,7 @@ def derivation_vectors(): ? choices.filter((label) => outputs.some( (item) => - item.resolution === label && item.durations.includes(duration), + item.resolution === label && durationMatches(duration, item.durations, input.fractionalDuration), ), ) : choices; diff --git a/packages/core/src/opendirector/domain/video_capabilities.py b/packages/core/src/opendirector/domain/video_capabilities.py index 3d5f1e306..751160693 100644 --- a/packages/core/src/opendirector/domain/video_capabilities.py +++ b/packages/core/src/opendirector/domain/video_capabilities.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator from opendirector.domain.projects import ProjectError +from opendirector.domain.rule_facts import duration_matches from opendirector.domain.minimax_models import MINIMAX_MODELS, minimax_capabilities from opendirector.domain.kling_models import kling_capabilities from opendirector.domain.bigmodel_video import cogvideo_capabilities, validate_cogvideo_outputs @@ -97,6 +98,8 @@ def capability_fields(config, rules): validate_cogvideo_outputs(capabilities) if rules['profile'] == 'kling-official': validate_kling_outputs(capabilities, config['modelId']) + if rules['profile'] in {'vllm-omni-h3', 'fasth3-omni'}: + rules['fractionalDuration'] = True rules['videoCapabilities'] = capabilities rules['outputResolutions'] = [item['resolution'] for item in capabilities['outputs']] if capabilities else [] return rules @@ -153,9 +156,12 @@ def check_video_config(kind, protocol, adapter, capabilities): def validate_video_duration(config, seconds, resolution=None): from opendirector.domain.input_rules import input_rules - capabilities = input_rules(config).get('videoCapabilities') + rules = input_rules(config) + if config.get('protocol') == 'vllm_omni' and not rules.get('fractionalDuration', False) and not float(seconds).is_integer(): + raise ProjectError('当前模型的分辨率与时长组合不受支持,请重新选择;未提交生成。') + capabilities = rules.get('videoCapabilities') if capabilities: choices = [item for item in capabilities['outputs'] if not resolution or resolution in {'auto', '自动'} or item['resolution'] == resolution.upper()] - if not any(seconds in item['durations'] for item in choices): + if not any(duration_matches(seconds, item['durations'], rules.get('fractionalDuration', False)) for item in choices): raise ProjectError('当前模型的分辨率与时长组合不受支持,请重新选择;未提交生成。') diff --git a/packages/core/src/opendirector/integrations/fasth3.py b/packages/core/src/opendirector/integrations/fasth3.py index 4cf9984af..5067f1021 100644 --- a/packages/core/src/opendirector/integrations/fasth3.py +++ b/packages/core/src/opendirector/integrations/fasth3.py @@ -1,16 +1,10 @@ -"""Compile H3 references and preserve a direct source-video edit's time/audio.""" +"""Compile H3 references and generation parameters.""" import math -from pathlib import Path import re -import shutil -import subprocess -import tempfile -from opendirector.domain.assets import MAX_ASSET_BYTES from opendirector.domain.projects import ProjectError from opendirector.domain.references import project_content -MP4_AUDIO_CODECS = {'aac', 'mp3', 'opus'} def native_prompt(task, sources): @@ -65,100 +59,4 @@ def video_parameters(node, records, *, duration_seconds=None): result = {'seconds': str(math.ceil(seconds)), 'duration': seconds, 'size': '480x832' if portrait else '832x480', 'fps': 24, 'num_frames': frames, 'num_inference_steps': 8, 'guidance_scale': 1} - if edit: - # Original audio is retained unless the user asks to change it. This is - # local muxing, independent of the model's generative audio prediction. - change_audio = re.search(r'(?:修改|替换|生成|更换|去除|去掉|移除|添加|改变|不要保留).{0,8}(?:音频|音乐|音轨|配音|声音|原音|BGM)|降噪|去噪|\b(?:change|replace|remove|add|generate)\b.{0,25}\b(?:audio|music|sound|voice)\b', text, re.I) - result['videoEdit'] = {'sourceAssetId': basis['id'], 'sourceDurationMs': basis['durationMs'], - 'preserveDuration': duration_seconds is None and explicit is None, - 'preserveAudio': not bool(change_audio)} return result - - -def _mp4_audio_arguments(path): - """Keep MP4-playable soundtrack packets and transcode the rest, such as MOV PCM.""" - process = subprocess.run(['ffprobe', '-v', 'error', '-protocol_whitelist', 'file,pipe', '-select_streams', 'a:0', - '-show_entries', 'stream=codec_name', '-of', 'csv=p=0', str(path)], - check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=15) - codec = process.stdout.decode(errors='replace').strip() - return ['-c:a', 'copy'] if not codec or codec in MP4_AUDIO_CODECS else ['-c:a', 'aac', '-b:a', '192k'] - - -def _trim_plain_fasth3(task, suffix, data): - """FastH3 rounds frames up to its 17n+5 grid; a 7 s request returns 175 frames (7.29 s).""" - seconds = float(task['parameters']['duration']) - if not shutil.which('ffmpeg') or not shutil.which('ffprobe'): - raise ProjectError('裁剪 FastH3 视频到所选时长需要 FFmpeg;生成结果可继续下载恢复,无需重新生成。', 409) - with tempfile.TemporaryDirectory(prefix='opendirector-video-trim-') as folder: - generated = Path(folder) / ('generated.' + suffix) - output = Path(folder) / 'output.mp4' - generated.write_bytes(data) - try: - probe = subprocess.run(['ffprobe', '-v', 'error', '-protocol_whitelist', 'file,pipe', '-select_streams', 'v:0', - '-show_entries', 'stream=duration', '-of', 'csv=p=0', str(generated)], - check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=15) - actual = float(probe.stdout.decode(errors='replace').strip()) - # Within one frame already matches; re-encoding it would only cost quality. - if actual <= seconds + 1 / 24: - return suffix, data - command = ['ffmpeg', '-nostdin', '-v', 'error', '-y', '-protocol_whitelist', 'file,pipe', '-i', str(generated), - '-map', '0:v:0', '-map', '0:a:0?', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', - '-pix_fmt', 'yuv420p', '-map_metadata', '-1', '-t', str(seconds), *_mp4_audio_arguments(generated), - '-movflags', '+faststart', str(output)] - subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=60) - except (subprocess.SubprocessError, OSError, ValueError): - raise ProjectError('裁剪 FastH3 视频到所选时长失败;请检查 FFmpeg,稍后继续下载原任务,无需重新生成。', 409) from None - if output.stat().st_size > MAX_ASSET_BYTES: - raise ProjectError('处理后的视频超过 50 MB,无法保存到项目。', 413) - return 'mp4', output.read_bytes() - - -def finalize_video_edit(store, task, suffix, data): - policy = task.get('parameters', {}).get('videoEdit') - if task.get('protocol') != 'vllm_omni' or task['kind'] != 'video': - return suffix, data - if not policy: - # Native H3 continuation resumes from the server's untrimmed clip, so only FastH3 is trimmed. - if (task.get('referenceInputs') or {}).get('profile') == 'fasth3-omni' and task.get('parameters', {}).get('duration'): - return _trim_plain_fasth3(task, suffix, data) - return suffix, data - if not shutil.which('ffmpeg') or not shutil.which('ffprobe'): - raise ProjectError('保留原片音轨和时长需要 FFmpeg;生成结果可继续下载恢复,无需重新生成。', 409) - # An edit that changes the audio and sets its own length only trims the result, - # so a later change to the source must not block a paid download. - uses_source = policy.get('preserveAudio') or policy.get('preserveDuration') - if uses_source: - # Reuse the same fingerprint validation as submission when recovering a download. - from opendirector.integrations.reference_inputs import references - source, meta, _ = next(references(store, task['projectId'], [policy['sourceAssetId']])) - expected = task.get('inputSnapshot', {}).get('references', {}).get(policy['sourceAssetId']) - if expected and meta.get('fingerprint') != expected: - raise ProjectError('原视频已变化,无法保留其音轨;请恢复原素材后继续下载原任务。', 409) - with tempfile.TemporaryDirectory(prefix='opendirector-video-edit-') as folder: - generated = Path(folder) / ('generated.' + suffix) - output = Path(folder) / 'output.mp4' - generated.write_bytes(data) - # Stream-copy trimming is not frame-accurate with reordered B-frames: - # packets after the target timestamp can extend the output duration. - # Re-encode video for an exact boundary; audio packets stay intact when MP4 can carry them. - # Chrome also plays High 10 and High 4:4:4, but 8-bit 4:2:0 is the widely supported H.264 baseline. - command = ['ffmpeg', '-nostdin', '-v', 'error', '-y', '-protocol_whitelist', 'file,pipe', '-i', str(generated)] - if uses_source: - original = Path(folder) / ('source' + Path(meta['name']).suffix.lower()) - original.write_bytes(source) - command += ['-protocol_whitelist', 'file,pipe', '-i', str(original)] - command += ['-map', '0:v:0', '-map', '1:a:0?' if policy.get('preserveAudio') else '0:a:0?', - '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', '-pix_fmt', 'yuv420p', - '-map_metadata', '-1'] - # A longer source soundtrack must never extend an explicitly shorter edit. - duration = policy['sourceDurationMs'] / 1000 if policy.get('preserveDuration') else float(task['parameters'].get('duration', task['parameters'].get('seconds', 5))) - command += ['-t', str(duration)] - try: - command += _mp4_audio_arguments(original if policy.get('preserveAudio') else generated) - command += ['-movflags', '+faststart', str(output)] - subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=60) - except (subprocess.SubprocessError, OSError): - raise ProjectError('保留原片音轨或时长失败;请检查 FFmpeg,稍后继续下载原任务,无需重新生成。', 409) from None - if output.stat().st_size > MAX_ASSET_BYTES: - raise ProjectError('处理后的视频超过 50 MB,无法保存到项目。', 413) - return 'mp4', output.read_bytes() diff --git a/packages/core/src/opendirector/integrations/providers.py b/packages/core/src/opendirector/integrations/providers.py index 4cc3cdb65..2954101fd 100644 --- a/packages/core/src/opendirector/integrations/providers.py +++ b/packages/core/src/opendirector/integrations/providers.py @@ -10,6 +10,7 @@ from opendirector.domain.assets import MAX_ASSET_BYTES from opendirector.domain.input_rules import input_rules +from opendirector.domain.video_capabilities import validate_video_duration from opendirector.domain.minimax_speech import is_minimax_speech from opendirector.domain.projects import ProjectError from opendirector.integrations.reference_inputs import generation_references, references, image_type @@ -18,7 +19,7 @@ from opendirector.integrations.minimax_video import MinimaxVideoGeneration from opendirector.integrations.kling_video import KlingVideoGeneration from opendirector.integrations.bigmodel_video import BigModelVideoGeneration -from opendirector.integrations.vllm_omni import VllmOmniGeneration +from opendirector.integrations.vllm_omni import FPS, VllmOmniGeneration, video_frame_parameters from opendirector.integrations.vllm_omni_image import VllmOmniImageGeneration from opendirector.integrations.model_adapters import BigModelImages, MinimaxH3Images, MinimaxSpeech, MossImages, MossSpeech, OpenAISpeech, VllmOmniMusic, VllmOmniScore, ZhipuSpeech from opendirector.domain.capabilities import ImageRequest, MusicRequest, ScoreRequest, SpeechRequest @@ -179,14 +180,23 @@ async def generate(self, config, task, submitted, *, result_ready=None): files.append(('input_references' if fast else 'input_reference', (meta['name'], data, image_type(data) if source_kind == 'image' else meta['type']))) parameters = task['parameters'] form = {'model': config['modelId'], 'prompt': prepared['prompt']} - if 'duration' in parameters: form['seconds'] = format(parameters['duration'], 'g') + if fast: + if 'duration' in parameters or 'seconds' in parameters or 'num_frames' in parameters: + frames = video_frame_parameters(float(parameters.get('duration', parameters.get('seconds', 5))), + parameters.get('fps', FPS), parameters.get('num_frames')) + form.update({key: str(value) for key, value in frames.items()}) + elif 'duration' in parameters: + seconds = parameters['duration'] + if config['protocol'] == 'vllm_omni': + validate_video_duration(config, seconds) + form['seconds'] = str(int(seconds)) if float(seconds).is_integer() else str(seconds) if 'size' in parameters: form['size'] = parameters['size'] if fast: form.update({ - 'seconds': str(parameters.get('seconds', '5')), 'size': parameters.get('size', '832x480'), + 'size': parameters.get('size', '832x480'), 'num_inference_steps': '8', 'guidance_scale': '1', 'fps': '24', 'seed': str(parameters.get('seed', 1234)), - 'extra_params': json.dumps({'task': 'ref2va', 'duration': float(parameters.get('duration', parameters.get('seconds', 5))), 'audio_flow_shift': 3})}) + 'extra_params': json.dumps({'task': 'ref2va', 'audio_flow_shift': 3})}) # Force multipart even for an empty reference list. value = await self._json(client, 'POST', url, config, data=form, files=files or {'_multipart': (None, '1')}) external_id = value['id'] diff --git a/packages/core/src/opendirector/integrations/vllm_omni.py b/packages/core/src/opendirector/integrations/vllm_omni.py index b8271ea0e..28ca989ee 100644 --- a/packages/core/src/opendirector/integrations/vllm_omni.py +++ b/packages/core/src/opendirector/integrations/vllm_omni.py @@ -25,10 +25,8 @@ RATIOS = {'16:9', '9:16', '1:1', '4:3', '3:4'} -def h3_frames(seconds): - """Nearest H3 length on the 17n+5 frame grid, within 4-15 seconds.""" - steps = round((seconds * FPS - 5) / 17) - return 17 * min(max(steps, 6), 20) + 5 +def video_frame_parameters(seconds: float, fps: int = FPS, num_frames: int | None = None) -> dict[str, int]: + return {'fps': fps, 'num_frames': num_frames if num_frames is not None else round(seconds * fps)} def _upload(data_uri, name): @@ -66,9 +64,9 @@ def form(self, config, task, prepared): mode = task.get('videoMode', 'first_frame') seconds = task['parameters'].get('duration', 5) ratio = task['parameters'].get('ratio') + frames = video_frame_parameters(seconds, task['parameters'].get('fps', FPS), task['parameters'].get('num_frames')) fields = {'model': config['modelId'], 'prompt': prepared['prompt'], - 'fps': str(task['parameters'].get('fps', FPS)), - 'num_frames': str(task['parameters'].get('num_frames', h3_frames(seconds)))} + **{key: str(value) for key, value in frames.items()}} if task['parameters'].get('resolution') and task['parameters'].get('size'): fields['size'] = task['parameters']['size'] files, extra = [], {} diff --git a/src/backend/rules.ts b/src/backend/rules.ts index 09d49adde..14748b5b7 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -3,6 +3,7 @@ import { nodeByteSize } from "../canvas/uploadPolicy.ts"; import type { CanvasNode, GenerationSettings } from "../canvas/model.ts"; import { RULE_FACTS, + durationMatches, aspectRatioChoices, effectiveVideoMode, generateAudioOptions, @@ -33,6 +34,7 @@ export interface VideoCapabilities { } export interface InputRules { + fractionalDuration?: boolean; profile: string; allowedKinds: string[]; maxImages: number; @@ -369,6 +371,7 @@ export function backendResolutions( ): string[] { const rules = config?.inputRules; return resolutionChoices({ + fractionalDuration: rules?.fractionalDuration, kind: settings.mode, videoMode: settings.videoMode, aspectRatio: settings.aspectRatio, @@ -483,7 +486,7 @@ export function serviceVideoSettings( const next = { ...settings }; const caps = config.inputRules.videoCapabilities; const durations = backendDurations(config); - if (durations.length && !durations.includes(next.durationSeconds)) + if (durations.length && !durationMatches(next.durationSeconds, durations, config.inputRules.fractionalDuration)) next.durationSeconds = caps!.defaultDuration; if ( next.videoMode === "continuation" && @@ -637,7 +640,7 @@ export function backendBlocker( const references = backendReferences(settings, [...incoming, ...pinned], config); if (settings.mode === "video") { const durations = backendDurations(config); - if (durations.length && !durations.includes(settings.durationSeconds)) + if (durations.length && !durationMatches(settings.durationSeconds, durations, rules.fractionalDuration)) return tNow("errors.rules.duration"); if ( rules.outputResolutions && diff --git a/src/canvas/DurationControl.tsx b/src/canvas/DurationControl.tsx index 6b1349d0d..c8e2adadc 100644 --- a/src/canvas/DurationControl.tsx +++ b/src/canvas/DurationControl.tsx @@ -9,7 +9,11 @@ export function DurationControl({ disabled, onChange, scale, + fractional = false, + outputFrames, }: { + fractional?: boolean; + outputFrames?: number; values: readonly number[]; value: number; disabled: boolean; @@ -50,16 +54,16 @@ export function DurationControl({ ? t("canvasNodes.duration.only", { minimum }) : t("canvasNodes.duration.range", { minimum, maximum }); const hint = (seconds: number) => - seconds <= minimum || seconds >= maximum + seconds < minimum || seconds > maximum ? limit - : !values.includes(seconds) + : !fractional && !values.includes(seconds) ? t("canvasNodes.duration.unsupported", { values: values.join(t("canvasNodes.duration.separator")), }) : ""; const choose = (seconds: number) => { if (!Number.isFinite(seconds) || !values.length || disabled) return; - const allowed = values.reduce((nearest, candidate) => + const allowed = fractional ? Math.max(minimum, Math.min(maximum, seconds)) : values.reduce((nearest, candidate) => Math.abs(candidate - seconds) < Math.abs(nearest - seconds) ? candidate : nearest, @@ -92,6 +96,11 @@ export function DurationControl({ {notice} )} + {outputFrames !== undefined && ( + + {t("generation.duration.aligned", { frames: outputFrames, seconds: Number((outputFrames / 24).toFixed(6)) })} + + )}