From d297e5569ffd3c3a6a68cee9d3eee5554e658506 Mon Sep 17 00:00:00 2001 From: ubuntu Date: Mon, 19 Oct 2026 08:07:50 +0800 Subject: [PATCH 1/2] feat: add multimodal embedding engine support --- internal/adapter/factory/factory.go | 2 +- internal/adapter/factory/factory_test.go | 2 +- internal/adapter/proxy/rewrite.go | 2 +- internal/config/config_enums.go | 28 ++++++++++++---------- internal/config/config_load_engine.go | 23 ++++++++++-------- internal/config/config_load_engine_test.go | 25 +++++++++---------- internal/config/context_size_test.go | 2 +- internal/config/engine_args.go | 2 +- internal/config/engine_args_test.go | 6 ++--- internal/config/max_concurrency_test.go | 2 +- internal/config/modelspec.go | 7 +++++- internal/config/modelspec_test.go | 4 ++++ internal/controlplane/endpoints.go | 10 ++++++-- internal/controlplane/endpoints_test.go | 3 ++- internal/moderoutes/moderoutes.go | 4 ++++ internal/moderoutes/moderoutes_test.go | 6 +++++ 16 files changed, 80 insertions(+), 48 deletions(-) diff --git a/internal/adapter/factory/factory.go b/internal/adapter/factory/factory.go index 15aca02..41e2d1f 100644 --- a/internal/adapter/factory/factory.go +++ b/internal/adapter/factory/factory.go @@ -36,7 +36,7 @@ func New(cfg config.Config, metrics *obs.Metrics) (adapter.Adapter, error) { return nullengine.New(), nil case config.EngineOllama: return ollama.NewAdapter(cfg), nil - case config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata, config.EngineEmbed, config.EngineClipEmbed, config.EngineAudio, config.EngineOCR, config.EngineRerank, config.EngineMusic, config.EngineSystemOne: + case config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata, config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed, config.EngineAudio, config.EngineOCR, config.EngineRerank, config.EngineMusic, config.EngineSystemOne: a, err := proxy.NewAdapter(cfg, cfg.Engine.Kind) if err != nil { return nil, err diff --git a/internal/adapter/factory/factory_test.go b/internal/adapter/factory/factory_test.go index 9ca6702..ca71375 100644 --- a/internal/adapter/factory/factory_test.go +++ b/internal/adapter/factory/factory_test.go @@ -42,7 +42,7 @@ func TestNew_ProxyKinds(t *testing.T) { t.Parallel() for _, k := range []config.EngineKind{ config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata, - config.EngineEmbed, config.EngineClipEmbed, config.EngineOCR, config.EngineRerank, + config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed, config.EngineOCR, config.EngineRerank, config.EngineSystemOne, } { k := k diff --git a/internal/adapter/proxy/rewrite.go b/internal/adapter/proxy/rewrite.go index 5d6aba2..4d332ca 100644 --- a/internal/adapter/proxy/rewrite.go +++ b/internal/adapter/proxy/rewrite.go @@ -108,7 +108,7 @@ func rewriteBodyCap(path string, kind config.EngineKind) int { case pathChatCompletions, pathAPIChatCompletions, pathResponses: return maxVisionRewriteBody case pathEmbeddings: - if kind == config.EngineClipEmbed { + if kind == config.EngineClipEmbed || kind == config.EngineMultimodalEmbed { return maxVisionRewriteBody } return maxRewriteBody diff --git a/internal/config/config_enums.go b/internal/config/config_enums.go index 53aaf10..c0e05bb 100644 --- a/internal/config/config_enums.go +++ b/internal/config/config_enums.go @@ -8,19 +8,20 @@ type EngineKind string // EngineKind values. The set is closed; AllEngineKinds is the canonical // slice form used for env validation. const ( - EngineOllama EngineKind = "ollama" - EngineVLLM EngineKind = "vllm" - EngineLlamaCpp EngineKind = "llamacpp" - EngineSGLang EngineKind = "sglang" - EngineFreeToken EngineKind = "freetoken" - EngineStrata EngineKind = "strata" - EngineEmbed EngineKind = "embed" - EngineClipEmbed EngineKind = "clipembed" - EngineAudio EngineKind = "audio" - EngineOCR EngineKind = "ocr" - EngineRerank EngineKind = "rerank" - EngineMusic EngineKind = "music" - EngineSystemOne EngineKind = "systemone" + EngineOllama EngineKind = "ollama" + EngineVLLM EngineKind = "vllm" + EngineLlamaCpp EngineKind = "llamacpp" + EngineSGLang EngineKind = "sglang" + EngineFreeToken EngineKind = "freetoken" + EngineStrata EngineKind = "strata" + EngineEmbed EngineKind = "embed" + EngineClipEmbed EngineKind = "clipembed" + EngineMultimodalEmbed EngineKind = "multimodalembed" + EngineAudio EngineKind = "audio" + EngineOCR EngineKind = "ocr" + EngineRerank EngineKind = "rerank" + EngineMusic EngineKind = "music" + EngineSystemOne EngineKind = "systemone" ) // AllEngineKinds lists every supported engine; used for env validation. @@ -33,6 +34,7 @@ var AllEngineKinds = []string{ string(EngineStrata), string(EngineEmbed), string(EngineClipEmbed), + string(EngineMultimodalEmbed), string(EngineAudio), string(EngineOCR), string(EngineRerank), diff --git a/internal/config/config_load_engine.go b/internal/config/config_load_engine.go index de5e471..1fb2227 100644 --- a/internal/config/config_load_engine.go +++ b/internal/config/config_load_engine.go @@ -53,16 +53,17 @@ func (l *loader) loadEngine() { // Conventional sibling-engine URLs derived from ENGINE_KIND. They match // the deploy/compose/.yml service names and upstream default ports. const ( - defaultOllamaURL = "http://ollama:11434" - defaultVLLMURL = "http://vllm:8000" - defaultLlamaCppURL = "http://llamacpp:8081" - defaultSGLangURL = "http://sglang:30000" - defaultFreeTokenURL = "http://freetoken:1919" - defaultStrataURL = "http://strata:8080" - defaultEmbedURL = "http://embed:8080" - defaultClipEmbedURL = "http://clipembed:8080" - defaultAudioURL = "http://audio-engine:8000" - defaultMusicURL = "http://music-engine:8001" + defaultOllamaURL = "http://ollama:11434" + defaultVLLMURL = "http://vllm:8000" + defaultLlamaCppURL = "http://llamacpp:8081" + defaultSGLangURL = "http://sglang:30000" + defaultFreeTokenURL = "http://freetoken:1919" + defaultStrataURL = "http://strata:8080" + defaultEmbedURL = "http://embed:8080" + defaultClipEmbedURL = "http://clipembed:8080" + defaultMultimodalEmbedURL = "http://multimodalembed:8080" + defaultAudioURL = "http://audio-engine:8000" + defaultMusicURL = "http://music-engine:8001" // OCRAdapter (not llamacpp): proxy Ready probes GET /v1/models here. defaultOCRAdapterURL = "http://ocradapter:8080" defaultRerankURL = "http://rerank:8080" @@ -90,6 +91,8 @@ func deriveEngineURL(kind EngineKind) string { return defaultEmbedURL case EngineClipEmbed: return defaultClipEmbedURL + case EngineMultimodalEmbed: + return defaultMultimodalEmbedURL case EngineAudio: return defaultAudioURL case EngineOCR: diff --git a/internal/config/config_load_engine_test.go b/internal/config/config_load_engine_test.go index 3e28545..587d6a4 100644 --- a/internal/config/config_load_engine_test.go +++ b/internal/config/config_load_engine_test.go @@ -9,18 +9,19 @@ import "testing" // proxy adapter probe the wrong port and never reach phase=ready. func TestDeriveEngineURL(t *testing.T) { cases := map[EngineKind]string{ - EngineOllama: "http://ollama:11434", - EngineVLLM: "http://vllm:8000", - EngineLlamaCpp: "http://llamacpp:8081", - EngineSGLang: "http://sglang:30000", - EngineFreeToken: "http://freetoken:1919", - EngineStrata: "http://strata:8080", - EngineEmbed: "http://embed:8080", - EngineClipEmbed: "http://clipembed:8080", - EngineOCR: "http://ocradapter:8080", - EngineRerank: "http://rerank:8080", - EngineMusic: "http://music-engine:8001", - EngineSystemOne: "http://systemone:8000", + EngineOllama: "http://ollama:11434", + EngineVLLM: "http://vllm:8000", + EngineLlamaCpp: "http://llamacpp:8081", + EngineSGLang: "http://sglang:30000", + EngineFreeToken: "http://freetoken:1919", + EngineStrata: "http://strata:8080", + EngineEmbed: "http://embed:8080", + EngineClipEmbed: "http://clipembed:8080", + EngineMultimodalEmbed: "http://multimodalembed:8080", + EngineOCR: "http://ocradapter:8080", + EngineRerank: "http://rerank:8080", + EngineMusic: "http://music-engine:8001", + EngineSystemOne: "http://systemone:8000", } for kind, want := range cases { if got := deriveEngineURL(kind); got != want { diff --git a/internal/config/context_size_test.go b/internal/config/context_size_test.go index fec95a6..0929dc5 100644 --- a/internal/config/context_size_test.go +++ b/internal/config/context_size_test.go @@ -126,7 +126,7 @@ func TestDeriveContextSize_LlamacppZeroMeansFromModel(t *testing.T) { // launch flags here, so nothing can be derived for them. func TestDeriveContextSize_NonLLMEnginesDeriveNothing(t *testing.T) { t.Parallel() - for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, ""} { + for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, ""} { args, err := ParseEngineArgs(kind, "") if err != nil && kind != "" { t.Fatalf("kind=%s: %v", kind, err) diff --git a/internal/config/engine_args.go b/internal/config/engine_args.go index ff9dc2e..bf90dba 100644 --- a/internal/config/engine_args.go +++ b/internal/config/engine_args.go @@ -66,7 +66,7 @@ func ParseEngineArgs(kind EngineKind, raw string) (EngineArgs, error) { return parseCmdlineArgs(raw, sglangKnownFlags) case EngineLlamaCpp: return parseLlamacppArgs(raw) - case EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne: + case EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne: // ENGINE_ARGS are set on the engine container (llamacpp for ocr), not llm-init. args := EngineArgs{Raw: raw, Known: map[string]string{}} if strings.TrimSpace(raw) == "" { diff --git a/internal/config/engine_args_test.go b/internal/config/engine_args_test.go index 8f7383c..fbc7f9f 100644 --- a/internal/config/engine_args_test.go +++ b/internal/config/engine_args_test.go @@ -176,7 +176,7 @@ func TestParseEngineArgs_SGLangCmdline(t *testing.T) { func TestParseEngineArgs_EmptyRaw(t *testing.T) { t.Parallel() - for _, kind := range []EngineKind{EngineOllama, EngineVLLM, EngineLlamaCpp, EngineSGLang, EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank} { + for _, kind := range []EngineKind{EngineOllama, EngineVLLM, EngineLlamaCpp, EngineSGLang, EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank} { got, err := ParseEngineArgs(kind, "") if err != nil { t.Errorf("kind=%s: %v", kind, err) @@ -204,7 +204,7 @@ func TestParseEngineArgs_UnbalancedQuoteFailFast(t *testing.T) { func TestParseEngineArgs_EmbedEmpty(t *testing.T) { t.Parallel() - for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} { + for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} { got, err := ParseEngineArgs(kind, "") if err != nil { t.Fatalf("kind=%s err: %v", kind, err) @@ -217,7 +217,7 @@ func TestParseEngineArgs_EmbedEmpty(t *testing.T) { func TestParseEngineArgs_EmbedRejectsNonEmpty(t *testing.T) { t.Parallel() - for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} { + for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} { _, err := ParseEngineArgs(kind, "EMBED_DEVICE=cpu") if err == nil { t.Fatalf("kind=%s: expected error when ENGINE_ARGS set", kind) diff --git a/internal/config/max_concurrency_test.go b/internal/config/max_concurrency_test.go index 925f6bf..14a4557 100644 --- a/internal/config/max_concurrency_test.go +++ b/internal/config/max_concurrency_test.go @@ -62,7 +62,7 @@ func TestDeriveMaxConcurrency_PerEngineFlag(t *testing.T) { // launch flags here, so nothing can be derived for them. func TestDeriveMaxConcurrency_NonLLMEnginesDeriveNothing(t *testing.T) { t.Parallel() - for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, ""} { + for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, ""} { args, err := ParseEngineArgs(kind, "") if err != nil && kind != "" { t.Fatalf("kind=%s: %v", kind, err) diff --git a/internal/config/modelspec.go b/internal/config/modelspec.go index e4a5933..bfe79b0 100644 --- a/internal/config/modelspec.go +++ b/internal/config/modelspec.go @@ -132,10 +132,15 @@ var allSupports = []string{ "supports_response_format", "supports_logit_bias", "supports_user", - // Embedding (1). An embedding model that places images in the same + // Embedding (5). These flags describe embedding inputs and must not be + // confused with chat-model multimodal input capabilities above. // vector space as text (Jina CLIP, Cohere embed-*-image). Not // supports_vision, which is a chat model looking at an image. "supports_embedding_image_input", + "supports_embedding_audio_input", + "supports_embedding_video_input", + "supports_embedding_interleaved_input", + "supports_embedding_async", } // MIRROR OF: diff --git a/internal/config/modelspec_test.go b/internal/config/modelspec_test.go index e731cae..06e1587 100644 --- a/internal/config/modelspec_test.go +++ b/internal/config/modelspec_test.go @@ -349,6 +349,10 @@ func TestAllSupports_RouterOrder(t *testing.T) { "supports_logit_bias", "supports_user", "supports_embedding_image_input", + "supports_embedding_audio_input", + "supports_embedding_video_input", + "supports_embedding_interleaved_input", + "supports_embedding_async", }) } diff --git a/internal/controlplane/endpoints.go b/internal/controlplane/endpoints.go index 25ed6b2..b75cad1 100644 --- a/internal/controlplane/endpoints.go +++ b/internal/controlplane/endpoints.go @@ -115,6 +115,7 @@ const ( // spelling here would let a row describe a path nothing serves. pathModels = moderoutes.PathModels pathEmbeddings = moderoutes.PathEmbeddings + pathEmbeddingTasks = moderoutes.PathEmbeddingTasks pathRerank = moderoutes.PathRerank pathSystemOne = moderoutes.PathSystemOne pathOCR = moderoutes.PathOCR @@ -200,8 +201,8 @@ func (s *Server) buildEndpointCatalog() []EndpointInfo { // The async task contract belongs to whichever engine runs // work a request cannot wait out — ocr, audio and tts today, image next. llm-init proxies it // and knows nothing about the documents. - tasksOn := dataPlaneOn && (mode == config.ModelOCR || mode == config.ModelAudio || mode == config.ModelTTS) - tasksOffReasons := []string{"not shown unless MODEL_MODE=ocr|audio|tts (catalog only)"} + tasksOn := dataPlaneOn && (mode == config.ModelEmbedding || mode == config.ModelOCR || mode == config.ModelAudio || mode == config.ModelTTS) + tasksOffReasons := []string{"not shown unless MODEL_MODE=embedding|ocr|audio|tts (catalog only)"} if !dataPlaneOn { tasksOffReasons = dataPlaneReason } @@ -294,6 +295,11 @@ func (s *Server) buildEndpointCatalog() []EndpointInfo { Group: groupOpenAI, Description: "Embeddings (catalog highlight when MODEL_MODE=embedding).", Available: dataPlaneOn, Reasons: unlessOn(dataPlaneOn, dataPlaneReason)}, + {Method: mPOST, Path: pathEmbeddingTasks, Category: categoryOpenAI, + Group: groupTasks, + Description: "Submit an asynchronous multimodal embedding task (multipart audio/video/interleaved input).", + Available: tasksOn && mode == config.ModelEmbedding, Reasons: unlessOn(tasksOn && mode == config.ModelEmbedding, tasksOffReasons), + CurlHint: "curl -sS -X POST http://HOST:PORT/v1/embedding-tasks -F 'media=@clip.mp4'"}, {Method: mPOST, Path: pathRerank, Category: categoryOpenAI, Group: groupOpenAI, Description: "Rerank documents (catalog highlight when MODEL_MODE=rerank).", diff --git a/internal/controlplane/endpoints_test.go b/internal/controlplane/endpoints_test.go index 6af85de..d69d30b 100644 --- a/internal/controlplane/endpoints_test.go +++ b/internal/controlplane/endpoints_test.go @@ -38,7 +38,7 @@ func endpointsFixtureMode(t *testing.T, kind config.EngineKind, mode string, mut env["MODEL_SOURCE"] = "ollama://llama3:8b" case config.EngineLlamaCpp: env["MODEL_SOURCE"] = "hf://Qwen/Qwen2.5-7B-Instruct --include demo.gguf --revision 0123456789abcdef0123456789abcdef01234567" - case config.EngineEmbed, config.EngineClipEmbed: + case config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed: env["MODEL_MODE"] = "embedding" env["MODEL_SOURCE"] = "https://example.com/model.tgz#sha256=" + strings.Repeat("a", 64) env["MODEL_SOURCE_LOCAL"] = "/data/model" @@ -628,6 +628,7 @@ func TestEndpoints_CatalogIsTheModeRouteTable(t *testing.T) { mode config.ModelType }{ {config.EngineEmbed, config.ModelEmbedding}, + {config.EngineMultimodalEmbed, config.ModelEmbedding}, {config.EngineRerank, config.ModelRerank}, {config.EngineOCR, config.ModelOCR}, {config.EngineSystemOne, config.ModelSystemOne}, diff --git a/internal/moderoutes/moderoutes.go b/internal/moderoutes/moderoutes.go index a62bccc..9fe97b4 100644 --- a/internal/moderoutes/moderoutes.go +++ b/internal/moderoutes/moderoutes.go @@ -59,6 +59,7 @@ type Route struct { const ( PathModels = "/v1/models" PathEmbeddings = "/v1/embeddings" + PathEmbeddingTasks = "/v1/embedding-tasks" PathRerank = "/v1/rerank" PathSystemOne = "/v1/systemone" PathOCR = "/v1/ocr" @@ -80,6 +81,9 @@ var declared = map[config.ModelType][]Route{ config.ModelEmbedding: { {http.MethodGet, PathModels}, {http.MethodPost, PathEmbeddings}, + {http.MethodPost, PathEmbeddingTasks}, + {AnyMethod, PathTask}, + {AnyMethod, PathTaskResult}, }, config.ModelRerank: { {http.MethodGet, PathModels}, diff --git a/internal/moderoutes/moderoutes_test.go b/internal/moderoutes/moderoutes_test.go index 7a8d976..448b93b 100644 --- a/internal/moderoutes/moderoutes_test.go +++ b/internal/moderoutes/moderoutes_test.go @@ -99,6 +99,12 @@ func TestDeclares_MethodIsPartOfTheRow(t *testing.T) { if !Declares(config.ModelEmbedding, "POST", PathEmbeddings) { t.Error("POST /v1/embeddings should be declared") } + if !Declares(config.ModelEmbedding, "POST", PathEmbeddingTasks) { + t.Error("POST /v1/embedding-tasks should be declared") + } + if !Declares(config.ModelEmbedding, "DELETE", PathTask) { + t.Error("embedding task routes should pass through engine-declared methods") + } // AnyMethod exists because the async task contract is the engine's: // a verb it declares on a task path is that mode's route even though // the catalogue never listed it. From cabe786a8a9c2bdb3ae05201fa7d1e8b6da709b5 Mon Sep 17 00:00:00 2001 From: ubuntu Date: Mon, 19 Oct 2026 08:29:00 +0800 Subject: [PATCH 2/2] fix: download exact multimodal ONNX asset lists --- internal/adapter/hfwrap/runner.go | 26 ++++++++++++++++++++++---- internal/adapter/hfwrap/runner_test.go | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/adapter/hfwrap/runner.go b/internal/adapter/hfwrap/runner.go index 1b8868f..e77466a 100644 --- a/internal/adapter/hfwrap/runner.go +++ b/internal/adapter/hfwrap/runner.go @@ -331,17 +331,35 @@ func buildArgs(cfg Config) []string { if cfg.HFFile != "" { args = append(args, cfg.HFFile) } + // Exact allow-list entries are filenames, not glob patterns. Pass them as + // positional filenames so this works with both generations of the hf CLI: + // the argparse implementation treats `--include` as one multi-value option, + // while the newer Click implementation accepts repeated scalar options. + // Repeating `--include` therefore silently keeps only the last pattern on + // the former, which is especially dangerous for sharded ONNX models. + exactIncludes := len(cfg.AllowPatterns) > 0 + for _, p := range cfg.AllowPatterns { + if p == "" || strings.ContainsAny(p, "*?[") { + exactIncludes = false + break + } + } + if exactIncludes { + args = append(args, cfg.AllowPatterns...) + } if cfg.CacheDir != "" { args = append(args, "--cache-dir", cfg.CacheDir) } if cfg.Revision != "" && cfg.Revision != defaultHFRef { args = append(args, "--revision", cfg.Revision) } - for _, p := range cfg.AllowPatterns { - if p == "" { - continue + if !exactIncludes { + for _, p := range cfg.AllowPatterns { + if p == "" { + continue + } + args = append(args, "--include", p) } - args = append(args, "--include", p) } for _, p := range cfg.ExcludePatterns { if p == "" { diff --git a/internal/adapter/hfwrap/runner_test.go b/internal/adapter/hfwrap/runner_test.go index 5d8130d..4e001f5 100644 --- a/internal/adapter/hfwrap/runner_test.go +++ b/internal/adapter/hfwrap/runner_test.go @@ -366,6 +366,22 @@ func TestBuildArgs_AllOptionsForwarded(t *testing.T) { } } +func TestBuildArgs_ExactIncludesArePositionalFilenames(t *testing.T) { + args := buildArgs(Config{ + Repo: "onnx-community/embeddinggemma-2-ONNX", + Revision: "abc123", + AllowPatterns: []string{"onnx/model.onnx", "onnx/model.onnx_data", "tokenizer.json"}, + }) + want := []string{ + "download", "onnx-community/embeddinggemma-2-ONNX", + "onnx/model.onnx", "onnx/model.onnx_data", "tokenizer.json", + "--revision", "abc123", + } + if strings.Join(args, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("got %v want %v", args, want) + } +} + func TestBuildArgs_ExcludeOnly(t *testing.T) { args := buildArgs(Config{ Repo: "beclab/embeddinggemma-300m",