feat: add multimodal embedding engine support

internal/adapter/factory/factory.go

@@ -36,7 +36,7 @@ func New(cfg config.Config, metrics *obs.Metrics) (adapter.Adapter, error) {
 		return nullengine.New(), nil
 	case config.EngineOllama:
 		return ollama.NewAdapter(cfg), nil
-	case config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata, config.EngineEmbed, config.EngineClipEmbed, config.EngineAudio, config.EngineOCR, config.EngineRerank, config.EngineMusic, config.EngineSystemOne:
+	case config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata, config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed, config.EngineAudio, config.EngineOCR, config.EngineRerank, config.EngineMusic, config.EngineSystemOne:
 		a, err := proxy.NewAdapter(cfg, cfg.Engine.Kind)
 		if err != nil {
 			return nil, err

internal/adapter/factory/factory_test.go

@@ -42,7 +42,7 @@ func TestNew_ProxyKinds(t *testing.T) {
 	t.Parallel()
 	for _, k := range []config.EngineKind{
 		config.EngineVLLM, config.EngineLlamaCpp, config.EngineSGLang, config.EngineFreeToken, config.EngineStrata,
-		config.EngineEmbed, config.EngineClipEmbed, config.EngineOCR, config.EngineRerank,
+		config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed, config.EngineOCR, config.EngineRerank,
 		config.EngineSystemOne,
 	} {
 		k := k

internal/adapter/hfwrap/runner.go

@@ -331,17 +331,35 @@ func buildArgs(cfg Config) []string {
 	if cfg.HFFile != "" {
 		args = append(args, cfg.HFFile)
 	}
+	// Exact allow-list entries are filenames, not glob patterns. Pass them as
+	// positional filenames so this works with both generations of the hf CLI:
+	// the argparse implementation treats `--include` as one multi-value option,
+	// while the newer Click implementation accepts repeated scalar options.
+	// Repeating `--include` therefore silently keeps only the last pattern on
+	// the former, which is especially dangerous for sharded ONNX models.
+	exactIncludes := len(cfg.AllowPatterns) > 0
+	for _, p := range cfg.AllowPatterns {
+		if p == "" || strings.ContainsAny(p, "*?[") {
+			exactIncludes = false
+			break
+		}
+	}
+	if exactIncludes {
+		args = append(args, cfg.AllowPatterns...)
+	}
 	if cfg.CacheDir != "" {
 		args = append(args, "--cache-dir", cfg.CacheDir)
 	}
 	if cfg.Revision != "" && cfg.Revision != defaultHFRef {
 		args = append(args, "--revision", cfg.Revision)
 	}
-	for _, p := range cfg.AllowPatterns {
-		if p == "" {
-			continue
+	if !exactIncludes {
+		for _, p := range cfg.AllowPatterns {
+			if p == "" {
+				continue
+			}
+			args = append(args, "--include", p)
 		}
-		args = append(args, "--include", p)
 	}
 	for _, p := range cfg.ExcludePatterns {
 		if p == "" {

internal/adapter/hfwrap/runner_test.go

@@ -366,6 +366,22 @@ func TestBuildArgs_AllOptionsForwarded(t *testing.T) {
 	}
 }
 
+func TestBuildArgs_ExactIncludesArePositionalFilenames(t *testing.T) {
+	args := buildArgs(Config{
+		Repo:          "onnx-community/embeddinggemma-2-ONNX",
+		Revision:      "abc123",
+		AllowPatterns: []string{"onnx/model.onnx", "onnx/model.onnx_data", "tokenizer.json"},
+	})
+	want := []string{
+		"download", "onnx-community/embeddinggemma-2-ONNX",
+		"onnx/model.onnx", "onnx/model.onnx_data", "tokenizer.json",
+		"--revision", "abc123",
+	}
+	if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
+		t.Fatalf("got %v want %v", args, want)
+	}
+}
+
 func TestBuildArgs_ExcludeOnly(t *testing.T) {
 	args := buildArgs(Config{
 		Repo:            "beclab/embeddinggemma-300m",

internal/adapter/proxy/rewrite.go

@@ -108,7 +108,7 @@ func rewriteBodyCap(path string, kind config.EngineKind) int {
 	case pathChatCompletions, pathAPIChatCompletions, pathResponses:
 		return maxVisionRewriteBody
 	case pathEmbeddings:
-		if kind == config.EngineClipEmbed {
+		if kind == config.EngineClipEmbed || kind == config.EngineMultimodalEmbed {
 			return maxVisionRewriteBody
 		}
 		return maxRewriteBody

internal/config/config_enums.go

@@ -8,19 +8,20 @@ type EngineKind string
 // EngineKind values. The set is closed; AllEngineKinds is the canonical
 // slice form used for env validation.
 const (
-	EngineOllama    EngineKind = "ollama"
-	EngineVLLM      EngineKind = "vllm"
-	EngineLlamaCpp  EngineKind = "llamacpp"
-	EngineSGLang    EngineKind = "sglang"
-	EngineFreeToken EngineKind = "freetoken"
-	EngineStrata    EngineKind = "strata"
-	EngineEmbed     EngineKind = "embed"
-	EngineClipEmbed EngineKind = "clipembed"
-	EngineAudio     EngineKind = "audio"
-	EngineOCR       EngineKind = "ocr"
-	EngineRerank    EngineKind = "rerank"
-	EngineMusic     EngineKind = "music"
-	EngineSystemOne EngineKind = "systemone"
+	EngineOllama          EngineKind = "ollama"
+	EngineVLLM            EngineKind = "vllm"
+	EngineLlamaCpp        EngineKind = "llamacpp"
+	EngineSGLang          EngineKind = "sglang"
+	EngineFreeToken       EngineKind = "freetoken"
+	EngineStrata          EngineKind = "strata"
+	EngineEmbed           EngineKind = "embed"
+	EngineClipEmbed       EngineKind = "clipembed"
+	EngineMultimodalEmbed EngineKind = "multimodalembed"
+	EngineAudio           EngineKind = "audio"
+	EngineOCR             EngineKind = "ocr"
+	EngineRerank          EngineKind = "rerank"
+	EngineMusic           EngineKind = "music"
+	EngineSystemOne       EngineKind = "systemone"
 )
 
 // AllEngineKinds lists every supported engine; used for env validation.
@@ -33,6 +34,7 @@ var AllEngineKinds = []string{
 	string(EngineStrata),
 	string(EngineEmbed),
 	string(EngineClipEmbed),
+	string(EngineMultimodalEmbed),
 	string(EngineAudio),
 	string(EngineOCR),
 	string(EngineRerank),

internal/config/config_load_engine.go

@@ -53,16 +53,17 @@ func (l *loader) loadEngine() {
 // Conventional sibling-engine URLs derived from ENGINE_KIND. They match
 // the deploy/compose/<kind>.yml service names and upstream default ports.
 const (
-	defaultOllamaURL    = "http://ollama:11434"
-	defaultVLLMURL      = "http://vllm:8000"
-	defaultLlamaCppURL  = "http://llamacpp:8081"
-	defaultSGLangURL    = "http://sglang:30000"
-	defaultFreeTokenURL = "http://freetoken:1919"
-	defaultStrataURL    = "http://strata:8080"
-	defaultEmbedURL     = "http://embed:8080"
-	defaultClipEmbedURL = "http://clipembed:8080"
-	defaultAudioURL     = "http://audio-engine:8000"
-	defaultMusicURL     = "http://music-engine:8001"
+	defaultOllamaURL          = "http://ollama:11434"
+	defaultVLLMURL            = "http://vllm:8000"
+	defaultLlamaCppURL        = "http://llamacpp:8081"
+	defaultSGLangURL          = "http://sglang:30000"
+	defaultFreeTokenURL       = "http://freetoken:1919"
+	defaultStrataURL          = "http://strata:8080"
+	defaultEmbedURL           = "http://embed:8080"
+	defaultClipEmbedURL       = "http://clipembed:8080"
+	defaultMultimodalEmbedURL = "http://multimodalembed:8080"
+	defaultAudioURL           = "http://audio-engine:8000"
+	defaultMusicURL           = "http://music-engine:8001"
 	// OCRAdapter (not llamacpp): proxy Ready probes GET /v1/models here.
 	defaultOCRAdapterURL = "http://ocradapter:8080"
 	defaultRerankURL     = "http://rerank:8080"
@@ -90,6 +91,8 @@ func deriveEngineURL(kind EngineKind) string {
 		return defaultEmbedURL
 	case EngineClipEmbed:
 		return defaultClipEmbedURL
+	case EngineMultimodalEmbed:
+		return defaultMultimodalEmbedURL
 	case EngineAudio:
 		return defaultAudioURL
 	case EngineOCR:

internal/config/config_load_engine_test.go

@@ -9,18 +9,19 @@ import "testing"
 // proxy adapter probe the wrong port and never reach phase=ready.
 func TestDeriveEngineURL(t *testing.T) {
 	cases := map[EngineKind]string{
-		EngineOllama:    "http://ollama:11434",
-		EngineVLLM:      "http://vllm:8000",
-		EngineLlamaCpp:  "http://llamacpp:8081",
-		EngineSGLang:    "http://sglang:30000",
-		EngineFreeToken: "http://freetoken:1919",
-		EngineStrata:    "http://strata:8080",
-		EngineEmbed:     "http://embed:8080",
-		EngineClipEmbed: "http://clipembed:8080",
-		EngineOCR:       "http://ocradapter:8080",
-		EngineRerank:    "http://rerank:8080",
-		EngineMusic:     "http://music-engine:8001",
-		EngineSystemOne: "http://systemone:8000",
+		EngineOllama:          "http://ollama:11434",
+		EngineVLLM:            "http://vllm:8000",
+		EngineLlamaCpp:        "http://llamacpp:8081",
+		EngineSGLang:          "http://sglang:30000",
+		EngineFreeToken:       "http://freetoken:1919",
+		EngineStrata:          "http://strata:8080",
+		EngineEmbed:           "http://embed:8080",
+		EngineClipEmbed:       "http://clipembed:8080",
+		EngineMultimodalEmbed: "http://multimodalembed:8080",
+		EngineOCR:             "http://ocradapter:8080",
+		EngineRerank:          "http://rerank:8080",
+		EngineMusic:           "http://music-engine:8001",
+		EngineSystemOne:       "http://systemone:8000",
 	}
 	for kind, want := range cases {
 		if got := deriveEngineURL(kind); got != want {

internal/config/context_size_test.go

@@ -126,7 +126,7 @@ func TestDeriveContextSize_LlamacppZeroMeansFromModel(t *testing.T) {
 // launch flags here, so nothing can be derived for them.
 func TestDeriveContextSize_NonLLMEnginesDeriveNothing(t *testing.T) {
 	t.Parallel()
-	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, ""} {
+	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, ""} {
 		args, err := ParseEngineArgs(kind, "")
 		if err != nil && kind != "" {
 			t.Fatalf("kind=%s: %v", kind, err)

internal/config/engine_args.go

@@ -66,7 +66,7 @@ func ParseEngineArgs(kind EngineKind, raw string) (EngineArgs, error) {
 		return parseCmdlineArgs(raw, sglangKnownFlags)
 	case EngineLlamaCpp:
 		return parseLlamacppArgs(raw)
-	case EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne:
+	case EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne:
 		// ENGINE_ARGS are set on the engine container (llamacpp for ocr), not llm-init.
 		args := EngineArgs{Raw: raw, Known: map[string]string{}}
 		if strings.TrimSpace(raw) == "" {

internal/config/engine_args_test.go

@@ -176,7 +176,7 @@ func TestParseEngineArgs_SGLangCmdline(t *testing.T) {
 
 func TestParseEngineArgs_EmptyRaw(t *testing.T) {
 	t.Parallel()
-	for _, kind := range []EngineKind{EngineOllama, EngineVLLM, EngineLlamaCpp, EngineSGLang, EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank} {
+	for _, kind := range []EngineKind{EngineOllama, EngineVLLM, EngineLlamaCpp, EngineSGLang, EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank} {
 		got, err := ParseEngineArgs(kind, "")
 		if err != nil {
 			t.Errorf("kind=%s: %v", kind, err)
@@ -204,7 +204,7 @@ func TestParseEngineArgs_UnbalancedQuoteFailFast(t *testing.T) {
 
 func TestParseEngineArgs_EmbedEmpty(t *testing.T) {
 	t.Parallel()
-	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} {
+	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} {
 		got, err := ParseEngineArgs(kind, "")
 		if err != nil {
 			t.Fatalf("kind=%s err: %v", kind, err)
@@ -217,7 +217,7 @@ func TestParseEngineArgs_EmbedEmpty(t *testing.T) {
 
 func TestParseEngineArgs_EmbedRejectsNonEmpty(t *testing.T) {
 	t.Parallel()
-	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} {
+	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineOCR, EngineRerank, EngineMusic, EngineSystemOne} {
 		_, err := ParseEngineArgs(kind, "EMBED_DEVICE=cpu")
 		if err == nil {
 			t.Fatalf("kind=%s: expected error when ENGINE_ARGS set", kind)

internal/config/max_concurrency_test.go

@@ -62,7 +62,7 @@ func TestDeriveMaxConcurrency_PerEngineFlag(t *testing.T) {
 // launch flags here, so nothing can be derived for them.
 func TestDeriveMaxConcurrency_NonLLMEnginesDeriveNothing(t *testing.T) {
 	t.Parallel()
-	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineAudio, EngineOCR, EngineRerank, ""} {
+	for _, kind := range []EngineKind{EngineEmbed, EngineClipEmbed, EngineMultimodalEmbed, EngineAudio, EngineOCR, EngineRerank, ""} {
 		args, err := ParseEngineArgs(kind, "")
 		if err != nil && kind != "" {
 			t.Fatalf("kind=%s: %v", kind, err)

internal/config/modelspec.go

@@ -132,10 +132,15 @@ var allSupports = []string{
 	"supports_response_format",
 	"supports_logit_bias",
 	"supports_user",
-	// Embedding (1). An embedding model that places images in the same
+	// Embedding (5). These flags describe embedding inputs and must not be
+	// confused with chat-model multimodal input capabilities above.
 	// vector space as text (Jina CLIP, Cohere embed-*-image). Not
 	// supports_vision, which is a chat model looking at an image.
 	"supports_embedding_image_input",
+	"supports_embedding_audio_input",
+	"supports_embedding_video_input",
+	"supports_embedding_interleaved_input",
+	"supports_embedding_async",
 }
 
 // MIRROR OF:

internal/config/modelspec_test.go

@@ -349,6 +349,10 @@ func TestAllSupports_RouterOrder(t *testing.T) {
 		"supports_logit_bias",
 		"supports_user",
 		"supports_embedding_image_input",
+		"supports_embedding_audio_input",
+		"supports_embedding_video_input",
+		"supports_embedding_interleaved_input",
+		"supports_embedding_async",
 	})
 }
 

internal/controlplane/endpoints.go

@@ -115,6 +115,7 @@ const (
 	// spelling here would let a row describe a path nothing serves.
 	pathModels           = moderoutes.PathModels
 	pathEmbeddings       = moderoutes.PathEmbeddings
+	pathEmbeddingTasks   = moderoutes.PathEmbeddingTasks
 	pathRerank           = moderoutes.PathRerank
 	pathSystemOne        = moderoutes.PathSystemOne
 	pathOCR              = moderoutes.PathOCR
@@ -200,8 +201,8 @@ func (s *Server) buildEndpointCatalog() []EndpointInfo {
 	// The async task contract belongs to whichever engine runs
 	// work a request cannot wait out — ocr, audio and tts today, image next. llm-init proxies it
 	// and knows nothing about the documents.
-	tasksOn := dataPlaneOn && (mode == config.ModelOCR || mode == config.ModelAudio || mode == config.ModelTTS)
-	tasksOffReasons := []string{"not shown unless MODEL_MODE=ocr|audio|tts (catalog only)"}
+	tasksOn := dataPlaneOn && (mode == config.ModelEmbedding || mode == config.ModelOCR || mode == config.ModelAudio || mode == config.ModelTTS)
+	tasksOffReasons := []string{"not shown unless MODEL_MODE=embedding|ocr|audio|tts (catalog only)"}
 	if !dataPlaneOn {
 		tasksOffReasons = dataPlaneReason
 	}
@@ -294,6 +295,11 @@ func (s *Server) buildEndpointCatalog() []EndpointInfo {
 			Group:       groupOpenAI,
 			Description: "Embeddings (catalog highlight when MODEL_MODE=embedding).",
 			Available:   dataPlaneOn, Reasons: unlessOn(dataPlaneOn, dataPlaneReason)},
+		{Method: mPOST, Path: pathEmbeddingTasks, Category: categoryOpenAI,
+			Group:       groupTasks,
+			Description: "Submit an asynchronous multimodal embedding task (multipart audio/video/interleaved input).",
+			Available:   tasksOn && mode == config.ModelEmbedding, Reasons: unlessOn(tasksOn && mode == config.ModelEmbedding, tasksOffReasons),
+			CurlHint: "curl -sS -X POST http://HOST:PORT/v1/embedding-tasks -F 'media=@clip.mp4'"},
 		{Method: mPOST, Path: pathRerank, Category: categoryOpenAI,
 			Group:       groupOpenAI,
 			Description: "Rerank documents (catalog highlight when MODEL_MODE=rerank).",

internal/controlplane/endpoints_test.go

@@ -38,7 +38,7 @@ func endpointsFixtureMode(t *testing.T, kind config.EngineKind, mode string, mut
 		env["MODEL_SOURCE"] = "ollama://llama3:8b"
 	case config.EngineLlamaCpp:
 		env["MODEL_SOURCE"] = "hf://Qwen/Qwen2.5-7B-Instruct --include demo.gguf --revision 0123456789abcdef0123456789abcdef01234567"
-	case config.EngineEmbed, config.EngineClipEmbed:
+	case config.EngineEmbed, config.EngineClipEmbed, config.EngineMultimodalEmbed:
 		env["MODEL_MODE"] = "embedding"
 		env["MODEL_SOURCE"] = "https://example.com/model.tgz#sha256=" + strings.Repeat("a", 64)
 		env["MODEL_SOURCE_LOCAL"] = "/data/model"
@@ -628,6 +628,7 @@ func TestEndpoints_CatalogIsTheModeRouteTable(t *testing.T) {
 		mode config.ModelType
 	}{
 		{config.EngineEmbed, config.ModelEmbedding},
+		{config.EngineMultimodalEmbed, config.ModelEmbedding},
 		{config.EngineRerank, config.ModelRerank},
 		{config.EngineOCR, config.ModelOCR},
 		{config.EngineSystemOne, config.ModelSystemOne},

internal/moderoutes/moderoutes.go

@@ -59,6 +59,7 @@ type Route struct {
 const (
 	PathModels           = "/v1/models"
 	PathEmbeddings       = "/v1/embeddings"
+	PathEmbeddingTasks   = "/v1/embedding-tasks"
 	PathRerank           = "/v1/rerank"
 	PathSystemOne        = "/v1/systemone"
 	PathOCR              = "/v1/ocr"
@@ -80,6 +81,9 @@ var declared = map[config.ModelType][]Route{
 	config.ModelEmbedding: {
 		{http.MethodGet, PathModels},
 		{http.MethodPost, PathEmbeddings},
+		{http.MethodPost, PathEmbeddingTasks},
+		{AnyMethod, PathTask},
+		{AnyMethod, PathTaskResult},
 	},
 	config.ModelRerank: {
 		{http.MethodGet, PathModels},

internal/moderoutes/moderoutes_test.go

@@ -99,6 +99,12 @@ func TestDeclares_MethodIsPartOfTheRow(t *testing.T) {
 	if !Declares(config.ModelEmbedding, "POST", PathEmbeddings) {
 		t.Error("POST /v1/embeddings should be declared")
 	}
+	if !Declares(config.ModelEmbedding, "POST", PathEmbeddingTasks) {
+		t.Error("POST /v1/embedding-tasks should be declared")
+	}
+	if !Declares(config.ModelEmbedding, "DELETE", PathTask) {
+		t.Error("embedding task routes should pass through engine-declared methods")
+	}
 	// AnyMethod exists because the async task contract is the engine's:
 	// a verb it declares on a task path is that mode's route even though
 	// the catalogue never listed it.