From 0ee32f51a35d423e2c1fc27c6f2331aa2f51be24 Mon Sep 17 00:00:00 2001 From: abetlen Date: Sat, 12 Sep 2026 15:49:06 -0700 Subject: [PATCH] feat: update llama.cpp to v0.4.0 --- CHANGELOG.md | 2 + CMakeLists.txt | 23 ++---- examples/server/server.py | 1 + llama_cpp/llama_cpp.py | 27 +++++-- llama_cpp/mtmd_cpp.py | 148 ++++++++++++++++++++++++++++++++++---- vendor/llama.cpp | 2 +- 6 files changed, 164 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 247ff2c23..6e13a4b5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@v0.4.0 + ## [0.3.35] - feat: update llama.cpp to ggml-org/llama.cpp@4df29be4f diff --git a/CMakeLists.txt b/CMakeLists.txt index 5feaaca5b..89c12958c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,25 +187,10 @@ if (LLAMA_BUILD) add_compile_definitions(GGML_USE_METAL) endif() - # Upstream mtmd expects LLAMA_INSTALL_VERSION to be set by llama.cpp's - # top-level CMakeLists.txt. When we include tools/mtmd directly from the - # Python package build, that directory scope is skipped. - if (NOT DEFINED LLAMA_INSTALL_VERSION OR "${LLAMA_INSTALL_VERSION}" STREQUAL "") - set(LLAMA_INSTALL_VERSION 0.0.0) - find_package(Git QUIET) - if (Git_FOUND) - execute_process( - COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp - OUTPUT_VARIABLE LLAMA_MTMD_BUILD_NUMBER - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE LLAMA_MTMD_BUILD_NUMBER_RESULT - ) - if (LLAMA_MTMD_BUILD_NUMBER_RESULT EQUAL 0) - set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_MTMD_BUILD_NUMBER}) - endif() - endif() - endif() + # tools/mtmd is added outside llama.cpp's directory scope, so inherit + # its library version variables explicitly. + get_directory_property(LLAMA_VERSION_BASE DIRECTORY vendor/llama.cpp DEFINITION LLAMA_VERSION_BASE) + get_directory_property(LLAMA_VERSION_MAJOR DIRECTORY vendor/llama.cpp DEFINITION LLAMA_VERSION_MAJOR) # Building llava add_subdirectory(vendor/llama.cpp/tools/mtmd) diff --git a/examples/server/server.py b/examples/server/server.py index 2eb31aed3..70a4a6e04 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -10748,6 +10748,7 @@ def _create_loaded_media( buffer, len(media_bytes), False, + mtmd_cpp.mtmd_helper_init_opt_default(), ) bitmap = wrapper.bitmap if bitmap is None: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c5387a3df..4aa9c6ec4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -162,13 +162,13 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 9 -LLAMA_SESSION_VERSION = 9 +# define LLAMA_SESSION_VERSION 10 +LLAMA_SESSION_VERSION = 10 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ -# define LLAMA_STATE_SEQ_VERSION 2 -LLAMA_STATE_SEQ_VERSION = 2 +# define LLAMA_STATE_SEQ_VERSION 3 +LLAMA_STATE_SEQ_VERSION = 3 # struct llama_vocab; llama_vocab_p = NewType("llama_vocab_p", int) @@ -536,6 +536,16 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_LOAD_MODE_DIRECT_IO = 4 +# enum llama_lazy_mode { +# LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front +# LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap) +# LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap) +# }; +LLAMA_LAZY_MODE_OFF = 0 +LLAMA_LAZY_MODE_AUTO = 1 +LLAMA_LAZY_MODE_ON = 2 + + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -809,6 +819,8 @@ class llama_model_imatrix_data(ctypes.Structure): # enum llama_split_mode split_mode; // how to split the model across multiple GPUs # enum llama_load_mode load_mode; // how to load the model +# enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch + # // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; @@ -844,6 +856,7 @@ class llama_model_params(ctypes.Structure): n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs load_mode (int): how to load the model + lazy_mode (int): on-demand reading of tensors marked by the arch main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. @@ -864,6 +877,7 @@ class llama_model_params(ctypes.Structure): n_gpu_layers: int split_mode: int load_mode: int + lazy_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] @@ -882,6 +896,7 @@ class llama_model_params(ctypes.Structure): ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("load_mode", ctypes.c_int), + ("lazy_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), @@ -1126,6 +1141,7 @@ class llama_context_params(ctypes.Structure): # const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides # const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides # const int32_t * prune_layers; // pointer to layer indices to prune +# size_t max_buf_size; // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB) # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -1145,6 +1161,7 @@ class llama_model_quantize_params(ctypes.Structure): kv_overrides (ctypes.Array[llama_model_kv_override]): pointer to kv overrides tt_overrides (ctypes.Array[llama_model_tensor_override]): pointer to tensor overrides prune_layers (ctypes.Array[ctypes.c_int32]): pointer to layer indices to prune + max_buf_size (int): max bytes of tensor rows kept in memory at once, 0 = default (8 GiB) """ if TYPE_CHECKING: @@ -1162,6 +1179,7 @@ class llama_model_quantize_params(ctypes.Structure): kv_overrides: CtypesPointer[llama_model_kv_override] tt_overrides: CtypesPointer[llama_model_tensor_override] prune_layers: CtypesPointer[ctypes.c_int32] + max_buf_size: int _fields_ = [ ("nthread", ctypes.c_int32), @@ -1178,6 +1196,7 @@ class llama_model_quantize_params(ctypes.Structure): ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("tt_overrides", ctypes.POINTER(llama_model_tensor_override)), ("prune_layers", ctypes.POINTER(ctypes.c_int32)), + ("max_buf_size", ctypes.c_size_t), ] diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 79f417cba..a6fb187f8 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -114,6 +114,7 @@ class mtmd_context_params(Structure): if TYPE_CHECKING: use_gpu: bool + device: c_void_p print_timings: bool n_threads: int image_marker: Optional[bytes] @@ -130,6 +131,7 @@ class mtmd_context_params(Structure): _fields_ = [ ("use_gpu", c_bool), + ("device", c_void_p), ("print_timings", c_bool), ("n_threads", c_int), ("image_marker", c_char_p), @@ -163,6 +165,24 @@ class mtmd_input_text(Structure): ] +# struct mtmd_input_part { +# // only text or bitmap can be set, not both +# const struct mtmd_input_text * text; +# const struct mtmd_bitmap * bitmap; +# }; +class mtmd_input_part(Structure): + """An input part with only text or bitmap set, not both.""" + + if TYPE_CHECKING: + text: _Pointer[mtmd_input_text] + bitmap: Optional[mtmd_bitmap_p] + + _fields_ = [ + ("text", POINTER(mtmd_input_text)), + ("bitmap", mtmd_bitmap_p_ctypes), + ] + + class mtmd_decoder_pos(Structure): """Decoder attention position for M-RoPE models.""" @@ -367,6 +387,17 @@ class mtmd_helper_video_init_params(Structure): ] +# // opt for mtmd_helper_bitmap_init_from_*() +# struct mtmd_helper_init_opt { +# struct mtmd_helper_video_init_params video_params; +# }; +class mtmd_helper_init_opt(Structure): + if TYPE_CHECKING: + video_params: mtmd_helper_video_init_params + + _fields_ = [("video_params", mtmd_helper_video_init_params)] + + # struct mtmd_helper_gen_audio_inp { # llama_seq_id seq_id; # @@ -588,7 +619,15 @@ def mtmd_bitmap_set_id(bitmap: mtmd_bitmap_p, id: Optional[bytes], /): ... -# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, +# // if true, this bitmap can be merged (temporal merge) with an adjacent mergeable bitmap by certain video input models +# MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable); +@ctypes_function("mtmd_bitmap_set_mergeable", [mtmd_bitmap_p_ctypes, c_bool], None) +def mtmd_bitmap_set_mergeable(bitmap: mtmd_bitmap_p, mergeable: bool, /): + """Allow temporal merging with an adjacent mergeable bitmap.""" + ... + + +# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx, # const char * id, # void * user_data, # mtmd_bitmap_lazy_callback callback); @@ -634,10 +673,10 @@ def mtmd_input_chunks_get( ) -> Optional[mtmd_input_chunk_p]: ... -# MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, +# MTMD_API int32_t mtmd_tokenize(const mtmd_context * ctx, # mtmd_input_chunks * output, # const mtmd_input_text * text, -# const mtmd_bitmap ** bitmaps, +# const mtmd_bitmap * const * bitmaps, # size_t n_bitmaps); @ctypes_function( "mtmd_tokenize", @@ -660,6 +699,40 @@ def mtmd_tokenize( ) -> int: ... +# // same as mtmd_tokenize(), but takes an array of mtmd_input_part +# // use cases: +# // - when you don't want to use media markers (they will be tokenized as normal text) +# // - when you want to control parse_special for each text part +# // note: per-part add_special will be ignored +# // return 1 if a part has both text and bitmap set (or neither) +# MTMD_API int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx, +# mtmd_input_chunks * output, +# const mtmd_input_part * const * parts, +# size_t n_parts, +# bool add_special); +@ctypes_function( + "mtmd_tokenize_from_parts", + [ + mtmd_context_p_ctypes, + mtmd_input_chunks_p_ctypes, + POINTER(POINTER(mtmd_input_part)), + c_size_t, + c_bool, + ], + c_int32, +) +def mtmd_tokenize_from_parts( + ctx: mtmd_context_p, + output: mtmd_input_chunks_p, + parts: CtypesArray[POINTER(mtmd_input_part)], + n_parts: Union[c_size_t, int], + add_special: bool, + /, +) -> int: + """Tokenize text and bitmap parts without substituting media markers.""" + ... + + # MTMD_API size_t mtmd_input_chunk_get_n_tokens(const mtmd_input_chunk * chunk); @ctypes_function("mtmd_input_chunk_get_n_tokens", [mtmd_input_chunk_p_ctypes], c_size_t) def mtmd_input_chunk_get_n_tokens(chunk: mtmd_input_chunk_p, /) -> int: ... @@ -726,6 +799,20 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p, /): ... +# // similar to mtmd_input_chunk_copy, but returns a placeholder chunk +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_input_chunk_get_placeholder", + [mtmd_input_chunk_p_ctypes], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_get_placeholder( + chunk: mtmd_input_chunk_p, / +) -> Optional[mtmd_input_chunk_p]: + """Like mtmd_input_chunk_copy, but returns a placeholder chunk.""" + ... + + # // save/load an input chunk to/from a buffer (useful for KV save/load) # // important: only chunk's metadata will be saved, the actual image/audio data will not be saved # // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() @@ -963,7 +1050,7 @@ def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: ################################################ -# MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); +# MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx); @ctypes_function( "mtmd_helper_support_video", [mtmd_context_p_ctypes], @@ -974,30 +1061,57 @@ def mtmd_helper_support_video(ctx: mtmd_context_p, /) -> bool: ... -# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); +# MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void); +@ctypes_function("mtmd_helper_init_opt_default", [], mtmd_helper_init_opt) +def mtmd_helper_init_opt_default() -> mtmd_helper_init_opt: + """Get default options for initializing media from files or buffers.""" + ... + + +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( +# const mtmd_context * ctx, +# const char * fname, +# bool placeholder, +# struct mtmd_helper_init_opt opt); @ctypes_function( "mtmd_helper_bitmap_init_from_file", - [mtmd_context_p_ctypes, c_char_p, c_bool], + [mtmd_context_p_ctypes, c_char_p, c_bool, mtmd_helper_init_opt], mtmd_helper_bitmap_wrapper, ) def mtmd_helper_bitmap_init_from_file_wrapper( - ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / + ctx: mtmd_context_p, + fname: bytes, + placeholder: Union[c_bool, bool], + opt: mtmd_helper_init_opt, + /, ) -> mtmd_helper_bitmap_wrapper: """Initialize an MTMD bitmap wrapper from a file.""" ... def mtmd_helper_bitmap_init_from_file( - ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / + ctx: mtmd_context_p, + fname: bytes, + placeholder: Union[c_bool, bool], + opt: Optional[mtmd_helper_init_opt] = None, + /, ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a file.""" - return mtmd_helper_bitmap_init_from_file_wrapper(ctx, fname, placeholder).bitmap + if opt is None: + opt = mtmd_helper_init_opt_default() + return mtmd_helper_bitmap_init_from_file_wrapper( + ctx, fname, placeholder, opt + ).bitmap -# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( +# const mtmd_context * ctx, +# const unsigned char * buf, size_t len, +# bool placeholder, +# struct mtmd_helper_init_opt opt); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", - [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool], + [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool, mtmd_helper_init_opt], mtmd_helper_bitmap_wrapper, ) def mtmd_helper_bitmap_init_from_buf_wrapper( @@ -1005,6 +1119,7 @@ def mtmd_helper_bitmap_init_from_buf_wrapper( buf: CtypesArray[c_uint8], length: Union[c_size_t, int], placeholder: Union[c_bool, bool], + opt: mtmd_helper_init_opt, /, ) -> mtmd_helper_bitmap_wrapper: ... @@ -1014,11 +1129,14 @@ def mtmd_helper_bitmap_init_from_buf( buf: CtypesArray[c_uint8], length: Union[c_size_t, int], placeholder: Union[c_bool, bool], + opt: Optional[mtmd_helper_init_opt] = None, /, ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a buffer.""" + if opt is None: + opt = mtmd_helper_init_opt_default() return mtmd_helper_bitmap_init_from_buf_wrapper( - ctx, buf, length, placeholder + ctx, buf, length, placeholder, opt ).bitmap @@ -1178,7 +1296,7 @@ def mtmd_helper_video_init_params_default() -> mtmd_helper_video_init_params: # MTMD_API mtmd_helper_video * mtmd_helper_video_init( -# struct mtmd_context * mctx, +# const struct mtmd_context * mctx, # const char * path, # struct mtmd_helper_video_init_params params); @ctypes_function( @@ -1197,7 +1315,7 @@ def mtmd_helper_video_init( # MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( -# struct mtmd_context * mctx, +# const struct mtmd_context * mctx, # const unsigned char * buf, size_t len, # struct mtmd_helper_video_init_params params); @ctypes_function( @@ -1257,7 +1375,7 @@ def mtmd_helper_video_read_next( # // return true if model can be used for chat -# MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); +# MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx); @ctypes_function( "mtmd_helper_model_can_chat", [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4df29be4f..5266f24da 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4df29be4f4c3673f428170fda944a5b19f743bb8 +Subproject commit 5266f24da75dc449bd56cbed7addb9c8e4a6a73e