feat : Add Support of Qwen2.5Omni model (without talker)

.gitignore

@@ -15,6 +15,7 @@ tasks/mllmteam*
 build*/
 install*/
 mllm-sdk-*/
+mllm-install-*/
 
 # Pymllm related
 stubs/

CMakeLists.txt

@@ -262,6 +262,7 @@ add_subdirectory(third_party/fmt)
 add_subdirectory(third_party/xxHash)
 set(FLATBUFFERS_BUILD_TESTS OFF)
 add_subdirectory(third_party/flatbuffers EXCLUDE_FROM_ALL)
+set_target_properties(flatbuffers PROPERTIES POSITION_INDEPENDENT_CODE ON)
 add_subdirectory(mllm)
 
 if(MLLM_ENABLE_TEST)
@@ -332,6 +333,13 @@ install(
   ARCHIVE DESTINATION lib
   RUNTIME DESTINATION bin)
 
+install(
+  TARGETS flatbuffers
+  EXPORT MllmTargets
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  RUNTIME DESTINATION bin)
+
 if(MLLM_BUILD_SDK_C_BINDING)
   install(
     TARGETS MllmSdkC

examples/CMakeLists.txt

@@ -2,6 +2,7 @@ add_subdirectory(qwen2vl)
 add_subdirectory(qwen2vl_tracer)
 add_subdirectory(qwen2_5vl)
 add_subdirectory(qwen2_5vl_tracer)
+add_subdirectory(qwen2_5omni)
 add_subdirectory(llama)
 add_subdirectory(minicpm_o)
 add_subdirectory(minicpm4)

examples/qwen2_5omni/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_executable(mllm-qwen2_5-omni-text-runner text_infer.cpp)
+target_link_libraries(mllm-qwen2_5-omni-text-runner PRIVATE MllmRT MllmCPUBackend)
+target_include_directories(mllm-qwen2_5-omni-text-runner PRIVATE ${MLLM_INCLUDE_DIR})
+
+add_executable(mllm-qwen2_5-omni-image-runner image_infer.cpp)
+target_link_libraries(mllm-qwen2_5-omni-image-runner PRIVATE MllmRT MllmCPUBackend)
+target_include_directories(mllm-qwen2_5-omni-image-runner PRIVATE ${MLLM_INCLUDE_DIR})
+
+add_executable(mllm-qwen2_5-omni-audio-runner audio_infer.cpp)
+target_link_libraries(mllm-qwen2_5-omni-audio-runner PRIVATE MllmRT MllmCPUBackend)
+target_include_directories(mllm-qwen2_5-omni-audio-runner PRIVATE ${MLLM_INCLUDE_DIR})

examples/qwen2_5omni/audio_infer.cpp

@@ -0,0 +1,84 @@
+// Copyright (c) MLLM Team.
+// Licensed under the MIT License.
+
+#include <iostream>
+#include <fmt/core.h>
+#include <mllm/mllm.hpp>
+#include <mllm/models/qwen2_5omni/configuration_qwen2_5omni.hpp>
+#include <mllm/models/qwen2_5omni/modeling_qwen2_5omni.hpp>
+#include <mllm/models/qwen2_5omni/tokenization_qwen2_5omni.hpp>
+
+using mllm::Argparse;
+
+MLLM_MAIN({
+  mllm::Logger::level() = mllm::LogLevel::kError;
+
+  auto& help = Argparse::add<bool>("-h|--help").help("Show help message");
+  auto& model_path = Argparse::add<std::string>("-m|--model_path").help("Model path").required(true);
+  auto& model_version = Argparse::add<std::string>("-mv|--model_version").help("Model version").required(true);
+  auto& tokenizer_path = Argparse::add<std::string>("-t|--tokenizer_path").help("Tokenizer directory").required(true);
+  auto& config_path = Argparse::add<std::string>("-c|--config_path").help("Config path").required(true);
+
+  Argparse::parse(argc, argv);
+
+  mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV1;
+  if (model_version.get() == "v1") {
+    file_version = mllm::ModelFileVersion::kV1;
+  } else if (model_version.get() == "v2") {
+    file_version = mllm::ModelFileVersion::kV2;
+  }
+
+  if (help.isSet()) {
+    Argparse::printHelp();
+    mllm::shutdownContext();
+    return 0;
+  }
+
+  {
+    auto qwen2_5omni_cfg = mllm::models::qwen2_5omni::Qwen2_5OmniConfig(config_path.get());
+    auto qwen2_5omni_tokenizer = mllm::models::qwen2_5omni::Qwen2_5OmniTokenizer(tokenizer_path.get());
+    auto qwen2_5omni = mllm::models::qwen2_5omni::Qwen2_5OmniForCausalLM(qwen2_5omni_cfg);
+
+    auto param = mllm::load(model_path.get(), file_version);
+    qwen2_5omni.thinker_.load(param);
+
+    fmt::print("\n{:*^60}\n", " Qwen2.5-Omni Audio CLI ");
+    fmt::print("Enter 'exit' or 'quit' to end the session\n\n");
+
+    std::string audio_path;
+    std::string prompt_text;
+
+    fmt::print("Audio path (or 'exit/quit'): ");
+    //std::getline(std::cin, audio_path);
+    //if (audio_path == "exit" || audio_path == "quit") { return 0; }
+    audio_path = "";
+
+    fmt::print("Prompt text: ");
+    //std::getline(std::cin, prompt_text);
+    //if (prompt_text.empty()) { prompt_text = "Please describe the audio."; }
+    prompt_text = "";
+
+    try {
+      fmt::print("Processing...\n");
+      auto inputs = qwen2_5omni_tokenizer.convertAudioMessage({.prompt = prompt_text, .audio_file_path = audio_path});
+
+      fmt::print("\nResponse: ");
+      qwen2_5omni.streamGenerate(inputs,
+                                 {
+                                     {"do_sample", mllm::AnyValue(false)},
+                                     {"max_length", mllm::AnyValue(qwen2_5omni_cfg.max_cache_length)},
+                                 },
+                                 [&](int64_t token_id) {
+                                   auto str = qwen2_5omni_tokenizer.detokenize(token_id);
+                                   std::wcout << str << std::flush;
+                                 });
+
+      fmt::print("\n{}\n", std::string(60, '-'));
+    } catch (const std::exception& e) { fmt::print("\nError: {}\n{}\n", e.what(), std::string(60, '-')); }
+
+    qwen2_5omni.perfSummary();
+  }
+
+  mllm::print("\n");
+  mllm::memoryReport();
+})