[Example] Add Qwen2.5-Omni offline inference example

.gitignore

@@ -3,6 +3,9 @@ magi_compiler/_version.py
 magi_dump_src_dir/
 *.nsys-rep
 *.ncu-rep
+nsys_reports/
+output_audio/
+example/inference/*/output_audio/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

example/inference/qwen2.5-omni/README.md

@@ -0,0 +1,45 @@
+# Qwen2.5-Omni Offline Inference
+
+This example runs Qwen2.5-Omni offline inference through vLLM-Omni
+(thinker, talker, token2wav). MagiCompiler is enabled inside vLLM-Omni when
+`VLLM_OMNI_MAGI_COMPILER=1` (default in `infer.sh`); this example does not call
+`magi_compile` directly.
+
+Install MagiCompiler and vLLM-Omni first. Use two GPUs for the default deploy
+layout (`CUDA_VISIBLE_DEVICES=0,1`). Set `VLLM_OMNI_MODEL` to a local checkpoint
+if needed. Without `DEPLOY_CONFIG`, vLLM-Omni uses
+`vllm_omni/deploy/qwen2_5_omni.yaml`.
+
+## Usage
+
+Text query:
+
+```bash
+VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B \
+CUDA_VISIBLE_DEVICES=0,1 \
+bash example/inference/qwen2.5-omni/infer.sh
+```
+
+Video query:
+
+```bash
+VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B \
+CUDA_VISIBLE_DEVICES=0,1 \
+QUERY_TYPE=use_video \
+VIDEO_PATH=/path/to/video.mp4 \
+bash example/inference/qwen2.5-omni/infer.sh
+```
+
+Skip nsys:
+
+```bash
+NSYS_PROFILE=false bash example/inference/qwen2.5-omni/infer.sh
+```
+
+`infer.py` warms up once (writes text/audio under `OUTPUT_DIR`), then runs the
+NVTX profile loop without writing files. Printed times are end-to-end wall
+clock for the multi-process Omni pipeline.
+
+Useful env vars: `QUERY_TYPE` (`text` / `use_video` / `use_audio_in_video`),
+`OUTPUT_DIR`, `VIDEO_PATH`, `NUM_FRAMES`, `PROFILE_CNT`,
+`VLLM_OMNI_MAGI_COMPILER`, `DEPLOY_CONFIG`, `VLLM_OMNI_ROOT`.

example/inference/qwen2.5-omni/infer.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2026 SandAI. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+import torch
+from modeling import DEFAULT_MODEL, Qwen2_5_OmniInference
+
+import magi_compiler.utils.nvtx as nvtx
+
+# Set VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B for a local checkpoint.
+MODEL = os.environ.get("VLLM_OMNI_MODEL", DEFAULT_MODEL)
+QUERY_TYPE = os.environ.get("QUERY_TYPE", "text")
+OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "output_audio")
+VIDEO_PATH = os.environ.get("VIDEO_PATH")
+NUM_FRAMES = int(os.environ.get("NUM_FRAMES", "16"))
+DEPLOY_CONFIG = os.environ.get("DEPLOY_CONFIG")
+PROFILE_CNT = int(os.environ.get("PROFILE_CNT", "3"))
+
+
+def main():
+    if not torch.cuda.is_available():
+        raise RuntimeError("Qwen2.5-Omni inference example requires CUDA.")
+
+    print(f"Query type: {QUERY_TYPE}")
+    print(f"Model: {MODEL}")
+    print(f"VLLM_OMNI_MAGI_COMPILER: {os.environ.get('VLLM_OMNI_MAGI_COMPILER', '0')}")
+    print(f"Deploy config: {DEPLOY_CONFIG}")
+    print(f"Output dir: {OUTPUT_DIR}")
+
+    pipeline = Qwen2_5_OmniInference(model_name=MODEL, deploy_config=DEPLOY_CONFIG)
+    infer_kwargs = dict(
+        query_type=QUERY_TYPE,
+        output_dir=OUTPUT_DIR,
+        video_path=VIDEO_PATH,
+        num_frames=NUM_FRAMES,
+    )
+
+    try:
+        # Warm up and trigger compilation.
+        outputs = pipeline.infer(**infer_kwargs, write_outputs=True)
+
+        for i in range(PROFILE_CNT + 1):
+            nvtx.switch_profile(i, 0, PROFILE_CNT)
+            start = time.perf_counter()
+            pipeline.infer(**infer_kwargs, write_outputs=False)
+            # Wall time for the full Omni pipeline (workers finish before generate returns).
+            print(f"{QUERY_TYPE} {i}-th iter: {time.perf_counter() - start:.4f}s")
+
+        print(f"outputs: {outputs}")
+    finally:
+        pipeline.close()
+
+
+if __name__ == "__main__":
+    main()

example/inference/qwen2.5-omni/infer.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) 2026 SandAI. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+PROJECT_ROOT=$(cd "$SCRIPT_DIR/../../.." &> /dev/null && pwd)
+VLLM_OMNI_ROOT=${VLLM_OMNI_ROOT:-$(cd "$PROJECT_ROOT/../vllm-omni" 2>/dev/null && pwd || true)}
+
+QUERY_TYPE=${QUERY_TYPE:-text}
+
+if [ "${NSYS_PROFILE:-true}" = "true" ]; then
+    mkdir -p "$PROJECT_ROOT/nsys_reports"
+
+    NSYS_OUTPUT="$PROJECT_ROOT/nsys_reports/nsys_qwen2_5_omni_${QUERY_TYPE}_$(date +%Y%m%d_%H%M%S)"
+    echo "${QUERY_TYPE} nsys report: ${NSYS_OUTPUT}.nsys-rep"
+
+    NSYS_CMD="nsys profile --force-overwrite true -o $NSYS_OUTPUT --trace=cuda,nvtx --capture-range=cudaProfilerApi"
+fi
+
+export MAGI_COMPILE_CACHE_ROOT_DIR=${MAGI_COMPILE_CACHE_ROOT_DIR:-"$PROJECT_ROOT/.cache"}
+export PYTHONPATH="$PROJECT_ROOT${VLLM_OMNI_ROOT:+:$VLLM_OMNI_ROOT}${PYTHONPATH:+:$PYTHONPATH}"
+export VLLM_OMNI_MAGI_COMPILER=${VLLM_OMNI_MAGI_COMPILER:-1}
+
+$NSYS_CMD python -u "$SCRIPT_DIR/infer.py" "$@"

example/inference/qwen2.5-omni/modeling.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2026 SandAI. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+import soundfile as sf
+from vllm.assets.video import VideoAsset, video_to_ndarrays
+from vllm.multimodal.media.audio import load_audio
+from vllm.sampling_params import SamplingParams
+
+from vllm_omni.entrypoints.omni import Omni
+
+__all__ = ["DEFAULT_MODEL", "Qwen2_5_OmniInference"]
+
+SEED = 42
+DEFAULT_MODEL = "Qwen/Qwen2.5-Omni-3B"
+DEFAULT_SYSTEM = (
+    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
+    "Group, capable of perceiving auditory and visual inputs, as well as "
+    "generating text and speech."
+)
+
+
+def load_video_frames(video_path=None, num_frames=16):
+    if video_path is None:
+        return VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+    return video_to_ndarrays(video_path, num_frames=num_frames)
+
+
+def build_prompt(query_type, video_path=None, num_frames=16):
+    def chat(user_body):
+        return (
+            f"<|im_start|>system\n{DEFAULT_SYSTEM}<|im_end|>\n"
+            f"<|im_start|>user\n{user_body}<|im_end|>\n"
+            f"<|im_start|>assistant\n"
+        )
+
+    if query_type == "text":
+        question = (
+            "Explain the system architecture for a scalable audio "
+            "generation pipeline. Answer in 15 words."
+        )
+        return {"prompt": chat(question)}
+
+    if query_type == "use_video":
+        return {
+            "prompt": chat("<|vision_bos|><|VIDEO|><|vision_eos|>Why is this video funny?"),
+            "multi_modal_data": {"video": load_video_frames(video_path, num_frames)},
+        }
+
+    if query_type == "use_audio_in_video":
+        question = "Describe the content of the video, then convert what the baby say into text."
+        if video_path is None:
+            asset = VideoAsset(name="baby_reading", num_frames=num_frames)
+            video = asset.np_ndarrays
+            audio = asset.get_audio(sampling_rate=16000)
+        else:
+            video = load_video_frames(video_path, num_frames)
+            audio_signal, sr = load_audio(video_path, sr=16000)
+            audio = (audio_signal.astype(np.float32), sr)
+        return {
+            "prompt": chat(
+                "<|vision_bos|><|VIDEO|><|vision_eos|>"
+                "<|audio_bos|><|AUDIO|><|audio_eos|>"
+                f"{question}"
+            ),
+            "multi_modal_data": {"video": video, "audio": audio},
+            "mm_processor_kwargs": {"use_audio_in_video": True},
+        }
+
+    raise ValueError(
+        f"Unsupported query_type={query_type!r}. "
+        "Use QUERY_TYPE=text, use_video, or use_audio_in_video."
+    )
+
+
+def default_sampling_params():
+    # thinker / talker / token2wav
+    return [
+        SamplingParams(
+            temperature=0.0,
+            top_p=1.0,
+            top_k=-1,
+            max_tokens=2048,
+            seed=SEED,
+            detokenize=True,
+            repetition_penalty=1.1,
+        ),
+        SamplingParams(
+            temperature=0.9,
+            top_p=0.8,
+            top_k=40,
+            max_tokens=2048,
+            seed=SEED,
+            detokenize=True,
+            repetition_penalty=1.05,
+            stop_token_ids=[8294],
+        ),
+        SamplingParams(
+            temperature=0.0,
+            top_p=1.0,
+            top_k=-1,
+            max_tokens=2048,
+            seed=SEED,
+            detokenize=True,
+            repetition_penalty=1.1,
+        ),
+    ]
+
+
+class Qwen2_5_OmniInference:
+    """Offline Qwen2.5-Omni via vLLM-Omni. Magi is toggled by VLLM_OMNI_MAGI_COMPILER."""
+
+    def __init__(self, model_name=DEFAULT_MODEL, deploy_config=None):
+        omni_kwargs = {
+            "stage_init_timeout": 300,
+            "init_timeout": 300,
+            "worker_backend": "multi_process",
+        }
+        if deploy_config:
+            omni_kwargs["deploy_config"] = deploy_config
+        self.omni = Omni(model=model_name, **omni_kwargs)
+        self.sampling_params = default_sampling_params()
+
+    def infer(
+        self,
+        query_type="text",
+        output_dir="output_audio",
+        video_path=None,
+        num_frames=16,
+        write_outputs=True,
+    ):
+        prompts = [build_prompt(query_type, video_path=video_path, num_frames=num_frames)]
+        if write_outputs:
+            os.makedirs(output_dir, exist_ok=True)
+
+        text_path = None
+        audio_path = None
+        for stage_outputs in self.omni.generate(prompts, self.sampling_params):
+            output = stage_outputs.request_output
+            request_id = output.request_id
+            if stage_outputs.final_output_type == "text":
+                if write_outputs:
+                    text_path = os.path.join(output_dir, f"{request_id}.txt")
+                    with open(text_path, "w", encoding="utf-8") as f:
+                        f.write("Prompt:\n")
+                        f.write(str(output.prompt) + "\n")
+                        f.write("vllm_text_output:\n")
+                        f.write(str(output.outputs[0].text).strip() + "\n")
+                    print(f"Request ID: {request_id}, Text saved to {text_path}")
+            elif stage_outputs.final_output_type == "audio" and write_outputs:
+                audio_path = os.path.join(output_dir, f"output_{request_id}.wav")
+                sf.write(
+                    audio_path,
+                    output.outputs[0].multimodal_output["audio"].detach().cpu().numpy(),
+                    samplerate=24000,
+                )
+                print(f"Request ID: {request_id}, Saved audio to {audio_path}")
+
+        return {"text_path": text_path, "audio_path": audio_path}
+
+    def close(self):
+        self.omni.close()