diff --git a/.gitignore b/.gitignore index 28aff8f..9329258 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ magi_compiler/_version.py magi_dump_src_dir/ *.nsys-rep *.ncu-rep +nsys_reports/ +output_audio/ +example/inference/*/output_audio/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/example/inference/qwen2.5-omni/README.md b/example/inference/qwen2.5-omni/README.md new file mode 100644 index 0000000..f592df8 --- /dev/null +++ b/example/inference/qwen2.5-omni/README.md @@ -0,0 +1,45 @@ +# Qwen2.5-Omni Offline Inference + +This example runs Qwen2.5-Omni offline inference through vLLM-Omni +(thinker, talker, token2wav). MagiCompiler is enabled inside vLLM-Omni when +`VLLM_OMNI_MAGI_COMPILER=1` (default in `infer.sh`); this example does not call +`magi_compile` directly. + +Install MagiCompiler and vLLM-Omni first. Use two GPUs for the default deploy +layout (`CUDA_VISIBLE_DEVICES=0,1`). Set `VLLM_OMNI_MODEL` to a local checkpoint +if needed. Without `DEPLOY_CONFIG`, vLLM-Omni uses +`vllm_omni/deploy/qwen2_5_omni.yaml`. + +## Usage + +Text query: + +```bash +VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B \ +CUDA_VISIBLE_DEVICES=0,1 \ +bash example/inference/qwen2.5-omni/infer.sh +``` + +Video query: + +```bash +VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B \ +CUDA_VISIBLE_DEVICES=0,1 \ +QUERY_TYPE=use_video \ +VIDEO_PATH=/path/to/video.mp4 \ +bash example/inference/qwen2.5-omni/infer.sh +``` + +Skip nsys: + +```bash +NSYS_PROFILE=false bash example/inference/qwen2.5-omni/infer.sh +``` + +`infer.py` warms up once (writes text/audio under `OUTPUT_DIR`), then runs the +NVTX profile loop without writing files. Printed times are end-to-end wall +clock for the multi-process Omni pipeline. + +Useful env vars: `QUERY_TYPE` (`text` / `use_video` / `use_audio_in_video`), +`OUTPUT_DIR`, `VIDEO_PATH`, `NUM_FRAMES`, `PROFILE_CNT`, +`VLLM_OMNI_MAGI_COMPILER`, `DEPLOY_CONFIG`, `VLLM_OMNI_ROOT`. diff --git a/example/inference/qwen2.5-omni/infer.py b/example/inference/qwen2.5-omni/infer.py new file mode 100644 index 0000000..cd4e9e6 --- /dev/null +++ b/example/inference/qwen2.5-omni/infer.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026 SandAI. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import torch +from modeling import DEFAULT_MODEL, Qwen2_5_OmniInference + +import magi_compiler.utils.nvtx as nvtx + +# Set VLLM_OMNI_MODEL=/path/to/Qwen2.5-Omni-3B for a local checkpoint. +MODEL = os.environ.get("VLLM_OMNI_MODEL", DEFAULT_MODEL) +QUERY_TYPE = os.environ.get("QUERY_TYPE", "text") +OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "output_audio") +VIDEO_PATH = os.environ.get("VIDEO_PATH") +NUM_FRAMES = int(os.environ.get("NUM_FRAMES", "16")) +DEPLOY_CONFIG = os.environ.get("DEPLOY_CONFIG") +PROFILE_CNT = int(os.environ.get("PROFILE_CNT", "3")) + + +def main(): + if not torch.cuda.is_available(): + raise RuntimeError("Qwen2.5-Omni inference example requires CUDA.") + + print(f"Query type: {QUERY_TYPE}") + print(f"Model: {MODEL}") + print(f"VLLM_OMNI_MAGI_COMPILER: {os.environ.get('VLLM_OMNI_MAGI_COMPILER', '0')}") + print(f"Deploy config: {DEPLOY_CONFIG}") + print(f"Output dir: {OUTPUT_DIR}") + + pipeline = Qwen2_5_OmniInference(model_name=MODEL, deploy_config=DEPLOY_CONFIG) + infer_kwargs = dict( + query_type=QUERY_TYPE, + output_dir=OUTPUT_DIR, + video_path=VIDEO_PATH, + num_frames=NUM_FRAMES, + ) + + try: + # Warm up and trigger compilation. + outputs = pipeline.infer(**infer_kwargs, write_outputs=True) + + for i in range(PROFILE_CNT + 1): + nvtx.switch_profile(i, 0, PROFILE_CNT) + start = time.perf_counter() + pipeline.infer(**infer_kwargs, write_outputs=False) + # Wall time for the full Omni pipeline (workers finish before generate returns). + print(f"{QUERY_TYPE} {i}-th iter: {time.perf_counter() - start:.4f}s") + + print(f"outputs: {outputs}") + finally: + pipeline.close() + + +if __name__ == "__main__": + main() diff --git a/example/inference/qwen2.5-omni/infer.sh b/example/inference/qwen2.5-omni/infer.sh new file mode 100755 index 0000000..0eec0e2 --- /dev/null +++ b/example/inference/qwen2.5-omni/infer.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (c) 2026 SandAI. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +PROJECT_ROOT=$(cd "$SCRIPT_DIR/../../.." &> /dev/null && pwd) +VLLM_OMNI_ROOT=${VLLM_OMNI_ROOT:-$(cd "$PROJECT_ROOT/../vllm-omni" 2>/dev/null && pwd || true)} + +QUERY_TYPE=${QUERY_TYPE:-text} + +if [ "${NSYS_PROFILE:-true}" = "true" ]; then + mkdir -p "$PROJECT_ROOT/nsys_reports" + + NSYS_OUTPUT="$PROJECT_ROOT/nsys_reports/nsys_qwen2_5_omni_${QUERY_TYPE}_$(date +%Y%m%d_%H%M%S)" + echo "${QUERY_TYPE} nsys report: ${NSYS_OUTPUT}.nsys-rep" + + NSYS_CMD="nsys profile --force-overwrite true -o $NSYS_OUTPUT --trace=cuda,nvtx --capture-range=cudaProfilerApi" +fi + +export MAGI_COMPILE_CACHE_ROOT_DIR=${MAGI_COMPILE_CACHE_ROOT_DIR:-"$PROJECT_ROOT/.cache"} +export PYTHONPATH="$PROJECT_ROOT${VLLM_OMNI_ROOT:+:$VLLM_OMNI_ROOT}${PYTHONPATH:+:$PYTHONPATH}" +export VLLM_OMNI_MAGI_COMPILER=${VLLM_OMNI_MAGI_COMPILER:-1} + +$NSYS_CMD python -u "$SCRIPT_DIR/infer.py" "$@" diff --git a/example/inference/qwen2.5-omni/modeling.py b/example/inference/qwen2.5-omni/modeling.py new file mode 100644 index 0000000..067f2d8 --- /dev/null +++ b/example/inference/qwen2.5-omni/modeling.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026 SandAI. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np +import soundfile as sf +from vllm.assets.video import VideoAsset, video_to_ndarrays +from vllm.multimodal.media.audio import load_audio +from vllm.sampling_params import SamplingParams + +from vllm_omni.entrypoints.omni import Omni + +__all__ = ["DEFAULT_MODEL", "Qwen2_5_OmniInference"] + +SEED = 42 +DEFAULT_MODEL = "Qwen/Qwen2.5-Omni-3B" +DEFAULT_SYSTEM = ( + "You are Qwen, a virtual human developed by the Qwen Team, Alibaba " + "Group, capable of perceiving auditory and visual inputs, as well as " + "generating text and speech." +) + + +def load_video_frames(video_path=None, num_frames=16): + if video_path is None: + return VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + return video_to_ndarrays(video_path, num_frames=num_frames) + + +def build_prompt(query_type, video_path=None, num_frames=16): + def chat(user_body): + return ( + f"<|im_start|>system\n{DEFAULT_SYSTEM}<|im_end|>\n" + f"<|im_start|>user\n{user_body}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + + if query_type == "text": + question = ( + "Explain the system architecture for a scalable audio " + "generation pipeline. Answer in 15 words." + ) + return {"prompt": chat(question)} + + if query_type == "use_video": + return { + "prompt": chat("<|vision_bos|><|VIDEO|><|vision_eos|>Why is this video funny?"), + "multi_modal_data": {"video": load_video_frames(video_path, num_frames)}, + } + + if query_type == "use_audio_in_video": + question = "Describe the content of the video, then convert what the baby say into text." + if video_path is None: + asset = VideoAsset(name="baby_reading", num_frames=num_frames) + video = asset.np_ndarrays + audio = asset.get_audio(sampling_rate=16000) + else: + video = load_video_frames(video_path, num_frames) + audio_signal, sr = load_audio(video_path, sr=16000) + audio = (audio_signal.astype(np.float32), sr) + return { + "prompt": chat( + "<|vision_bos|><|VIDEO|><|vision_eos|>" + "<|audio_bos|><|AUDIO|><|audio_eos|>" + f"{question}" + ), + "multi_modal_data": {"video": video, "audio": audio}, + "mm_processor_kwargs": {"use_audio_in_video": True}, + } + + raise ValueError( + f"Unsupported query_type={query_type!r}. " + "Use QUERY_TYPE=text, use_video, or use_audio_in_video." + ) + + +def default_sampling_params(): + # thinker / talker / token2wav + return [ + SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=2048, + seed=SEED, + detokenize=True, + repetition_penalty=1.1, + ), + SamplingParams( + temperature=0.9, + top_p=0.8, + top_k=40, + max_tokens=2048, + seed=SEED, + detokenize=True, + repetition_penalty=1.05, + stop_token_ids=[8294], + ), + SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=2048, + seed=SEED, + detokenize=True, + repetition_penalty=1.1, + ), + ] + + +class Qwen2_5_OmniInference: + """Offline Qwen2.5-Omni via vLLM-Omni. Magi is toggled by VLLM_OMNI_MAGI_COMPILER.""" + + def __init__(self, model_name=DEFAULT_MODEL, deploy_config=None): + omni_kwargs = { + "stage_init_timeout": 300, + "init_timeout": 300, + "worker_backend": "multi_process", + } + if deploy_config: + omni_kwargs["deploy_config"] = deploy_config + self.omni = Omni(model=model_name, **omni_kwargs) + self.sampling_params = default_sampling_params() + + def infer( + self, + query_type="text", + output_dir="output_audio", + video_path=None, + num_frames=16, + write_outputs=True, + ): + prompts = [build_prompt(query_type, video_path=video_path, num_frames=num_frames)] + if write_outputs: + os.makedirs(output_dir, exist_ok=True) + + text_path = None + audio_path = None + for stage_outputs in self.omni.generate(prompts, self.sampling_params): + output = stage_outputs.request_output + request_id = output.request_id + if stage_outputs.final_output_type == "text": + if write_outputs: + text_path = os.path.join(output_dir, f"{request_id}.txt") + with open(text_path, "w", encoding="utf-8") as f: + f.write("Prompt:\n") + f.write(str(output.prompt) + "\n") + f.write("vllm_text_output:\n") + f.write(str(output.outputs[0].text).strip() + "\n") + print(f"Request ID: {request_id}, Text saved to {text_path}") + elif stage_outputs.final_output_type == "audio" and write_outputs: + audio_path = os.path.join(output_dir, f"output_{request_id}.wav") + sf.write( + audio_path, + output.outputs[0].multimodal_output["audio"].detach().cpu().numpy(), + samplerate=24000, + ) + print(f"Request ID: {request_id}, Saved audio to {audio_path}") + + return {"text_path": text_path, "audio_path": audio_path} + + def close(self): + self.omni.close()