Merge branch 'main' into dependabot/github_actions/docker/login-action-3.7.0

XkunW · web-flow · commit 566215fb1a1d · 2026-03-18T13:33:33.000-04:00
diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml
@@ -28,7 +28,7 @@ jobs:
   run-code-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.1
+      - uses: actions/checkout@v6.0.2
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -31,7 +31,7 @@ jobs:
         backend: [vllm, sglang]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6.0.1
+        uses: actions/checkout@v6.0.2
 
       - name: Extract backend version
         id: backend-version
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6.0.1
+        uses: actions/checkout@v6.0.2
         with:
           fetch-depth: 0  # Fetch all history for proper versioning
 
@@ -88,7 +88,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6.0.1
+        uses: actions/checkout@v6.0.2
         with:
           fetch-depth: 0  # Fetch all history for proper versioning
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -13,7 +13,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev
 
-      - uses: actions/checkout@v6.0.1
+      - uses: actions/checkout@v6.0.2
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -43,7 +43,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v6.0.1
+      - uses: actions/checkout@v6.0.2
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,7 +17,7 @@ repos:
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.14.10'
+    rev: 'v0.14.14'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
diff --git a/MODEL_TRACKING.md b/MODEL_TRACKING.md
@@ -175,8 +175,9 @@ This document tracks all model weights available in the `/model-weights` directo
 ### Qwen: Qwen3
 | Model | Configuration |
 |:------|:-------------|
-| `Qwen3-14B` | ✅ |
+| `Qwen3-0.6B` | ✅ |
 | `Qwen3-8B` | ✅ |
+| `Qwen3-14B` | ✅ |
 | `Qwen3-32B` | ✅ |
 | `Qwen3-235B-A22B` | ❌ |
 | `Qwen3-Embedding-8B` | ❌ |
@@ -233,7 +234,8 @@ This document tracks all model weights available in the `/model-weights` directo
 #### Moonshot AI: Kimi
 | Model | Configuration |
 |:------|:-------------|
-| `Kimi-K2-Instruct` | ❌ |
+| `Kimi-K2-Instruct` | ✅ |
+| `Kimi-K2.5` | ✅ |
 
 #### Mistral AI: Ministral
 | Model | Configuration |
diff --git a/README.md b/README.md
@@ -7,11 +7,11 @@
 [![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
 [![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml)
 [![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main)
-[![vLLM](https://img.shields.io/badge/vLLM-0.12.0-blue)](https://docs.vllm.ai/en/v0.12.0/)
-[![SGLang](https://img.shields.io/badge/SGLang-0.5.5.post3-blue)](https://docs.sglang.io/index.html)
+[![vLLM](https://img.shields.io/badge/vLLM-0.15.0-blue)](https://docs.vllm.ai/en/v0.15.0/)
+[![SGLang](https://img.shields.io/badge/SGLang-0.5.8-blue)](https://docs.sglang.io/index.html)
 ![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
 
-This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.12.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.15.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
 
 **NOTE**: Supported models on Killarney are tracked [here](./MODEL_TRACKING.md)
 
@@ -49,7 +49,7 @@ You should see an output like the following:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
-Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command)
+Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command). During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache.
 
 #### Other commands
 
@@ -138,7 +138,7 @@ The example provided above is for the Vector Killarney cluster, change the varia
 If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template:
 ```
 @software{vector_inference,
-  title        = {Vector Inference: Efficient LLM inference on Slurm clusters using vLLM},
+  title        = {Vector Inference: Efficient LLM inference on Slurm clusters},
   author       = {Wang, Marshall},
   organization = {Vector Institute},
   year         = {<YEAR_OF_RELEASE>},
diff --git a/docs/index.md b/docs/index.md
@@ -1,6 +1,6 @@
 # Vector Inference: Easy inference on Slurm clusters
 
-This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.12.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.15.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
 
 
 **NOTE**: Supported models on Killarney are tracked [here](https://github.com/VectorInstitute/vector-inference/blob/main/MODEL_TRACKING.md)
diff --git a/docs/user_guide.md b/docs/user_guide.md
@@ -110,6 +110,8 @@ export VEC_INF_MODEL_CONFIG=/h/<username>/my-model-config.yaml
 
 **NOTE**: There are other parameters that can also be added to the config but not shown in this example, check the [`ModelConfig`](https://github.com/VectorInstitute/vector-inference/blob/main/vec_inf/client/config.py) for details.
 
+During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache.
+
 ### `batch-launch` command
 
 The `batch-launch` command allows users to launch multiple inference servers at once, here is an example of launching 2 models:
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "vec-inf"
-version = "0.8.0"
+version = "0.8.1"
 description = "Efficient LLM inference on Slurm clusters using vLLM."
 readme = "README.md"
 authors = [{name = "Marshall Wang", email = "marshall.wang@vectorinstitute.ai"}]
@@ -42,6 +42,9 @@ inference = [
     "torch>=2.7.0",
     "cupy-cuda12x>=12.3.0",
     "flashinfer-python>=0.4.0",
+    "ax-platform>=1.1.0",
+    "py3nvml",
+    "wandb>=0.17.0",
 ]
 
 [project.optional-dependencies]
@@ -50,6 +53,9 @@ inference = [
 vllm = [
     "vllm>=0.11.2",
     "ray[default]>=2.51.0",
+    "vllm[audio]",
+    "vllm[bench]",
+    "torchcodec>=0.9.0,<0.10.0",
 ]
 # SGLang inference backend (conflicts with vllm due to dependency version conflicts)
 # Install with: uv sync --extra sglang --group inference
diff --git a/sglang.Dockerfile b/sglang.Dockerfile
@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
     wget build-essential libssl-dev zlib1g-dev libbz2-dev \
     libreadline-dev libsqlite3-dev libffi-dev libncursesw5-dev \
     xz-utils tk-dev libxml2-dev libxmlsec1-dev liblzma-dev libnuma1 \
-    git vim \
+    git vim ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Python
diff --git a/uv.lock b/uv.lock
diff --git a/vec_inf/client/_helper.py b/vec_inf/client/_helper.py
diff --git a/vec_inf/client/_slurm_script_generator.py b/vec_inf/client/_slurm_script_generator.py
diff --git a/vec_inf/client/_slurm_templates.py b/vec_inf/client/_slurm_templates.py
diff --git a/vec_inf/config/models.yaml b/vec_inf/config/models.yaml
diff --git a/vllm.Dockerfile b/vllm.Dockerfile
