Merge branch 'main' into dependabot/github_actions/codecov/codecov-action-6.0.0

Author: XkunW

.github/ISSUE_TEMPLATE/model-request.md

@@ -1,7 +1,7 @@
 ---
 name: Model request
 about: Request for new model weights or model config
-title: New model request for [MODEL_NAME]
+title: New model request for [MODEL_NAME] on [CLUSTER_NAME]
 labels: new model
 assignees: XkunW
 
@@ -11,4 +11,7 @@ assignees: XkunW
 Model weights | Model config | Both
 
 ### Model Name
-Name of the model requested
+Name of the model requested, preferably with model repo URL as well (HuggingFace, ModelScope, etc.)
+
+### Cluster Name
+On which cluster do you need this model

.github/workflows/docker.yml

@@ -58,7 +58,7 @@ jobs:
           df -h
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
@@ -68,12 +68,12 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf
         with:
           images: vectorinstitute/vector-inference-${{ matrix.backend }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294
         with:
           context: .
           file: ./${{ matrix.backend }}.Dockerfile

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.15.6'
+    rev: 'v0.15.9'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
@@ -26,7 +26,7 @@ repos:
       types_or: [python, jupyter]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.19.1
+    rev: v1.20.0
     hooks:
     - id: mypy
       entry: python3 -m mypy --config-file pyproject.toml

uv.lock

@@ -118,12 +118,22 @@ def _get_model_configuration(self, config_path: str | None = None) -> ModelConfi
 
         model_weights_path = Path(model_weights_parent_dir, self.model_name)
 
-        # Only give a warning if weights exist but config missing
+        if self.kwargs.get("hf_model"):
+            return ModelConfig(
+                model_name=self.model_name,
+                model_family="model_family_placeholder",
+                model_type="LLM",
+                gpus_per_node=1,
+                num_nodes=1,
+                vocab_size=1000,
+                model_weights_parent_dir=Path(str(model_weights_parent_dir)),
+            )
+
         if model_weights_path.exists():
             self._warn(
-                f"Warning: '{self.model_name}' configuration not found in config, please ensure model configuration are properly set in command arguments",
+                f"Warning: '{self.model_name}' configuration not found in config, "
+                "please ensure model configuration are properly set in command arguments",
             )
-            # Return a dummy model config object with model name and weights parent dir
             return ModelConfig(
                 model_name=self.model_name,
                 model_family="model_family_placeholder",

vec_inf/client/_helper.py

@@ -186,10 +186,15 @@ class SlurmScriptTemplate(TypedDict):
     ],
     "write_to_json": [
         '\njson_path="{log_dir}/{model_name}.$SLURM_JOB_ID/{model_name}.$SLURM_JOB_ID.json"',
-        'jq --arg server_addr "$server_address" \\',
-        "    '. + {{\"server_address\": $server_addr}}' \\",
-        '    "$json_path" > temp.json \\',
-        '    && mv temp.json "$json_path"',
+        'tmp_json="${{json_path}}.tmp.$$"',
+        "for _attempt in 1 2 3 4 5; do",
+        '    jq --arg server_addr "$server_address" \\',
+        "        '. + {{\"server_address\": $server_addr}}' \\",
+        '        "$json_path" > "$tmp_json" \\',
+        '        && mv "$tmp_json" "$json_path" \\',
+        "        && break",
+        "    sleep 2",
+        "done",
     ],
     "launch_cmd": {
         "vllm": [
@@ -303,10 +308,15 @@ class BatchModelLaunchScriptTemplate(TypedDict):
     "write_to_json": [
         "het_job_id=$(($SLURM_JOB_ID+{het_group_id}))",
         'json_path="{log_dir}/{slurm_job_name}.$het_job_id/{model_name}.$het_job_id.json"',
-        'jq --arg server_addr "$server_address" \\',
-        "    '. + {{\"server_address\": $server_addr}}' \\",
-        '    "$json_path" > temp_{model_name}.json \\',
-        '    && mv temp_{model_name}.json "$json_path"\n',
+        'tmp_json="${{json_path}}.tmp.$$"',
+        "for _attempt in 1 2 3 4 5; do",
+        '    jq --arg server_addr "$server_address" \\',
+        "        '. + {{\"server_address\": $server_addr}}' \\",
+        '        "$json_path" > "$tmp_json" \\',
+        '        && mv "$tmp_json" "$json_path" \\',
+        "        && break",
+        "    sleep 2",
+        "done\n",
     ],
     "container_command": f"{CONTAINER_MODULE_NAME} exec --nv --containall {{image_path}} \\",
     "launch_cmd": {

vec_inf/client/_slurm_templates.py

@@ -24,7 +24,7 @@ required_args:
 
 python_version: "python3.12"
 
-model_types: ["LLM", "VLM", "Text_Embedding", "Reward_Modeling", "OCR"] # Derived from models.yaml
+model_types: ["LLM", "VLM", "Text_Embedding", "Reward_Modeling", "OCR", "Audio"] # Derived from models.yaml
 
 default_args:
   cpus_per_task: "16"