From fcea8aa2a942257aeae564623748f4d7b3664755 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Fri, 11 Sep 2026 06:02:53 +0000 Subject: [PATCH] Default the CI model to Qwen, the model this build is measured on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task-battery and a11y-audit workflows fell back to Llama-3.3-70B when no ELM_MODEL secret was set, and the README's .env.local example named it too — both leftovers from before v4.0.0 moved the app to Qwen. With the repository secret about to carry only the gateway URL and key, the default is what CI runs, and it has to be the model production runs: the startup report in every recent battery job said "Model resolution is off the measured profile", and the L2/L3/L5 language tasks fail on the fallback model while passing on Qwen. --- .github/workflows/a11y-audit.yml | 4 ++-- .github/workflows/task-battery.yml | 4 ++-- README.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/a11y-audit.yml b/.github/workflows/a11y-audit.yml index 0daa769..70e6972 100644 --- a/.github/workflows/a11y-audit.yml +++ b/.github/workflows/a11y-audit.yml @@ -86,8 +86,8 @@ jobs: ELM_API_KEY: ${{ secrets.ELM_API_KEY || 'a11y-audit-no-model-calls' }} ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} APPROVED_ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} - ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} - APPROVED_ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} + ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'Qwen/Qwen3.5-397B-A17B-FP8' }} + APPROVED_ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'Qwen/Qwen3.5-397B-A17B-FP8' }} VFB_MCP_URL: ${{ secrets.VFB_MCP_URL || vars.VFB_MCP_URL || 'https://vfb3-mcp-preview.virtualflybrain.org/' }} RATE_LIMIT_PER_IP: '10000' GA_MEASUREMENT_ID: '' diff --git a/.github/workflows/task-battery.yml b/.github/workflows/task-battery.yml index 24a5278..5e75baf 100644 --- a/.github/workflows/task-battery.yml +++ b/.github/workflows/task-battery.yml @@ -115,8 +115,8 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} APPROVED_ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} - ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} - APPROVED_ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} + ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'Qwen/Qwen3.5-397B-A17B-FP8' }} + APPROVED_ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'Qwen/Qwen3.5-397B-A17B-FP8' }} VFB_MCP_URL: ${{ secrets.VFB_MCP_URL || vars.VFB_MCP_URL || 'https://vfb3-mcp-preview.virtualflybrain.org/' }} RATE_LIMIT_PER_IP: '10000' TASK_BATTERY_REPETITIONS: ${{ github.event.inputs.repetitions || '1' }} diff --git a/README.md b/README.md index 81a140a..fee6dc4 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Create `.env.local` with explicit values: ```bash ELM_API_KEY=elm-xxxxxxxx-xxxxxxxxxxxxxxxx ELM_BASE_URL=https://elm.edina.ac.uk/api/v1 -ELM_MODEL=meta-llama/Llama-3.3-70B-Instruct +ELM_MODEL=Qwen/Qwen3.5-397B-A17B-FP8 LOG_ROOT_DIR=./logs ```