From c0383ce01b06c2508c94d9eaa76cbc841e4199df Mon Sep 17 00:00:00 2001 From: "Ahmed, Mushtaq" Date: Sat, 26 Sep 2026 00:52:49 -0600 Subject: [PATCH] fix(litellm): edge rate limit + HSTS on the public LiteLLM route The litellm. route exposes the LiteLLM UI and its admin API to the internet (the UI must stay reachable for user login). Harden that route with two edge controls: - HSTS response header so the UI login password / virtual key is never sent over plaintext :80 before the HTTPS redirect. - Local per-client-IP rate limit (Distinct match on x-forwarded-for, no Redis or external ratelimit service) to blunt brute-force against the login/admin API and cap runaway abuse. Both are toggleable/tunable via role defaults, self-heal on disable, and are cleaned up on teardown. Also corrects stale /key/verify extAuth references in the role docs (superseded: the gateway does TLS+routing only, LiteLLM is the sole auth boundary). --- roles/litellm/defaults/main.yaml | 17 +++++++-- roles/litellm/tasks/install.yaml | 60 ++++++++++++++++++++++++++++++- roles/litellm/tasks/teardown.yaml | 9 +++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/roles/litellm/defaults/main.yaml b/roles/litellm/defaults/main.yaml index 8bb44b9..4b4dd0e 100644 --- a/roles/litellm/defaults/main.yaml +++ b/roles/litellm/defaults/main.yaml @@ -6,12 +6,13 @@ # spend tracking, rate limiting, and model routing on top of inference backends. # # When auth_provider=litellm: -# - Envoy Gateway uses LiteLLM's /key/verify as extAuth (replaces Keycloak JWT) +# - The gateway does TLS + routing only; LiteLLM is the sole auth boundary +# and validates its own virtual keys (no gateway JWT, no extAuth) # - Keycloak is disabled (not deployed) # - Virtual keys are the auth mechanism (Bearer ) # # LiteLLM sits in front of KServe inference endpoints: -# Client → Envoy Gateway (extAuth→LiteLLM) → LiteLLM proxy → KServe +# Client → Envoy Gateway (TLS+routing) → LiteLLM proxy (auth) → KServe # ============================================================================= litellm_version: "v1.83.14-stable.patch.2" # Docker image tag to deploy @@ -22,6 +23,18 @@ litellm_valkey_tag: "8.0-alpine" # valkey/valkey (official upstre gateway_create: "{{ envoy_gateway_enabled | default(false) }}" litellm_hostname: "litellm.{{ base_domain_name | default('inference-example.com') }}" +# Edge protection on the public LiteLLM route (litellm.). +# HSTS forces HTTPS so the UI login password / virtual key is never sent on :80. +litellm_hsts_enabled: true +litellm_hsts_max_age: 31536000 # 1 year (seconds) +# Local per-client-IP rate limit (no Redis). Generous enough for legitimate +# inference, low enough to blunt automated brute-force; tune per deployment. +# Note: clients behind a shared NAT/egress IP share one counter. +litellm_rate_limit_enabled: true +litellm_rate_limit_requests: 600 # requests per unit, per client IP +litellm_rate_limit_unit: "Minute" # Second | Minute | Hour | Day +litellm_rate_limit_client_header: "x-forwarded-for" + # PostgreSQL — stores virtual keys, spend logs, team/org data litellm_db_name: "litellm" litellm_db_user: "litellm" diff --git a/roles/litellm/tasks/install.yaml b/roles/litellm/tasks/install.yaml index 79aef0c..dcf948d 100644 --- a/roles/litellm/tasks/install.yaml +++ b/roles/litellm/tasks/install.yaml @@ -382,6 +382,20 @@ # ── Gateway integration (runs even on re-runs to stay idempotent) ───────────── +# HSTS on the public LiteLLM host: browsers/SDKs must never send the UI login +# password or a virtual key over plaintext :80 before the 301 to HTTPS. Built +# as a fact so disabling it re-renders the route with no filter (self-healing). +- name: "litellm | Build HTTPRoute response filters (HSTS)" + ansible.builtin.set_fact: + _litellm_route_filters: >- + {{ + [{'type': 'ResponseHeaderModifier', + 'responseHeaderModifier': {'set': [{ + 'name': 'Strict-Transport-Security', + 'value': 'max-age=' ~ (litellm_hsts_max_age | int) ~ '; includeSubDomains'}]}}] + if litellm_hsts_enabled | default(true) | bool else [] + }} + - name: "litellm | Create HTTPRoute" when: gateway_create | bool kubernetes.core.k8s: @@ -404,6 +418,7 @@ - path: type: PathPrefix value: / + filters: "{{ _litellm_route_filters }}" backendRefs: - name: litellm port: 4000 @@ -424,6 +439,49 @@ kind: HTTPRoute name: litellm +# ── Edge rate limit ─────────────────────────────────────────────────────────── +# Local (per-gateway) rate limit — no external ratelimit service or Redis. A +# Distinct match on the client-IP header gives each source IP its own counter, +# blunting brute-force against the internet-facing UI login/admin API and capping +# runaway abuse. Coarse by design; per-identity limits come with the auth work. +- name: "litellm | Create edge rate-limit BackendTrafficPolicy" + when: gateway_create | bool and litellm_rate_limit_enabled | default(true) | bool + kubernetes.core.k8s: + state: present + definition: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: litellm + rateLimit: + type: Local + local: + rules: + - clientSelectors: + - headers: + - name: "{{ litellm_rate_limit_client_header | default('x-forwarded-for') }}" + type: Distinct + limit: + requests: "{{ litellm_rate_limit_requests | int }}" + unit: "{{ litellm_rate_limit_unit | default('Minute') }}" + +# Remove the policy if rate limiting is turned off on a re-run (drift cleanup). +- name: "litellm | Remove edge rate-limit BackendTrafficPolicy when disabled" + when: gateway_create | bool and not (litellm_rate_limit_enabled | default(true) | bool) + kubernetes.core.k8s: + state: absent + api_version: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + failed_when: false + # ── Print credentials ───────────────────────────────────────────────────────── - name: "litellm | Store credentials summary" @@ -450,4 +508,4 @@ - "UI: https://{{ litellm_hostname }}" - "Admin username: admin" - "Master key: kubectl get secret litellm-master-key -n {{ litellm_namespace }} -o jsonpath='{.data.master_key}' | base64 -d" - - "Auth mode: extAuth via /key/verify (virtual keys required for all inference requests)" + - "Auth mode: LiteLLM virtual keys (Bearer ); LiteLLM is the sole auth boundary" diff --git a/roles/litellm/tasks/teardown.yaml b/roles/litellm/tasks/teardown.yaml index 5470623..3cb9b2b 100644 --- a/roles/litellm/tasks/teardown.yaml +++ b/roles/litellm/tasks/teardown.yaml @@ -29,6 +29,15 @@ namespace: "{{ litellm_namespace }}" failed_when: false +- name: "litellm | teardown | Remove rate-limit BackendTrafficPolicy" + kubernetes.core.k8s: + state: absent + api_version: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + failed_when: false + - name: "litellm | teardown | Remove ConfigMaps" kubernetes.core.k8s: state: absent