diff --git a/roles/litellm/defaults/main.yaml b/roles/litellm/defaults/main.yaml index 8bb44b9..4b4dd0e 100644 --- a/roles/litellm/defaults/main.yaml +++ b/roles/litellm/defaults/main.yaml @@ -6,12 +6,13 @@ # spend tracking, rate limiting, and model routing on top of inference backends. # # When auth_provider=litellm: -# - Envoy Gateway uses LiteLLM's /key/verify as extAuth (replaces Keycloak JWT) +# - The gateway does TLS + routing only; LiteLLM is the sole auth boundary +# and validates its own virtual keys (no gateway JWT, no extAuth) # - Keycloak is disabled (not deployed) # - Virtual keys are the auth mechanism (Bearer ) # # LiteLLM sits in front of KServe inference endpoints: -# Client → Envoy Gateway (extAuth→LiteLLM) → LiteLLM proxy → KServe +# Client → Envoy Gateway (TLS+routing) → LiteLLM proxy (auth) → KServe # ============================================================================= litellm_version: "v1.83.14-stable.patch.2" # Docker image tag to deploy @@ -22,6 +23,18 @@ litellm_valkey_tag: "8.0-alpine" # valkey/valkey (official upstre gateway_create: "{{ envoy_gateway_enabled | default(false) }}" litellm_hostname: "litellm.{{ base_domain_name | default('inference-example.com') }}" +# Edge protection on the public LiteLLM route (litellm.). +# HSTS forces HTTPS so the UI login password / virtual key is never sent on :80. +litellm_hsts_enabled: true +litellm_hsts_max_age: 31536000 # 1 year (seconds) +# Local per-client-IP rate limit (no Redis). Generous enough for legitimate +# inference, low enough to blunt automated brute-force; tune per deployment. +# Note: clients behind a shared NAT/egress IP share one counter. +litellm_rate_limit_enabled: true +litellm_rate_limit_requests: 600 # requests per unit, per client IP +litellm_rate_limit_unit: "Minute" # Second | Minute | Hour | Day +litellm_rate_limit_client_header: "x-forwarded-for" + # PostgreSQL — stores virtual keys, spend logs, team/org data litellm_db_name: "litellm" litellm_db_user: "litellm" diff --git a/roles/litellm/tasks/install.yaml b/roles/litellm/tasks/install.yaml index 79aef0c..dcf948d 100644 --- a/roles/litellm/tasks/install.yaml +++ b/roles/litellm/tasks/install.yaml @@ -382,6 +382,20 @@ # ── Gateway integration (runs even on re-runs to stay idempotent) ───────────── +# HSTS on the public LiteLLM host: browsers/SDKs must never send the UI login +# password or a virtual key over plaintext :80 before the 301 to HTTPS. Built +# as a fact so disabling it re-renders the route with no filter (self-healing). +- name: "litellm | Build HTTPRoute response filters (HSTS)" + ansible.builtin.set_fact: + _litellm_route_filters: >- + {{ + [{'type': 'ResponseHeaderModifier', + 'responseHeaderModifier': {'set': [{ + 'name': 'Strict-Transport-Security', + 'value': 'max-age=' ~ (litellm_hsts_max_age | int) ~ '; includeSubDomains'}]}}] + if litellm_hsts_enabled | default(true) | bool else [] + }} + - name: "litellm | Create HTTPRoute" when: gateway_create | bool kubernetes.core.k8s: @@ -404,6 +418,7 @@ - path: type: PathPrefix value: / + filters: "{{ _litellm_route_filters }}" backendRefs: - name: litellm port: 4000 @@ -424,6 +439,49 @@ kind: HTTPRoute name: litellm +# ── Edge rate limit ─────────────────────────────────────────────────────────── +# Local (per-gateway) rate limit — no external ratelimit service or Redis. A +# Distinct match on the client-IP header gives each source IP its own counter, +# blunting brute-force against the internet-facing UI login/admin API and capping +# runaway abuse. Coarse by design; per-identity limits come with the auth work. +- name: "litellm | Create edge rate-limit BackendTrafficPolicy" + when: gateway_create | bool and litellm_rate_limit_enabled | default(true) | bool + kubernetes.core.k8s: + state: present + definition: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: litellm + rateLimit: + type: Local + local: + rules: + - clientSelectors: + - headers: + - name: "{{ litellm_rate_limit_client_header | default('x-forwarded-for') }}" + type: Distinct + limit: + requests: "{{ litellm_rate_limit_requests | int }}" + unit: "{{ litellm_rate_limit_unit | default('Minute') }}" + +# Remove the policy if rate limiting is turned off on a re-run (drift cleanup). +- name: "litellm | Remove edge rate-limit BackendTrafficPolicy when disabled" + when: gateway_create | bool and not (litellm_rate_limit_enabled | default(true) | bool) + kubernetes.core.k8s: + state: absent + api_version: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + failed_when: false + # ── Print credentials ───────────────────────────────────────────────────────── - name: "litellm | Store credentials summary" @@ -450,4 +508,4 @@ - "UI: https://{{ litellm_hostname }}" - "Admin username: admin" - "Master key: kubectl get secret litellm-master-key -n {{ litellm_namespace }} -o jsonpath='{.data.master_key}' | base64 -d" - - "Auth mode: extAuth via /key/verify (virtual keys required for all inference requests)" + - "Auth mode: LiteLLM virtual keys (Bearer ); LiteLLM is the sole auth boundary" diff --git a/roles/litellm/tasks/teardown.yaml b/roles/litellm/tasks/teardown.yaml index 5470623..3cb9b2b 100644 --- a/roles/litellm/tasks/teardown.yaml +++ b/roles/litellm/tasks/teardown.yaml @@ -29,6 +29,15 @@ namespace: "{{ litellm_namespace }}" failed_when: false +- name: "litellm | teardown | Remove rate-limit BackendTrafficPolicy" + kubernetes.core.k8s: + state: absent + api_version: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + name: litellm-rate-limit + namespace: "{{ litellm_namespace }}" + failed_when: false + - name: "litellm | teardown | Remove ConfigMaps" kubernetes.core.k8s: state: absent