diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index e6f97c36f..75aeb8b23 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -79,8 +79,9 @@ global: invocationServiceURL: "" # gRPC proxy worker CONNECT endpoint advertised by the proxy. grpcProxyWorkerConnectURL: "" - # Optional LLM request-router address override advertised to LLM worker - # sidecars. Empty uses llm-request-router.nvcf.svc.cluster.local:50071. + # Pylon discovery endpoint. Empty uses + # llm-request-router-backend-router.nvcf.svc.cluster.local:50071 when backend + # routing is enabled, otherwise llm-request-router.nvcf.svc.cluster.local:50071. llmRequestRouterAddress: "" nodeSelectors: @@ -401,6 +402,8 @@ addons: # enabled: defaults to addons.llm.enabled # replicaCount defaults to the request-router chart value. # replicaCount: 2 + # service: + # grpcPort: 50071 # Addresses workers dial to reach the router. Both default to the # backend-router Service in-cluster, which is correct when workers run # alongside the control plane. Set both to externally reachable diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 29417f8a4..064009254 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -362,7 +362,13 @@ natsAuthCalloutService: {{- $grpcProxyWorkerConnectURL := dig "grpcProxyWorkerConnectURL" "" $workerEndpoints }} {{- $grpcProxyWorkerConnectBaseURL := dig "grpcproxy" "workerConnectBaseURL" "" .Values | default $grpcProxyWorkerConnectURL }} {{- $llmRequestRouterGrpcPort := dig "addons" "llm" "requestRouter" "service" "grpcPort" 50071 .Values }} +{{- $backendRouterGrpcPort := dig "addons" "llm" "requestRouter" "backendRouter" "service" "grpcPort" 50071 .Values }} +{{- $llmEnabled := dig "addons" "llm" "enabled" false .Values }} +{{- $backendRouterEnabled := dig "addons" "llm" "requestRouter" "backendRouter" "enabled" $llmEnabled .Values }} {{- $llmRequestRouterDefaultAddress := printf "llm-request-router.nvcf.svc.cluster.local:%v" $llmRequestRouterGrpcPort }} +{{- if $backendRouterEnabled }} +{{- $llmRequestRouterDefaultAddress = printf "llm-request-router-backend-router.nvcf.svc.cluster.local:%v" $backendRouterGrpcPort }} +{{- end }} {{- $llmRequestRouterWorkerAddress := dig "llmRequestRouterAddress" "" $workerEndpoints | trim | default $llmRequestRouterDefaultAddress }} {{- $llmRequestRouterWorkerAuthority := $llmRequestRouterWorkerAddress }} {{- $llmRequestRouterDNSAddressPattern := `^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(\.([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*:[0-9]{1,5}$` }} @@ -371,7 +377,6 @@ natsAuthCalloutService: {{- else if hasPrefix "http://" $llmRequestRouterWorkerAuthority }} {{- $llmRequestRouterWorkerAuthority = trimPrefix "http://" $llmRequestRouterWorkerAuthority }} {{- end }} -{{- $llmEnabled := dig "addons" "llm" "enabled" false .Values }} {{- if $llmEnabled }} {{- $llmRequestRouterWorkerAddressError := "global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535" }} {{- $llmRequestRouterBracketedIPv6Pattern := `^\[[0-9A-Fa-f:.]+\]:[0-9]{1,5}$` }} @@ -1045,7 +1050,6 @@ llmApiGateway: {{- $pylonGrpcDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonGrpcDialAddress" "" .Values | default "" | toString | trim }} {{- $pylonReverseTunnelDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonReverseTunnelDialAddress" "" .Values | default "" | toString | trim }} -{{- $backendRouterEnabled := dig "addons" "llm" "requestRouter" "backendRouter" "enabled" $llmEnabled .Values }} {{- $llmWorkerRouteEnabled := dig "ingress" "gatewayApi" "routes" "llmWorker" "enabled" false .Values }} {{- $grpcTlsEnabled := dig "addons" "llm" "requestRouter" "grpcTls" "enabled" false .Values }} {{- $grpcTlsAllowInsecureHttp := dig "addons" "llm" "requestRouter" "grpcTls" "allowInsecureHttp" false .Values }} @@ -1129,6 +1133,8 @@ llmRequestRouter: */}} backendRouter: enabled: {{ $backendRouterEnabled }} + service: + grpcPort: {{ $backendRouterGrpcPort }} {{- $backendRouterValues := dig "addons" "llm" "requestRouter" "backendRouter" dict .Values }} {{- if hasKey $backendRouterValues "replicaCount" }} replicaCount: {{ index $backendRouterValues "replicaCount" }} diff --git a/deploy/stacks/self-managed/tests/api-env-wiring.sh b/deploy/stacks/self-managed/tests/api-env-wiring.sh index 0562fe388..479eae465 100755 --- a/deploy/stacks/self-managed/tests/api-env-wiring.sh +++ b/deploy/stacks/self-managed/tests/api-env-wiring.sh @@ -120,7 +120,7 @@ assert_yaml_value "$explicit_values" '.api.remoteConfig.configData.custom.retain assert_yaml_value "$explicit_values" '.api.remoteConfig.configData.nvcf.sidecars.retained-setting' \ keep-inside-sidecars "nested sidecar remote config" assert_yaml_value "$explicit_values" "$remote_worker_address_expression" \ - llm-request-router.nvcf.svc.cluster.local:50071 "stack-owned worker address" + llm-request-router-backend-router.nvcf.svc.cluster.local:50071 "stack-owned worker address" assert_yaml_value "$explicit_values" '.api.env.CUSTOM_API_ENV' \ configured "generic API env" assert_yaml_value "$explicit_values" '.api.env.LITERAL_TEMPLATE_VALUE' \ diff --git a/deploy/stacks/self-managed/tests/llm-router-worker-address.sh b/deploy/stacks/self-managed/tests/llm-router-worker-address.sh index 32b8ae95c..78dc27217 100755 --- a/deploy/stacks/self-managed/tests/llm-router-worker-address.sh +++ b/deploy/stacks/self-managed/tests/llm-router-worker-address.sh @@ -160,6 +160,14 @@ assert_llm_request_router_grpc_port() { test "$actual_port" = "$expected_port" } +assert_backend_router_grpc_port() { + local values_file="$1" + local expected_port="$2" + + test "$(yq -r '.llmRequestRouter.backendRouter.service.grpcPort' "$values_file")" = \ + "$expected_port" +} + invalid_worker_address_error='global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535' assert_worker_address_rejected() { @@ -218,6 +226,9 @@ printf '%s\n' \ 'addons:' \ ' llm:' \ ' enabled: true' \ + ' requestRouter:' \ + ' backendRouter:' \ + ' enabled: false' \ >"$environment_file" render_api_values "$work_dir/local-api-values.yaml" >/dev/null assert_remote_config_address "$work_dir/local-api-values.yaml" \ @@ -230,11 +241,6 @@ fi custom_router_grpc_port='51071' custom_port_worker_address="llm-request-router.nvcf.svc.cluster.local:$custom_router_grpc_port" -printf '%s\n' \ - 'addons:' \ - ' llm:' \ - ' enabled: true' \ - >"$environment_file" render_api_values \ "$work_dir/custom-port-api-values.yaml" \ --state-values-set \ @@ -242,11 +248,38 @@ render_api_values \ >/dev/null assert_remote_config_address "$work_dir/custom-port-api-values.yaml" \ "$custom_port_worker_address" || - fail "enabled local LLM did not use the configured request-router gRPC port" + fail "disabled backend routing did not use the configured request-router gRPC port" assert_llm_request_router_grpc_port "$work_dir/custom-port-api-values.yaml" \ "$custom_router_grpc_port" || fail "enabled LLM did not pass the configured gRPC port to the request-router chart" +backend_router_address='llm-request-router-backend-router.nvcf.svc.cluster.local:50071' +printf '%s\n' \ + 'addons:' \ + ' llm:' \ + ' enabled: true' \ + >"$environment_file" +render_api_values "$work_dir/backend-api-values.yaml" >/dev/null +assert_remote_config_address "$work_dir/backend-api-values.yaml" \ + "$backend_router_address" || + fail "enabled backend routing did not use the backend-router address" +assert_backend_router_grpc_port "$work_dir/backend-api-values.yaml" 50071 || + fail "enabled backend routing did not pass the default gRPC port to the chart" + +custom_backend_router_grpc_port='51072' +custom_backend_router_address="llm-request-router-backend-router.nvcf.svc.cluster.local:$custom_backend_router_grpc_port" +render_api_values \ + "$work_dir/backend-custom-port-api-values.yaml" \ + --state-values-set \ + "addons.llm.requestRouter.backendRouter.service.grpcPort=$custom_backend_router_grpc_port" \ + >/dev/null +assert_remote_config_address "$work_dir/backend-custom-port-api-values.yaml" \ + "$custom_backend_router_address" || + fail "enabled backend routing did not use the configured backend-router gRPC port" +assert_backend_router_grpc_port "$work_dir/backend-custom-port-api-values.yaml" \ + "$custom_backend_router_grpc_port" || + fail "enabled LLM did not pass the configured backend-router gRPC port to the chart" + external_worker_address='router.example.com:443' render_api_values \ "$work_dir/external-api-values.yaml" \ @@ -317,8 +350,8 @@ assert_remote_config_address "$work_dir/maximum-port-api-values.yaml" \ write_environment true '' render_api_values "$work_dir/default-api-values.yaml" >/dev/null assert_remote_config_address "$work_dir/default-api-values.yaml" \ - "$local_worker_address" || - fail "enabled LLM did not default the worker address to the cluster-local service" + "$backend_router_address" || + fail "enabled LLM did not default the worker address to the backend-router service" invalid_address_cases=( 'missing-port|router' diff --git a/docs/user/csp-end-to-end-example-installation.md b/docs/user/csp-end-to-end-example-installation.md index fa25199ff..62b64e23f 100644 --- a/docs/user/csp-end-to-end-example-installation.md +++ b/docs/user/csp-end-to-end-example-installation.md @@ -206,8 +206,9 @@ global: nvctServiceURL: "" # CHANGE (multi-cluster): "http://tasks.${GATEWAY_ADDR}". Worker env NVCT_FQDN. nvctGrpcServiceURL: "" # CHANGE (multi-cluster): "http://worker-tasks.${GATEWAY_ADDR}". Worker env NVCT_FQDN_GRPC. invocationServiceURL: "" # Empty = in-cluster default. Workers use this for the invocation stream address. - # CHANGE (multi-cluster): worker-reachable request-router host:port. Empty - # uses llm-request-router.nvcf.svc.cluster.local:50071. + # CHANGE (multi-cluster): worker-reachable request-router host:port. Empty uses + # llm-request-router-backend-router.nvcf.svc.cluster.local:50071 when backend routing is enabled, + # otherwise llm-request-router.nvcf.svc.cluster.local:50071. llmRequestRouterAddress: "" nodeSelectors: diff --git a/docs/user/helmfile-installation.md b/docs/user/helmfile-installation.md index 66d861367..5295f7d42 100644 --- a/docs/user/helmfile-installation.md +++ b/docs/user/helmfile-installation.md @@ -356,8 +356,9 @@ ingress: When `addons.llm` is enabled, the stack defaults `global.workerEndpoints.llmRequestRouterAddress` to -`llm-request-router.nvcf.svc.cluster.local:50071`. Colocated workers require no -additional configuration. For a split deployment, this address alone is not +`llm-request-router-backend-router.nvcf.svc.cluster.local:50071` when backend +routing is enabled, otherwise `llm-request-router.nvcf.svc.cluster.local:50071`. +For a split deployment, this address alone is not enough. Configure the paired backend-router gRPC and reverse QUIC dial addresses, Gateway routes, DNS, and trust described in [Remote compute clusters and regions](./llm-function-enablement.md#remote-compute-clusters-and-regions). diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index d09332344..3f184391c 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -76,11 +76,12 @@ pod SANs. The QUIC identity remains separate. The request router presents a certificate issued by cert-manager, or one you issue and supply in a pre-created Secret. -For a single-cluster deployment, workers dial -`llm-request-router.nvcf.svc.cluster.local:50071`. For remote workers, the -backend router preserves the advertised request-router pod hostname as the -QUIC SNI while it sends traffic through an external UDP endpoint. The QUIC -certificate must cover that advertised hostname. The default wildcard SAN is +For a single-cluster deployment, initial gRPC discovery defaults to +`llm-request-router-backend-router.nvcf.svc.cluster.local:50071`. For remote +workers, the backend router preserves the advertised request-router pod +hostname as the QUIC SNI while it sends traffic through an external UDP +endpoint. The QUIC certificate must cover that advertised hostname. The +default wildcard SAN is `*.llm-request-router-headless.nvcf.svc.cluster.local`. It does not need the external UDP load-balancer hostname. Do not reuse the gRPC NLB leaf certificate as the QUIC leaf certificate. @@ -182,10 +183,10 @@ constraint in the external issuer's own configuration. When `addons.llm.enabled` is `true`, the stack defaults `global.workerEndpoints.llmRequestRouterAddress` to -`llm-request-router.nvcf.svc.cluster.local:50071`. Colocated workers require no -additional configuration. For a split control-plane and compute-plane -deployment, override this value with a host and port that worker pods can -reach. +`llm-request-router-backend-router.nvcf.svc.cluster.local:50071` when backend +routing is enabled, otherwise `llm-request-router.nvcf.svc.cluster.local:50071`. +For a split control-plane and compute-plane deployment, override this value +with a host and port that worker pods can reach. The stack maps the configured or default address to `api.remoteConfig.configData.nvcf.llm-request-router.worker-address`. The NVCF @@ -709,11 +710,11 @@ kubectl -n nvcf-backend get pod \ ``` The worker args must contain -`--stargate-address=llm-request-router.nvcf.svc.cluster.local:50071`, or the -configured routable DNS name, and must not contain `--quic-insecure`. The -external address is the initial gRPC dial endpoint. The reverse tunnel verifies -the advertised request-router pod hostname instead. The environment must -contain: +`--stargate-address=llm-request-router-backend-router.nvcf.svc.cluster.local:50071` +with default backend routing, or the configured routable endpoint, and must +not contain `--quic-insecure`. The external address is the initial gRPC dial +endpoint. The reverse tunnel verifies the advertised request-router pod +hostname instead. The environment must contain: ```text STARGATE_TLS_CERT_PATH=/etc/ssl/certs/ca-certificates.crt diff --git a/tests/bdd/features/single-cluster-helmfile-llm-pki.feature b/tests/bdd/features/single-cluster-helmfile-llm-pki.feature index 0ec78e7ac..9398eecc4 100644 --- a/tests/bdd/features/single-cluster-helmfile-llm-pki.feature +++ b/tests/bdd/features/single-cluster-helmfile-llm-pki.feature @@ -71,10 +71,11 @@ Feature: Install a local single-cluster NVCF stack with PKI-secured LLM transpor | name: ADDONS_LLM_ENABLED | | value: "true" | | llm-request-router.nvcf.svc.cluster.local | + | worker-address: llm-request-router-backend-router.nvcf.svc.cluster.local:50071 | | name: NVCF_SERVICE_PKI_ALLOWED_DOMAINS | | value: "nvcf.svc.cluster.local" | | nvcf-openbao-migrations: | - # A colocated worker uses the in-cluster h2c Service directly. The + # A colocated worker uses the in-cluster backend-router Service directly. The # dedicated HTTPS identity and route belong only to an explicitly # enabled remote-worker ingress. And the rendered manifests in "deploy/stacks/self-managed/out" should not contain: @@ -113,7 +114,7 @@ Feature: Install a local single-cluster NVCF stack with PKI-secured LLM transpor When I run command "kubectl --context k3d-ncp-local get configmap/nvcf-api-remote-config -n nvcf -o yaml" Then the command exit code should be 0 - And the command output should contain "worker-address: llm-request-router.nvcf.svc.cluster.local:50071" + And the command output should contain "worker-address: llm-request-router-backend-router.nvcf.svc.cluster.local:50071" And the command output should contain "llm-router-client-image: nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/pylon:" Then these Kubernetes resources should not exist in namespace "envoy-gateway-system" using context "k3d-ncp-local": diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 0460bdb67..0114fea7f 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -682,7 +682,8 @@ func TestSingleClusterHelmfileLLMPKIFeatureFileWiresToSteps(t *testing.T) { "kubectl --context k3d-ncp-local get configmap/nvcf-api-remote-config -n nvcf -o yaml": { ExitCode: 0, Stdout: "data:\n nvcf-api.yaml: |\n nvcf:\n" + - " llm-request-router:\n worker-address: llm-request-router.nvcf.svc.cluster.local:50071\n" + + " llm-request-router:\n" + + " worker-address: llm-request-router-backend-router.nvcf.svc.cluster.local:50071\n" + " sidecars:\n llm-router-client-image: nvcr.io/test-org/test-team/pylon:test\n", }, "helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o yaml": { @@ -2099,6 +2100,7 @@ metadata: spec: dnsNames: - llm-request-router.nvcf.svc.cluster.local +worker-address: llm-request-router-backend-router.nvcf.svc.cluster.local:50071 env: - name: ADDONS_LLM_ENABLED value: "true"