fix(llm): propagate deployment concurrency to Pylon

src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsClient.java

@@ -323,11 +323,12 @@ public UUID createInstance(
             String cacheHandle,
             String secretsAssertionToken) {
         var telemetries = base64EncodeTelemetryDetails(function);
-        var env = getEnvironment(function, gpuSpec, secretsAssertionToken);
+        var maxRequestConcurrency = requireNonNullElse(gpuSpec.getMaxRequestConcurrency(), 1);
+        var env = getEnvironment(function, maxRequestConcurrency, secretsAssertionToken);
         var helmValidationPolicy = getHelmValidationPolicy(gpuSpec);
         return scheduleSingleInstanceType(function, deploymentId, env, count, gpuSpec,
                                           artifactSize, telemetries, cacheHandle,
-                                          helmValidationPolicy);
+                                          helmValidationPolicy, maxRequestConcurrency);
     }
 
     private String base64EncodeTelemetryDetails(FunctionEntity function) {
@@ -349,7 +350,8 @@ private UUID scheduleSingleInstanceType(
             long cacheSize,
             String telemetries,
             String cacheHandle,
-            String helmValidationPolicy) {
+            String helmValidationPolicy,
+            int maxRequestConcurrency) {
         var functionId = function.getFunctionId();
         var versionId = function.getFunctionVersionId();
         var functionName = function.getFunctionName();
@@ -396,6 +398,7 @@ private UUID scheduleSingleInstanceType(
                                               cacheSize != 0 ? cacheSize : null,
                                               deploymentId,
                                               gpuSpecificationId,
+                                              maxRequestConcurrency,
                                               functionId,
                                               versionId,
                                               ownerNcaId,
@@ -415,7 +418,7 @@ private UUID scheduleSingleInstanceType(
 
     private String getEnvironment(
             FunctionEntity function,
-            GpuSpecificationEntity gpuSpec,
+            int maxRequestConcurrency,
             String secretsAssertionToken) {
         var functionId = function.getFunctionId();
         var versionId = function.getFunctionVersionId();
@@ -426,7 +429,6 @@ private String getEnvironment(
         var cenv = isNotBlank(containerEnv) ? containerEnv : DEFAULT_CONTAINER_ENV;
         var inferencePort = function.getInferencePort() != null ?
                 function.getInferencePort() : DEFAULT_INFERENCE_PORT;
-        var maxRequestConcurrency = requireNonNullElse(gpuSpec.getMaxRequestConcurrency(), 1);
         var helmChartServiceName = getHelmChartServiceName(function);
         var functionSecretsPresent = function.hasSecrets();
         var containerRegistryCredentialsEncoded =

src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsStubService.java

@@ -85,6 +85,7 @@ CreateInstancesResponse createInstance(
                           required = false) Long cacheSize,
             @RequestParam("LaunchSpecification.DeploymentId") UUID deploymentId,
             @RequestParam("LaunchSpecification.GpuSpecificationId") UUID gpuSpecificationId,
+            @RequestParam("LaunchSpecification.MaxRequestConcurrency") int maxRequestConcurrency,
             @RequestParam("FunctionDetails.FunctionId") UUID functionId,
             @RequestParam("FunctionDetails.FunctionVersionId") UUID functionVersionId,
             @RequestParam("FunctionDetails.OwnerNcaId") String ownerNcaId,

src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/icms/client/IcmsClientTest.java

@@ -628,6 +628,7 @@ private Map<String, String> validateEnvironmentVariables(String formUrlEncodedBo
         var tags = new HashSet<>(Arrays.asList(env.get("FUNCTION_TAGS").split(",")));
         assertThat(tags).containsAll(TEST_TAGS);
         assertThat(env.get("MAX_REQUEST_CONCURRENCY")).isEqualTo("9");
+        assertThat(params.get("LaunchSpecification.MaxRequestConcurrency")).isEqualTo("9");
         assertThat(env.get("SECRETS_ASSERTION_TOKEN")).isEqualTo("");
         assertThat(env.get("ESS_AGENT_CONTAINER")).isEqualTo(
                 "stg.nvcr.io/nv-cf/nvcf-core/ess-agent:0.0.4");

src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverter.java

@@ -74,7 +74,9 @@ public SpotInstanceRequestSchema toSpotInstanceRequestSchema(
                     .terminationGracePeriodDuration(createRequest.getLaunchSpecification()
                                                             .getTerminationGracePeriodDuration())
                     .resultHandlingStrategy(createRequest.getLaunchSpecification()
-                                                    .getResultHandlingStrategy());
+                                                    .getResultHandlingStrategy())
+                    .maxRequestConcurrency(createRequest.getLaunchSpecification()
+                                                   .getMaxRequestConcurrency());
 
             if (createRequest.getLaunchSpecification().getDeploymentId() != null) {
                 builder.deploymentId(createRequest.getLaunchSpecification().getDeploymentId());

src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/CreateSpotInstanceLaunchSpecificationApiModel.java

@@ -104,4 +104,7 @@ public class CreateSpotInstanceLaunchSpecificationApiModel {
     @Schema(hidden = true)
     UUID gpuSpecificationId;
 
+    @Schema(hidden = true)
+    Integer maxRequestConcurrency;
+
 }

src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/swagger/schema/SpotInstanceRequestSchema.java

@@ -167,6 +167,10 @@ public class SpotInstanceRequestSchema {
     @Schema(name = "LaunchSpecification.GpuSpecificationId", description = "Function or Task Gpu Specification id")
     UUID gpuSpecificationId;
 
+    @Schema(name = "LaunchSpecification.MaxRequestConcurrency",
+            description = "Maximum concurrent requests configured for each function instance")
+    Integer maxRequestConcurrency;
+
     public String getLoggingId() {
         if (taskId != null) {
             return String.format("TaskId %s | DeploymentId %s | NcaId %s", this.getTaskId(),  this.getDeploymentId(), this.getNcaId());

src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModel.java

@@ -220,6 +220,12 @@ public static class ByocLaunchSpecification {
         @Schema(description = "NVCF GPU Specification Id", nullable = true)
         private UUID gpuSpecificationId;
 
+        @Nullable
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @Schema(description = "Maximum concurrent requests configured for each function instance",
+                nullable = true)
+        private Integer maxRequestConcurrency;
+
         // Bring Your Own Observability (BYOO) fields
         @Nullable
         @JsonInclude(JsonInclude.Include.NON_NULL)

src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/service/byoc/ByocMessageGenerator.java

@@ -155,6 +155,7 @@ private ByocLaunchSpecification generateByocLaunchSpecification(@NotNull SpotIns
                 .attributes(instanceRequest.getAttributes())
                 .deploymentId(instanceRequest.getDeploymentId())
                 .gpuSpecificationId(instanceRequest.getGpuSpecificationId())
+                .maxRequestConcurrency(instanceRequest.getMaxRequestConcurrency())
                 .build();
     }
 
@@ -183,4 +184,4 @@ InstanceRequestV2Entity generateInstanceRequestEntity(String customer, SpotInsta
                 .gpuSpecificationId(instanceRequest.getGpuSpecificationId())
                 .build();
     }
-} 
+}

src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverterTest.java

@@ -97,6 +97,7 @@ void testConvertWithLaunchSpecification() {
         launchSpec.setResultHandlingStrategy(ResultHandlingStrategy.UPLOAD);
         launchSpec.setDeploymentId(UUID.randomUUID());
         launchSpec.setGpuSpecificationId(UUID.randomUUID());
+        launchSpec.setMaxRequestConcurrency(37);
         launchSpec.setCacheArtifacts(true);
         launchSpec.setCacheSize(100L);
         launchSpec.setCacheHandle("test-cache-handle");
@@ -127,6 +128,7 @@ void testConvertWithLaunchSpecification() {
         assertEquals(ResultHandlingStrategy.UPLOAD, result.getResultHandlingStrategy());
         assertEquals(launchSpec.getDeploymentId(), result.getDeploymentId());
         assertEquals(launchSpec.getGpuSpecificationId(), result.getGpuSpecificationId());
+        assertEquals(37, result.getMaxRequestConcurrency());
         assertTrue(result.isCacheArtifacts());
         assertEquals(100L, result.getCacheSize());
         assertEquals("test-cache-handle", result.getCacheHandle());

src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModelTest.java

@@ -59,6 +59,7 @@ void testByocSqsMessageModelBuilder() throws Exception {
                         .cloudProvider("test")
                         .deploymentId(UUID.randomUUID())
                         .gpuSpecificationId(UUID.randomUUID())
+                        .maxRequestConcurrency(37)
                         .build())
                 .accountName("test-account-name")
                 .functionDetails(FunctionDetails.builder()
@@ -101,6 +102,7 @@ void testByocSqsMessageModelBuilder() throws Exception {
         assertEquals("test", launchSpecNode.get("cloudProvider").asString());
         assertNotNull(launchSpecNode.get("deploymentId"));
         assertNotNull(launchSpecNode.get("gpuSpecificationId"));
+        assertEquals(37, launchSpecNode.get("maxRequestConcurrency").asInt());
 
         // Validate functionDetails object
         JsonNode functionDetailsNode = jsonNode.get("functionDetails");
@@ -205,4 +207,4 @@ void testByocLaunchSpecificationBuilder() {
         assertNotNull(byocLaunchSpecification.getDeploymentId());
         assertNotNull(byocLaunchSpecification.getGpuSpecificationId());
     }
-}
+}

src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/service/byoc/ByocMessageGeneratorTest.java

@@ -69,6 +69,7 @@ void generateSqsMessageModelForByocFunction_forwardsModelsUnchanged() {
                 .gpu("A100")
                 .containerImage(DUMMY_CONTAINER_IMAGE)
                 .environment(DUMMY_ENVIRONMENT_VALUE)
+                .maxRequestConcurrency(37)
                 .models("[{\"name\":\"model-1\"}]")
                 .functionType(FunctionType.LLM)
                 .functionId(UUID.randomUUID())
@@ -91,6 +92,7 @@ void generateSqsMessageModelForByocFunction_forwardsModelsUnchanged() {
                 "request-id", 1, instanceRequest, destination, "customer");
 
         assertEquals(instanceRequest.getModels(), message.getLaunchSpecification().getModels());
+        assertEquals(37, message.getLaunchSpecification().getMaxRequestConcurrency());
     }
 
     @Test

src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go

@@ -41,6 +41,7 @@ const (
 	llmRouterClientImageDefault      = "nvcr.io/0651155215864979/ncp-dev/stargate-client:0.4.0"
 	llmRequestRouterAddressEnv       = "LLM_REQUEST_ROUTER_ADDRESS"
 	legacyStargateAddressEnv         = "STARGATE_ADDRESS"
+	maxRequestConcurrencyEnv         = "MAX_REQUEST_CONCURRENCY"
 
 	llmDirMountPath    = "/var/run/llm"
 	llmWorkerTokenPath = llmDirMountPath + "/worker-token"
@@ -154,6 +155,15 @@ func newLLMRouterClientContainer(
 		"--backend-connectivity=reverse",
 		"--initial-input-tps=100",
 	}
+	maxRequestConcurrency := ""
+	if ls.MaxRequestConcurrency > 0 {
+		maxRequestConcurrency = strconv.Itoa(ls.MaxRequestConcurrency)
+	} else {
+		maxRequestConcurrency = allEnvSet[maxRequestConcurrencyEnv]
+	}
+	if maxRequestConcurrency != "" {
+		args = append(args, fmt.Sprintf("--max-engine-concurrency=%s", maxRequestConcurrency))
+	}
 	if healthPath := upstreamHealthPath(allEnvSet); healthPath != "" {
 		args = append(args, fmt.Sprintf("--upstream-health-path=%s", healthPath))
 	}

src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go

@@ -57,6 +57,16 @@ func healthPathArgs(args []string) []string {
 	return healthPaths
 }
 
+func maxEngineConcurrencyArgs(args []string) []string {
+	var maxEngineConcurrency []string
+	for _, arg := range args {
+		if strings.HasPrefix(arg, "--max-engine-concurrency=") {
+			maxEngineConcurrency = append(maxEngineConcurrency, arg)
+		}
+	}
+	return maxEngineConcurrency
+}
+
 // TestNewLLMRouterClientContainer verifies Pylon configuration and validation.
 func TestNewLLMRouterClientContainer(t *testing.T) {
 	type spec struct {
@@ -72,11 +82,14 @@ func TestNewLLMRouterClientContainer(t *testing.T) {
 
 	cases := []spec{
 		{
-			name: "container mode with env-set LLM request router address injects legacy stargate env",
-			ls:   &LaunchSpecification{},
+			name: "deployment concurrency overrides legacy environment value",
+			ls: &LaunchSpecification{
+				MaxRequestConcurrency: 64,
+			},
 			allEnvSet: map[string]string{
 				"LLM_REQUEST_ROUTER_ADDRESS": "llm-router.example.com:443",
 				"INFERENCE_PORT":             "8080",
+				maxRequestConcurrencyEnv:     "11",
 			},
 			tcfg:       TranslateConfig{},
 			instanceID: "inst-123",
@@ -94,6 +107,7 @@ func TestNewLLMRouterClientContainer(t *testing.T) {
 				assert.Equal(t, "llm-router.example.com:443", envMap["STARGATE_ADDRESS"])
 				assert.Contains(t, c.Args, "--inference-server-id=inst-123")
 				assert.Contains(t, c.Args, "--auth-token-file=/var/run/llm/worker-token")
+				assert.Equal(t, []string{"--max-engine-concurrency=64"}, maxEngineConcurrencyArgs(c.Args))
 				assertCanonicalPylonBootstrapArgs(t, c.Args)
 				assert.NotContains(t, c.Args, "--quic-insecure")
 			},
@@ -129,18 +143,20 @@ func TestNewLLMRouterClientContainer(t *testing.T) {
 			},
 		},
 		{
-			name: "helm mode without namespace uses service name",
+			name: "legacy environment value is used when deployment field is absent",
 			ls:   &LaunchSpecification{},
 			allEnvSet: map[string]string{
 				"STARGATE_ADDRESS":                  "stargate.example.com:443",
 				"INFERENCE_PORT":                    "8080",
 				"HELM_CHART_INFERENCE_SERVICE_NAME": "my-inference-svc",
+				maxRequestConcurrencyEnv:            "32",
 			},
 			tcfg:       TranslateConfig{},
 			instanceID: "inst-789",
 			isHelm:     true,
 			validate: func(t *testing.T, c corev1.Container) {
 				assert.Contains(t, c.Args, "--upstream-http-base-url=http://my-inference-svc:8080")
+				assert.Equal(t, []string{"--max-engine-concurrency=32"}, maxEngineConcurrencyArgs(c.Args))
 			},
 		},
 		{
@@ -229,6 +245,7 @@ func TestNewLLMRouterClientContainer(t *testing.T) {
 				for _, arg := range c.Args {
 					assert.NotContains(t, arg, "--model-name=")
 				}
+				assert.Empty(t, maxEngineConcurrencyArgs(c.Args))
 			},
 		},
 		{

src/libraries/go/lib/pkg/icms-translate/translate/function/types.go

@@ -79,6 +79,8 @@ type LaunchSpecification struct {
 	ICMSEnvironment string `json:"icmsEnvironment"`
 	CloudProvider   string `json:"cloudProvider"`
 	GPUName         string `json:"gpuName"`
+	// MaxRequestConcurrency is the per-instance limit from the NVCF deployment GPU specification.
+	MaxRequestConcurrency int `json:"maxRequestConcurrency,omitempty"`
 
 	// Helm chart function components of the launch spec.
 	*common.HelmChartLaunchSpecification `json:",inline"`

src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml

@@ -187,6 +187,7 @@ spec:
     - --auth-token-file=/var/run/llm/worker-token
     - --backend-connectivity=reverse
     - --initial-input-tps=100
+    - --max-engine-concurrency=37
     - --upstream-health-path=/v1/health/ready
     - --quic-insecure
     - --model-name=model-gamma

src/libraries/go/lib/testdata/icms-translate/function/container/llm/message.json

@@ -24,6 +24,7 @@
     "cloudProvider": "DGXCLOUD",
     "deploymentId": "5a3d4a7e-9ee3-4762-8d37-d3b40a6f84c6",
     "gpuSpecificationId": "2c948d9b-db5d-4f93-8c29-f5d8a5d89cb9",
+    "maxRequestConcurrency": 37,
     "models": "W3sibmFtZSI6Im1vZGVsLWJldGEiLCJ2ZXJzaW9uIjoiMS4wLjAiLCJ1cmkiOiJodHRwczovL2FwaS5uZ2MubnZpZGlhLmNvbS92Mi9vcmcvbXktb3JnL3RlYW0vbXktdGVhbS9tb2RlbHMvbW9kZWwtYmV0YS8xLjAuMC9maWxlcyJ9LHsibmFtZSI6Im1vZGVsLWdhbW1hIiwidmVyc2lvbiI6IjEuMC4wIiwidXJpIjoiaHR0cHM6Ly9hcGkubmdjLm52aWRpYS5jb20vdjIvb3JnL215LW9yZy90ZWFtL215LXRlYW0vbW9kZWxzL21vZGVsLWdhbW1hLzEuMC4wL2ZpbGVzIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS9jaGF0L2NvbXBsZXRpb25zIiwiaHR0cDovLy92MS9yZXNwb25zZXMiXSwidG9rZW5pemVyIjoibGxhbWEzIiwidG9rZW5SYXRlTGltaXQiOiIxMDAwIiwicm91dGluZ01ldGhvZCI6ImZvbyJ9fSx7Im5hbWUiOiJtb2RlbC1lcHNpbG9uIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS8vY2hhdC9jb21wbGV0aW9ucyJdLCJ0b2tlbml6ZXIiOiJsbGFtYTMiLCJ0b2tlblJhdGVMaW1pdCI6IjIwMDAiLCJyb3V0aW5nTWV0aG9kIjoiZm9vIn19XQo="
   }
 }

src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml

@@ -150,6 +150,7 @@ spec:
     - --auth-token-file=/var/run/llm/worker-token
     - --backend-connectivity=reverse
     - --initial-input-tps=100
+    - --max-engine-concurrency=37
     - --upstream-health-path=/v2/health/ready
     - --quic-insecure
     - --model-name=model-gamma

src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/message.json

@@ -26,6 +26,7 @@
     "configuration": "eyJmb28iOnsiYmFyIjoiYmF6In19",
     "deploymentId": "5a3d4a7e-9ee3-4762-8d37-d3b40a6f84c6",
     "gpuSpecificationId": "2c948d9b-db5d-4f93-8c29-f5d8a5d89cb9",
+    "maxRequestConcurrency": 37,
     "models": "W3sibmFtZSI6Im1vZGVsLWJldGEiLCJ2ZXJzaW9uIjoiMS4wLjAiLCJ1cmkiOiJodHRwczovL2FwaS5uZ2MubnZpZGlhLmNvbS92Mi9vcmcvbXktb3JnL3RlYW0vbXktdGVhbS9tb2RlbHMvbW9kZWwtYmV0YS8xLjAuMC9maWxlcyJ9LHsibmFtZSI6Im1vZGVsLWdhbW1hIiwidmVyc2lvbiI6IjEuMC4wIiwidXJpIjoiaHR0cHM6Ly9hcGkubmdjLm52aWRpYS5jb20vdjIvb3JnL215LW9yZy90ZWFtL215LXRlYW0vbW9kZWxzL21vZGVsLWdhbW1hLzEuMC4wL2ZpbGVzIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS9jaGF0L2NvbXBsZXRpb25zIiwiaHR0cDovLy92MS9yZXNwb25zZXMiXSwidG9rZW5pemVyIjoibGxhbWEzIiwidG9rZW5SYXRlTGltaXQiOiIxMDAwIiwicm91dGluZ01ldGhvZCI6ImZvbyJ9fSx7Im5hbWUiOiJtb2RlbC1lcHNpbG9uIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS8vY2hhdC9jb21wbGV0aW9ucyJdLCJ0b2tlbml6ZXIiOiJsbGFtYTMiLCJ0b2tlblJhdGVMaW1pdCI6IjIwMDAiLCJyb3V0aW5nTWV0aG9kIjoiZm9vIn19XQo="
   }
 }