diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsClient.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsClient.java index e205b8de74..2dc0578c81 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsClient.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsClient.java @@ -323,11 +323,12 @@ public UUID createInstance( String cacheHandle, String secretsAssertionToken) { var telemetries = base64EncodeTelemetryDetails(function); - var env = getEnvironment(function, gpuSpec, secretsAssertionToken); + var maxRequestConcurrency = requireNonNullElse(gpuSpec.getMaxRequestConcurrency(), 1); + var env = getEnvironment(function, maxRequestConcurrency, secretsAssertionToken); var helmValidationPolicy = getHelmValidationPolicy(gpuSpec); return scheduleSingleInstanceType(function, deploymentId, env, count, gpuSpec, artifactSize, telemetries, cacheHandle, - helmValidationPolicy); + helmValidationPolicy, maxRequestConcurrency); } private String base64EncodeTelemetryDetails(FunctionEntity function) { @@ -349,7 +350,8 @@ private UUID scheduleSingleInstanceType( long cacheSize, String telemetries, String cacheHandle, - String helmValidationPolicy) { + String helmValidationPolicy, + int maxRequestConcurrency) { var functionId = function.getFunctionId(); var versionId = function.getFunctionVersionId(); var functionName = function.getFunctionName(); @@ -396,6 +398,7 @@ private UUID scheduleSingleInstanceType( cacheSize != 0 ? cacheSize : null, deploymentId, gpuSpecificationId, + maxRequestConcurrency, functionId, versionId, ownerNcaId, @@ -415,7 +418,7 @@ private UUID scheduleSingleInstanceType( private String getEnvironment( FunctionEntity function, - GpuSpecificationEntity gpuSpec, + int maxRequestConcurrency, String secretsAssertionToken) { var functionId = function.getFunctionId(); var versionId = function.getFunctionVersionId(); @@ -426,7 +429,6 @@ private String getEnvironment( var cenv = isNotBlank(containerEnv) ? containerEnv : DEFAULT_CONTAINER_ENV; var inferencePort = function.getInferencePort() != null ? function.getInferencePort() : DEFAULT_INFERENCE_PORT; - var maxRequestConcurrency = requireNonNullElse(gpuSpec.getMaxRequestConcurrency(), 1); var helmChartServiceName = getHelmChartServiceName(function); var functionSecretsPresent = function.hasSecrets(); var containerRegistryCredentialsEncoded = diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsStubService.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsStubService.java index ec1390580b..a6dcea31b5 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsStubService.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/icms/client/IcmsStubService.java @@ -85,6 +85,7 @@ CreateInstancesResponse createInstance( required = false) Long cacheSize, @RequestParam("LaunchSpecification.DeploymentId") UUID deploymentId, @RequestParam("LaunchSpecification.GpuSpecificationId") UUID gpuSpecificationId, + @RequestParam("LaunchSpecification.MaxRequestConcurrency") int maxRequestConcurrency, @RequestParam("FunctionDetails.FunctionId") UUID functionId, @RequestParam("FunctionDetails.FunctionVersionId") UUID functionVersionId, @RequestParam("FunctionDetails.OwnerNcaId") String ownerNcaId, diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/icms/client/IcmsClientTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/icms/client/IcmsClientTest.java index dce82eb60a..e15a9a1929 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/icms/client/IcmsClientTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/icms/client/IcmsClientTest.java @@ -628,6 +628,7 @@ private Map validateEnvironmentVariables(String formUrlEncodedBo var tags = new HashSet<>(Arrays.asList(env.get("FUNCTION_TAGS").split(","))); assertThat(tags).containsAll(TEST_TAGS); assertThat(env.get("MAX_REQUEST_CONCURRENCY")).isEqualTo("9"); + assertThat(params.get("LaunchSpecification.MaxRequestConcurrency")).isEqualTo("9"); assertThat(env.get("SECRETS_ASSERTION_TOKEN")).isEqualTo(""); assertThat(env.get("ESS_AGENT_CONTAINER")).isEqualTo( "stg.nvcr.io/nv-cf/nvcf-core/ess-agent:0.0.4"); diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverter.java b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverter.java index 4605af3e02..29ac906cb7 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverter.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverter.java @@ -74,7 +74,9 @@ public SpotInstanceRequestSchema toSpotInstanceRequestSchema( .terminationGracePeriodDuration(createRequest.getLaunchSpecification() .getTerminationGracePeriodDuration()) .resultHandlingStrategy(createRequest.getLaunchSpecification() - .getResultHandlingStrategy()); + .getResultHandlingStrategy()) + .maxRequestConcurrency(createRequest.getLaunchSpecification() + .getMaxRequestConcurrency()); if (createRequest.getLaunchSpecification().getDeploymentId() != null) { builder.deploymentId(createRequest.getLaunchSpecification().getDeploymentId()); diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/CreateSpotInstanceLaunchSpecificationApiModel.java b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/CreateSpotInstanceLaunchSpecificationApiModel.java index 489354fb2f..62b2200150 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/CreateSpotInstanceLaunchSpecificationApiModel.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/CreateSpotInstanceLaunchSpecificationApiModel.java @@ -104,4 +104,7 @@ public class CreateSpotInstanceLaunchSpecificationApiModel { @Schema(hidden = true) UUID gpuSpecificationId; + @Schema(hidden = true) + Integer maxRequestConcurrency; + } diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/swagger/schema/SpotInstanceRequestSchema.java b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/swagger/schema/SpotInstanceRequestSchema.java index ea214c2b06..7e60311683 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/swagger/schema/SpotInstanceRequestSchema.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/inbound/rest/model/swagger/schema/SpotInstanceRequestSchema.java @@ -167,6 +167,10 @@ public class SpotInstanceRequestSchema { @Schema(name = "LaunchSpecification.GpuSpecificationId", description = "Function or Task Gpu Specification id") UUID gpuSpecificationId; + @Schema(name = "LaunchSpecification.MaxRequestConcurrency", + description = "Maximum concurrent requests configured for each function instance") + Integer maxRequestConcurrency; + public String getLoggingId() { if (taskId != null) { return String.format("TaskId %s | DeploymentId %s | NcaId %s", this.getTaskId(), this.getDeploymentId(), this.getNcaId()); diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModel.java b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModel.java index f04282e565..38baa5e610 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModel.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModel.java @@ -220,6 +220,12 @@ public static class ByocLaunchSpecification { @Schema(description = "NVCF GPU Specification Id", nullable = true) private UUID gpuSpecificationId; + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @Schema(description = "Maximum concurrent requests configured for each function instance", + nullable = true) + private Integer maxRequestConcurrency; + // Bring Your Own Observability (BYOO) fields @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/service/byoc/ByocMessageGenerator.java b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/service/byoc/ByocMessageGenerator.java index be2160f091..e1816837d9 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/service/byoc/ByocMessageGenerator.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/main/java/com/nvidia/icms/service/byoc/ByocMessageGenerator.java @@ -155,6 +155,7 @@ private ByocLaunchSpecification generateByocLaunchSpecification(@NotNull SpotIns .attributes(instanceRequest.getAttributes()) .deploymentId(instanceRequest.getDeploymentId()) .gpuSpecificationId(instanceRequest.getGpuSpecificationId()) + .maxRequestConcurrency(instanceRequest.getMaxRequestConcurrency()) .build(); } @@ -183,4 +184,4 @@ InstanceRequestV2Entity generateInstanceRequestEntity(String customer, SpotInsta .gpuSpecificationId(instanceRequest.getGpuSpecificationId()) .build(); } -} \ No newline at end of file +} diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverterTest.java b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverterTest.java index b69b67b529..3c0260ba86 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverterTest.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/inbound/rest/converters/CreateInstanceApiModelConverterTest.java @@ -97,6 +97,7 @@ void testConvertWithLaunchSpecification() { launchSpec.setResultHandlingStrategy(ResultHandlingStrategy.UPLOAD); launchSpec.setDeploymentId(UUID.randomUUID()); launchSpec.setGpuSpecificationId(UUID.randomUUID()); + launchSpec.setMaxRequestConcurrency(37); launchSpec.setCacheArtifacts(true); launchSpec.setCacheSize(100L); launchSpec.setCacheHandle("test-cache-handle"); @@ -127,6 +128,7 @@ void testConvertWithLaunchSpecification() { assertEquals(ResultHandlingStrategy.UPLOAD, result.getResultHandlingStrategy()); assertEquals(launchSpec.getDeploymentId(), result.getDeploymentId()); assertEquals(launchSpec.getGpuSpecificationId(), result.getGpuSpecificationId()); + assertEquals(37, result.getMaxRequestConcurrency()); assertTrue(result.isCacheArtifacts()); assertEquals(100L, result.getCacheSize()); assertEquals("test-cache-handle", result.getCacheHandle()); diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModelTest.java b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModelTest.java index c27f9ec42b..44b1ab3864 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModelTest.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/outbound/sqs/model/byoc/ByocSqsMessageModelTest.java @@ -59,6 +59,7 @@ void testByocSqsMessageModelBuilder() throws Exception { .cloudProvider("test") .deploymentId(UUID.randomUUID()) .gpuSpecificationId(UUID.randomUUID()) + .maxRequestConcurrency(37) .build()) .accountName("test-account-name") .functionDetails(FunctionDetails.builder() @@ -101,6 +102,7 @@ void testByocSqsMessageModelBuilder() throws Exception { assertEquals("test", launchSpecNode.get("cloudProvider").asString()); assertNotNull(launchSpecNode.get("deploymentId")); assertNotNull(launchSpecNode.get("gpuSpecificationId")); + assertEquals(37, launchSpecNode.get("maxRequestConcurrency").asInt()); // Validate functionDetails object JsonNode functionDetailsNode = jsonNode.get("functionDetails"); @@ -205,4 +207,4 @@ void testByocLaunchSpecificationBuilder() { assertNotNull(byocLaunchSpecification.getDeploymentId()); assertNotNull(byocLaunchSpecification.getGpuSpecificationId()); } -} \ No newline at end of file +} diff --git a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/service/byoc/ByocMessageGeneratorTest.java b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/service/byoc/ByocMessageGeneratorTest.java index 2b85844f63..160d9372be 100644 --- a/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/service/byoc/ByocMessageGeneratorTest.java +++ b/src/control-plane-services/instance-cluster-management/icms-core/src/test/java/com/nvidia/icms/service/byoc/ByocMessageGeneratorTest.java @@ -69,6 +69,7 @@ void generateSqsMessageModelForByocFunction_forwardsModelsUnchanged() { .gpu("A100") .containerImage(DUMMY_CONTAINER_IMAGE) .environment(DUMMY_ENVIRONMENT_VALUE) + .maxRequestConcurrency(37) .models("[{\"name\":\"model-1\"}]") .functionType(FunctionType.LLM) .functionId(UUID.randomUUID()) @@ -91,6 +92,7 @@ void generateSqsMessageModelForByocFunction_forwardsModelsUnchanged() { "request-id", 1, instanceRequest, destination, "customer"); assertEquals(instanceRequest.getModels(), message.getLaunchSpecification().getModels()); + assertEquals(37, message.getLaunchSpecification().getMaxRequestConcurrency()); } @Test diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go index 337ef70965..4bb5023b25 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go @@ -41,6 +41,7 @@ const ( llmRouterClientImageDefault = "nvcr.io/0651155215864979/ncp-dev/stargate-client:0.4.0" llmRequestRouterAddressEnv = "LLM_REQUEST_ROUTER_ADDRESS" legacyStargateAddressEnv = "STARGATE_ADDRESS" + maxRequestConcurrencyEnv = "MAX_REQUEST_CONCURRENCY" llmDirMountPath = "/var/run/llm" llmWorkerTokenPath = llmDirMountPath + "/worker-token" @@ -154,6 +155,15 @@ func newLLMRouterClientContainer( "--backend-connectivity=reverse", "--initial-input-tps=100", } + maxRequestConcurrency := "" + if ls.MaxRequestConcurrency > 0 { + maxRequestConcurrency = strconv.Itoa(ls.MaxRequestConcurrency) + } else { + maxRequestConcurrency = allEnvSet[maxRequestConcurrencyEnv] + } + if maxRequestConcurrency != "" { + args = append(args, fmt.Sprintf("--max-engine-concurrency=%s", maxRequestConcurrency)) + } if healthPath := upstreamHealthPath(allEnvSet); healthPath != "" { args = append(args, fmt.Sprintf("--upstream-health-path=%s", healthPath)) } diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go index d7f91fa34a..1040aa15c1 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go @@ -57,6 +57,16 @@ func healthPathArgs(args []string) []string { return healthPaths } +func maxEngineConcurrencyArgs(args []string) []string { + var maxEngineConcurrency []string + for _, arg := range args { + if strings.HasPrefix(arg, "--max-engine-concurrency=") { + maxEngineConcurrency = append(maxEngineConcurrency, arg) + } + } + return maxEngineConcurrency +} + // TestNewLLMRouterClientContainer verifies Pylon configuration and validation. func TestNewLLMRouterClientContainer(t *testing.T) { type spec struct { @@ -72,11 +82,14 @@ func TestNewLLMRouterClientContainer(t *testing.T) { cases := []spec{ { - name: "container mode with env-set LLM request router address injects legacy stargate env", - ls: &LaunchSpecification{}, + name: "deployment concurrency overrides legacy environment value", + ls: &LaunchSpecification{ + MaxRequestConcurrency: 64, + }, allEnvSet: map[string]string{ "LLM_REQUEST_ROUTER_ADDRESS": "llm-router.example.com:443", "INFERENCE_PORT": "8080", + maxRequestConcurrencyEnv: "11", }, tcfg: TranslateConfig{}, instanceID: "inst-123", @@ -94,6 +107,7 @@ func TestNewLLMRouterClientContainer(t *testing.T) { assert.Equal(t, "llm-router.example.com:443", envMap["STARGATE_ADDRESS"]) assert.Contains(t, c.Args, "--inference-server-id=inst-123") assert.Contains(t, c.Args, "--auth-token-file=/var/run/llm/worker-token") + assert.Equal(t, []string{"--max-engine-concurrency=64"}, maxEngineConcurrencyArgs(c.Args)) assertCanonicalPylonBootstrapArgs(t, c.Args) assert.NotContains(t, c.Args, "--quic-insecure") }, @@ -129,18 +143,20 @@ func TestNewLLMRouterClientContainer(t *testing.T) { }, }, { - name: "helm mode without namespace uses service name", + name: "legacy environment value is used when deployment field is absent", ls: &LaunchSpecification{}, allEnvSet: map[string]string{ "STARGATE_ADDRESS": "stargate.example.com:443", "INFERENCE_PORT": "8080", "HELM_CHART_INFERENCE_SERVICE_NAME": "my-inference-svc", + maxRequestConcurrencyEnv: "32", }, tcfg: TranslateConfig{}, instanceID: "inst-789", isHelm: true, validate: func(t *testing.T, c corev1.Container) { assert.Contains(t, c.Args, "--upstream-http-base-url=http://my-inference-svc:8080") + assert.Equal(t, []string{"--max-engine-concurrency=32"}, maxEngineConcurrencyArgs(c.Args)) }, }, { @@ -229,6 +245,7 @@ func TestNewLLMRouterClientContainer(t *testing.T) { for _, arg := range c.Args { assert.NotContains(t, arg, "--model-name=") } + assert.Empty(t, maxEngineConcurrencyArgs(c.Args)) }, }, { diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/types.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/types.go index 19f61d27c0..095093e60f 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/types.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/types.go @@ -79,6 +79,8 @@ type LaunchSpecification struct { ICMSEnvironment string `json:"icmsEnvironment"` CloudProvider string `json:"cloudProvider"` GPUName string `json:"gpuName"` + // MaxRequestConcurrency is the per-instance limit from the NVCF deployment GPU specification. + MaxRequestConcurrency int `json:"maxRequestConcurrency,omitempty"` // Helm chart function components of the launch spec. *common.HelmChartLaunchSpecification `json:",inline"` diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml index f0e004f89b..60f3fe8746 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml @@ -187,6 +187,7 @@ spec: - --auth-token-file=/var/run/llm/worker-token - --backend-connectivity=reverse - --initial-input-tps=100 + - --max-engine-concurrency=37 - --upstream-health-path=/v1/health/ready - --quic-insecure - --model-name=model-gamma diff --git a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/message.json b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/message.json index 4824d2f44e..58833bd1f3 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/container/llm/message.json +++ b/src/libraries/go/lib/testdata/icms-translate/function/container/llm/message.json @@ -24,6 +24,7 @@ "cloudProvider": "DGXCLOUD", "deploymentId": "5a3d4a7e-9ee3-4762-8d37-d3b40a6f84c6", "gpuSpecificationId": "2c948d9b-db5d-4f93-8c29-f5d8a5d89cb9", + "maxRequestConcurrency": 37, "models": "W3sibmFtZSI6Im1vZGVsLWJldGEiLCJ2ZXJzaW9uIjoiMS4wLjAiLCJ1cmkiOiJodHRwczovL2FwaS5uZ2MubnZpZGlhLmNvbS92Mi9vcmcvbXktb3JnL3RlYW0vbXktdGVhbS9tb2RlbHMvbW9kZWwtYmV0YS8xLjAuMC9maWxlcyJ9LHsibmFtZSI6Im1vZGVsLWdhbW1hIiwidmVyc2lvbiI6IjEuMC4wIiwidXJpIjoiaHR0cHM6Ly9hcGkubmdjLm52aWRpYS5jb20vdjIvb3JnL215LW9yZy90ZWFtL215LXRlYW0vbW9kZWxzL21vZGVsLWdhbW1hLzEuMC4wL2ZpbGVzIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS9jaGF0L2NvbXBsZXRpb25zIiwiaHR0cDovLy92MS9yZXNwb25zZXMiXSwidG9rZW5pemVyIjoibGxhbWEzIiwidG9rZW5SYXRlTGltaXQiOiIxMDAwIiwicm91dGluZ01ldGhvZCI6ImZvbyJ9fSx7Im5hbWUiOiJtb2RlbC1lcHNpbG9uIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS8vY2hhdC9jb21wbGV0aW9ucyJdLCJ0b2tlbml6ZXIiOiJsbGFtYTMiLCJ0b2tlblJhdGVMaW1pdCI6IjIwMDAiLCJyb3V0aW5nTWV0aG9kIjoiZm9vIn19XQo=" } } diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml index dfe1569af9..7c8c8b0030 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/exp.yaml @@ -150,6 +150,7 @@ spec: - --auth-token-file=/var/run/llm/worker-token - --backend-connectivity=reverse - --initial-input-tps=100 + - --max-engine-concurrency=37 - --upstream-health-path=/v2/health/ready - --quic-insecure - --model-name=model-gamma diff --git a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/message.json b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/message.json index 85def0830a..8a5e5de602 100644 --- a/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/message.json +++ b/src/libraries/go/lib/testdata/icms-translate/function/helmchart/llm/message.json @@ -26,6 +26,7 @@ "configuration": "eyJmb28iOnsiYmFyIjoiYmF6In19", "deploymentId": "5a3d4a7e-9ee3-4762-8d37-d3b40a6f84c6", "gpuSpecificationId": "2c948d9b-db5d-4f93-8c29-f5d8a5d89cb9", + "maxRequestConcurrency": 37, "models": "W3sibmFtZSI6Im1vZGVsLWJldGEiLCJ2ZXJzaW9uIjoiMS4wLjAiLCJ1cmkiOiJodHRwczovL2FwaS5uZ2MubnZpZGlhLmNvbS92Mi9vcmcvbXktb3JnL3RlYW0vbXktdGVhbS9tb2RlbHMvbW9kZWwtYmV0YS8xLjAuMC9maWxlcyJ9LHsibmFtZSI6Im1vZGVsLWdhbW1hIiwidmVyc2lvbiI6IjEuMC4wIiwidXJpIjoiaHR0cHM6Ly9hcGkubmdjLm52aWRpYS5jb20vdjIvb3JnL215LW9yZy90ZWFtL215LXRlYW0vbW9kZWxzL21vZGVsLWdhbW1hLzEuMC4wL2ZpbGVzIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS9jaGF0L2NvbXBsZXRpb25zIiwiaHR0cDovLy92MS9yZXNwb25zZXMiXSwidG9rZW5pemVyIjoibGxhbWEzIiwidG9rZW5SYXRlTGltaXQiOiIxMDAwIiwicm91dGluZ01ldGhvZCI6ImZvbyJ9fSx7Im5hbWUiOiJtb2RlbC1lcHNpbG9uIiwibGxtQ29uZmlnIjp7InVyaXMiOlsiaHR0cDovLy92MS8vY2hhdC9jb21wbGV0aW9ucyJdLCJ0b2tlbml6ZXIiOiJsbGFtYTMiLCJ0b2tlblJhdGVMaW1pdCI6IjIwMDAiLCJyb3V0aW5nTWV0aG9kIjoiZm9vIn19XQo=" } }