diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml new file mode 100644 index 000000000000..aa07fe72defb --- /dev/null +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -0,0 +1,437 @@ +# Copyright 2026 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: OnlineEvaluator +description: | + An OnlineEvaluator contains the configuration for an Online Evaluation. It + periodically samples traces or sessions emitted by an agent and evaluates them + using the configured metric sources. +references: + guides: + 'Official Documentation': 'https://docs.cloud.google.com/gemini-enterprise-agent-platform/optimize/evaluation/evaluate-online' + api: 'https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/rest#rest-resource:-v1beta1.projects.locations.onlineevaluators' +docs: +base_url: projects/{{project}}/locations/{{region}}/onlineEvaluators +self_link: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}} +create_url: projects/{{project}}/locations/{{region}}/onlineEvaluators +update_verb: PATCH +update_mask: true +generate_list_resource: true +id_format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}} +import_format: + - projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}} +timeouts: + insert_minutes: 20 + update_minutes: 20 + delete_minutes: 60 +async: + type: OpAsync + operation: + base_url: '{{op_id}}' + actions: + - create + - update + - delete + result: + resource_inside_response: true +custom_code: +samples: + - name: vertex_ai_online_evaluator_basic + primary_resource_id: evaluator + steps: + - name: vertex_ai_online_evaluator_basic + resource_id_vars: + evaluator_name: online-evaluator + engine_name: reasoning-engine + test_env_vars: + project: PROJECT_NAME + - name: vertex_ai_online_evaluator_full + primary_resource_id: evaluator + steps: + - name: vertex_ai_online_evaluator_full + resource_id_vars: + evaluator_name: online-evaluator + engine_name: reasoning-engine + metric_name: eval-metric + test_env_vars: + project: PROJECT_NAME +parameters: + - name: region + type: String + description: The region of the OnlineEvaluator (e.g. us-central1). + url_param_only: true + required: true + immutable: true +properties: + - name: name + type: String + description: | + The resource name of the OnlineEvaluator. This is the final component of + the resource path. + Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}. + output: true + custom_flatten: templates/terraform/custom_flatten/name_from_self_link.tmpl + - name: displayName + type: String + description: | + Human-readable name for the OnlineEvaluator. The name doesn't have to be + unique. The name can consist of any UTF-8 characters. The maximum length + is 63 characters. + - name: agentResource + type: String + required: true + immutable: true + description: | + The name of the agent that the OnlineEvaluator evaluates periodically. + This value is used to filter the traces with a matching cloud.resource_id + and link the evaluation results with relevant dashboards/UIs. This field + is immutable; once set, it cannot be changed. + - name: config + type: NestedObject + required: true + description: | + Configuration for sampling behavior of the OnlineEvaluator. The + OnlineEvaluator runs at a fixed interval of 10 minutes. + properties: + - name: maxEvaluatedSamplesPerRun + type: String + description: | + The maximum number of evaluations to perform per run. If set to 0, the + number is unbounded. + - name: randomSampling + type: NestedObject + description: | + Configuration for random sampling. This is the sampling method used to + select traces for evaluation. + properties: + - name: percentage + type: Integer + required: true + description: | + The percentage of traces to sample for evaluation. Must be an + integer between 1 and 100. + - name: cloudObservability + type: NestedObject + required: true + description: | + Data source for the OnlineEvaluator, based on the Google Cloud + Observability stack (Cloud Trace & Cloud Logging). The data source type is + immutable once set. Within `cloud_observability`, `log_view` and + `trace_view` can be updated; the eval scope (`trace_scope` / + `session_scope`) and the convention (`open_telemetry`) are immutable. + properties: + - name: logView + type: String + description: | + Optional log view that will be used to query logs. If empty, the + project's default view (`projects/{{project_id}}`) will be used. + - name: traceView + type: String + description: | + Optional trace view that will be used to query traces. If empty, the + `_AllSpans` view from the `_Trace` US bucket will be used. + - name: openTelemetry + type: NestedObject + required: true + immutable: true + ignore_read: true + description: | + Configuration for a data source following the OpenTelemetry + convention. This is the convention that the data source follows. + properties: + - name: semconvVersion + type: String + required: true + immutable: true + description: | + Defines which version of the OTel Semantic Convention the data + follows. Can be "1.39.0" or newer. + - name: traceScope + type: NestedObject + immutable: true + description: | + If chosen, the OnlineEvaluator will evaluate single traces matching + the specified filter. Exactly one of `trace_scope` or `session_scope` + must be set. + properties: + - name: filter + type: Array + immutable: true + description: | + A list of predicates to filter traces. Multiple predicates are + combined using AND. The maximum number of predicates is 10. + item_type: + type: NestedObject + properties: + - name: duration + type: NestedObject + immutable: true + description: Filter on the duration of a trace (in seconds). + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: totalTokenUsage + type: NestedObject + immutable: true + description: Filter on the total token usage within a trace. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: sessionScope + type: NestedObject + immutable: true + description: | + If chosen, the OnlineEvaluator will evaluate sessions matching the + specified filter. A session is a group of traces with a common + `gen_ai.conversation.id`. Exactly one of `trace_scope` or + `session_scope` must be set. + properties: + - name: inactivityTrigger + type: NestedObject + immutable: true + description: | + Session is considered ready for evaluation when there are no new + traces for a specified period of inactivity. + properties: + - name: threshold + type: String + required: true + immutable: true + description: | + The amount of time that must pass with no new traces before a + session is considered ready for evaluation. Must be a positive + duration no greater than 7 days (604800 seconds), e.g. "3600s". + - name: filter + type: Array + immutable: true + description: | + A list of predicates to filter sessions. Multiple predicates are + combined using AND. The maximum number of predicates is 10. + item_type: + type: NestedObject + properties: + - name: totalTokenUsage + type: NestedObject + immutable: true + description: Filter on the total token usage within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: userTurns + type: NestedObject + immutable: true + description: Filter on the number of user turns within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: modelCalls + type: NestedObject + immutable: true + description: Filter on the number of underlying LLM calls within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: modelCallErrors + type: NestedObject + immutable: true + description: Filter on the number of LLM call errors within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: toolCalls + type: NestedObject + immutable: true + description: Filter on the number of underlying tool calls within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: toolCallErrors + type: NestedObject + immutable: true + description: Filter on the number of tool call errors within a session. + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: duration + type: NestedObject + immutable: true + description: Filter on the duration of a session (in seconds). + properties: + - name: comparisonOperator + type: String + required: true + immutable: true + description: | + The comparison operator to apply. Possible values are: + LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, + GREATER. + - name: value + type: Double + required: true + immutable: true + description: The value to compare against. + - name: metricSources + type: Array + required: true + immutable: true + description: | + A list of metric sources to be used for evaluating samples. At least one + metric source must be provided. Only predefined metrics (inline `metric`) + and registered metrics (`metric_resource_name`) are supported. Every + registered metric must have a `display_name` (or `title`) and a + `score_range` defined, otherwise the evaluations will fail. The maximum + number of metric sources is 25. Exactly one of `metric` or + `metric_resource_name` must be set per metric source. + item_type: + type: NestedObject + properties: + - name: metric + type: String + immutable: true + description: | + Inline metric config, provided as a JSON-formatted string using + camelCase field names to match the API format (see the + [Metric API documentation](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/Metric)). + Suggested predefined `metricSpecName` values: + `final_response_quality_v1`, `tool_use_quality_v1`, + `hallucination_v1`, `safety_v1`, `multi_turn_task_success_v1`, + `multi_turn_tool_use_quality_v1`, `multi_turn_trajectory_quality_v1`. + state_func: 'func(v interface{}) string { s, _ := structure.NormalizeJsonString(v); return s }' + custom_flatten: templates/terraform/custom_flatten/json_schema.tmpl + custom_expand: templates/terraform/custom_expand/json_schema.tmpl + validation: + function: validation.StringIsJSON + - name: metricResourceName + type: String + immutable: true + description: | + Resource name of a registered metric (a + `google_vertex_ai_evaluation_metric`), e.g. + projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}. + - name: state + type: String + output: true + description: | + The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, + FAILED, WARNING. + - name: stateDetails + type: Array + output: true + description: | + Contains additional information about the state of the OnlineEvaluator. + This is used to provide more details in the event of a failure. + item_type: + type: NestedObject + properties: + - name: message + type: String + output: true + description: | + Human-readable message describing the state of the OnlineEvaluator. + - name: createTime + type: String + output: true + description: Timestamp when the OnlineEvaluator was created. + - name: updateTime + type: String + output: true + description: Timestamp when the OnlineEvaluator was last updated. diff --git a/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl new file mode 100644 index 000000000000..2dcc44713979 --- /dev/null +++ b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl @@ -0,0 +1,49 @@ +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "{{index $.ResourceIdVars "engine_name"}}" + description = "Reasoning engine evaluated by the online evaluator" + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "{{$.PrimaryResourceId}}" { + region = "us-central1" + display_name = "{{index $.ResourceIdVars "evaluator_name"}}" + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "100" + random_sampling { + percentage = 10 + } + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + } + } + + metric_sources { + metric = jsonencode({ + predefinedMetricSpec = { + metricSpecName = "safety_v1" + } + }) + } + + metric_sources { + metric = jsonencode({ + predefinedMetricSpec = { + metricSpecName = "final_response_quality_v1" + } + }) + } +} diff --git a/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_full.tf.tmpl b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_full.tf.tmpl new file mode 100644 index 000000000000..fe365c041b8c --- /dev/null +++ b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_full.tf.tmpl @@ -0,0 +1,60 @@ +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "{{index $.ResourceIdVars "engine_name"}}" + description = "Reasoning engine evaluated by the online evaluator" + region = "us-central1" +} + +resource "google_vertex_ai_evaluation_metric" "metric" { + evaluation_metric_id = "{{index $.ResourceIdVars "metric_name"}}" + region = "us-central1" + display_name = "Custom response quality" + + metric = jsonencode({ + llmBasedMetricSpec = { + metricPromptTemplate = "Rate the quality of the following response on a scale of 1 to 5. Response: {response}" + } + metadata = { + title = "custom-response-quality" + scoreRange = { + min = 1 + max = 5 + } + } + }) +} + +resource "google_vertex_ai_online_evaluator" "{{$.PrimaryResourceId}}" { + region = "us-central1" + display_name = "{{index $.ResourceIdVars "evaluator_name"}}" + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + random_sampling { + percentage = 100 + } + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + session_scope { + inactivity_trigger { + threshold = "3600s" + } + + filter { + total_token_usage { + comparison_operator = "GREATER" + value = 0 + } + } + } + } + + # Registered metric, referencing a separate google_vertex_ai_evaluation_metric. + metric_sources { + metric_resource_name = google_vertex_ai_evaluation_metric.metric.id + } +} diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go new file mode 100644 index 000000000000..f2ccbc67abd0 --- /dev/null +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go @@ -0,0 +1,101 @@ +package vertexai_test + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" +) + +// Exercises the mutable fields of the OnlineEvaluator (display_name and the +// config sampling settings), which the generated example tests do not cover. +func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { + t.Parallel() + + suffix := acctest.RandString(t, 10) + + first := map[string]interface{}{ + "random_suffix": suffix, + "display_name": "tf-test-online-evaluator" + suffix, + "max_samples": "100", + "percentage": 10, + } + second := map[string]interface{}{ + "random_suffix": suffix, + "display_name": "tf-test-online-evaluator-updated" + suffix, + "max_samples": "200", + "percentage": 25, + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccVertexAIOnlineEvaluator_update(first), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"cloud_observability.0.open_telemetry", "region"}, + }, + { + Config: testAccVertexAIOnlineEvaluator_update(second), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"cloud_observability.0.open_telemetry", "region"}, + }, + }, + }) +} + +func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "tf-test-reasoning-engine%{random_suffix}" + description = "Reasoning engine evaluated by the online evaluator" + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + region = "us-central1" + display_name = "%{display_name}" + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "%{max_samples}" + random_sampling { + percentage = %{percentage} + } + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + } + } + + metric_sources { + metric = jsonencode({ + predefinedMetricSpec = { + metricSpecName = "safety_v1" + } + }) + } +} +`, context) +} diff --git a/tools/issue-labeler/labeler/enrolled_teams.yml b/tools/issue-labeler/labeler/enrolled_teams.yml index 67328fd3470a..79f71eeba964 100755 --- a/tools/issue-labeler/labeler/enrolled_teams.yml +++ b/tools/issue-labeler/labeler/enrolled_teams.yml @@ -28,6 +28,7 @@ service/aiplatform-dataset: service/aiplatform-evaluation: resources: - google_vertex_ai_evaluation_.* + - google_vertex_ai_online_evaluator service/aiplatform-featurestore: resources: - google_vertex_ai_featurestore.*