From 4efa86f23ad2a8716b6a3ab5421b407841931997 Mon Sep 17 00:00:00 2001 From: Tejas Sovani Date: Thu, 10 Sep 2026 09:38:55 +0000 Subject: [PATCH 1/3] [Dataproc] Add instance_flexibility_policy to worker_config in google_dataproc_workflow_template --- ...esource_dataproc_workflow_template.go.tmpl | 265 +++++++++- ..._dataproc_workflow_template_meta.yaml.tmpl | 8 + ...ce_dataproc_workflow_template_test.go.tmpl | 85 ++++ .../dataproc/workflow_template.go.tmpl | 158 +++++- .../workflow_template_internal.go.tmpl | 480 ++++++++++++++++++ .../dataproc_workflow_template.html.markdown | 111 +++- 6 files changed, 1084 insertions(+), 23 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl index 96e354b52f58..50b4a17b1205 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl @@ -2012,6 +2012,16 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigSchema() * Description: "Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.", Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfigSchema(), }, + + "instance_flexibility_policy": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Instance flexibility Policy allowing a mixture of VM shapes.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicySchema(), + }, }, } } @@ -2041,6 +2051,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig Schema: map[string]*schema.Schema{ "boot_disk_size_gb": { Type: schema.TypeInt, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Size in GB of the boot disk (default is 500GB).", @@ -2048,6 +2059,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig "boot_disk_type": { Type: schema.TypeString, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Type of the boot disk (default is \"pd-standard\"). Valid values: \"pd-balanced\" (Persistent Disk Balanced Solid State Drive), \"pd-ssd\" (Persistent Disk Solid State Drive), or \"pd-standard\" (Persistent Disk Hard Disk Drive). See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types).", @@ -2082,6 +2094,88 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGro } } +func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicySchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_machine_types": { + Type: schema.TypeMap, + Computed: true, + Description: "Output only. A map of instance names to their machine type.", + Elem: &schema.Schema{Type: schema.TypeString}, + }, + + "instance_selection_list": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list", + }, + Description: "Optional. List of instance selection options that the group will use when creating new VMs.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSchema(), + }, + + "instance_selection_results": { + Type: schema.TypeList, + Computed: true, + Description: "Output only. A list of instance selection results that were successfully allocated.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema(), + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "disk_config": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Optional. Disk configuration to apply to the instances in this instance selection.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigSchema(), + }, + + "machine_types": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: "Optional. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "rank": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.", + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_type": { + Type: schema.TypeString, + Computed: true, + Description: "Output only. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "vm_count": { + Type: schema.TypeInt, + Computed: true, + Description: "Output only. Number of VM provisioned with the corresponding machine_type.", + }, + }, + } +} + func DataprocWorkflowTemplateEncryptionConfigSchema() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -4044,13 +4138,14 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(o i } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigWorkerConfig{ - Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigAcceleratorsArray(obj["accelerators"]), - DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(obj["disk_config"]), - Image: dcl.String(obj["image"].(string)), - MachineType: dcl.String(obj["machine_type"].(string)), - MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), - NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), - Preemptibility: WorkflowTemplatePlacementManagedClusterConfigWorkerConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigAcceleratorsArray(obj["accelerators"]), + DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(obj["disk_config"]), + Image: dcl.String(obj["image"].(string)), + MachineType: dcl.String(obj["machine_type"].(string)), + MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), + NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), + Preemptibility: WorkflowTemplatePlacementManagedClusterConfigWorkerConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + InstanceFlexibilityPolicy: expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(obj["instance_flexibility_policy"]), } } @@ -4059,16 +4154,17 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(ob return nil } transformed := map[string]interface{}{ - "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigAcceleratorsArray(obj.Accelerators), - "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(obj.DiskConfig), - "image": obj.Image, - "machine_type": obj.MachineType, - "min_cpu_platform": obj.MinCpuPlatform, - "num_instances": obj.NumInstances, - "preemptibility": obj.Preemptibility, - "instance_names": obj.InstanceNames, - "is_preemptible": obj.IsPreemptible, - "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig(obj.ManagedGroupConfig), + "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigAcceleratorsArray(obj.Accelerators), + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(obj.DiskConfig), + "image": obj.Image, + "machine_type": obj.MachineType, + "min_cpu_platform": obj.MinCpuPlatform, + "num_instances": obj.NumInstances, + "preemptibility": obj.Preemptibility, + "instance_names": obj.InstanceNames, + "is_preemptible": obj.IsPreemptible, + "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig(obj.ManagedGroupConfig), + "instance_flexibility_policy": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(obj.InstanceFlexibilityPolicy), } return []interface{}{transformed} @@ -4142,8 +4238,8 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDisk } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig{ - BootDiskSizeGb: dcl.Int64(int64(obj["boot_disk_size_gb"].(int))), - BootDiskType: dcl.String(obj["boot_disk_type"].(string)), + BootDiskSizeGb: dcl.Int64OrNil(int64(obj["boot_disk_size_gb"].(int))), + BootDiskType: dcl.StringOrNil(obj["boot_disk_type"].(string)), NumLocalSsds: dcl.Int64OrNil(int64(obj["num_local_ssds"].(int))), } } @@ -4175,6 +4271,137 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigMan } +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy { + if o == nil { + return nil + } + objArr := o.([]interface{}) + if len(objArr) == 0 || objArr[0] == nil { + return nil + } + obj := objArr[0].(map[string]interface{}) + isl := expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj["instance_selection_list"]) + if len(isl) == 0 { + return nil + } + return &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy{ + InstanceSelectionList: isl, + } +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(obj *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "instance_machine_types": obj.InstanceMachineTypes, + "instance_selection_list": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj.InstanceSelectionList), + "instance_selection_results": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(obj.InstanceSelectionResults), + } + + return []interface{}{transformed} +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(o interface{}) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + + objs := o.([]interface{}) + if len(objs) == 0 || objs[0] == nil { + return nil + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(objs)) + for _, item := range objs { + i := expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(item) + if i != nil { + items = append(items, *i) + } + } + + return items +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + obj := o.(map[string]interface{}) + m := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + if v, ok := obj["machine_types"].([]interface{}); ok { + for _, item := range v { + if s, ok := item.(string); ok && s != "" { + m.MachineTypes = append(m.MachineTypes, s) + } + } + } + if v, ok := obj["rank"].(int); ok { + m.Rank = dcl.Int64(int64(v)) + } + if v, ok := obj["disk_config"]; ok { + m.DiskConfig = expandDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(v) + } + return m +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(objs []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(obj *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(obj.DiskConfig), + "machine_types": obj.MachineTypes, + "rank": obj.Rank, + } + + return transformed +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(objs []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(obj *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "machine_type": obj.MachineType, + "vm_count": obj.VMCount, + } + + return transformed +} + func expandDataprocWorkflowTemplateEncryptionConfig(o interface{}) *WorkflowTemplateEncryptionConfig { if o == nil { return EmptyWorkflowTemplateEncryptionConfig diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl index d5f866670807..b50a0f8f0949 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl @@ -176,6 +176,14 @@ fields: - api_field: 'placement.managedCluster.config.workerConfig.instanceNames' - api_field: 'placement.managedCluster.config.workerConfig.isPreemptible' - field: 'placement.managed_cluster.config.worker_config.machine_type' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceMachineTypes' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskSizeGb' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskType' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.numLocalSsds' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionList.machineTypes' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionList.rank' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionResults.machineType' + - api_field: 'placement.managedCluster.config.workerConfig.instanceFlexibilityPolicy.instanceSelectionResults.vmCount' - api_field: 'placement.managedCluster.config.workerConfig.managedGroupConfig.instanceGroupManagerName' - api_field: 'placement.managedCluster.config.workerConfig.managedGroupConfig.instanceTemplateName' - api_field: 'placement.managedCluster.config.workerConfig.minCpuPlatform' diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl index 3f7f380c8807..33e134e1f3e8 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl @@ -294,6 +294,91 @@ resource "google_kms_crypto_key_iam_member" "crypto_key" { `, context) } +func TestAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + "project": envvar.GetTestProjectFromEnv(), + "version": "2.0.35-debian10", + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: funcAccTestDataprocWorkflowTemplateCheckDestroy(t), + ExternalProviders: map[string]resource.ExternalProvider{ + "random": {}, + }, + Steps: []resource.TestStep{ + { + Config: testAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(context), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "n2d-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "1"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_type", "pd-ssd"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_size_gb", "35"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.num_local_ssds", "1"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.machine_types.0", "e2-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.rank", "2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.disk_config.0.boot_disk_type", "pd-standard"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.disk_config.0.boot_disk_size_gb", "35"), + ), + }, + { + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"labels", "terraform_labels"}, + ResourceName: "google_dataproc_workflow_template.template", + }, + }, + }) +} + +func testAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_dataproc_workflow_template" "template" { + name = "tf-test-workflow-%{random_suffix}" + location = "us-central1" + placement { + managed_cluster { + cluster_name = "tf-test-cluster-%{random_suffix}" + config { + software_config { + image_version = "%{version}" + } + master_config { + num_instances = 1 + } + worker_config { + num_instances = 2 + instance_flexibility_policy { + instance_selection_list { + machine_types = ["n2d-standard-2"] + rank = 1 + disk_config { + boot_disk_type = "pd-ssd" + boot_disk_size_gb = 35 + num_local_ssds = 1 + } + } + instance_selection_list { + machine_types = ["e2-standard-2"] + rank = 2 + disk_config { + boot_disk_type = "pd-standard" + boot_disk_size_gb = 35 + } + } + } + } + } + } + } +} +`, context) +} + func funcAccTestDataprocWorkflowTemplateCheckDestroy(t *testing.T) func(s *terraform.State) error { return func(s *terraform.State) error { for name, rs := range s.RootModule().Resources { diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl index d15930a266a4..c53371895db0 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl @@ -895,7 +895,8 @@ type WorkflowTemplatePlacementManagedClusterConfigWorkerConfig struct { Preemptibility *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigPreemptibilityEnum `json:"preemptibility"` ManagedGroupConfig *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig `json:"managedGroupConfig"` Accelerators []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerators `json:"accelerators"` - MinCpuPlatform *string `json:"minCpuPlatform"` + MinCpuPlatform *string `json:"minCpuPlatform"` + InstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy `json:"instanceFlexibilityPolicy"` } type jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfig @@ -933,6 +934,8 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig) UnmarshalJSO r.MinCpuPlatform = res.MinCpuPlatform + r.InstanceFlexibilityPolicy = res.InstanceFlexibilityPolicy + } return nil } @@ -1107,6 +1110,159 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerators) return fmt.Sprintf("%x", hash) } +type WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy struct { + empty bool `json:"-"` + InstanceMachineTypes map[string]string `json:"instanceMachineTypes"` + InstanceSelectionList []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection `json:"instanceSelectionList"` + InstanceSelectionResults []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult `json:"instanceSelectionResults"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy + } else { + + r.InstanceMachineTypes = res.InstanceMachineTypes + + r.InstanceSelectionList = res.InstanceSelectionList + + r.InstanceSelectionResults = res.InstanceSelectionResults + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy = &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection struct { + empty bool `json:"-"` + MachineTypes []string `json:"machineTypes"` + Rank *int64 `json:"rank"` + DiskConfig *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig `json:"diskConfig"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection + } else { + + r.MachineTypes = res.MachineTypes + + r.Rank = res.Rank + + r.DiskConfig = res.DiskConfig + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection = &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult struct { + empty bool `json:"-"` + MachineType *string `json:"machineType"` + VMCount *int64 `json:"vmCount"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + } else { + + r.MachineType = res.MachineType + + r.VMCount = res.VMCount + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult = &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig struct { empty bool `json:"-"` NumInstances *int64 `json:"numInstances"` diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl index 661a68dd3e34..f7f3ea92c337 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl @@ -198,6 +198,11 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig) validate() e return err } } + if !dcl.IsEmptyValueIndirect(r.InstanceFlexibilityPolicy) { + if err := r.InstanceFlexibilityPolicy.validate(); err != nil { + return err + } + } return nil } func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig) validate() error { @@ -209,6 +214,15 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupCo func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerators) validate() error { return nil } +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) validate() error { + return nil +} func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig) validate() error { if !dcl.IsEmptyValueIndirect(r.DiskConfig) { if err := r.DiskConfig.validate(); err != nil { @@ -2514,6 +2528,7 @@ func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcc return items } + func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(des, initial *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig { if des == nil { return initial @@ -2558,6 +2573,7 @@ func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(des, } else { cDes.MinCpuPlatform = des.MinCpuPlatform } + cDes.InstanceFlexibilityPolicy = canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(des.InstanceFlexibilityPolicy, initial.InstanceFlexibilityPolicy, opts...) return cDes } @@ -2619,10 +2635,148 @@ func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(c if dcl.StringCanonicalize(des.MinCpuPlatform, nw.MinCpuPlatform) { nw.MinCpuPlatform = des.MinCpuPlatform } + nw.InstanceFlexibilityPolicy = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c, des.InstanceFlexibilityPolicy, nw.InstanceFlexibilityPolicy) + + return nw +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(des, initial *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy{} + + cDes.InstanceSelectionList = canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des.InstanceSelectionList, initial.InstanceSelectionList, opts...) + + return cDes +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + nw.InstanceSelectionList = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, des.InstanceSelectionList, nw.InstanceSelectionList) return nw } +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(des, initial *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.StringArrayCanonicalize(des.MachineTypes, initial.MachineTypes) { + cDes.MachineTypes = initial.MachineTypes + } else { + cDes.MachineTypes = des.MachineTypes + } + if dcl.IsZeroValue(des.Rank) || (dcl.IsEmptyValueIndirect(des.Rank) && dcl.IsEmptyValueIndirect(initial.Rank)) { + cDes.Rank = initial.Rank + } else { + cDes.Rank = des.Rank + } + cDes.DiskConfig = canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(des.DiskConfig, initial.DiskConfig, opts...) + + return cDes +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des, initial []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if dcl.IsEmptyValueIndirect(des) { + return initial + } + + if len(des) != len(initial) { + items := make([]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for _, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&d, nil, opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for i, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&d, &initial[i], opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + if dcl.StringArrayCanonicalize(des.MachineTypes, nw.MachineTypes) { + nw.MachineTypes = des.MachineTypes + } + nw.DiskConfig = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(c, des.DiskConfig, nw.DiskConfig) + + return nw +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, des, nw []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + var items []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection + for i, d := range des { + n := nw[i] + items = append(items, *canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, &d, &n)) + } + + return items +} + + + func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigWorkerConfigSet(c *DclClient, des, nw []WorkflowTemplatePlacementManagedClusterConfigWorkerConfig) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfig { if des == nil { return nw @@ -9248,9 +9402,145 @@ func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigNewStyle(d, } diffs = append(diffs, ds...) } + + if ds, err := dcl.Diff(desired.InstanceFlexibilityPolicy, actual.InstanceFlexibilityPolicy, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceFlexibilityPolicy")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } return diffs, nil } +func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy or *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.InstanceMachineTypes, actual.InstanceMachineTypes, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceMachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionList, actual.InstanceSelectionList, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionList")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionResults, actual.InstanceSelectionResults, dcl.DiffInfo{OutputOnly: true, ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionResults")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection or *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineTypes, actual.MachineTypes, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.Rank, actual.Rank, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("Rank")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.DiskConfig, actual.DiskConfig, dcl.DiffInfo{ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("DiskConfig")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult or *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineType, actual.MachineType, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineType")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.VMCount, actual.VMCount, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("VmCount")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + + + func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { var diffs []*dcl.FieldDiff @@ -13074,6 +13364,11 @@ func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(c *DclClien if v := f.MinCpuPlatform; !dcl.IsEmptyValueIndirect(v) { m["minCpuPlatform"] = v } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c, f.InstanceFlexibilityPolicy, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceFlexibilityPolicy into instanceFlexibilityPolicy: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["instanceFlexibilityPolicy"] = v + } return m, nil } @@ -13101,10 +13396,195 @@ func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(c *DclClie r.ManagedGroupConfig = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig(c, m["managedGroupConfig"], res) r.Accelerators = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigAcceleratorsSlice(c, m["accelerators"], res) r.MinCpuPlatform = dcl.FlattenString(m["minCpuPlatform"]) + r.InstanceFlexibilityPolicy = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c, m["instanceFlexibilityPolicy"], res) + + return r +} + +// expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy expands an instance of WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy into a JSON request object. +func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, f.InstanceSelectionList, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionList into instanceSelectionList: %w", err) + } else if v != nil { + m["instanceSelectionList"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, f.InstanceSelectionResults, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionResults into instanceSelectionResults: %w", err) + } else if v != nil { + m["instanceSelectionResults"] = v + } + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicy + } + r.InstanceMachineTypes = dcl.FlattenKeyValuePairs(m["instanceMachineTypes"]) + r.InstanceSelectionList = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, m["instanceSelectionList"], res) + r.InstanceSelectionResults = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, m["instanceSelectionResults"], res) + + return r +} + +func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, &item, res) + if err != nil { + return nil, err + } + + items = append(items, i) + } + + return items, nil +} + +func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineTypes; !dcl.IsEmptyValueIndirect(v) { + m["machineTypes"] = v + } + if v := f.Rank; v != nil { + m["rank"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(c, f.DiskConfig, res); err != nil { + return nil, fmt.Errorf("error expanding DiskConfig into diskConfig: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["diskConfig"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, item.(map[string]interface{}), res)) + } + + return items +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelection + } + r.MachineTypes = dcl.FlattenStringSlice(m["machineTypes"]) + r.Rank = dcl.FlattenInteger(m["rank"]) + r.DiskConfig = flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig(c, m["diskConfig"], res) return r } +func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, &item, res) + if err != nil { + return nil, err + } + + items = append(items, i) + } + + return items, nil +} + +func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineType; !dcl.IsEmptyValueIndirect(v) { + m["machineType"] = v + } + if v := f.VMCount; !dcl.IsEmptyValueIndirect(v) { + m["vmCount"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, item.(map[string]interface{}), res)) + } + + return items +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + } + r.MachineType = dcl.FlattenString(m["machineType"]) + r.VMCount = dcl.FlattenInteger(m["vmCount"]) + + return r +} + + + // expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigMap expands the contents of WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig into a JSON // request object. func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigMap(c *DclClient, f map[string]WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig, res *WorkflowTemplate) (map[string]interface{}, error) { diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown index 42b1eaa72069..b33e6e14c495 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown @@ -136,6 +136,38 @@ resource "google_dataproc_workflow_template" "example" { } ``` +## Example Usage - Dataproc Workflow Template Instance Flexibility Policy + +```hcl +resource "google_dataproc_workflow_template" "template" { + name = "template-flexible-vms" + location = "us-central1" + placement { + managed_cluster { + cluster_name = "my-flexible-cluster" + config { + software_config { + image_version = "2.0.35-debian10" + } + master_config { + num_instances = 1 + machine_type = "e2-standard-2" + } + worker_config { + num_instances = 2 + instance_flexibility_policy { + instance_selection_list { + machine_types = ["e2-standard-2"] + rank = 1 + } + } + } + } + } + } +} +``` + ## Argument Reference The following arguments are supported: @@ -300,7 +332,7 @@ The `config` block supports: * `worker_config` - (Optional) - The Compute Engine config settings for additional worker instances in a cluster. + The Compute Engine config settings for additional worker instances in a cluster. Structure is [documented below](#nested_worker_config). - - - @@ -745,7 +777,46 @@ The `master_config` block supports: * `managed_group_config` - Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups. -The `accelerators` block supports: +The `worker_config` block supports: + +* `accelerators` - + (Optional) + The Compute Engine accelerator configuration for these instances. Structure is [documented below](#nested_accelerators). + +* `disk_config` - + (Optional) + Disk option config settings. Structure is [documented below](#nested_disk_config). + +* `image` - + (Optional) + The Compute Engine image resource used for cluster instances. + +* `machine_type` - + (Optional) + The Compute Engine machine type used for cluster instances. + +* `min_cpu_platform` - + (Optional) + The minimum CPU platform and architecture for the cluster. + +* `num_instances` - + (Optional) + The number of worker instances in the cluster. + +* `instance_names` - + Output only. The list of instance names. + +* `is_preemptible` - + Output only. Specifies that this instance group contains preemptible instances. + +* `managed_group_config` - + Output only. The config for Compute Engine Instance Group Manager that manages this group. + +* `instance_flexibility_policy` - + (Optional) + Instance flexibility Policy allowing a mixture of VM shapes. Structure is [documented below](#nested_instance_flexibility_policy). + +The `accelerators` block supports: * `accelerator_count` - (Optional) @@ -755,7 +826,7 @@ The `accelerators` block supports: (Optional) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://docs.cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, `nvidia-tesla-k80`. -The `disk_config` block supports: +The `disk_config` block supports: * `boot_disk_size_gb` - (Optional) @@ -769,6 +840,40 @@ The `disk_config` block supports: (Optional) Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries. +The `instance_flexibility_policy` block supports: + +* `instance_selection_list` - + (Optional) + List of instance selection options that the group will use when creating new VMs. Structure is [documented below](#nested_instance_selection_list). + +* `instance_selection_results` - + Output only. A list of instance selection results that were successfully allocated. Structure is [documented below](#nested_instance_selection_results). + +* `instance_machine_types` - + Output only. A map of instance names to their machine types. + +The `instance_selection_list` block supports: + +* `machine_types` - + (Optional) + Full machine-type names, e.g. `n1-standard-16`. + +* `rank` - + (Optional) + Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference. + +* `disk_config` - + (Optional) + Disk option for the instance group. Structure is [documented above](#nested_disk_config). + +The `instance_selection_results` block supports: + +* `machine_type` - + Output only. Full machine-type names, e.g. `n1-standard-16`. + +* `vm_count` - + Output only. Number of VM provisioned with the corresponding machine_type. + The `autoscaling_config` block supports: * `policy` - From 3d812be40369ff744310c85d288b6a3fde4994e2 Mon Sep 17 00:00:00 2001 From: Tejas Sovani Date: Thu, 10 Sep 2026 09:38:58 +0000 Subject: [PATCH 2/3] [Dataproc] Add instance_flexibility_policy to master_config in google_dataproc_workflow_template --- ...esource_dataproc_workflow_template.go.tmpl | 265 +++++++++- ..._dataproc_workflow_template_meta.yaml.tmpl | 8 + ...ce_dataproc_workflow_template_test.go.tmpl | 16 + .../dataproc/workflow_template.go.tmpl | 156 ++++++ .../workflow_template_internal.go.tmpl | 491 ++++++++++++++++++ .../dataproc_workflow_template.html.markdown | 4 + 6 files changed, 921 insertions(+), 19 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl index 50b4a17b1205..a77b8adf3776 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl @@ -1544,6 +1544,16 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigSchema() * Description: "Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.", Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfigSchema(), }, + + "instance_flexibility_policy": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Instance flexibility Policy allowing a mixture of VM shapes.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicySchema(), + }, }, } } @@ -1573,6 +1583,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig Schema: map[string]*schema.Schema{ "boot_disk_size_gb": { Type: schema.TypeInt, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Size in GB of the boot disk (default is 500GB).", @@ -1580,6 +1591,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig "boot_disk_type": { Type: schema.TypeString, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Type of the boot disk (default is \"pd-standard\"). Valid values: \"pd-balanced\" (Persistent Disk Balanced Solid State Drive), \"pd-ssd\" (Persistent Disk Solid State Drive), or \"pd-standard\" (Persistent Disk Hard Disk Drive). See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types).", @@ -1596,6 +1608,88 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig } } +func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicySchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_machine_types": { + Type: schema.TypeMap, + Computed: true, + Description: "Output only. A map of instance names to their machine type.", + Elem: &schema.Schema{Type: schema.TypeString}, + }, + + "instance_selection_list": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list", + }, + Description: "Optional. List of instance selection options that the group will use when creating new VMs.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSchema(), + }, + + "instance_selection_results": { + Type: schema.TypeList, + Computed: true, + Description: "Output only. A list of instance selection results that were successfully allocated.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema(), + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "disk_config": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Optional. Disk configuration to apply to the instances in this instance selection.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigSchema(), + }, + + "machine_types": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: "Optional. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "rank": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.", + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_type": { + Type: schema.TypeString, + Computed: true, + Description: "Output only. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "vm_count": { + Type: schema.TypeInt, + Computed: true, + Description: "Output only. Number of VM provisioned with the corresponding machine_type.", + }, + }, + } +} + func DataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfigSchema() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -3718,13 +3812,14 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfig(o i } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigMasterConfig{ - Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcceleratorsArray(obj["accelerators"]), - DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(obj["disk_config"]), - Image: dcl.String(obj["image"].(string)), - MachineType: dcl.String(obj["machine_type"].(string)), - MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), - NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), - Preemptibility: WorkflowTemplatePlacementManagedClusterConfigMasterConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcceleratorsArray(obj["accelerators"]), + DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(obj["disk_config"]), + Image: dcl.String(obj["image"].(string)), + MachineType: dcl.String(obj["machine_type"].(string)), + MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), + NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), + Preemptibility: WorkflowTemplatePlacementManagedClusterConfigMasterConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + InstanceFlexibilityPolicy: expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(obj["instance_flexibility_policy"]), } } @@ -3733,16 +3828,17 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfig(ob return nil } transformed := map[string]interface{}{ - "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcceleratorsArray(obj.Accelerators), - "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(obj.DiskConfig), - "image": obj.Image, - "machine_type": obj.MachineType, - "min_cpu_platform": obj.MinCpuPlatform, - "num_instances": obj.NumInstances, - "preemptibility": obj.Preemptibility, - "instance_names": obj.InstanceNames, - "is_preemptible": obj.IsPreemptible, - "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig(obj.ManagedGroupConfig), + "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcceleratorsArray(obj.Accelerators), + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(obj.DiskConfig), + "image": obj.Image, + "machine_type": obj.MachineType, + "min_cpu_platform": obj.MinCpuPlatform, + "num_instances": obj.NumInstances, + "preemptibility": obj.Preemptibility, + "instance_names": obj.InstanceNames, + "is_preemptible": obj.IsPreemptible, + "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig(obj.ManagedGroupConfig), + "instance_flexibility_policy": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(obj.InstanceFlexibilityPolicy), } return []interface{}{transformed} @@ -3816,8 +3912,8 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDisk } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig{ - BootDiskSizeGb: dcl.Int64(int64(obj["boot_disk_size_gb"].(int))), - BootDiskType: dcl.String(obj["boot_disk_type"].(string)), + BootDiskSizeGb: dcl.Int64OrNil(int64(obj["boot_disk_size_gb"].(int))), + BootDiskType: dcl.StringOrNil(obj["boot_disk_type"].(string)), NumLocalSsds: dcl.Int64OrNil(int64(obj["num_local_ssds"].(int))), } } @@ -3836,6 +3932,137 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDis } +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy { + if o == nil { + return nil + } + objArr := o.([]interface{}) + if len(objArr) == 0 || objArr[0] == nil { + return nil + } + obj := objArr[0].(map[string]interface{}) + isl := expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj["instance_selection_list"]) + if len(isl) == 0 { + return nil + } + return &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy{ + InstanceSelectionList: isl, + } +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(obj *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "instance_machine_types": obj.InstanceMachineTypes, + "instance_selection_list": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj.InstanceSelectionList), + "instance_selection_results": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(obj.InstanceSelectionResults), + } + + return []interface{}{transformed} + +} +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionArray(o interface{}) []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + + objs := o.([]interface{}) + if len(objs) == 0 || objs[0] == nil { + return nil + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(objs)) + for _, item := range objs { + i := expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(item) + if i != nil { + items = append(items, *i) + } + } + + return items +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + obj := o.(map[string]interface{}) + m := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{} + if v, ok := obj["machine_types"].([]interface{}); ok { + for _, item := range v { + if s, ok := item.(string); ok && s != "" { + m.MachineTypes = append(m.MachineTypes, s) + } + } + } + if v, ok := obj["rank"].(int); ok { + m.Rank = dcl.Int64(int64(v)) + } + if v, ok := obj["disk_config"]; ok { + m.DiskConfig = expandDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(v) + } + return m +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionArray(objs []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(obj *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(obj.DiskConfig), + "machine_types": obj.MachineTypes, + "rank": obj.Rank, + } + + return transformed +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(objs []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(obj *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "machine_type": obj.MachineType, + "vm_count": obj.VMCount, + } + + return transformed +} + func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig(obj *WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig) interface{} { if obj == nil || obj.Empty() { return nil diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl index b50a0f8f0949..913f26fc47eb 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl @@ -125,6 +125,14 @@ fields: - api_field: 'placement.managedCluster.config.masterConfig.instanceNames' - api_field: 'placement.managedCluster.config.masterConfig.isPreemptible' - field: 'placement.managed_cluster.config.master_config.machine_type' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceMachineTypes' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskSizeGb' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskType' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.numLocalSsds' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionList.machineTypes' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionList.rank' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionResults.machineType' + - api_field: 'placement.managedCluster.config.masterConfig.instanceFlexibilityPolicy.instanceSelectionResults.vmCount' - api_field: 'placement.managedCluster.config.masterConfig.managedGroupConfig.instanceGroupManagerName' - api_field: 'placement.managedCluster.config.masterConfig.managedGroupConfig.instanceTemplateName' - api_field: 'placement.managedCluster.config.masterConfig.minCpuPlatform' diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl index 33e134e1f3e8..2287746bb2bc 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl @@ -314,6 +314,11 @@ func TestAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(t *testing.T) { { Config: testAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(context), Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "e2-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "0"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_type", "pd-ssd"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_size_gb", "35"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.master_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.num_local_ssds", "1"), resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "n2d-standard-2"), resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "1"), resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_type", "pd-ssd"), @@ -349,6 +354,17 @@ resource "google_dataproc_workflow_template" "template" { } master_config { num_instances = 1 + instance_flexibility_policy { + instance_selection_list { + machine_types = ["e2-standard-2"] + rank = 0 + disk_config { + boot_disk_type = "pd-ssd" + boot_disk_size_gb = 35 + num_local_ssds = 1 + } + } + } } worker_config { num_instances = 2 diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl index c53371895db0..27c41cb28bd9 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl @@ -673,6 +673,7 @@ type WorkflowTemplatePlacementManagedClusterConfigMasterConfig struct { ManagedGroupConfig *WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig `json:"managedGroupConfig"` Accelerators []WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerators `json:"accelerators"` MinCpuPlatform *string `json:"minCpuPlatform"` + InstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy `json:"instanceFlexibilityPolicy"` } type jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfig @@ -710,6 +711,8 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfig) UnmarshalJSO r.MinCpuPlatform = res.MinCpuPlatform + r.InstanceFlexibilityPolicy = res.InstanceFlexibilityPolicy + } return nil } @@ -884,6 +887,159 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerators) return fmt.Sprintf("%x", hash) } +type WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy struct { + empty bool `json:"-"` + InstanceMachineTypes map[string]string `json:"instanceMachineTypes"` + InstanceSelectionList []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection `json:"instanceSelectionList"` + InstanceSelectionResults []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult `json:"instanceSelectionResults"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy + } else { + + r.InstanceMachineTypes = res.InstanceMachineTypes + + r.InstanceSelectionList = res.InstanceSelectionList + + r.InstanceSelectionResults = res.InstanceSelectionResults + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy = &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection struct { + empty bool `json:"-"` + MachineTypes []string `json:"machineTypes"` + Rank *int64 `json:"rank"` + DiskConfig *WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig `json:"diskConfig"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection + } else { + + r.MachineTypes = res.MachineTypes + + r.Rank = res.Rank + + r.DiskConfig = res.DiskConfig + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection = &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult struct { + empty bool `json:"-"` + MachineType *string `json:"machineType"` + VMCount *int64 `json:"vmCount"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult + } else { + + r.MachineType = res.MachineType + + r.VMCount = res.VMCount + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult = &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + type WorkflowTemplatePlacementManagedClusterConfigWorkerConfig struct { empty bool `json:"-"` NumInstances *int64 `json:"numInstances"` diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl index f7f3ea92c337..e8bd752c55fe 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl @@ -176,6 +176,11 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfig) validate() e return err } } + if !dcl.IsEmptyValueIndirect(r.InstanceFlexibilityPolicy) { + if err := r.InstanceFlexibilityPolicy.validate(); err != nil { + return err + } + } return nil } func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig) validate() error { @@ -187,6 +192,15 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupCo func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerators) validate() error { return nil } +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) validate() error { + return nil +} func (r *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig) validate() error { if !dcl.IsEmptyValueIndirect(r.DiskConfig) { if err := r.DiskConfig.validate(); err != nil { @@ -2048,6 +2062,7 @@ func canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfig(des, } else { cDes.MinCpuPlatform = des.MinCpuPlatform } + cDes.InstanceFlexibilityPolicy = canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(des.InstanceFlexibilityPolicy, initial.InstanceFlexibilityPolicy, opts...) return cDes } @@ -2109,6 +2124,7 @@ func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfig(c if dcl.StringCanonicalize(des.MinCpuPlatform, nw.MinCpuPlatform) { nw.MinCpuPlatform = des.MinCpuPlatform } + nw.InstanceFlexibilityPolicy = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c, des.InstanceFlexibilityPolicy, nw.InstanceFlexibilityPolicy) return nw } @@ -2528,6 +2544,139 @@ func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcc return items } +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(des, initial *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy{} + + cDes.InstanceSelectionList = canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des.InstanceSelectionList, initial.InstanceSelectionList, opts...) + + return cDes +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + nw.InstanceSelectionList = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, des.InstanceSelectionList, nw.InstanceSelectionList) + + return nw +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(des, initial *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.StringArrayCanonicalize(des.MachineTypes, initial.MachineTypes) { + cDes.MachineTypes = initial.MachineTypes + } else { + cDes.MachineTypes = des.MachineTypes + } + if dcl.IsZeroValue(des.Rank) || (dcl.IsEmptyValueIndirect(des.Rank) && dcl.IsEmptyValueIndirect(initial.Rank)) { + cDes.Rank = initial.Rank + } else { + cDes.Rank = des.Rank + } + cDes.DiskConfig = canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(des.DiskConfig, initial.DiskConfig, opts...) + + return cDes +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des, initial []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if dcl.IsEmptyValueIndirect(des) { + return initial + } + + if len(des) != len(initial) { + items := make([]WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for _, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(&d, nil, opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for i, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(&d, &initial[i], opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + if dcl.StringArrayCanonicalize(des.MachineTypes, nw.MachineTypes) { + nw.MachineTypes = des.MachineTypes + } + nw.DiskConfig = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(c, des.DiskConfig, nw.DiskConfig) + + return nw +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, des, nw []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + // Lengths are unequal. A diff will occur later, so we shouldn't canonicalize. + // Return the original array. + if len(des) != len(nw) { + return nw + } + + var items []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection + for i, d := range des { + n := nw[i] + items = append(items, *canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c, &d, &n)) + } + + return items +} + func canonicalizeWorkflowTemplatePlacementManagedClusterConfigWorkerConfig(des, initial *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigWorkerConfig { if des == nil { @@ -9195,6 +9344,13 @@ func compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigNewStyle(d, } diffs = append(diffs, ds...) } + + if ds, err := dcl.Diff(desired.InstanceFlexibilityPolicy, actual.InstanceFlexibilityPolicy, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceFlexibilityPolicy")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } return diffs, nil } @@ -9313,6 +9469,131 @@ func compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator return diffs, nil } +func compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy or *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.InstanceMachineTypes, actual.InstanceMachineTypes, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceMachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionList, actual.InstanceSelectionList, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionList")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionResults, actual.InstanceSelectionResults, dcl.DiffInfo{OutputOnly: true, ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionResults")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection or *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineTypes, actual.MachineTypes, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.Rank, actual.Rank, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("Rank")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.DiskConfig, actual.DiskConfig, dcl.DiffInfo{ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("DiskConfig")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult or *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineType, actual.MachineType, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineType")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.VMCount, actual.VMCount, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("VmCount")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + func compareWorkflowTemplatePlacementManagedClusterConfigWorkerConfigNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { var diffs []*dcl.FieldDiff @@ -12867,6 +13148,11 @@ func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfig(c *DclClien if v := f.MinCpuPlatform; !dcl.IsEmptyValueIndirect(v) { m["minCpuPlatform"] = v } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c, f.InstanceFlexibilityPolicy, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceFlexibilityPolicy into instanceFlexibilityPolicy: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["instanceFlexibilityPolicy"] = v + } return m, nil } @@ -12894,6 +13180,7 @@ func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfig(c *DclClie r.ManagedGroupConfig = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig(c, m["managedGroupConfig"], res) r.Accelerators = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigAcceleratorsSlice(c, m["accelerators"], res) r.MinCpuPlatform = dcl.FlattenString(m["minCpuPlatform"]) + r.InstanceFlexibilityPolicy = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c, m["instanceFlexibilityPolicy"], res) return r } @@ -13250,6 +13537,210 @@ func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator return r } +// expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy expands an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy into a JSON +// request object. +func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, f.InstanceSelectionList, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionList into instanceSelectionList: %w", err) + } else if v != nil { + m["instanceSelectionList"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, f.InstanceSelectionResults, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionResults into instanceSelectionResults: %w", err) + } else if v != nil { + m["instanceSelectionResults"] = v + } + + return m, nil +} + +// flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy flattens an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy from a JSON +// response object. +func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicy + } + r.InstanceMachineTypes = dcl.FlattenKeyValuePairs(m["instanceMachineTypes"]) + r.InstanceSelectionList = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, m["instanceSelectionList"], res) + r.InstanceSelectionResults = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, m["instanceSelectionResults"], res) + + return r +} + +// expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice expands the contents of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection into a JSON +// request object. +func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c, &item, res) + if err != nil { + return nil, err + } + if i != nil { + items = append(items, i) + } + } + + return items, nil +} + +// expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection expands an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection into a JSON +// request object. +func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineTypes; !dcl.IsEmptyValueIndirect(v) { + m["machineTypes"] = v + } + if v := f.Rank; v != nil { + m["rank"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(c, f.DiskConfig, res); err != nil { + return nil, fmt.Errorf("error expanding DiskConfig into diskConfig: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["diskConfig"] = v + } + + return m, nil +} + +// flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice flattens the contents of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection from a JSON +// response object. +func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c, item.(map[string]interface{}), res)) + } + + return items +} + +// flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection flattens an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection from a JSON +// response object. +func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelection + } + r.MachineTypes = dcl.FlattenStringSlice(m["machineTypes"]) + r.Rank = dcl.FlattenInteger(m["rank"]) + r.DiskConfig = flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig(c, m["diskConfig"], res) + + return r +} + +// expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice expands the contents of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult into a JSON +// request object. +func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, &item, res) + if err != nil { + return nil, err + } + if i != nil { + items = append(items, i) + } + } + + return items, nil +} + +// expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult expands an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult into a JSON +// request object. +func expandWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineType; !dcl.IsEmptyValueIndirect(v) { + m["machineType"] = v + } + if v := f.VMCount; !dcl.IsEmptyValueIndirect(v) { + m["vmCount"] = v + } + + return m, nil +} + +// flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice flattens the contents of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult from a JSON +// response object. +func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, item.(map[string]interface{}), res)) + } + + return items +} + +// flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult flattens an instance of WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult from a JSON +// response object. +func flattenWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigMasterConfigInstanceFlexibilityPolicyInstanceSelectionResult + } + r.MachineType = dcl.FlattenString(m["machineType"]) + r.VMCount = dcl.FlattenInteger(m["vmCount"]) + + return r +} + // expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigMap expands the contents of WorkflowTemplatePlacementManagedClusterConfigWorkerConfig into a JSON // request object. func expandWorkflowTemplatePlacementManagedClusterConfigWorkerConfigMap(c *DclClient, f map[string]WorkflowTemplatePlacementManagedClusterConfigWorkerConfig, res *WorkflowTemplate) (map[string]interface{}, error) { diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown index b33e6e14c495..6cf4c51ae6e2 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown @@ -777,6 +777,10 @@ The `master_config` block supports: * `managed_group_config` - Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups. +* `instance_flexibility_policy` - + (Optional) + Instance flexibility Policy allowing a mixture of VM shapes and provisioning models. Supported on `master_config`, `worker_config`, and `secondary_worker_config` (provisioning models are supported exclusively on `secondary_worker_config`). Structure is [documented below](#nested_instance_flexibility_policy). + The `worker_config` block supports: * `accelerators` - From 9129add41f5ba694a82743948f043fe4141e448d Mon Sep 17 00:00:00 2001 From: Tejas Sovani Date: Thu, 10 Sep 2026 09:39:03 +0000 Subject: [PATCH 3/3] [Dataproc] Add instance_flexibility_policy with provisioning_model_mix to secondary_worker_config in google_dataproc_workflow_template --- ...esource_dataproc_workflow_template.go.tmpl | 335 +++++++++- ..._dataproc_workflow_template_meta.yaml.tmpl | 10 + ...ce_dataproc_workflow_template_test.go.tmpl | 38 ++ .../dataproc/workflow_template.go.tmpl | 210 +++++- .../workflow_template_internal.go.tmpl | 609 ++++++++++++++++++ .../dataproc_workflow_template.html.markdown | 94 ++- 6 files changed, 1264 insertions(+), 32 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl index a77b8adf3776..f7c9fe0ac038 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template.go.tmpl @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" dcl "github.com/hashicorp/terraform-provider-google/google/tpgdclresource" "github.com/hashicorp/terraform-provider-google/google/tpgresource" transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" @@ -1802,6 +1803,16 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigS Description: "Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.", Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfigSchema(), }, + + "instance_flexibility_policy": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicySchema(), + }, }, } } @@ -1831,6 +1842,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigD Schema: map[string]*schema.Schema{ "boot_disk_size_gb": { Type: schema.TypeInt, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Size in GB of the boot disk (default is 500GB).", @@ -1838,6 +1850,7 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigD "boot_disk_type": { Type: schema.TypeString, + Computed: true, Optional: true, ForceNew: true, Description: "Optional. Type of the boot disk (default is \"pd-standard\"). Valid values: \"pd-balanced\" (Persistent Disk Balanced Solid State Drive), \"pd-ssd\" (Persistent Disk Solid State Drive), or \"pd-standard\" (Persistent Disk Hard Disk Drive). See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types).", @@ -1872,6 +1885,127 @@ func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigM } } +func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicySchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_machine_types": { + Type: schema.TypeMap, + Computed: true, + Description: "Output only. A map of instance names to their machine type.", + Elem: &schema.Schema{Type: schema.TypeString}, + }, + + "instance_selection_list": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list", + "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.provisioning_model_mix", + }, + Description: "Optional. List of instance selection options that the group will use when creating new VMs.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSchema(), + }, + + "instance_selection_results": { + Type: schema.TypeList, + Computed: true, + Description: "Output only. A list of instance selection results that were successfully allocated.", + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema(), + }, + + "provisioning_model_mix": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list", + "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.provisioning_model_mix", + }, + Description: "Optional. Strategy for provisioning model mix for secondary worker instances.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMixSchema(), + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "disk_config": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Optional. Disk configuration to apply to the instances in this instance selection.", + MaxItems: 1, + Elem: DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfigSchema(), + }, + + "machine_types": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: "Optional. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "rank": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.", + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_type": { + Type: schema.TypeString, + Computed: true, + Description: "Output only. Full machine-type names, e.g. \"n1-standard-16\".", + }, + + "vm_count": { + Type: schema.TypeInt, + Computed: true, + Description: "Output only. Number of VM provisioned with the corresponding machine_type.", + }, + }, + } +} + +func DataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMixSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "standard_capacity_base": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. The base capacity that will always use Standard VMs to avoid risk of premature allocation.", + ValidateFunc: validation.IntAtLeast(0), + }, + + "standard_capacity_percent_above_base": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Description: "Optional. The percentage of target capacity that will use Standard VMs above standardCapacityBase.", + ValidateFunc: validation.IntBetween(0, 100), + }, + }, + } +} + func DataprocWorkflowTemplatePlacementManagedClusterConfigSecurityConfigSchema() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -4114,13 +4248,14 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerC } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig{ - Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAcceleratorsArray(obj["accelerators"]), - DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(obj["disk_config"]), - Image: dcl.String(obj["image"].(string)), - MachineType: dcl.String(obj["machine_type"].(string)), - MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), - NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), - Preemptibility: WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + Accelerators: expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAcceleratorsArray(obj["accelerators"]), + DiskConfig: expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(obj["disk_config"]), + Image: dcl.String(obj["image"].(string)), + MachineType: dcl.String(obj["machine_type"].(string)), + MinCpuPlatform: dcl.StringOrNil(obj["min_cpu_platform"].(string)), + NumInstances: dcl.Int64(int64(obj["num_instances"].(int))), + Preemptibility: WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigPreemptibilityEnumRef(obj["preemptibility"].(string)), + InstanceFlexibilityPolicy: expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(obj["instance_flexibility_policy"]), } } @@ -4129,16 +4264,17 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorker return nil } transformed := map[string]interface{}{ - "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAcceleratorsArray(obj.Accelerators), - "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(obj.DiskConfig), - "image": obj.Image, - "machine_type": obj.MachineType, - "min_cpu_platform": obj.MinCpuPlatform, - "num_instances": obj.NumInstances, - "preemptibility": obj.Preemptibility, - "instance_names": obj.InstanceNames, - "is_preemptible": obj.IsPreemptible, - "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig(obj.ManagedGroupConfig), + "accelerators": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAcceleratorsArray(obj.Accelerators), + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(obj.DiskConfig), + "image": obj.Image, + "machine_type": obj.MachineType, + "min_cpu_platform": obj.MinCpuPlatform, + "num_instances": obj.NumInstances, + "preemptibility": obj.Preemptibility, + "instance_names": obj.InstanceNames, + "is_preemptible": obj.IsPreemptible, + "managed_group_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig(obj.ManagedGroupConfig), + "instance_flexibility_policy": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(obj.InstanceFlexibilityPolicy), } return []interface{}{transformed} @@ -4212,8 +4348,8 @@ func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerC } obj := objArr[0].(map[string]interface{}) return &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig{ - BootDiskSizeGb: dcl.Int64(int64(obj["boot_disk_size_gb"].(int))), - BootDiskType: dcl.String(obj["boot_disk_type"].(string)), + BootDiskSizeGb: dcl.Int64OrNil(int64(obj["boot_disk_size_gb"].(int))), + BootDiskType: dcl.StringOrNil(obj["boot_disk_type"].(string)), NumLocalSsds: dcl.Int64OrNil(int64(obj["num_local_ssds"].(int))), } } @@ -4245,6 +4381,167 @@ func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorker } +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy { + if o == nil { + return nil + } + objArr := o.([]interface{}) + if len(objArr) == 0 || objArr[0] == nil { + return nil + } + obj := objArr[0].(map[string]interface{}) + isl := expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj["instance_selection_list"]) + pmm := expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(obj["provisioning_model_mix"]) + if len(isl) == 0 && pmm == nil { + return nil + } + return &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy{ + InstanceSelectionList: isl, + ProvisioningModelMix: pmm, + } +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(obj *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "instance_machine_types": obj.InstanceMachineTypes, + "instance_selection_list": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(obj.InstanceSelectionList), + "instance_selection_results": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(obj.InstanceSelectionResults), + "provisioning_model_mix": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(obj.ProvisioningModelMix), + } + + return []interface{}{transformed} +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(o interface{}) []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + + objs := o.([]interface{}) + if len(objs) == 0 || objs[0] == nil { + return nil + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(objs)) + for _, item := range objs { + i := expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(item) + if i != nil { + items = append(items, *i) + } + } + + return items +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if o == nil { + return nil + } + obj := o.(map[string]interface{}) + m := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + if v, ok := obj["machine_types"].([]interface{}); ok { + for _, item := range v { + if s, ok := item.(string); ok && s != "" { + m.MachineTypes = append(m.MachineTypes, s) + } + } + } + if v, ok := obj["rank"].(int); ok { + m.Rank = dcl.Int64(int64(v)) + } + if v, ok := obj["disk_config"]; ok { + m.DiskConfig = expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(v) + } + return m +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionArray(objs []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(obj *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "disk_config": flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(obj.DiskConfig), + "machine_types": obj.MachineTypes, + "rank": obj.Rank, + } + + return transformed +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultArray(objs []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) []interface{} { + if objs == nil { + return nil + } + + items := []interface{}{} + for _, item := range objs { + i := flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(&item) + if i != nil { + items = append(items, i) + } + } + + return items +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(obj *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "machine_type": obj.MachineType, + "vm_count": obj.VMCount, + } + + return transformed +} + +func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix { + if o == nil { + return nil + } + objArr := o.([]interface{}) + if len(objArr) == 0 || objArr[0] == nil { + return nil + } + obj := objArr[0].(map[string]interface{}) + return &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix{ + StandardCapacityBase: dcl.Int64(int64(obj["standard_capacity_base"].(int))), + StandardCapacityPercentAboveBase: dcl.Int64(int64(obj["standard_capacity_percent_above_base"].(int))), + } +} + +func flattenDataprocWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(obj *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) interface{} { + if obj == nil || obj.Empty() { + return nil + } + transformed := map[string]interface{}{ + "standard_capacity_base": obj.StandardCapacityBase, + "standard_capacity_percent_above_base": obj.StandardCapacityPercentAboveBase, + } + + return []interface{}{transformed} +} + func expandDataprocWorkflowTemplatePlacementManagedClusterConfigSecurityConfig(o interface{}) *WorkflowTemplatePlacementManagedClusterConfigSecurityConfig { if o == nil { return EmptyWorkflowTemplatePlacementManagedClusterConfigSecurityConfig diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl index 913f26fc47eb..8830b9f63474 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_meta.yaml.tmpl @@ -150,6 +150,16 @@ fields: - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceNames' - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.isPreemptible' - field: 'placement.managed_cluster.config.secondary_worker_config.machine_type' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceMachineTypes' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskSizeGb' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.bootDiskType' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionList.diskConfig.numLocalSsds' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionList.machineTypes' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionList.rank' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionResults.machineType' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.instanceSelectionResults.vmCount' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.provisioningModelMix.standardCapacityBase' + - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.instanceFlexibilityPolicy.provisioningModelMix.standardCapacityPercentAboveBase' - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.managedGroupConfig.instanceGroupManagerName' - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.managedGroupConfig.instanceTemplateName' - api_field: 'placement.managedCluster.config.secondaryWorkerConfig.minCpuPlatform' diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl index 2287746bb2bc..34135ca741be 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_workflow_template_test.go.tmpl @@ -130,6 +130,7 @@ resource "google_dataproc_workflow_template" "template" { num_instances = 3 machine_type = "n1-standard-2" disk_config { + boot_disk_type = "pd-standard" boot_disk_size_gb = 10 num_local_ssds = 2 } @@ -137,6 +138,10 @@ resource "google_dataproc_workflow_template" "template" { secondary_worker_config { num_instances = 2 + disk_config { + boot_disk_type = "pd-standard" + boot_disk_size_gb = 15 + } } software_config { image_version = "%{version}" @@ -328,6 +333,14 @@ func TestAccDataprocWorkflowTemplate_instanceFlexibilityPolicy(t *testing.T) { resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.rank", "2"), resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.disk_config.0.boot_disk_type", "pd-standard"), resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.worker_config.0.instance_flexibility_policy.0.instance_selection_list.1.disk_config.0.boot_disk_size_gb", "35"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "n1-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.1", "n2-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "1"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_type", "pd-ssd"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.boot_disk_size_gb", "35"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.disk_config.0.num_local_ssds", "1"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.provisioning_model_mix.0.standard_capacity_base", "0"), + resource.TestCheckResourceAttr("google_dataproc_workflow_template.template", "placement.0.managed_cluster.0.config.0.secondary_worker_config.0.instance_flexibility_policy.0.provisioning_model_mix.0.standard_capacity_percent_above_base", "50"), ), }, { @@ -388,9 +401,34 @@ resource "google_dataproc_workflow_template" "template" { } } } + secondary_worker_config { + num_instances = 2 + instance_flexibility_policy { + instance_selection_list { + machine_types = ["n1-standard-2", "n2-standard-2"] + rank = 1 + disk_config { + boot_disk_type = "pd-ssd" + boot_disk_size_gb = 35 + num_local_ssds = 1 + } + } + provisioning_model_mix { + standard_capacity_base = 0 + standard_capacity_percent_above_base = 50 + } + } + } } } } + + jobs { + step_id = "someJob" + spark_job { + main_class = "SomeClass" + } + } } `, context) } diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl index 27c41cb28bd9..053f2970b6a8 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template.go.tmpl @@ -1430,7 +1430,8 @@ type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig struct { Preemptibility *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigPreemptibilityEnum `json:"preemptibility"` ManagedGroupConfig *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig `json:"managedGroupConfig"` Accelerators []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerators `json:"accelerators"` - MinCpuPlatform *string `json:"minCpuPlatform"` + MinCpuPlatform *string `json:"minCpuPlatform"` + InstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy `json:"instanceFlexibilityPolicy"` } type jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig @@ -1468,6 +1469,8 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig) Unm r.MinCpuPlatform = res.MinCpuPlatform + r.InstanceFlexibilityPolicy = res.InstanceFlexibilityPolicy + } return nil } @@ -1642,6 +1645,211 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccel return fmt.Sprintf("%x", hash) } +type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy struct { + empty bool `json:"-"` + InstanceMachineTypes map[string]string `json:"instanceMachineTypes"` + InstanceSelectionList []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection `json:"instanceSelectionList"` + InstanceSelectionResults []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult `json:"instanceSelectionResults"` + ProvisioningModelMix *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix `json:"provisioningModelMix"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy + } else { + + r.InstanceMachineTypes = res.InstanceMachineTypes + + r.InstanceSelectionList = res.InstanceSelectionList + + r.InstanceSelectionResults = res.InstanceSelectionResults + + r.ProvisioningModelMix = res.ProvisioningModelMix + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy = &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection struct { + empty bool `json:"-"` + MachineTypes []string `json:"machineTypes"` + Rank *int64 `json:"rank"` + DiskConfig *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig `json:"diskConfig"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection + } else { + + r.MachineTypes = res.MachineTypes + + r.Rank = res.Rank + + r.DiskConfig = res.DiskConfig + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection = &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult struct { + empty bool `json:"-"` + MachineType *string `json:"machineType"` + VMCount *int64 `json:"vmCount"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + } else { + + r.MachineType = res.MachineType + + r.VMCount = res.VMCount + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult = &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + +type WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix struct { + empty bool `json:"-"` + StandardCapacityBase *int64 `json:"standardCapacityBase"` + StandardCapacityPercentAboveBase *int64 `json:"standardCapacityPercentAboveBase"` +} + +type jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) UnmarshalJSON(data []byte) error { + var res jsonWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix + if err := json.Unmarshal(data, &res); err != nil { + return err + } + + var m map[string]interface{} + json.Unmarshal(data, &m) + + if len(m) == 0 { + *r = *EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix + } else { + + r.StandardCapacityBase = res.StandardCapacityBase + + r.StandardCapacityPercentAboveBase = res.StandardCapacityPercentAboveBase + + } + return nil +} + +// This object is used to assert a desired state where this WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix is +// empty. Go lacks global const objects, but this object should be treated +// as one. Modifying this object will have undesirable results. +var EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix = &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix{empty: true} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) Empty() bool { + return r.empty +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) String() string { + return dcl.SprintResource(r) +} + +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) HashCode() string { + // Placeholder for a more complex hash method that handles ordering, etc + // Hash resource body for easy comparison later + hash := sha256.Sum256([]byte(r.String())) + return fmt.Sprintf("%x", hash) +} + type WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig struct { empty bool `json:"-"` ImageVersion *string `json:"imageVersion"` diff --git a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl index e8bd752c55fe..120b1bff6f64 100644 --- a/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/workflow_template_internal.go.tmpl @@ -248,6 +248,11 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig) val return err } } + if !dcl.IsEmptyValueIndirect(r.InstanceFlexibilityPolicy) { + if err := r.InstanceFlexibilityPolicy.validate(); err != nil { + return err + } + } return nil } func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig) validate() error { @@ -259,6 +264,23 @@ func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManag func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerators) validate() error { return nil } +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) validate() error { + if !dcl.IsEmptyValueIndirect(r.ProvisioningModelMix) { + if err := r.ProvisioningModelMix.validate(); err != nil { + return err + } + } + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) validate() error { + return nil +} +func (r *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) validate() error { + return nil +} func (r *WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig) validate() error { return nil } @@ -3385,6 +3407,7 @@ func canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerCon } else { cDes.MinCpuPlatform = des.MinCpuPlatform } + cDes.InstanceFlexibilityPolicy = canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(des.InstanceFlexibilityPolicy, initial.InstanceFlexibilityPolicy, opts...) return cDes } @@ -3446,6 +3469,188 @@ func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorker if dcl.StringCanonicalize(des.MinCpuPlatform, nw.MinCpuPlatform) { nw.MinCpuPlatform = des.MinCpuPlatform } + nw.InstanceFlexibilityPolicy = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c, des.InstanceFlexibilityPolicy, nw.InstanceFlexibilityPolicy) + + return nw +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(des, initial *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy{} + + cDes.InstanceSelectionList = canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des.InstanceSelectionList, initial.InstanceSelectionList, opts...) + cDes.ProvisioningModelMix = canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(des.ProvisioningModelMix, initial.ProvisioningModelMix, opts...) + + return cDes +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + nw.InstanceSelectionList = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, des.InstanceSelectionList, nw.InstanceSelectionList) + nw.ProvisioningModelMix = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c, des.ProvisioningModelMix, nw.ProvisioningModelMix) + + return nw +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(des, initial *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.StringArrayCanonicalize(des.MachineTypes, initial.MachineTypes) { + cDes.MachineTypes = initial.MachineTypes + } else { + cDes.MachineTypes = des.MachineTypes + } + if dcl.IsZeroValue(des.Rank) || (dcl.IsEmptyValueIndirect(des.Rank) && dcl.IsEmptyValueIndirect(initial.Rank)) { + cDes.Rank = initial.Rank + } else { + cDes.Rank = des.Rank + } + cDes.DiskConfig = canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(des.DiskConfig, initial.DiskConfig, opts...) + + return cDes +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(des, initial []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, opts ...dcl.ApplyOption) []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if dcl.IsEmptyValueIndirect(des) { + return initial + } + + if len(des) != len(initial) { + items := make([]WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for _, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&d, nil, opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(des)) + for i, d := range des { + cd := canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(&d, &initial[i], opts...) + if cd != nil { + items = append(items, *cd) + } + } + return items +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + if dcl.StringArrayCanonicalize(des.MachineTypes, nw.MachineTypes) { + nw.MachineTypes = des.MachineTypes + } + nw.DiskConfig = canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(c, des.DiskConfig, nw.DiskConfig) + + return nw +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, des, nw []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } + + var items []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection + for i, d := range des { + n := nw[i] + items = append(items, *canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, &d, &n)) + } + + return items +} + +func canonicalizeWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(des, initial *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix, opts ...dcl.ApplyOption) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix { + if des == nil { + return initial + } + if des.empty { + return des + } + + if initial == nil { + return des + } + + cDes := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix{} + + if dcl.IsZeroValue(des.StandardCapacityBase) || (dcl.IsEmptyValueIndirect(des.StandardCapacityBase) && dcl.IsEmptyValueIndirect(initial.StandardCapacityBase)) { + cDes.StandardCapacityBase = initial.StandardCapacityBase + } else { + cDes.StandardCapacityBase = des.StandardCapacityBase + } + if dcl.IsZeroValue(des.StandardCapacityPercentAboveBase) || (dcl.IsEmptyValueIndirect(des.StandardCapacityPercentAboveBase) && dcl.IsEmptyValueIndirect(initial.StandardCapacityPercentAboveBase)) { + cDes.StandardCapacityPercentAboveBase = initial.StandardCapacityPercentAboveBase + } else { + cDes.StandardCapacityPercentAboveBase = des.StandardCapacityPercentAboveBase + } + + return cDes +} + +func canonicalizeNewWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c *DclClient, des, nw *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix { + if des == nil { + return nw + } + + if nw == nil { + if dcl.IsEmptyValueIndirect(des) { + c.Config.Logger.Info("Found explicitly empty value for WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix while comparing non-nil desired to nil actual. Returning desired object.") + return des + } + return nil + } return nw } @@ -10026,6 +10231,182 @@ func compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigNe } diffs = append(diffs, ds...) } + + if ds, err := dcl.Diff(desired.InstanceFlexibilityPolicy, actual.InstanceFlexibilityPolicy, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceFlexibilityPolicy")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy or *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.InstanceMachineTypes, actual.InstanceMachineTypes, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceMachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionList, actual.InstanceSelectionList, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionList")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.InstanceSelectionResults, actual.InstanceSelectionResults, dcl.DiffInfo{OutputOnly: true, ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("InstanceSelectionResults")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.ProvisioningModelMix, actual.ProvisioningModelMix, dcl.DiffInfo{ServerDefault: true, ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMixNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("ProvisioningModelMix")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection or *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineTypes, actual.MachineTypes, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineTypes")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.Rank, actual.Rank, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("Rank")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.DiskConfig, actual.DiskConfig, dcl.DiffInfo{ObjectFunction: compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfigNewStyle, EmptyObject: EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("DiskConfig")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult or *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.MachineType, actual.MachineType, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("MachineType")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.VMCount, actual.VMCount, dcl.DiffInfo{OutputOnly: true, OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("VmCount")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + return diffs, nil +} + +func compareWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMixNewStyle(d, a interface{}, fn dcl.FieldName) ([]*dcl.FieldDiff, error) { + var diffs []*dcl.FieldDiff + + desired, ok := d.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) + if !ok { + desiredNotPointer, ok := d.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix or *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix", d) + } + desired = &desiredNotPointer + } + actual, ok := a.(*WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) + if !ok { + actualNotPointer, ok := a.(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix) + if !ok { + return nil, fmt.Errorf("obj %v is not a WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix", a) + } + actual = &actualNotPointer + } + + if ds, err := dcl.Diff(desired.StandardCapacityBase, actual.StandardCapacityBase, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("StandardCapacityBase")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + + if ds, err := dcl.Diff(desired.StandardCapacityPercentAboveBase, actual.StandardCapacityPercentAboveBase, dcl.DiffInfo{OperationSelector: dcl.RequiresRecreate()}, fn.AddNest("StandardCapacityPercentAboveBase")); len(ds) != 0 || err != nil { + if err != nil { + return nil, err + } + diffs = append(diffs, ds...) + } + return diffs, nil } @@ -14542,6 +14923,11 @@ func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig(c if v := f.MinCpuPlatform; !dcl.IsEmptyValueIndirect(v) { m["minCpuPlatform"] = v } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c, f.InstanceFlexibilityPolicy, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceFlexibilityPolicy into instanceFlexibilityPolicy: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["instanceFlexibilityPolicy"] = v + } return m, nil } @@ -14569,6 +14955,229 @@ func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig(c r.ManagedGroupConfig = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig(c, m["managedGroupConfig"], res) r.Accelerators = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAcceleratorsSlice(c, m["accelerators"], res) r.MinCpuPlatform = dcl.FlattenString(m["minCpuPlatform"]) + r.InstanceFlexibilityPolicy = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c, m["instanceFlexibilityPolicy"], res) + + return r +} + +// expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy expands an instance of WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy into a JSON request object. +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, f.InstanceSelectionList, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionList into instanceSelectionList: %w", err) + } else if v != nil { + m["instanceSelectionList"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, f.InstanceSelectionResults, res); err != nil { + return nil, fmt.Errorf("error expanding InstanceSelectionResults into instanceSelectionResults: %w", err) + } else if v != nil { + m["instanceSelectionResults"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c, f.ProvisioningModelMix, res); err != nil { + return nil, fmt.Errorf("error expanding ProvisioningModelMix into provisioningModelMix: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["provisioningModelMix"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicy + } + r.InstanceMachineTypes = dcl.FlattenKeyValuePairs(m["instanceMachineTypes"]) + r.InstanceSelectionList = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c, m["instanceSelectionList"], res) + r.InstanceSelectionResults = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c, m["instanceSelectionResults"], res) + r.ProvisioningModelMix = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c, m["provisioningModelMix"], res) + + return r +} + +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, &item, res) + if err != nil { + return nil, err + } + + items = append(items, i) + } + + return items, nil +} + +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineTypes; !dcl.IsEmptyValueIndirect(v) { + m["machineTypes"] = v + } + if v := f.Rank; v != nil { + m["rank"] = v + } + if v, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(c, f.DiskConfig, res); err != nil { + return nil, fmt.Errorf("error expanding DiskConfig into diskConfig: %w", err) + } else if !dcl.IsEmptyValueIndirect(v) { + m["diskConfig"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c, item.(map[string]interface{}), res)) + } + + return items +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelection + } + r.MachineTypes = dcl.FlattenStringSlice(m["machineTypes"]) + r.Rank = dcl.FlattenInteger(m["rank"]) + r.DiskConfig = flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig(c, m["diskConfig"], res) + + return r +} + +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, f []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) ([]map[string]interface{}, error) { + if f == nil { + return nil, nil + } + + items := []map[string]interface{}{} + for _, item := range f { + i, err := expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, &item, res) + if err != nil { + return nil, err + } + + items = append(items, i) + } + + return items, nil +} + +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.MachineType; !dcl.IsEmptyValueIndirect(v) { + m["machineType"] = v + } + if v := f.VMCount; !dcl.IsEmptyValueIndirect(v) { + m["vmCount"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResultSlice(c *DclClient, i interface{}, res *WorkflowTemplate) []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult { + a, ok := i.([]interface{}) + if !ok { + return []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + if len(a) == 0 { + return []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + } + + items := make([]WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult, 0, len(a)) + for _, item := range a { + items = append(items, *flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c, item.(map[string]interface{}), res)) + } + + return items +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyInstanceSelectionResult + } + r.MachineType = dcl.FlattenString(m["machineType"]) + r.VMCount = dcl.FlattenInteger(m["vmCount"]) + + return r +} + +func expandWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c *DclClient, f *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix, res *WorkflowTemplate) (map[string]interface{}, error) { + if dcl.IsEmptyValueIndirect(f) { + return nil, nil + } + + m := make(map[string]interface{}) + if v := f.StandardCapacityBase; v != nil { + m["standardCapacityBase"] = v + } + if v := f.StandardCapacityPercentAboveBase; v != nil { + m["standardCapacityPercentAboveBase"] = v + } + + return m, nil +} + +func flattenWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix(c *DclClient, i interface{}, res *WorkflowTemplate) *WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix { + m, ok := i.(map[string]interface{}) + if !ok { + return nil + } + + r := &WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix{} + + if dcl.IsEmptyValueIndirect(i) { + return EmptyWorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigInstanceFlexibilityPolicyProvisioningModelMix + } + r.StandardCapacityBase = dcl.FlattenInteger(m["standardCapacityBase"]) + r.StandardCapacityPercentAboveBase = dcl.FlattenInteger(m["standardCapacityPercentAboveBase"]) return r } diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown index 6cf4c51ae6e2..a6096f1c1698 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_workflow_template.html.markdown @@ -162,6 +162,19 @@ resource "google_dataproc_workflow_template" "template" { } } } + secondary_worker_config { + num_instances = 2 + instance_flexibility_policy { + instance_selection_list { + machine_types = ["n1-standard-2"] + rank = 1 + } + provisioning_model_mix { + standard_capacity_base = 1 + standard_capacity_percent_above_base = 50 + } + } + } } } } @@ -180,7 +193,7 @@ The following arguments are supported: (Required) The resource name of the workflow template, as described in https://docs.cloud.google.com/apis/design/resource_names. * For `projects.regions.workflowTemplates`, the resource name of the template has the following format: `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` * For `projects.locations.workflowTemplates`, the resource name of the template has the following format: `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` * `placement` - - (Required) WorkflowTemplate scheduling information. + (Required) WorkflowTemplate scheduling information. Structure is [documented below](#nested_placement). * `jobs` - (Required) The Directed Acyclic Graph of Jobs to submit. Structure is [documented below](#nested_jobs) @@ -261,18 +274,18 @@ When this this key is provided, the following workflow template job arguments, i Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job `goog-dataproc-workflow-step-id` label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters. -The `placement` block supports: +The `placement` block supports: * `cluster_selector` - (Optional) - A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted. + A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted. Structure is [documented below](#nested_cluster_selector). * `managed_cluster` - (Optional) - A cluster that is managed by the workflow. + A cluster that is managed by the workflow. Structure is [documented below](#nested_managed_cluster). -The `config` block supports: +The `config` block supports: * `autoscaling_config` - (Optional) @@ -304,7 +317,7 @@ The `config` block supports: * `master_config` - (Optional) - The Compute Engine config settings for additional worker instances in a cluster. + The Compute Engine config settings for additional worker instances in a cluster. Structure is [documented below](#nested_master_config). * `metastore_config` - (Optional) @@ -312,7 +325,7 @@ The `config` block supports: * `secondary_worker_config` - (Optional) - The Compute Engine config settings for additional worker instances in a cluster. + The Compute Engine config settings for additional worker instances in a cluster. Structure is [documented below](#nested_secondary_worker_config). * `security_config` - (Optional) @@ -714,7 +727,7 @@ The `values` block supports: (Required) Required. List of allowed values for the parameter. -The `cluster_selector` block supports: +The `cluster_selector` block supports: * `cluster_labels` - (Required) @@ -724,7 +737,7 @@ The `cluster_selector` block supports: (Optional) The zone where workflow process executes. This parameter does not affect the selection of the cluster. If unspecified, the zone of the first cluster matching the selector is used. -The `managed_cluster` block supports: +The `managed_cluster` block supports: * `cluster_name` - (Required) @@ -732,13 +745,13 @@ The `managed_cluster` block supports: * `config` - (Required) - Required. The cluster configuration. + Required. The cluster configuration. Structure is [documented below](#nested_config). * `labels` - (Optional) The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster. -The `master_config` block supports: +The `master_config` block supports: * `accelerators` - (Optional) @@ -781,6 +794,49 @@ The `master_config` block supports: (Optional) Instance flexibility Policy allowing a mixture of VM shapes and provisioning models. Supported on `master_config`, `worker_config`, and `secondary_worker_config` (provisioning models are supported exclusively on `secondary_worker_config`). Structure is [documented below](#nested_instance_flexibility_policy). +The `secondary_worker_config` block supports: + +* `accelerators` - + (Optional) + The Compute Engine accelerator configuration for these instances. Structure is [documented below](#nested_accelerators). + +* `disk_config` - + (Optional) + Disk option config settings. Structure is [documented below](#nested_disk_config). + +* `image` - + (Optional) + The Compute Engine image resource used for cluster instances. + +* `machine_type` - + (Optional) + The Compute Engine machine type used for cluster instances. + +* `min_cpu_platform` - + (Optional) + The minimum CPU platform and architecture for the cluster. + +* `num_instances` - + (Optional) + The number of worker instances in the cluster. + +* `preemptibility` - + (Optional) + Specifies the preemptibility of the secondary worker group. + +* `instance_names` - + Output only. The list of instance names. + +* `is_preemptible` - + Output only. Specifies that this instance group contains preemptible instances. + +* `managed_group_config` - + Output only. The config for Compute Engine Instance Group Manager that manages this group. + +* `instance_flexibility_policy` - + (Optional) + Instance flexibility Policy allowing a mixture of VM shapes and provisioning models. Structure is [documented below](#nested_instance_flexibility_policy). + The `worker_config` block supports: * `accelerators` - @@ -856,6 +912,10 @@ The `master_config` block supports: * `instance_machine_types` - Output only. A map of instance names to their machine types. +* `provisioning_model_mix` - + (Optional) + Strategy for provisioning model mix for secondary worker instances. Supported only for `secondary_worker_config`. Structure is [documented below](#nested_provisioning_model_mix). + The `instance_selection_list` block supports: * `machine_types` - @@ -878,7 +938,17 @@ The `master_config` block supports: * `vm_count` - Output only. Number of VM provisioned with the corresponding machine_type. -The `autoscaling_config` block supports: +The `provisioning_model_mix` block supports: + +* `standard_capacity_base` - + (Optional) + The base capacity that will always use Standard VMs to avoid risk of premature allocation. + +* `standard_capacity_percent_above_base` - + (Optional) + The percentage of target capacity that will use Standard VMs above standardCapacityBase. + +The `autoscaling_config` block supports: * `policy` - (Optional)