diff --git a/quickstart/101-aks-automatic-custom-network-azapi/main.tf b/quickstart/101-aks-automatic-custom-network-azapi/main.tf new file mode 100644 index 000000000..31170c3ce --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network-azapi/main.tf @@ -0,0 +1,113 @@ +# Create a random name for the resource group using random_pet +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +# Create a resource group using the generated random name +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create the custom virtual network that hosts the cluster +resource "azurerm_virtual_network" "vnet" { + name = var.virtual_network_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + address_space = var.virtual_network_address_space +} + +# Create the subnet delegated to AKS for API Server VNet Integration +resource "azurerm_subnet" "api_server" { + name = "apiServerSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.api_server_subnet_address_prefixes + + delegation { + name = "aks-delegation" + + service_delegation { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +# Create the subnet that hosts the user node pools +resource "azurerm_subnet" "user_nodes" { + name = "userNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.user_node_subnet_address_prefixes +} + +# Create the subnet that hosts the managed system node pool +resource "azurerm_subnet" "system_nodes" { + name = "managedSystemNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.system_node_subnet_address_prefixes + + lifecycle { + # AKS adds its own managed cluster delegation to this subnet after the + # cluster is created. + ignore_changes = [delegation] + } +} + +# Create the user-assigned managed identity used by the cluster +resource "azurerm_user_assigned_identity" "aks" { + name = var.identity_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Grant the cluster identity Network Contributor on the virtual network +resource "azurerm_role_assignment" "network_contributor" { + scope = azurerm_virtual_network.vnet.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks.principal_id + principal_type = "ServicePrincipal" +} + +# Create a random name for the AKS Automatic cluster +resource "random_pet" "cluster_name" { + prefix = var.cluster_name_prefix +} + +# Create the AKS Automatic cluster in the custom virtual network. +# The AzAPI provider gives you direct control over the managed cluster API +# payload, including properties that the AzureRM provider doesn't expose yet. +resource "azapi_resource" "aks_automatic" { + type = "Microsoft.ContainerService/managedClusters@2026-04-01" + name = random_pet.cluster_name.id + parent_id = azurerm_resource_group.rg.id + location = azurerm_resource_group.rg.location + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.aks.id] + } + + body = { + sku = { + name = "Automatic" + } + + properties = { + apiServerAccessProfile = { + subnetId = azurerm_subnet.api_server.id + } + + hostedSystemProfile = { + nodeSubnetID = azurerm_subnet.user_nodes.id + systemNodeSubnetID = azurerm_subnet.system_nodes.id + } + } + } + + response_export_values = ["properties.nodeResourceGroup", "properties.fqdn"] + + depends_on = [azurerm_role_assignment.network_contributor] +} diff --git a/quickstart/101-aks-automatic-custom-network-azapi/outputs.tf b/quickstart/101-aks-automatic-custom-network-azapi/outputs.tf new file mode 100644 index 000000000..69c2038c3 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network-azapi/outputs.tf @@ -0,0 +1,19 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "cluster_name" { + value = azapi_resource.aks_automatic.name +} + +output "cluster_id" { + value = azapi_resource.aks_automatic.id +} + +output "node_resource_group" { + value = azapi_resource.aks_automatic.output.properties.nodeResourceGroup +} + +output "virtual_network_name" { + value = azurerm_virtual_network.vnet.name +} diff --git a/quickstart/101-aks-automatic-custom-network-azapi/providers.tf b/quickstart/101-aks-automatic-custom-network-azapi/providers.tf new file mode 100644 index 000000000..2c375b07c --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network-azapi/providers.tf @@ -0,0 +1,24 @@ +terraform { + required_version = ">= 1.0" + required_providers { + azapi = { + source = "Azure/azapi" + version = "~>2.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~>5.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} + +provider "azapi" { +} diff --git a/quickstart/101-aks-automatic-custom-network-azapi/readme.md b/quickstart/101-aks-automatic-custom-network-azapi/readme.md new file mode 100644 index 000000000..c3b100f9f --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network-azapi/readme.md @@ -0,0 +1,54 @@ +# Azure Kubernetes Service (AKS) Automatic cluster in a custom virtual network (AzAPI provider) + +This template deploys an AKS Automatic cluster into a custom virtual network, in a resource group with a random name beginning with "rg-". + +The virtual network contains a subnet delegated to the cluster API server, a subnet for the user node pools, and a subnet for the managed system node pool. The cluster uses a user-assigned managed identity that's granted the Network Contributor role on the virtual network, which is required when you bring your own network. + +The cluster is declared with the AzAPI provider's [`azapi_resource`](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/resource) against the `Microsoft.ContainerService/managedClusters` API. Use this pattern when you need direct control over the managed cluster API payload, or when you need an API version or property that the AzureRM provider doesn't expose yet. The cluster SKU is hard-coded to `Automatic`. + +For the equivalent sample that uses the AzureRM provider's `azurerm_kubernetes_automatic_cluster` resource, see [101-aks-automatic-custom-network](../101-aks-automatic-custom-network/). For a private cluster in a custom virtual network, see [101-aks-automatic-private-custom-network-azapi](../101-aks-automatic-private-custom-network-azapi/). + +## Terraform resource types + +- [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) +- [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +- [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) +- [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) +- [azurerm_user_assigned_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) +- [azurerm_role_assignment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) +- [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/resource) + +## Variables + +| Name | Description | Default | Validation | +|-|-|-|-| +| `resource_group_name_prefix` | Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription. | rg | | +| `resource_group_location` | Location of the resource group. | westus2 | | +| `cluster_name_prefix` | Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription. | aks-automatic | | +| `virtual_network_name` | Name of the custom virtual network that hosts the cluster. | aks-automatic-vnet | | +| `identity_name` | Name of the user-assigned managed identity that the cluster uses. | aks-automatic-identity | | +| `virtual_network_address_space` | Address space of the custom virtual network. | ["172.19.0.0/16"] | | +| `api_server_subnet_address_prefixes` | Address prefixes of the subnet delegated to the cluster API server. | ["172.19.0.0/28"] | | +| `user_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the user node pools. | ["172.19.1.0/24"] | | +| `system_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the managed system node pool. | ["172.19.0.64/26"] | | + + +## Network guardrails + +This sample creates its own virtual network, so the defaults are safe as written. Read these before pointing the variables at an existing network: + +- **Subnet containment and non-overlap.** Every subnet prefix must sit inside `virtual_network_address_space` and must not overlap another subnet in the same virtual network. The defaults carve `172.19.0.0/28`, `172.19.0.64/26`, and `172.19.1.0/24` out of `172.19.0.0/16`. +- **Subnet sizing.** The API server subnet must be at least a `/28`, and AKS reserves at least nine addresses in it. Size the node subnets for the pod and node scale you expect, because a subnet that runs out of addresses blocks scaling. +- **Role assignment blast radius.** The cluster identity is granted **Network Contributor** on the whole virtual network, which is what Node Autoprovisioning needs. If you retarget this sample at a shared virtual network, that grant reaches every subnet and resource in it. Scope it more tightly, or keep the cluster in a dedicated network. +- **Delegation drift is hidden.** The managed system node subnet sets `ignore_changes = [delegation]` because AKS adds its own managed cluster delegation after the cluster is created. That keeps plans clean, but it also means Terraform won't report if the delegation is later changed or removed outside Terraform. + +## Example + +```console +terraform init -upgrade +terraform plan -out main.tfplan +terraform apply main.tfplan +``` + + + diff --git a/quickstart/101-aks-automatic-custom-network-azapi/variables.tf b/quickstart/101-aks-automatic-custom-network-azapi/variables.tf new file mode 100644 index 000000000..8fe2b1ef4 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network-azapi/variables.tf @@ -0,0 +1,53 @@ +variable "resource_group_location" { + type = string + default = "westus2" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "cluster_name_prefix" { + type = string + default = "aks-automatic" + description = "Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription." +} + +variable "virtual_network_name" { + type = string + default = "aks-automatic-vnet" + description = "Name of the custom virtual network that hosts the cluster." +} + +variable "identity_name" { + type = string + default = "aks-automatic-identity" + description = "Name of the user-assigned managed identity that the cluster uses." +} + +variable "virtual_network_address_space" { + type = list(string) + default = ["172.19.0.0/16"] + description = "Address space of the custom virtual network." +} + +variable "api_server_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.0/28"] + description = "Address prefixes of the subnet delegated to the cluster API server." +} + +variable "user_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.1.0/24"] + description = "Address prefixes of the subnet that hosts the user node pools." +} + +variable "system_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.64/26"] + description = "Address prefixes of the subnet that hosts the managed system node pool." +} diff --git a/quickstart/101-aks-automatic-custom-network/main.tf b/quickstart/101-aks-automatic-custom-network/main.tf new file mode 100644 index 000000000..c8d3db86c --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network/main.tf @@ -0,0 +1,100 @@ +# Create a random name for the resource group using random_pet +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +# Create a resource group using the generated random name +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create the custom virtual network that hosts the cluster +resource "azurerm_virtual_network" "vnet" { + name = var.virtual_network_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + address_space = var.virtual_network_address_space +} + +# Create the subnet delegated to AKS for API Server VNet Integration +resource "azurerm_subnet" "api_server" { + name = "apiServerSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.api_server_subnet_address_prefixes + + delegation { + name = "aks-delegation" + + service_delegation { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +# Create the subnet that hosts the user node pools +resource "azurerm_subnet" "user_nodes" { + name = "userNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.user_node_subnet_address_prefixes +} + +# Create the subnet that hosts the managed system node pool +resource "azurerm_subnet" "system_nodes" { + name = "managedSystemNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.system_node_subnet_address_prefixes + + lifecycle { + # AKS adds its own managed cluster delegation to this subnet after the + # cluster is created. + ignore_changes = [delegation] + } +} + +# Create the user-assigned managed identity used by the cluster +resource "azurerm_user_assigned_identity" "aks" { + name = var.identity_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Grant the cluster identity Network Contributor on the virtual network +resource "azurerm_role_assignment" "network_contributor" { + scope = azurerm_virtual_network.vnet.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks.principal_id + principal_type = "ServicePrincipal" +} + +# Create a random name for the AKS Automatic cluster +resource "random_pet" "cluster_name" { + prefix = var.cluster_name_prefix +} + +# Create the AKS Automatic cluster in the custom virtual network +resource "azurerm_kubernetes_automatic_cluster" "aks_automatic" { + name = random_pet.cluster_name.id + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.aks.id] + } + + api_server_access { + subnet_id = azurerm_subnet.api_server.id + } + + hosted_system { + node_subnet_id = azurerm_subnet.user_nodes.id + system_node_subnet_id = azurerm_subnet.system_nodes.id + } + + depends_on = [azurerm_role_assignment.network_contributor] +} diff --git a/quickstart/101-aks-automatic-custom-network/outputs.tf b/quickstart/101-aks-automatic-custom-network/outputs.tf new file mode 100644 index 000000000..b735efa72 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network/outputs.tf @@ -0,0 +1,23 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "cluster_name" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.name +} + +output "cluster_id" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.id +} + +output "node_resource_group_id" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.node_resource_group_id +} + +output "virtual_network_name" { + value = azurerm_virtual_network.vnet.name +} + +output "fully_qualified_domain_name" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.fully_qualified_domain_name +} diff --git a/quickstart/101-aks-automatic-custom-network/providers.tf b/quickstart/101-aks-automatic-custom-network/providers.tf new file mode 100644 index 000000000..9e76a7ea8 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network/providers.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>5.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} diff --git a/quickstart/101-aks-automatic-custom-network/readme.md b/quickstart/101-aks-automatic-custom-network/readme.md new file mode 100644 index 000000000..ba812b094 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network/readme.md @@ -0,0 +1,53 @@ +# Azure Kubernetes Service (AKS) Automatic cluster in a custom virtual network (AzureRM provider) + +This template deploys an AKS Automatic cluster into a custom virtual network, in a resource group with a random name beginning with "rg-". + +The virtual network contains a subnet delegated to the cluster API server, a subnet for the user node pools, and a subnet for the managed system node pool. The cluster uses a user-assigned managed identity that's granted the Network Contributor role on the virtual network, which is required when you bring your own network. + +The cluster is created with the AzureRM provider's [`azurerm_kubernetes_automatic_cluster`](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_automatic_cluster) resource, which is the recommended way to declare an AKS Automatic cluster. This resource requires AzureRM provider `v4.81` or later. + +For an equivalent sample that declares the same cluster with the AzAPI provider, see [101-aks-automatic-custom-network-azapi](../101-aks-automatic-custom-network-azapi/). For a private cluster in a custom virtual network, see [101-aks-automatic-private-custom-network](../101-aks-automatic-private-custom-network/). + +## Terraform resource types + +- [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) +- [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +- [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) +- [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) +- [azurerm_user_assigned_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) +- [azurerm_role_assignment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) +- [azurerm_kubernetes_automatic_cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_automatic_cluster) + +## Variables + +| Name | Description | Default | +|-|-|-| +| `resource_group_name_prefix` | Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription. | rg | +| `resource_group_location` | Location of the resource group. | westus2 | +| `cluster_name_prefix` | Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription. | aks-automatic | +| `virtual_network_name` | Name of the custom virtual network that hosts the cluster. | aks-automatic-vnet | +| `identity_name` | Name of the user-assigned managed identity that the cluster uses. | aks-automatic-identity | +| `virtual_network_address_space` | Address space of the custom virtual network. | ["172.19.0.0/16"] | +| `api_server_subnet_address_prefixes` | Address prefixes of the subnet delegated to the cluster API server. | ["172.19.0.0/28"] | +| `user_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the user node pools. | ["172.19.1.0/24"] | +| `system_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the managed system node pool. | ["172.19.0.64/26"] | + + +## Network guardrails + +This sample creates its own virtual network, so the defaults are safe as written. Read these before pointing the variables at an existing network: + +- **Subnet containment and non-overlap.** Every subnet prefix must sit inside `virtual_network_address_space` and must not overlap another subnet in the same virtual network. The defaults carve `172.19.0.0/28`, `172.19.0.64/26`, and `172.19.1.0/24` out of `172.19.0.0/16`. +- **Subnet sizing.** The API server subnet must be at least a `/28`, and AKS reserves at least nine addresses in it. Size the node subnets for the pod and node scale you expect, because a subnet that runs out of addresses blocks scaling. +- **Role assignment blast radius.** The cluster identity is granted **Network Contributor** on the whole virtual network, which is what Node Autoprovisioning needs. If you retarget this sample at a shared virtual network, that grant reaches every subnet and resource in it. Scope it more tightly, or keep the cluster in a dedicated network. +- **Delegation drift is hidden.** The managed system node subnet sets `ignore_changes = [delegation]` because AKS adds its own managed cluster delegation after the cluster is created. That keeps plans clean, but it also means Terraform won't report if the delegation is later changed or removed outside Terraform. + +## Example + +```console +terraform init -upgrade +terraform plan -out main.tfplan +terraform apply main.tfplan +``` + + diff --git a/quickstart/101-aks-automatic-custom-network/variables.tf b/quickstart/101-aks-automatic-custom-network/variables.tf new file mode 100644 index 000000000..8fe2b1ef4 --- /dev/null +++ b/quickstart/101-aks-automatic-custom-network/variables.tf @@ -0,0 +1,53 @@ +variable "resource_group_location" { + type = string + default = "westus2" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "cluster_name_prefix" { + type = string + default = "aks-automatic" + description = "Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription." +} + +variable "virtual_network_name" { + type = string + default = "aks-automatic-vnet" + description = "Name of the custom virtual network that hosts the cluster." +} + +variable "identity_name" { + type = string + default = "aks-automatic-identity" + description = "Name of the user-assigned managed identity that the cluster uses." +} + +variable "virtual_network_address_space" { + type = list(string) + default = ["172.19.0.0/16"] + description = "Address space of the custom virtual network." +} + +variable "api_server_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.0/28"] + description = "Address prefixes of the subnet delegated to the cluster API server." +} + +variable "user_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.1.0/24"] + description = "Address prefixes of the subnet that hosts the user node pools." +} + +variable "system_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.64/26"] + description = "Address prefixes of the subnet that hosts the managed system node pool." +} diff --git a/quickstart/101-aks-automatic-private-custom-network-azapi/main.tf b/quickstart/101-aks-automatic-private-custom-network-azapi/main.tf new file mode 100644 index 000000000..9fd84d1c6 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network-azapi/main.tf @@ -0,0 +1,123 @@ +# Create a random name for the resource group using random_pet +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +# Create a resource group using the generated random name +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create the custom virtual network that hosts the cluster +resource "azurerm_virtual_network" "vnet" { + name = var.virtual_network_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + address_space = var.virtual_network_address_space +} + +# Create the subnet delegated to AKS for API Server VNet Integration +resource "azurerm_subnet" "api_server" { + name = "apiServerSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.api_server_subnet_address_prefixes + + delegation { + name = "aks-delegation" + + service_delegation { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +# Create the subnet that hosts the user node pools +resource "azurerm_subnet" "user_nodes" { + name = "userNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.user_node_subnet_address_prefixes +} + +# Create the subnet that hosts the managed system node pool +resource "azurerm_subnet" "system_nodes" { + name = "managedSystemNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.system_node_subnet_address_prefixes + + lifecycle { + # AKS adds its own managed cluster delegation to this subnet after the + # cluster is created. + ignore_changes = [delegation] + } +} + +# Create the user-assigned managed identity used by the cluster +resource "azurerm_user_assigned_identity" "aks" { + name = var.identity_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Grant the cluster identity Network Contributor on the virtual network +resource "azurerm_role_assignment" "network_contributor" { + scope = azurerm_virtual_network.vnet.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks.principal_id + principal_type = "ServicePrincipal" +} + +# Create a random name for the AKS Automatic cluster +resource "random_pet" "cluster_name" { + prefix = var.cluster_name_prefix +} + +# Create the private AKS Automatic cluster in the custom virtual network. +# The AzAPI provider gives you direct control over the managed cluster API +# payload, including properties that the AzureRM provider doesn't expose yet. +resource "azapi_resource" "aks_automatic" { + type = "Microsoft.ContainerService/managedClusters@2026-04-01" + name = random_pet.cluster_name.id + parent_id = azurerm_resource_group.rg.id + location = azurerm_resource_group.rg.location + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.aks.id] + } + + body = { + sku = { + name = "Automatic" + } + + properties = { + apiServerAccessProfile = { + # A private cluster API server is assigned a private IP address in the + # virtual network. Reaching it requires private connectivity and DNS + # resolution of the private FQDN, which peering, a VPN, or ExpressRoute + # can provide from outside this virtual network. + enablePrivateCluster = true + subnetId = azurerm_subnet.api_server.id + } + + hostedSystemProfile = { + nodeSubnetID = azurerm_subnet.user_nodes.id + systemNodeSubnetID = azurerm_subnet.system_nodes.id + } + } + } + + response_export_values = ["properties.nodeResourceGroup", "properties.privateFQDN"] + + timeouts { + create = "2h" + delete = "2h" + } + + depends_on = [azurerm_role_assignment.network_contributor] +} diff --git a/quickstart/101-aks-automatic-private-custom-network-azapi/outputs.tf b/quickstart/101-aks-automatic-private-custom-network-azapi/outputs.tf new file mode 100644 index 000000000..fce5003c5 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network-azapi/outputs.tf @@ -0,0 +1,23 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "cluster_name" { + value = azapi_resource.aks_automatic.name +} + +output "cluster_id" { + value = azapi_resource.aks_automatic.id +} + +output "node_resource_group" { + value = azapi_resource.aks_automatic.output.properties.nodeResourceGroup +} + +output "virtual_network_name" { + value = azurerm_virtual_network.vnet.name +} + +output "private_fully_qualified_domain_name" { + value = azapi_resource.aks_automatic.output.properties.privateFQDN +} diff --git a/quickstart/101-aks-automatic-private-custom-network-azapi/providers.tf b/quickstart/101-aks-automatic-private-custom-network-azapi/providers.tf new file mode 100644 index 000000000..2c375b07c --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network-azapi/providers.tf @@ -0,0 +1,24 @@ +terraform { + required_version = ">= 1.0" + required_providers { + azapi = { + source = "Azure/azapi" + version = "~>2.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~>5.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} + +provider "azapi" { +} diff --git a/quickstart/101-aks-automatic-private-custom-network-azapi/readme.md b/quickstart/101-aks-automatic-private-custom-network-azapi/readme.md new file mode 100644 index 000000000..d30690194 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network-azapi/readme.md @@ -0,0 +1,55 @@ +# Private Azure Kubernetes Service (AKS) Automatic cluster in a custom virtual network (AzAPI provider) + +This template deploys a private AKS Automatic cluster into a custom virtual network, in a resource group with a random name beginning with "rg-". + +The virtual network contains a subnet delegated to the cluster API server, a subnet for the user node pools, and a subnet for the managed system node pool. The cluster uses a user-assigned managed identity that's granted the Network Contributor role on the virtual network, which is required when you bring your own network. Because the cluster is private, the API server endpoint is assigned a private IP address in the virtual network instead of a public one. Reaching it requires private network connectivity to that virtual network and DNS resolution of the private FQDN, so access isn't limited to clients inside the virtual network itself: virtual network peering, a VPN gateway, or Azure ExpressRoute can all provide a path from outside it. Enabling a public FQDN changes only how the cluster is named in DNS. It doesn't make the private endpoint routable from the internet. + +The cluster is declared with the AzAPI provider's [`azapi_resource`](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/resource) against the `Microsoft.ContainerService/managedClusters` API. Use this pattern when you need direct control over the managed cluster API payload, or when you need an API version or property that the AzureRM provider doesn't expose yet. The cluster SKU is hard-coded to `Automatic`. + +For the equivalent sample that uses the AzureRM provider's `azurerm_kubernetes_automatic_cluster` resource, see [101-aks-automatic-private-custom-network](../101-aks-automatic-private-custom-network/). For a public cluster in a custom virtual network, see [101-aks-automatic-custom-network-azapi](../101-aks-automatic-custom-network-azapi/). + +## Terraform resource types + +- [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) +- [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +- [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) +- [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) +- [azurerm_user_assigned_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) +- [azurerm_role_assignment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) +- [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/resource) + +## Variables + +| Name | Description | Default | Validation | +|-|-|-|-| +| `resource_group_name_prefix` | Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription. | rg | | +| `resource_group_location` | Location of the resource group. | westus2 | | +| `cluster_name_prefix` | Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription. | aks-automatic | | +| `virtual_network_name` | Name of the custom virtual network that hosts the cluster. | aks-automatic-vnet | | +| `identity_name` | Name of the user-assigned managed identity that the cluster uses. | aks-automatic-identity | | +| `virtual_network_address_space` | Address space of the custom virtual network. | ["172.19.0.0/16"] | | +| `api_server_subnet_address_prefixes` | Address prefixes of the subnet delegated to the cluster API server. | ["172.19.0.0/28"] | | +| `user_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the user node pools. | ["172.19.1.0/24"] | | +| `system_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the managed system node pool. | ["172.19.0.64/26"] | | + + +## Network guardrails + +This sample creates its own virtual network, so the defaults are safe as written. Read these before pointing the variables at an existing network: + +- **Subnet containment and non-overlap.** Every subnet prefix must sit inside `virtual_network_address_space` and must not overlap another subnet in the same virtual network. The defaults carve `172.19.0.0/28`, `172.19.0.64/26`, and `172.19.1.0/24` out of `172.19.0.0/16`. +- **Subnet sizing.** The API server subnet must be at least a `/28`, and AKS reserves at least nine addresses in it. Size the node subnets for the pod and node scale you expect, because a subnet that runs out of addresses blocks scaling. +- **Role assignment blast radius.** The cluster identity is granted **Network Contributor** on the whole virtual network, which is what Node Autoprovisioning needs. If you retarget this sample at a shared virtual network, that grant reaches every subnet and resource in it. Scope it more tightly, or keep the cluster in a dedicated network. +- **Delegation drift is hidden.** The managed system node subnet sets `ignore_changes = [delegation]` because AKS adds its own managed cluster delegation after the cluster is created. That keeps plans clean, but it also means Terraform won't report if the delegation is later changed or removed outside Terraform. + +## Example + +```console +terraform init -upgrade +terraform plan -out main.tfplan +terraform apply main.tfplan +``` + + + + diff --git a/quickstart/101-aks-automatic-private-custom-network-azapi/variables.tf b/quickstart/101-aks-automatic-private-custom-network-azapi/variables.tf new file mode 100644 index 000000000..8fe2b1ef4 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network-azapi/variables.tf @@ -0,0 +1,53 @@ +variable "resource_group_location" { + type = string + default = "westus2" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "cluster_name_prefix" { + type = string + default = "aks-automatic" + description = "Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription." +} + +variable "virtual_network_name" { + type = string + default = "aks-automatic-vnet" + description = "Name of the custom virtual network that hosts the cluster." +} + +variable "identity_name" { + type = string + default = "aks-automatic-identity" + description = "Name of the user-assigned managed identity that the cluster uses." +} + +variable "virtual_network_address_space" { + type = list(string) + default = ["172.19.0.0/16"] + description = "Address space of the custom virtual network." +} + +variable "api_server_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.0/28"] + description = "Address prefixes of the subnet delegated to the cluster API server." +} + +variable "user_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.1.0/24"] + description = "Address prefixes of the subnet that hosts the user node pools." +} + +variable "system_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.64/26"] + description = "Address prefixes of the subnet that hosts the managed system node pool." +} diff --git a/quickstart/101-aks-automatic-private-custom-network/main.tf b/quickstart/101-aks-automatic-private-custom-network/main.tf new file mode 100644 index 000000000..a80e65589 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network/main.tf @@ -0,0 +1,108 @@ +# Create a random name for the resource group using random_pet +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +# Create a resource group using the generated random name +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create the custom virtual network that hosts the cluster +resource "azurerm_virtual_network" "vnet" { + name = var.virtual_network_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + address_space = var.virtual_network_address_space +} + +# Create the subnet delegated to AKS for API Server VNet Integration +resource "azurerm_subnet" "api_server" { + name = "apiServerSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.api_server_subnet_address_prefixes + + delegation { + name = "aks-delegation" + + service_delegation { + name = "Microsoft.ContainerService/managedClusters" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +# Create the subnet that hosts the user node pools +resource "azurerm_subnet" "user_nodes" { + name = "userNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.user_node_subnet_address_prefixes +} + +# Create the subnet that hosts the managed system node pool +resource "azurerm_subnet" "system_nodes" { + name = "managedSystemNodeSubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.vnet.name + address_prefixes = var.system_node_subnet_address_prefixes + + lifecycle { + # AKS adds its own managed cluster delegation to this subnet after the + # cluster is created. + ignore_changes = [delegation] + } +} + +# Create the user-assigned managed identity used by the cluster +resource "azurerm_user_assigned_identity" "aks" { + name = var.identity_name + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Grant the cluster identity Network Contributor on the virtual network +resource "azurerm_role_assignment" "network_contributor" { + scope = azurerm_virtual_network.vnet.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks.principal_id + principal_type = "ServicePrincipal" +} + +# Create a random name for the AKS Automatic cluster +resource "random_pet" "cluster_name" { + prefix = var.cluster_name_prefix +} + +# Create the private AKS Automatic cluster in the custom virtual network +resource "azurerm_kubernetes_automatic_cluster" "aks_automatic" { + name = random_pet.cluster_name.id + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.aks.id] + } + + api_server_access { + subnet_id = azurerm_subnet.api_server.id + } + + hosted_system { + node_subnet_id = azurerm_subnet.user_nodes.id + system_node_subnet_id = azurerm_subnet.system_nodes.id + } + + # The private_cluster block gives the API server a private IP address in the + # virtual network. Reaching it requires private connectivity and DNS + # resolution of the private FQDN, which peering, a VPN, or ExpressRoute can + # provide from outside this virtual network. + private_cluster { + public_fully_qualified_domain_name_enabled = var.public_fully_qualified_domain_name_enabled + } + + depends_on = [azurerm_role_assignment.network_contributor] +} diff --git a/quickstart/101-aks-automatic-private-custom-network/outputs.tf b/quickstart/101-aks-automatic-private-custom-network/outputs.tf new file mode 100644 index 000000000..56b82dc53 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network/outputs.tf @@ -0,0 +1,23 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "cluster_name" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.name +} + +output "cluster_id" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.id +} + +output "node_resource_group_id" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.node_resource_group_id +} + +output "virtual_network_name" { + value = azurerm_virtual_network.vnet.name +} + +output "private_fully_qualified_domain_name" { + value = azurerm_kubernetes_automatic_cluster.aks_automatic.private_fully_qualified_domain_name +} diff --git a/quickstart/101-aks-automatic-private-custom-network/providers.tf b/quickstart/101-aks-automatic-private-custom-network/providers.tf new file mode 100644 index 000000000..9e76a7ea8 --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network/providers.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>5.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} diff --git a/quickstart/101-aks-automatic-private-custom-network/readme.md b/quickstart/101-aks-automatic-private-custom-network/readme.md new file mode 100644 index 000000000..40462a6cc --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network/readme.md @@ -0,0 +1,55 @@ +# Private Azure Kubernetes Service (AKS) Automatic cluster in a custom virtual network (AzureRM provider) + +This template deploys a private AKS Automatic cluster into a custom virtual network, in a resource group with a random name beginning with "rg-". + +The virtual network contains a subnet delegated to the cluster API server, a subnet for the user node pools, and a subnet for the managed system node pool. The cluster uses a user-assigned managed identity that's granted the Network Contributor role on the virtual network, which is required when you bring your own network. Because the cluster is private, the API server endpoint is assigned a private IP address in the virtual network instead of a public one. Reaching it requires private network connectivity to that virtual network and DNS resolution of the private FQDN, so access isn't limited to clients inside the virtual network itself: virtual network peering, a VPN gateway, or Azure ExpressRoute can all provide a path from outside it. Enabling a public FQDN changes only how the cluster is named in DNS. It doesn't make the private endpoint routable from the internet. + +The cluster is created with the AzureRM provider's [`azurerm_kubernetes_automatic_cluster`](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_automatic_cluster) resource, which is the recommended way to declare an AKS Automatic cluster. This resource requires AzureRM provider `v4.81` or later. + +For an equivalent sample that declares the same cluster with the AzAPI provider, see [101-aks-automatic-private-custom-network-azapi](../101-aks-automatic-private-custom-network-azapi/). For a public cluster in a custom virtual network, see [101-aks-automatic-custom-network](../101-aks-automatic-custom-network/). + +## Terraform resource types + +- [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) +- [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +- [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) +- [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) +- [azurerm_user_assigned_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) +- [azurerm_role_assignment](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) +- [azurerm_kubernetes_automatic_cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_automatic_cluster) + +## Variables + +| Name | Description | Default | +|-|-|-| +| `resource_group_name_prefix` | Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription. | rg | +| `resource_group_location` | Location of the resource group. | westus2 | +| `cluster_name_prefix` | Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription. | aks-automatic | +| `public_fully_qualified_domain_name_enabled` | Whether to provision a public FQDN for the private cluster. | false | +| `virtual_network_name` | Name of the custom virtual network that hosts the cluster. | aks-automatic-vnet | +| `identity_name` | Name of the user-assigned managed identity that the cluster uses. | aks-automatic-identity | +| `virtual_network_address_space` | Address space of the custom virtual network. | ["172.19.0.0/16"] | +| `api_server_subnet_address_prefixes` | Address prefixes of the subnet delegated to the cluster API server. | ["172.19.0.0/28"] | +| `user_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the user node pools. | ["172.19.1.0/24"] | +| `system_node_subnet_address_prefixes` | Address prefixes of the subnet that hosts the managed system node pool. | ["172.19.0.64/26"] | + + +## Network guardrails + +This sample creates its own virtual network, so the defaults are safe as written. Read these before pointing the variables at an existing network: + +- **Subnet containment and non-overlap.** Every subnet prefix must sit inside `virtual_network_address_space` and must not overlap another subnet in the same virtual network. The defaults carve `172.19.0.0/28`, `172.19.0.64/26`, and `172.19.1.0/24` out of `172.19.0.0/16`. +- **Subnet sizing.** The API server subnet must be at least a `/28`, and AKS reserves at least nine addresses in it. Size the node subnets for the pod and node scale you expect, because a subnet that runs out of addresses blocks scaling. +- **Role assignment blast radius.** The cluster identity is granted **Network Contributor** on the whole virtual network, which is what Node Autoprovisioning needs. If you retarget this sample at a shared virtual network, that grant reaches every subnet and resource in it. Scope it more tightly, or keep the cluster in a dedicated network. +- **Delegation drift is hidden.** The managed system node subnet sets `ignore_changes = [delegation]` because AKS adds its own managed cluster delegation after the cluster is created. That keeps plans clean, but it also means Terraform won't report if the delegation is later changed or removed outside Terraform. + +## Example + +```console +terraform init -upgrade +terraform plan -out main.tfplan +terraform apply main.tfplan +``` + + + diff --git a/quickstart/101-aks-automatic-private-custom-network/variables.tf b/quickstart/101-aks-automatic-private-custom-network/variables.tf new file mode 100644 index 000000000..28e4a7d0a --- /dev/null +++ b/quickstart/101-aks-automatic-private-custom-network/variables.tf @@ -0,0 +1,59 @@ +variable "resource_group_location" { + type = string + default = "westus2" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "cluster_name_prefix" { + type = string + default = "aks-automatic" + description = "Prefix of the AKS Automatic cluster name that's combined with a random ID so the name is unique in your Azure subscription." +} + +variable "public_fully_qualified_domain_name_enabled" { + type = bool + default = false + description = "Whether to provision a public FQDN for the private cluster." +} + +variable "virtual_network_name" { + type = string + default = "aks-automatic-vnet" + description = "Name of the custom virtual network that hosts the cluster." +} + +variable "identity_name" { + type = string + default = "aks-automatic-identity" + description = "Name of the user-assigned managed identity that the cluster uses." +} + +variable "virtual_network_address_space" { + type = list(string) + default = ["172.19.0.0/16"] + description = "Address space of the custom virtual network." +} + +variable "api_server_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.0/28"] + description = "Address prefixes of the subnet delegated to the cluster API server." +} + +variable "user_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.1.0/24"] + description = "Address prefixes of the subnet that hosts the user node pools." +} + +variable "system_node_subnet_address_prefixes" { + type = list(string) + default = ["172.19.0.64/26"] + description = "Address prefixes of the subnet that hosts the managed system node pool." +}