{AKS} Add FlexNodes support

src/aks-preview/HISTORY.rst

@@ -12,6 +12,14 @@ To release a new version, please select a new version number (usually plus 1 to
 Pending
 +++++++
 
+21.0.0b15
++++++++++
+* `az aks nodepool add`: Add preview support for creating FlexNodes pools with `--vm-set-type FlexNodes`.
+* `az aks nodepool get-bootstrap-data`: Add preview support for retrieving FlexNodes bootstrap data.
+* `az aks nodepool update`: Add preview support for updating FlexNodes labels, taints, and `maxUnavailable`.
+* `az aks nodepool upgrade`: Add Kubernetes version upgrade support for FlexNodes pools. Node image-only upgrades are not supported.
+* `az aks machine add/update` and `az aks nodepool delete-machines`: Add FlexNode machine lifecycle support.
+
 21.0.0b14
 +++++++++
 * Skip SSH key generation/validation for `az aks create --sku automatic` so `--no-ssh-key` is no longer required for Automatic clusters.

src/aks-preview/azext_aks_preview/_consts.py

@@ -52,6 +52,7 @@
 CONST_VIRTUAL_MACHINE_SCALE_SETS = "VirtualMachineScaleSets"
 CONST_AVAILABILITY_SET = "AvailabilitySet"
 CONST_VIRTUAL_MACHINES = "VirtualMachines"
+CONST_FLEX_NODES = "FlexNodes"
 
 # vm size
 CONST_DEFAULT_NODE_VM_SIZE = ""

src/aks-preview/azext_aks_preview/_help.py

@@ -2362,6 +2362,14 @@
     short-summary: List node pools in the managed Kubernetes cluster.
 """
 
+helps['aks nodepool get-bootstrap-data'] = """
+    type: command
+    short-summary: Get bootstrap data for a FlexNodes pool.
+    examples:
+        - name: Get bootstrap data for a FlexNodes pool
+          text: az aks nodepool get-bootstrap-data -g MyResourceGroup --cluster-name MyManagedCluster -n flexpool
+"""
+
 helps['aks nodepool add'] = """
     type: command
     short-summary: Add a node pool to the managed Kubernetes cluster.
@@ -2459,7 +2467,7 @@
           short-summary: The mode for a node pool which defines a node pool's primary function. If set as "System", AKS prefers system pods scheduling to node pools with mode `System`. If set as "ManagedSystem", all other properties except name and mode will be reset and managed by AKS. Learn more at https://aka.ms/aks/nodepool/mode.
         - name: --vm-set-type
           type: string
-          short-summary: Agent pool vm set type. VirtualMachineScaleSets, AvailabilitySet or VirtualMachines(Preview).
+          short-summary: Agent pool vm set type. VirtualMachineScaleSets, AvailabilitySet, VirtualMachines(Preview) or FlexNodes(Preview).
         - name: --aks-custom-headers
           type: string
           short-summary: Send custom headers. When specified, format should be Key1=Value1,Key2=Value2
@@ -2615,6 +2623,8 @@
           text: az aks nodepool add -g MyResourceGroup -n nodepool1 --cluster-name MyManagedCluster  --os-sku Ubuntu --pod-subnet-id /subscriptions/00000/resourceGroups/AnotherResourceGroup/providers/Microsoft.Network/virtualNetworks/MyVnet/subnets/MySubnet --pod-ip-allocation-mode StaticBlock
         - name: Create a nodepool of type VirtualMachines
           text: az aks nodepool add -g MyResourceGroup -n nodepool1 --cluster-name MyManagedCluster --vm-set-type VirtualMachines --vm-sizes "Standard_D4s_v3,Standard_D8s_v3" --node-count 3
+        - name: Create a FlexNodes pool for customer-provided machines
+          text: az aks nodepool add -g MyResourceGroup -n flexpool --cluster-name MyManagedCluster --vm-set-type FlexNodes --kubernetes-version 1.32
         - name: Create a nodepool with ManagedSystem mode
           text: az aks nodepool add -g MyResourceGroup -n managedsystem1 --cluster-name MyManagedCluster --mode ManagedSystem
         - name: Create a node pool with blue-green upgrade strategy and default parameters
@@ -3085,6 +3095,15 @@
        - name: --kubernetes-version
          type: string
          short-summary: Version of Kubernetes to use for creating the machine, such as "1.7.12" or "1.8.7".
+       - name: --labels
+         type: string
+         short-summary: Node labels for a FlexNode machine.
+       - name: --node-taints
+         type: string
+         short-summary: Node taints for a FlexNode machine.
+       - name: --max-pods -m
+         type: int
+         short-summary: Maximum pods for a FlexNode machine.
        - name: --enable-fips-image
          type: bool
          short-summary: Switch to use FIPS-enabled OS on the machine.
@@ -3139,6 +3158,9 @@
        - name: --node-taints
          type: string
          short-summary: The taints of the machine.
+       - name: --kubernetes-version
+         type: string
+         short-summary: Kubernetes version for a FlexNode machine. This option is not supported for regular machines.
 """
 
 helps['aks machine list'] = """

src/aks-preview/azext_aks_preview/_helpers.py

@@ -9,7 +9,7 @@
 import stat
 import sys
 import tempfile
-from typing import List, TypeVar
+from typing import Dict, List, Mapping, TypeVar
 
 import yaml
 from azext_aks_preview._client_factory import (
@@ -43,6 +43,55 @@
 ManagedCluster = TypeVar("ManagedCluster")
 allowed_extensions = ["microsoft.dataprotection.kubernetes"]
 
+# Resource identifiers and command controls do not describe FlexNodes capabilities.
+_FLEXNODES_COMMON_PARAMETERS = {
+    "resource_group_name",
+    "cluster_name",
+    "nodepool_name",
+    "machine_name",
+    "vm_set_type",
+    "no_wait",
+    "aks_custom_headers",
+    "yes",
+    "if_match",
+    "if_none_match",
+}
+
+
+def get_user_supplied_argument_options(cmd) -> Dict[str, str]:
+    """Return explicitly supplied command arguments and their canonical option names."""
+    safe_params = set(cmd.cli_ctx.data.get("safe_params") or [])
+    supplied_options = {}
+    for argument_name, argument in getattr(cmd, "arguments", {}).items():
+        options = [option for option in (getattr(argument, "options_list", None) or [])
+                   if isinstance(option, str)]
+        if any(option in safe_params for option in options):
+            supplied_options[argument_name] = next(
+                (option for option in options if option.startswith("--")), options[0]
+            )
+    return supplied_options
+
+
+def validate_flexnodes_options(
+    cmd,
+    command_parameters: Mapping[str, object],
+    supported_parameters: Mapping[str, str],
+) -> None:
+    """Reject explicitly supplied options outside an operation's FlexNodes capabilities."""
+    supplied_options = get_user_supplied_argument_options(cmd)
+    allowed_parameters = _FLEXNODES_COMMON_PARAMETERS | set(supported_parameters)
+    unsupported_options = sorted({
+        option for name, option in supplied_options.items()
+        if name in command_parameters and name not in allowed_parameters
+    })
+    if unsupported_options:
+        raise InvalidArgumentValueError(
+            "The following options are not supported for FlexNodes pools: {}. "
+            "Supported FlexNodes pool options are: {}.".format(
+                ", ".join(unsupported_options), ", ".join(supported_parameters.values())
+            )
+        )
+
 
 def which(binary):
     path_var = os.getenv('PATH')

src/aks-preview/azext_aks_preview/_params.py

@@ -2700,6 +2700,9 @@ def load_arguments(self, _):
             arg_type=get_enum_type(node_eviction_policies),
             validator=validate_eviction_policy,
         )
+        c.argument("labels", nargs="*", validator=validate_nodepool_labels)
+        c.argument("node_taints", validator=validate_nodepool_taints)
+        c.argument("max_pods", type=int, options_list=["--max-pods", "-m"])
 
     with self.argument_context("aks machine update") as c:
         c.argument(
@@ -2708,6 +2711,12 @@ def load_arguments(self, _):
         c.argument("tags", tags_type, help="The tags to set on the machine.")
         c.argument("node_taints", validator=validate_nodepool_taints)
         c.argument("labels", nargs="*", help="Labels to set on the machine.")
+        c.argument(
+            "kubernetes_version",
+            options_list=["--kubernetes-version"],
+            validator=validate_k8s_version,
+            help="Kubernetes version to use for a FlexNode machine.",
+        )
 
     with self.argument_context("aks operation") as c:
         c.argument(

src/aks-preview/azext_aks_preview/_validators.py

@@ -244,9 +244,11 @@ def validate_vm_set_type(namespace):
             return
         if namespace.vm_set_type.lower() != "availabilityset" and \
             namespace.vm_set_type.lower() != "virtualmachines" and \
-                namespace.vm_set_type.lower() != "virtualmachinescalesets":
+            namespace.vm_set_type.lower() != "virtualmachinescalesets" and \
+                namespace.vm_set_type.lower() != "flexnodes":
             raise CLIError(
-                "--vm-set-type can only be VirtualMachineScaleSets, AvailabilitySet or VirtualMachines(Preview)")
+                "--vm-set-type can only be VirtualMachineScaleSets, AvailabilitySet, "
+                "VirtualMachines(Preview), or FlexNodes(Preview)")
 
 
 def validate_load_balancer_sku(namespace):

src/aks-preview/azext_aks_preview/agentpool_decorator.py

@@ -40,6 +40,7 @@
     CONST_VIRTUAL_MACHINE_SCALE_SETS,
     CONST_AVAILABILITY_SET,
     CONST_VIRTUAL_MACHINES,
+    CONST_FLEX_NODES,
     CONST_DEFAULT_NODE_VM_SIZE,
     CONST_DEFAULT_WINDOWS_NODE_VM_SIZE,
     CONST_DEFAULT_VMS_VM_SIZE,
@@ -57,6 +58,7 @@
     get_nodepool_snapshot_by_snapshot_id,
     filter_hard_taints,
     process_dns_overrides,
+    validate_flexnodes_options,
 )
 
 logger = get_logger(__name__)
@@ -140,9 +142,12 @@ def get_vm_set_type(self) -> str:
             vm_set_type = CONST_AVAILABILITY_SET
         elif vm_set_type.lower() == CONST_VIRTUAL_MACHINES.lower():
             vm_set_type = CONST_VIRTUAL_MACHINES
+        elif vm_set_type.lower() == CONST_FLEX_NODES.lower():
+            vm_set_type = CONST_FLEX_NODES
         else:
             raise InvalidArgumentValueError(
-                "--vm-set-type can only be VirtualMachineScaleSets, AvailabilitySet or VirtualMachines(Preview)"
+                "--vm-set-type can only be VirtualMachineScaleSets, AvailabilitySet, "
+                "VirtualMachines(Preview) or FlexNodes(Preview)"
             )
         # this parameter does not need validation
         return vm_set_type
@@ -1302,6 +1307,25 @@ def init_context(self) -> None:
             self.agentpool_decorator_mode,
         )
 
+    def _validate_flexnodes_add_options(self) -> None:
+        if self.agentpool_decorator_mode != AgentPoolDecoratorMode.STANDALONE:
+            return
+        vm_set_type = self.__raw_parameters.get("vm_set_type")
+        if not vm_set_type or vm_set_type.lower() != CONST_FLEX_NODES.lower():
+            return
+        validate_flexnodes_options(
+            self.cmd,
+            self.__raw_parameters,
+            {
+                "kubernetes_version": "--kubernetes-version",
+                "labels": "--labels",
+                "max_pods": "--max-pods",
+                "max_unavailable": "--max-unavailable",
+                "mode": "--mode",
+                "node_taints": "--node-taints",
+            },
+        )
+
     def set_up_preview_vm_properties(self, agentpool: AgentPool) -> AgentPool:
         """Set up preview vm related properties for the AgentPool object.
 
@@ -1314,6 +1338,33 @@ def set_up_preview_vm_properties(self, agentpool: AgentPool) -> AgentPool:
             agentpool.capacity_reservation_group_id = crg_id
         return agentpool
 
+    def _keep_supported_flexnodes_properties(self, agentpool: AgentPool) -> AgentPool:
+        """Keep only properties supported by FlexNodes pools."""
+        supported_properties = {
+            "name",
+            "orchestrator_version",
+            "max_pods",
+            "mode",
+            "node_labels",
+            "node_taints",
+            "type",
+            "type_properties_type",
+            "upgrade_settings",
+        }
+        properties = getattr(agentpool, "properties", None) or agentpool
+        for property_name in properties._attr_to_rest_field:  # pylint: disable=protected-access
+            if property_name not in supported_properties:
+                setattr(agentpool, property_name, None)
+
+        upgrade_settings = agentpool.upgrade_settings
+        if upgrade_settings is not None:
+            for property_name in upgrade_settings._attr_to_rest_field:  # pylint: disable=protected-access
+                if property_name != "max_unavailable":
+                    setattr(upgrade_settings, property_name, None)
+            if not upgrade_settings.as_dict():
+                agentpool.upgrade_settings = None
+        return agentpool
+
     def set_up_motd(self, agentpool: AgentPool) -> AgentPool:
         """Set up message of the day for the AgentPool object.
 
@@ -1686,7 +1737,8 @@ def construct_agentpool_profile_preview(self) -> AgentPool:
 
         :return: the AgentPool object
         """
-        # DO NOT MOVE: keep this on top, construct the default AgentPool profile
+        self._validate_flexnodes_add_options()
+        # DO NOT MOVE: construct the default AgentPool profile before applying preview properties
         agentpool = self.construct_agentpool_profile_default(bypass_restore_defaults=True)
 
         # Check if mode is ManagedSystem or Machines, if yes, reset all other properties
@@ -1745,6 +1797,9 @@ def construct_agentpool_profile_preview(self) -> AgentPool:
         agentpool = self.set_up_prepared_image_specification(agentpool)
         # DO NOT MOVE: keep this at the bottom, restore defaults
         agentpool = self._restore_defaults_in_agentpool(agentpool)
+        vm_set_type = getattr(agentpool, "type_properties_type", getattr(agentpool, "type", None))
+        if vm_set_type == CONST_FLEX_NODES:
+            agentpool = self._keep_supported_flexnodes_properties(agentpool)
         return agentpool
 
     def set_up_upgrade_strategy(self, agentpool: AgentPool) -> AgentPool:
@@ -1880,6 +1935,20 @@ def init_context(self) -> None:
             self.agentpool_decorator_mode,
         )
 
+    def _validate_flexnodes_update_options(self, agentpool: AgentPool) -> None:
+        if self.agentpool_decorator_mode != AgentPoolDecoratorMode.STANDALONE or \
+                agentpool.type_properties_type != CONST_FLEX_NODES:
+            return
+        validate_flexnodes_options(
+            self.cmd,
+            self.__raw_parameters,
+            {
+                "labels": "--labels",
+                "max_unavailable": "--max-unavailable",
+                "node_taints": "--node-taints",
+            },
+        )
+
     def update_network_profile(self, agentpool: AgentPool) -> AgentPool:
         self._ensure_agentpool(agentpool)
 
@@ -2116,6 +2185,8 @@ def update_agentpool_profile_preview(self, agentpools: List[AgentPool] = None) -
         """
         # DO NOT MOVE: keep this on top, fetch and update the default AgentPool profile
         agentpool = self.update_agentpool_profile_default(agentpools)
+        # Update has no --vm-set-type argument; the inherited path fetches the pool before we can validate its type.
+        self._validate_flexnodes_update_options(agentpool)
 
         # Check if agentpool is in ManagedSystem mode and handle special case
         if agentpool.mode == CONST_NODEPOOL_MODE_MANAGEDSYSTEM:

src/aks-preview/azext_aks_preview/commands.py

@@ -313,6 +313,13 @@ def load_command_table(self, _):
         g.custom_command("update", "aks_agentpool_update", supports_no_wait=True)
         g.custom_command("delete", "aks_agentpool_delete", supports_no_wait=True)
         g.custom_command("get-upgrades", "aks_agentpool_get_upgrade_profile")
+        g.custom_command(
+            "get-bootstrap-data",
+            "aks_agentpool_get_bootstrap_data",
+            sensitive_info=g.sensitive(
+                sensitive_keys=["bootstrapToken", "caCertData"]
+            ),
+        )
         g.custom_command(
             "get-rollback-versions",
             "aks_agentpool_get_rollback_versions",