From 9bf9aa77824f474ab41a1f9963e93f490dff52f6 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:24:36 +0000 Subject: [PATCH 01/10] aks-preview: Support BYO VNet for --enable-hosted-system automatic clusters + --disable-hosted-system Adds CLI surface for BYO VNet HOBO (hosted system pool) automatic clusters: * `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id` on `az aks create` to bring your own VNet for the hosted system pool and user node pool. Must be used together with `--apiserver-subnet-id` and `--enable-hosted-system`. * `--disable-hosted-system` on `az aks create` to deterministically opt out of HOBO on automatic clusters (mutually exclusive with `--enable-hosted-system`, both gated to `--sku automatic`). Supported scenarios: 1. az aks create --sku automatic --enable-hosted-system 2. ... + --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id (NATGW) 3. ... + --outbound-type loadBalancer for BYO VNet with SLB outbound 4. az aks create --sku automatic --disable-hosted-system 5. az aks update --sku base to downgrade an automatic+HOBO cluster Validation (client-side, before PATCH): * --enable-hosted-system and --disable-hosted-system are mutually exclusive. * Both require --sku automatic. * If --enable-hosted-system is set with any of the 3 BYO subnet flags, all three must be provided; otherwise a clear error lists the missing ones. * BYO subnet flags cannot be used without --enable-hosted-system. Live-only E2E tests cover BYO+NATGW, BYO+SLB with downgrade to base SKU, and the disable opt-out path. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/HISTORY.rst | 1 + src/aks-preview/azext_aks_preview/_help.py | 28 ++ src/aks-preview/azext_aks_preview/_params.py | 15 + .../azext_aks_preview/_validators.py | 8 + src/aks-preview/azext_aks_preview/custom.py | 3 + .../managed_cluster_decorator.py | 304 ++++++++++++++++-- .../tests/latest/test_aks_commands.py | 223 ++++++++++++- 7 files changed, 552 insertions(+), 30 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index 92bce607137..085a13fd80e 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,6 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id`; add `--disable-hosted-system` opt-out. 21.0.0b1 ++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index a06b6051ae8..63ea7a10f48 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -712,6 +712,28 @@ - name: --enable-hosted-system type: bool short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. + long-summary: | + Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. + Can be combined with BYO VNet via `--system-node-vnet-subnet-id`, `--node-vnet-subnet-id`, and `--apiserver-subnet-id` + (all three must be provided together and must belong to the same VNet). Cannot be used with `--disable-hosted-system`. + - name: --disable-hosted-system + type: bool + short-summary: Opt the automatic cluster out of hosted system components. + long-summary: | + Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. + Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. + - name: --system-node-vnet-subnet-id + type: string + short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--node-vnet-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. + - name: --node-vnet-subnet-id + type: string + short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-vnet-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - name: --control-plane-scaling-size --cp-scaling-size type: string short-summary: (PREVIEW) The control plane scaling size for the cluster. @@ -814,6 +836,12 @@ text: az aks create -g MyResourceGroup -n MyManagedCluster --control-plane-scaling-size H4 - name: Create an automatic cluster with hosted system components enabled. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system + - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id + - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id --outbound-type loadBalancer + - name: Create an automatic cluster and opt out of hosted system components. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system """ diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index f3dfa8af56b..05f41167c3b 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -235,6 +235,8 @@ validate_utc_offset, validate_vm_set_type, validate_vnet_subnet_id, + validate_system_node_vnet_subnet_id, + validate_node_vnet_subnet_id, validate_force_upgrade_disable_and_enable_parameters, validate_azure_service_mesh_revision, validate_artifact_streaming, @@ -1257,6 +1259,19 @@ def load_arguments(self, _): help="Enable Gateway API based ingress on App Routing via Istio" ) c.argument("enable_hosted_system", action="store_true", is_preview=True) + c.argument("disable_hosted_system", action="store_true", is_preview=True) + c.argument( + "system_node_vnet_subnet_id", + options_list=["--system-node-vnet-subnet-id"], + validator=validate_system_node_vnet_subnet_id, + is_preview=True, + ) + c.argument( + "node_vnet_subnet_id", + options_list=["--node-vnet-subnet-id"], + validator=validate_node_vnet_subnet_id, + is_preview=True, + ) c.argument( "control_plane_scaling_size", options_list=["--control-plane-scaling-size", "--cp-scaling-size"], diff --git a/src/aks-preview/azext_aks_preview/_validators.py b/src/aks-preview/azext_aks_preview/_validators.py index aa43b9052a2..1e4039e4822 100644 --- a/src/aks-preview/azext_aks_preview/_validators.py +++ b/src/aks-preview/azext_aks_preview/_validators.py @@ -355,6 +355,14 @@ def validate_apiserver_subnet_id(namespace): _validate_subnet_id(namespace.apiserver_subnet_id, "--apiserver-subnet-id") +def validate_system_node_vnet_subnet_id(namespace): + _validate_subnet_id(namespace.system_node_vnet_subnet_id, "--system-node-vnet-subnet-id") + + +def validate_node_vnet_subnet_id(namespace): + _validate_subnet_id(namespace.node_vnet_subnet_id, "--node-vnet-subnet-id") + + def _validate_subnet_id(subnet_id, name): if subnet_id is None or subnet_id == '': return diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index d73fd9b8b71..f36ba2abe80 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1181,6 +1181,9 @@ def aks_create( # app routing istio enable_app_routing_istio=False, enable_hosted_system=False, + disable_hosted_system=False, + system_node_vnet_subnet_id=None, + node_vnet_subnet_id=None, control_plane_scaling_size=None, # health monitor enable_continuous_control_plane_and_addon_monitor=False, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 60f22700aaa..5897d05abf2 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -602,12 +602,28 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_NONE, CONST_OUTBOUND_TYPE_BLOCK,] ): + # Preserve the user's explicit --outbound-type (if any) before default-completion + # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. + user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER skuName = self.get_sku_name() isVnetSubnetIdEmpty = self.get_vnet_subnet_id() in ["", None] - if skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and isVnetSubnetIdEmpty: - # outbound_type of Automatic SKU should be ManagedNATGateway if no subnet id provided. - outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY + # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / + # --node-vnet-subnet-id instead of --vnet-subnet-id. + hobo_byo_subnets = bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if ( + skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and + isVnetSubnetIdEmpty + ): + # Default outbound for Automatic SKU without a VNet is managedNATGateway. + # For BYO HOBO, only apply this default when the user did NOT explicitly pass + # --outbound-type; otherwise we would silently overwrite a user's explicit + # --outbound-type loadBalancer (regression seen in BYO SLB scenario). + if not (hobo_byo_subnets and user_supplied_outbound_type): + outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY # validation # Note: The parameters involved in the validation are not verified in their own getters. @@ -1925,8 +1941,18 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # validation if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: + # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse + # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) + # is rejected up front rather than silently dropped. + self._validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() - if apiserver_subnet_id and vnet_subnet_id is None: + # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and + # --node-vnet-subnet-id replace --vnet-subnet-id. + hobo_byo_subnets = ( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if apiserver_subnet_id and vnet_subnet_id is None and not hobo_byo_subnets: raise RequiredArgumentMissingError( '"--apiserver-subnet-id" requires "--vnet-subnet-id".') @@ -3951,10 +3977,94 @@ def get_enable_hosted_system(self) -> bool: :return: bool """ enable_hosted_system = self.raw_param.get("enable_hosted_system") + disable_hosted_system = self.raw_param.get("disable_hosted_system") + if enable_hosted_system and disable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) if enable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: raise RequiredArgumentMissingError('"--enable-hosted-system" requires "--sku automatic".') return enable_hosted_system + def get_disable_hosted_system(self) -> bool: + """Obtain the value of disable_hosted_system. + + :return: bool + """ + disable_hosted_system = self.raw_param.get("disable_hosted_system") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + if disable_hosted_system and enable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) + if disable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') + return disable_hosted_system + + def get_system_node_vnet_subnet_id(self) -> Union[str, None]: + """Obtain the value of system_node_vnet_subnet_id. + + Validates that BYO VNet subnet flags are used together with + --enable-hosted-system and that the full triple + (system-node / node / apiserver) is provided. + + :return: str or None + """ + system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + self._validate_byo_hobo_subnet_trio() + return system_node_vnet_subnet_id + + def get_node_vnet_subnet_id(self) -> Union[str, None]: + """Obtain the value of node_vnet_subnet_id. + + :return: str or None + """ + node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + self._validate_byo_hobo_subnet_trio() + return node_vnet_subnet_id + + def _validate_byo_hobo_subnet_trio(self) -> None: + """Cross-validate the BYO VNet HOBO subnet flags. + + Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or + --apiserver-subnet-id is set together with --enable-hosted-system, then + all three must be set (SDK requires all three subnets to share a VNet). + Any of these flags set without --enable-hosted-system is flagged so + the user doesn't silently get a non-HOBO cluster with the wrong shape. + """ + system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + + # The HOBO-specific flags only make sense with --enable-hosted-system. + # --apiserver-subnet-id is a general VNet-integration flag, so only + # require HOBO on it when one of the other two is also set. + hobo_byo_flags_set = bool(system_node_vnet_subnet_id) or bool(node_vnet_subnet_id) + if hobo_byo_flags_set and not enable_hosted_system: + raise RequiredArgumentMissingError( + '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' + 'require "--enable-hosted-system".' + ) + + if enable_hosted_system and ( + system_node_vnet_subnet_id or node_vnet_subnet_id or apiserver_subnet_id + ): + missing = [] + if not system_node_vnet_subnet_id: + missing.append("--system-node-vnet-subnet-id") + if not node_vnet_subnet_id: + missing.append("--node-vnet-subnet-id") + if not apiserver_subnet_id: + missing.append("--apiserver-subnet-id") + if missing: + raise RequiredArgumentMissingError( + "BYO VNet for hosted-system clusters requires all of " + "--system-node-vnet-subnet-id, --node-vnet-subnet-id, and " + "--apiserver-subnet-id to be provided together. " + f"Missing: {', '.join(missing)}." + ) + def get_control_plane_scaling_size(self) -> Union[str, None]: """Obtain the value of control_plane_scaling_size. @@ -4167,29 +4277,43 @@ def set_up_run_command(self, mc: ManagedCluster) -> ManagedCluster: def set_up_api_server_access_profile(self, mc: ManagedCluster) -> ManagedCluster: """Set up apiserverAccessProfile enableVnetIntegration and subnetId for the ManagedCluster object. - Note: Inherited and extended in aks-preview to set vnet integration configs. + Note: This is a full override (not calling super()) because the base acs module writes + `enableVnetIntegration` / `subnetId` via msrest-style `additional_properties`, which the + vendored 2026-02-02-preview SDK (azure.core Model) does not expose and would raise + AttributeError. The logic below mirrors the base implementation (authorized IP ranges, + private cluster, public FQDN, private DNS zone, fqdn subdomain) but writes + `enable_vnet_integration` and `subnet_id` as typed fields. :return: the ManagedCluster object """ - mc = super().set_up_api_server_access_profile(mc) + self._ensure_mc(mc) + + api_server_access_profile = None + api_server_authorized_ip_ranges = self.context.get_api_server_authorized_ip_ranges() + enable_private_cluster = self.context.get_enable_private_cluster() + disable_public_fqdn = self.context.get_disable_public_fqdn() + private_dns_zone = self.context.get_private_dns_zone() + if api_server_authorized_ip_ranges or enable_private_cluster: + # pylint: disable=no-member + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile( + authorized_ip_ranges=api_server_authorized_ip_ranges, + enable_private_cluster=True if enable_private_cluster else None, + enable_private_cluster_public_fqdn=False if disable_public_fqdn else None, + private_dns_zone=private_dns_zone, + ) if self.context.get_enable_apiserver_vnet_integration(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.enable_vnet_integration = True + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.enable_vnet_integration = True if self.context.get_apiserver_subnet_id(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + mc.api_server_access_profile = api_server_access_profile - if ( - mc.api_server_access_profile is not None and - hasattr(mc.api_server_access_profile, 'additional_properties') and - mc.api_server_access_profile.additional_properties is not None - ): - # remove the additional properties that are set in official azure-cli/acs - mc.api_server_access_profile.additional_properties = {} + mc.fqdn_subdomain = self.context.get_fqdn_subdomain() return mc def build_gitops_addon_profile(self) -> ManagedClusterAddonProfile: @@ -5143,17 +5267,123 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: self._ensure_mc(mc) enable_hosted_components = self.context.get_enable_hosted_system() + disable_hosted_components = self.context.get_disable_hosted_system() if enable_hosted_components: if mc.hosted_system_profile is None: mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member mc.hosted_system_profile.enabled = True + # BYO VNet: plumb subnet IDs through to the SDK model. All three + # subnets (system-node / node / apiserver) must share a VNet, but + # the server enforces that check. + system_node_vnet_subnet_id = self.context.get_system_node_vnet_subnet_id() + node_vnet_subnet_id = self.context.get_node_vnet_subnet_id() + if system_node_vnet_subnet_id: + mc.hosted_system_profile.system_node_subnet_id = system_node_vnet_subnet_id + if node_vnet_subnet_id: + mc.hosted_system_profile.node_subnet_id = node_vnet_subnet_id + # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: mc.agent_pool_profiles = None + elif disable_hosted_components: + # Deterministic opt-out: explicitly set enabled=False so customers + # don't get HOBO once the server flips the default in their region. + if mc.hosted_system_profile is None: + mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member + mc.hosted_system_profile.enabled = False return mc + def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> None: + """Extend base role assignment to also cover BYO VNet HOBO subnets. + + Base behavior: if ``--vnet-subnet-id`` is provided, grant Network Contributor on + that subnet to the cluster identity (SP or UAMI). BYO HOBO uses three separate + subnets (``--system-node-vnet-subnet-id``, ``--node-vnet-subnet-id``, + ``--apiserver-subnet-id``) instead of ``--vnet-subnet-id``; without this + override, cluster creation fails with ``ResourceMissingPermissionError`` on the + BYO subnets. + + Strategy: call super() so the original ``--vnet-subnet-id`` path still works, + then iterate over any HOBO BYO subnets and run the same role-assignment logic + for each. Skipping is honored via ``--skip-subnet-role-assignment``. + """ + # Preserve base behavior for the --vnet-subnet-id case. + super().process_add_role_assignment_for_vnet_subnet(mc) + + # Only extend for BYO VNet HOBO; outside that mode --apiserver-subnet-id keeps its + # generic apiserver-VNet-integration meaning and must NOT trigger an extra RBAC grant. + if not self.context.get_enable_hosted_system(): + return + + # Trio validation fires earlier via _get_apiserver_subnet_id on CREATE, so by the + # time we reach here we should have either all three HOBO subnets or none. Defend + # anyway — skip cleanly when no HOBO subnets are present. + hobo_subnets = [] + seen = set() + for raw_key in ( + "system_node_vnet_subnet_id", + "node_vnet_subnet_id", + "apiserver_subnet_id", + ): + subnet_id = self.context.raw_param.get(raw_key) + if subnet_id and subnet_id not in seen: + seen.add(subnet_id) + hobo_subnets.append(subnet_id) + + if not hobo_subnets: + return + + if self.context.get_skip_subnet_role_assignment(): + return + + service_principal_profile = mc.service_principal_profile + assign_identity = self.context.get_assign_identity() + + # For system-assigned identity clusters the SP does not exist yet and we can + # only grant after the cluster is created. Defer via the existing post-create + # flag AND stash the HOBO subnet list so the post-create handler can iterate it; + # base behavior only grants on --vnet-subnet-id, which is absent for BYO HOBO. + if service_principal_profile is None and not assign_identity: + self.context.set_intermediate( + "need_post_creation_vnet_permission_granting", + True, + overwrite_exists=True, + ) + self.context.set_intermediate( + "hobo_byo_subnets_pending_grant", + hobo_subnets, + overwrite_exists=True, + ) + return + + for subnet_id in hobo_subnets: + if self.context.external_functions.subnet_role_assignment_exists(self.cmd, subnet_id): + continue + if assign_identity: + identity_object_id = self.context.get_user_assigned_identity_object_id() + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + identity_object_id, + is_service_principal=False, + scope=subnet_id, + ) + else: + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + service_principal_profile.client_id, + scope=subnet_id, + ) + if not granted: + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + subnet_id, + ) + # pylint: disable=unused-argument def construct_mc_profile_preview(self, bypass_restore_defaults: bool = False) -> ManagedCluster: """The overall controller used to construct the default ManagedCluster profile. @@ -5322,16 +5552,32 @@ def immediate_processing_after_request(self, mc: ManagedCluster) -> None: # Grant vnet permission to system assigned identity RIGHT AFTER the cluster is put, this operation can # reduce latency for the role assignment take effect instant_cluster = self.client.get(self.context.get_resource_group_name(), self.context.get_name()) - if not self.context.external_functions.add_role_assignment( - self.cmd, - "Network Contributor", - instant_cluster.identity.principal_id, - scope=self.context.get_vnet_subnet_id(), - is_service_principal=False, - ): - logger.warning( - "Could not create a role assignment for subnet. Are you an Owner on this subscription?" - ) + # Determine the scopes to grant: base behavior uses --vnet-subnet-id only; BYO VNet HOBO + # uses the three HOBO subnets stashed by process_add_role_assignment_for_vnet_subnet. + # Iterate a list so both classic and HOBO cases share the same code path. + scopes = [] + vnet_subnet_id = self.context.get_vnet_subnet_id() + if vnet_subnet_id: + scopes.append(vnet_subnet_id) + hobo_subnets = self.context.get_intermediate( + "hobo_byo_subnets_pending_grant", default_value=[] + ) + for subnet in hobo_subnets or []: + if subnet and subnet not in scopes: + scopes.append(subnet) + for scope in scopes: + if not self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + instant_cluster.identity.principal_id, + scope=scope, + is_service_principal=False, + ): + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + scope, + ) # pylint: disable=too-many-locals,too-many-branches def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 32cbffb7fa2..862901cd1af 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -5984,7 +5984,7 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso # so we only need to test for cluster creation here create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--aks-custom-header AKSHTTPCustomFeatures=Microsoft.ContainerService/AutomaticSKUPreview," "AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-AutomaticHostedSystemProfilePreview " "--ssh-key-value={ssh_key_value}" @@ -6016,6 +6016,227 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso checks=[self.is_empty()], ) + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster (default outbound = NAT gateway) + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + ], + ) + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with loadBalancer outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type loadBalancer " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "loadBalancer"), + ], + ) + + # convert to Base SKU; expect the transition to succeed for the BYO VNet HOBO case. + # Wait for the cluster to become idle (no in-progress RP reconciliation) before starting + # the update; without this the RP returns 409 OperationNotAllowed for a few minutes after + # create completes while it finishes post-create reconciliation. + self.cmd( + "aks wait -g {resource_group} -n {name} --created --timeout 900", + checks=[self.is_empty()], + ) + time.sleep(180) + self.cmd( + "aks update -g {resource_group} -n {name} --sku base", + checks=[ + self.check("sku.name", "Base"), + ], + ) + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_with_hosted_system_disabled(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # opt out of HOBO even if the region defaults to it + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --disable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", False), + ], + ) + + # agent pool profiles should be present (non-HOBO automatic) + self.cmd( + "aks show --resource-group={resource_group} --name={name}", + checks=[self.greater_than("length(agentPoolProfiles)", 0)], + ) + @AllowLargeResponse() @AKSCustomResourceGroupPreparer( random_name_length=17, name_prefix="clitest", location="westus2" From ad595479339910cc3a6061b0ba38de31747bd17c Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 00:59:25 +0000 Subject: [PATCH 02/10] aks-preview: Address CI linter/style + Copilot review feedback * Add short alias --sys-node-subnet-id for --system-node-vnet-subnet-id to satisfy option_length_too_long linter rule. * Rename skuName/isVnetSubnetIdEmpty to snake_case per PEP 8. * Disable too-many-branches pylint warning on _get_outbound_type (overridden from base azure-cli and the preview-specific branches are necessary). * Replace fixed 180s sleep before aks update --sku base with a retry loop that handles the RP's post-create 409 OperationNotAllowed window more robustly. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_params.py | 2 +- .../managed_cluster_decorator.py | 9 +++--- .../tests/latest/test_aks_commands.py | 29 ++++++++++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index 05f41167c3b..c4fca1ed862 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -1262,7 +1262,7 @@ def load_arguments(self, _): c.argument("disable_hosted_system", action="store_true", is_preview=True) c.argument( "system_node_vnet_subnet_id", - options_list=["--system-node-vnet-subnet-id"], + options_list=["--system-node-vnet-subnet-id", "--sys-node-subnet-id"], validator=validate_system_node_vnet_subnet_id, is_preview=True, ) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 5897d05abf2..3028c0b209e 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -545,6 +545,7 @@ def _get_disable_local_accounts(self, enable_validation: bool = False) -> bool: ) return disable_local_accounts + # pylint: disable=too-many-branches def _get_outbound_type( self, enable_validation: bool = False, @@ -606,8 +607,8 @@ def _get_outbound_type( # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER - skuName = self.get_sku_name() - isVnetSubnetIdEmpty = self.get_vnet_subnet_id() in ["", None] + sku_name = self.get_sku_name() + is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / # --node-vnet-subnet-id instead of --vnet-subnet-id. hobo_byo_subnets = bool( @@ -615,8 +616,8 @@ def _get_outbound_type( self.raw_param.get("node_vnet_subnet_id") ) if ( - skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and - isVnetSubnetIdEmpty + sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and + is_vnet_subnet_id_empty ): # Default outbound for Automatic SKU without a VNet is managedNATGateway. # For BYO HOBO, only apply this default when the user did NOT explicitly pass diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 862901cd1af..f9b25ca6884 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6190,13 +6190,28 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou "aks wait -g {resource_group} -n {name} --created --timeout 900", checks=[self.is_empty()], ) - time.sleep(180) - self.cmd( - "aks update -g {resource_group} -n {name} --sku base", - checks=[ - self.check("sku.name", "Base"), - ], - ) + # Poll with retries: `aks wait --created` returns once provisioningState hits Succeeded, + # but the RP can still have an internal in-progress reconciliation for several minutes + # afterwards which surfaces as 409 OperationNotAllowed on `aks update`. Back off and retry. + update_deadline = time.time() + 900 + last_error = None + while time.time() < update_deadline: + try: + self.cmd( + "aks update -g {resource_group} -n {name} --sku base", + checks=[self.check("sku.name", "Base")], + ) + last_error = None + break + except Exception as ex: # pylint: disable=broad-except + message = str(ex) + if "OperationNotAllowed" in message or "in progress" in message or "409" in message: + last_error = ex + time.sleep(60) + continue + raise + if last_error is not None: + raise last_error @live_only() @AllowLargeResponse() From ce7055e58ad51d3ff2f1cbf2ec0939e53b335261 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 01:35:46 +0000 Subject: [PATCH 03/10] aks-preview: Fix help name for --system-node-vnet-subnet-id alias Include both --system-node-vnet-subnet-id and --sys-node-subnet-id in the help entry name so azdev linter recognizes all option aliases and does not report unrecognized_help_parameter_rule / missing_parameter_help. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 63ea7a10f48..70684ffb9f6 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -722,7 +722,7 @@ long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --system-node-vnet-subnet-id + - name: --system-node-vnet-subnet-id --sys-node-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | From a58ee076f4023ef62039213b492156cc283ba96b Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 01:43:24 +0000 Subject: [PATCH 04/10] aks-preview: Sort options in help name to match azdev linter expectation The azdev linter compares help parameter names against knack's HelpParameter.name which is built by sorting options alphabetically (knack/help.py line 349). Swap the order so --sys-node-subnet-id comes before --system-node-vnet-subnet-id. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/azext_aks_preview/_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index 70684ffb9f6..da96223fc93 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -722,7 +722,7 @@ long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --system-node-vnet-subnet-id --sys-node-subnet-id + - name: --sys-node-subnet-id --system-node-vnet-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | From 008a392bc7e4fa10ba6977fbf88b80ad1c73ed60 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:52:33 +0000 Subject: [PATCH 05/10] aks-preview: Allow userAssignedNATGateway outbound for BYO HOBO The _get_outbound_type validation previously required --vnet-subnet-id when outbound_type is userAssignedNATGateway / userDefinedRouting. For BYO HOBO automatic clusters the VNet is provided via --system-node-vnet-subnet-id / --node-vnet-subnet-id instead, so treat those as satisfying the requirement. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../azext_aks_preview/managed_cluster_decorator.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 3028c0b209e..4af3ff36b1e 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -645,7 +645,17 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_USER_DEFINED_ROUTING, CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, ]: - if self.get_vnet_subnet_id() in ["", None]: + # BYO HOBO scenarios satisfy the VNet requirement via + # --system-node-vnet-subnet-id / --node-vnet-subnet-id + # instead of --vnet-subnet-id. + hobo_byo_has_subnet = bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + if ( + self.get_vnet_subnet_id() in ["", None] and + not hobo_byo_has_subnet + ): raise RequiredArgumentMissingError( "--vnet-subnet-id must be specified for userDefinedRouting and it must " "be pre-configured with a route table with egress rules" From 4c26d7eae56248891e333866946053c4b245fb06 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:42:52 +0000 Subject: [PATCH 06/10] test: add BYO HOBO userAssignedNATGateway live test Adds test_aks_automatic_sku_hosted_system_byovnet_user_natgw covering BYO VNet hosted-system automatic clusters with userAssignedNATGateway outbound type, exercising the _get_outbound_type fix that treats BYO HOBO subnets as satisfying the VNet requirement. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../tests/latest/test_aks_commands.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index f9b25ca6884..2fb1cca1ebe 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6213,6 +6213,119 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou if last_error is not None: raise last_error + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + natgw_name = self.create_random_name("clinatgw", 16) + pip_name = self.create_random_name("clinatpip", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "natgw_name": natgw_name, + "pip_name": pip_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + + # Create a user-assigned NAT gateway and attach it to the node subnet + self.cmd( + "network public-ip create -g {resource_group} -n {pip_name} " + "--sku Standard --allocation-method Static" + ) + self.cmd( + "network nat gateway create -g {resource_group} -n {natgw_name} " + "--public-ip-addresses {pip_name} --idle-timeout 10" + ) + natgw_id = self.cmd( + "network nat gateway show -g {resource_group} -n {natgw_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"natgw_id": natgw_id}) + + # Attach the NAT gateway to both the system-node and node subnets (egress paths) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name systemnode --nat-gateway {natgw_id}" + ) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name node --nat-gateway {natgw_id}" + ) + + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with userAssignedNATGateway outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-vnet-subnet-id={system_node_subnet_id} " + "--node-vnet-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type userAssignedNATGateway " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "userAssignedNATGateway"), + ], + ) + @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer( From d226b68763ba4408006a6c6ebd11c8e7ca75dd47 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:10:21 +0000 Subject: [PATCH 07/10] aks-preview: Fail-fast BYO HOBO trio validation before RBAC grants Previously _validate_byo_hobo_subnet_trio only ran when set_up_api_server_access_profile invoked get_apiserver_subnet_id, which in the base construct_mc_profile_default flow runs AFTER process_add_role_assignment_for_vnet_subnet. A malformed BYO HOBO create (partial subnet trio, or HOBO subnet flags without --enable-hosted-system) could therefore leave residual Network Contributor grants on customer subnets before the CLI surface-level validation fired. Move trio validation to the start of the role-assignment override so misuse fails before any RBAC mutation happens. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../azext_aks_preview/managed_cluster_decorator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 4af3ff36b1e..de33813d103 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -5320,6 +5320,14 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non then iterate over any HOBO BYO subnets and run the same role-assignment logic for each. Skipping is honored via ``--skip-subnet-role-assignment``. """ + # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO + # create (e.g. partial subnet trio, --system-node-vnet-subnet-id without + # --enable-hosted-system) cannot leave residual Network Contributor grants on + # customer subnets. Trio validation is otherwise invoked later through + # set_up_api_server_access_profile, which executes AFTER this method in the base + # construct_mc_profile_default flow. + self.context._validate_byo_hobo_subnet_trio() # pylint: disable=protected-access + # Preserve base behavior for the --vnet-subnet-id case. super().process_add_role_assignment_for_vnet_subnet(mc) @@ -5328,9 +5336,8 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non if not self.context.get_enable_hosted_system(): return - # Trio validation fires earlier via _get_apiserver_subnet_id on CREATE, so by the - # time we reach here we should have either all three HOBO subnets or none. Defend - # anyway — skip cleanly when no HOBO subnets are present. + # By the time we reach here trio validation has already passed, so we have either + # all three HOBO subnets or none. Defend anyway — skip cleanly when absent. hobo_subnets = [] seen = set() for raw_key in ( From 846dd6d6f8bbdcc934ad91f0f8af940710e0ba09 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:16:52 +0000 Subject: [PATCH 08/10] aks-preview: Consolidate BYO HOBO subnet presence check into helper Refactor-only (no behavior change): introduce public AKSPreviewManagedClusterContext.has_byo_hobo_subnets() and replace three inline duplicate 'system_node_vnet_subnet_id or node_vnet_subnet_id' checks (in _get_outbound_type default-completion, _get_outbound_type validation, and get_api_server_access_profile validation) with calls to it. Also rename _validate_byo_hobo_subnet_trio to validate_byo_hobo_subnet_trio (drop the leading underscore) so the CreateDecorator override can call it directly without a pylint protected-access exception. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../managed_cluster_decorator.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index de33813d103..461d62d08f8 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -611,10 +611,7 @@ def _get_outbound_type( is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / # --node-vnet-subnet-id instead of --vnet-subnet-id. - hobo_byo_subnets = bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) + hobo_byo_subnets = self.has_byo_hobo_subnets() if ( sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and is_vnet_subnet_id_empty @@ -648,13 +645,9 @@ def _get_outbound_type( # BYO HOBO scenarios satisfy the VNet requirement via # --system-node-vnet-subnet-id / --node-vnet-subnet-id # instead of --vnet-subnet-id. - hobo_byo_has_subnet = bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) if ( self.get_vnet_subnet_id() in ["", None] and - not hobo_byo_has_subnet + not self.has_byo_hobo_subnets() ): raise RequiredArgumentMissingError( "--vnet-subnet-id must be specified for userDefinedRouting and it must " @@ -1955,15 +1948,15 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) # is rejected up front rather than silently dropped. - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and # --node-vnet-subnet-id replace --vnet-subnet-id. - hobo_byo_subnets = ( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") - ) - if apiserver_subnet_id and vnet_subnet_id is None and not hobo_byo_subnets: + if ( + apiserver_subnet_id and + vnet_subnet_id is None and + not self.has_byo_hobo_subnets() + ): raise RequiredArgumentMissingError( '"--apiserver-subnet-id" requires "--vnet-subnet-id".') @@ -4022,7 +4015,7 @@ def get_system_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() return system_node_vnet_subnet_id def get_node_vnet_subnet_id(self) -> Union[str, None]: @@ -4031,10 +4024,22 @@ def get_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") - self._validate_byo_hobo_subnet_trio() + self.validate_byo_hobo_subnet_trio() return node_vnet_subnet_id - def _validate_byo_hobo_subnet_trio(self) -> None: + def has_byo_hobo_subnets(self) -> bool: + """Return True when at least one of the BYO HOBO node subnet flags is set. + + The apiserver subnet flag is intentionally excluded here: it is a general + VNet-integration flag (not HOBO-specific) and including it would mis-classify + non-HOBO apiserver-vnet-integration clusters as BYO HOBO. + """ + return bool( + self.raw_param.get("system_node_vnet_subnet_id") or + self.raw_param.get("node_vnet_subnet_id") + ) + + def validate_byo_hobo_subnet_trio(self) -> None: """Cross-validate the BYO VNet HOBO subnet flags. Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or @@ -4051,8 +4056,7 @@ def _validate_byo_hobo_subnet_trio(self) -> None: # The HOBO-specific flags only make sense with --enable-hosted-system. # --apiserver-subnet-id is a general VNet-integration flag, so only # require HOBO on it when one of the other two is also set. - hobo_byo_flags_set = bool(system_node_vnet_subnet_id) or bool(node_vnet_subnet_id) - if hobo_byo_flags_set and not enable_hosted_system: + if self.has_byo_hobo_subnets() and not enable_hosted_system: raise RequiredArgumentMissingError( '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' 'require "--enable-hosted-system".' @@ -5326,7 +5330,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non # customer subnets. Trio validation is otherwise invoked later through # set_up_api_server_access_profile, which executes AFTER this method in the base # construct_mc_profile_default flow. - self.context._validate_byo_hobo_subnet_trio() # pylint: disable=protected-access + self.context.validate_byo_hobo_subnet_trio() # Preserve base behavior for the --vnet-subnet-id case. super().process_add_role_assignment_for_vnet_subnet(mc) From c0c33e4eafe73a993e31ed1a4f4297bf2a629965 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Wed, 22 Apr 2026 21:59:06 +0000 Subject: [PATCH 09/10] aks-preview: Rename BYO HOBO subnet flags per review feedback Rename --system-node-vnet-subnet-id -> --system-node-subnet-id and --node-vnet-subnet-id -> --node-subnet-id (with Python identifiers system_node_subnet_id / node_subnet_id) per @zqingqing1 review feedback on PR #9812. The --sys-node-subnet-id alias is retained. Also drop the BYO VNet combination paragraph from the --enable-hosted-system long-summary per PM guidance. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- src/aks-preview/HISTORY.rst | 2 +- src/aks-preview/azext_aks_preview/_help.py | 15 ++-- src/aks-preview/azext_aks_preview/_params.py | 16 ++--- .../azext_aks_preview/_validators.py | 8 +-- src/aks-preview/azext_aks_preview/custom.py | 4 +- .../managed_cluster_decorator.py | 72 +++++++++---------- .../tests/latest/test_aks_commands.py | 12 ++-- 7 files changed, 64 insertions(+), 65 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index 085a13fd80e..cdb72e0b9f9 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,7 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ -* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-vnet-subnet-id` and `--node-vnet-subnet-id`; add `--disable-hosted-system` opt-out. +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`; add `--disable-hosted-system` opt-out. 21.0.0b1 ++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index da96223fc93..486840c7f02 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -714,25 +714,24 @@ short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. long-summary: | Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. - Can be combined with BYO VNet via `--system-node-vnet-subnet-id`, `--node-vnet-subnet-id`, and `--apiserver-subnet-id` - (all three must be provided together and must belong to the same VNet). Cannot be used with `--disable-hosted-system`. + Cannot be used with `--disable-hosted-system`. - name: --disable-hosted-system type: bool short-summary: Opt the automatic cluster out of hosted system components. long-summary: | Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. - - name: --sys-node-subnet-id --system-node-vnet-subnet-id + - name: --sys-node-subnet-id --system-node-subnet-id type: string short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--node-vnet-subnet-id` + Only valid with `--enable-hosted-system`. Must be provided together with `--node-subnet-id` and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - - name: --node-vnet-subnet-id + - name: --node-subnet-id type: string short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. long-summary: | - Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-vnet-subnet-id` + Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-subnet-id` and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - name: --control-plane-scaling-size --cp-scaling-size type: string @@ -837,9 +836,9 @@ - name: Create an automatic cluster with hosted system components enabled. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. - text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-vnet-subnet-id --node-vnet-subnet-id --apiserver-subnet-id --outbound-type loadBalancer + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id --outbound-type loadBalancer - name: Create an automatic cluster and opt out of hosted system components. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index c4fca1ed862..3f82f517451 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -235,8 +235,8 @@ validate_utc_offset, validate_vm_set_type, validate_vnet_subnet_id, - validate_system_node_vnet_subnet_id, - validate_node_vnet_subnet_id, + validate_system_node_subnet_id, + validate_node_subnet_id, validate_force_upgrade_disable_and_enable_parameters, validate_azure_service_mesh_revision, validate_artifact_streaming, @@ -1261,15 +1261,15 @@ def load_arguments(self, _): c.argument("enable_hosted_system", action="store_true", is_preview=True) c.argument("disable_hosted_system", action="store_true", is_preview=True) c.argument( - "system_node_vnet_subnet_id", - options_list=["--system-node-vnet-subnet-id", "--sys-node-subnet-id"], - validator=validate_system_node_vnet_subnet_id, + "system_node_subnet_id", + options_list=["--system-node-subnet-id", "--sys-node-subnet-id"], + validator=validate_system_node_subnet_id, is_preview=True, ) c.argument( - "node_vnet_subnet_id", - options_list=["--node-vnet-subnet-id"], - validator=validate_node_vnet_subnet_id, + "node_subnet_id", + options_list=["--node-subnet-id"], + validator=validate_node_subnet_id, is_preview=True, ) c.argument( diff --git a/src/aks-preview/azext_aks_preview/_validators.py b/src/aks-preview/azext_aks_preview/_validators.py index 1e4039e4822..cdadab44b4c 100644 --- a/src/aks-preview/azext_aks_preview/_validators.py +++ b/src/aks-preview/azext_aks_preview/_validators.py @@ -355,12 +355,12 @@ def validate_apiserver_subnet_id(namespace): _validate_subnet_id(namespace.apiserver_subnet_id, "--apiserver-subnet-id") -def validate_system_node_vnet_subnet_id(namespace): - _validate_subnet_id(namespace.system_node_vnet_subnet_id, "--system-node-vnet-subnet-id") +def validate_system_node_subnet_id(namespace): + _validate_subnet_id(namespace.system_node_subnet_id, "--system-node-subnet-id") -def validate_node_vnet_subnet_id(namespace): - _validate_subnet_id(namespace.node_vnet_subnet_id, "--node-vnet-subnet-id") +def validate_node_subnet_id(namespace): + _validate_subnet_id(namespace.node_subnet_id, "--node-subnet-id") def _validate_subnet_id(subnet_id, name): diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index f36ba2abe80..c57e41bccba 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1182,8 +1182,8 @@ def aks_create( enable_app_routing_istio=False, enable_hosted_system=False, disable_hosted_system=False, - system_node_vnet_subnet_id=None, - node_vnet_subnet_id=None, + system_node_subnet_id=None, + node_subnet_id=None, control_plane_scaling_size=None, # health monitor enable_continuous_control_plane_and_addon_monitor=False, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 461d62d08f8..4c98d442b12 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -609,8 +609,8 @@ def _get_outbound_type( outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER sku_name = self.get_sku_name() is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] - # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-vnet-subnet-id / - # --node-vnet-subnet-id instead of --vnet-subnet-id. + # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-subnet-id / + # --node-subnet-id instead of --vnet-subnet-id. hobo_byo_subnets = self.has_byo_hobo_subnets() if ( sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and @@ -643,7 +643,7 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, ]: # BYO HOBO scenarios satisfy the VNet requirement via - # --system-node-vnet-subnet-id / --node-vnet-subnet-id + # --system-node-subnet-id / --node-subnet-id # instead of --vnet-subnet-id. if ( self.get_vnet_subnet_id() in ["", None] and @@ -1946,12 +1946,12 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse - # (e.g. passing --system-node-vnet-subnet-id without --enable-hosted-system) + # (e.g. passing --system-node-subnet-id without --enable-hosted-system) # is rejected up front rather than silently dropped. self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() - # For BYO VNet HOBO automatic clusters, --system-node-vnet-subnet-id and - # --node-vnet-subnet-id replace --vnet-subnet-id. + # For BYO VNet HOBO automatic clusters, --system-node-subnet-id and + # --node-subnet-id replace --vnet-subnet-id. if ( apiserver_subnet_id and vnet_subnet_id is None and @@ -4005,8 +4005,8 @@ def get_disable_hosted_system(self) -> bool: raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') return disable_hosted_system - def get_system_node_vnet_subnet_id(self) -> Union[str, None]: - """Obtain the value of system_node_vnet_subnet_id. + def get_system_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of system_node_subnet_id. Validates that BYO VNet subnet flags are used together with --enable-hosted-system and that the full triple @@ -4014,18 +4014,18 @@ def get_system_node_vnet_subnet_id(self) -> Union[str, None]: :return: str or None """ - system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") self.validate_byo_hobo_subnet_trio() - return system_node_vnet_subnet_id + return system_node_subnet_id - def get_node_vnet_subnet_id(self) -> Union[str, None]: - """Obtain the value of node_vnet_subnet_id. + def get_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of node_subnet_id. :return: str or None """ - node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + node_subnet_id = self.raw_param.get("node_subnet_id") self.validate_byo_hobo_subnet_trio() - return node_vnet_subnet_id + return node_subnet_id def has_byo_hobo_subnets(self) -> bool: """Return True when at least one of the BYO HOBO node subnet flags is set. @@ -4035,21 +4035,21 @@ def has_byo_hobo_subnets(self) -> bool: non-HOBO apiserver-vnet-integration clusters as BYO HOBO. """ return bool( - self.raw_param.get("system_node_vnet_subnet_id") or - self.raw_param.get("node_vnet_subnet_id") + self.raw_param.get("system_node_subnet_id") or + self.raw_param.get("node_subnet_id") ) def validate_byo_hobo_subnet_trio(self) -> None: """Cross-validate the BYO VNet HOBO subnet flags. - Rule: if any of --system-node-vnet-subnet-id, --node-vnet-subnet-id, or + Rule: if any of --system-node-subnet-id, --node-subnet-id, or --apiserver-subnet-id is set together with --enable-hosted-system, then all three must be set (SDK requires all three subnets to share a VNet). Any of these flags set without --enable-hosted-system is flagged so the user doesn't silently get a non-HOBO cluster with the wrong shape. """ - system_node_vnet_subnet_id = self.raw_param.get("system_node_vnet_subnet_id") - node_vnet_subnet_id = self.raw_param.get("node_vnet_subnet_id") + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") + node_subnet_id = self.raw_param.get("node_subnet_id") apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") enable_hosted_system = self.raw_param.get("enable_hosted_system") @@ -4058,24 +4058,24 @@ def validate_byo_hobo_subnet_trio(self) -> None: # require HOBO on it when one of the other two is also set. if self.has_byo_hobo_subnets() and not enable_hosted_system: raise RequiredArgumentMissingError( - '"--system-node-vnet-subnet-id" and "--node-vnet-subnet-id" ' + '"--system-node-subnet-id" and "--node-subnet-id" ' 'require "--enable-hosted-system".' ) if enable_hosted_system and ( - system_node_vnet_subnet_id or node_vnet_subnet_id or apiserver_subnet_id + system_node_subnet_id or node_subnet_id or apiserver_subnet_id ): missing = [] - if not system_node_vnet_subnet_id: - missing.append("--system-node-vnet-subnet-id") - if not node_vnet_subnet_id: - missing.append("--node-vnet-subnet-id") + if not system_node_subnet_id: + missing.append("--system-node-subnet-id") + if not node_subnet_id: + missing.append("--node-subnet-id") if not apiserver_subnet_id: missing.append("--apiserver-subnet-id") if missing: raise RequiredArgumentMissingError( "BYO VNet for hosted-system clusters requires all of " - "--system-node-vnet-subnet-id, --node-vnet-subnet-id, and " + "--system-node-subnet-id, --node-subnet-id, and " "--apiserver-subnet-id to be provided together. " f"Missing: {', '.join(missing)}." ) @@ -5291,12 +5291,12 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: # BYO VNet: plumb subnet IDs through to the SDK model. All three # subnets (system-node / node / apiserver) must share a VNet, but # the server enforces that check. - system_node_vnet_subnet_id = self.context.get_system_node_vnet_subnet_id() - node_vnet_subnet_id = self.context.get_node_vnet_subnet_id() - if system_node_vnet_subnet_id: - mc.hosted_system_profile.system_node_subnet_id = system_node_vnet_subnet_id - if node_vnet_subnet_id: - mc.hosted_system_profile.node_subnet_id = node_vnet_subnet_id + system_node_subnet_id = self.context.get_system_node_subnet_id() + node_subnet_id = self.context.get_node_subnet_id() + if system_node_subnet_id: + mc.hosted_system_profile.system_node_subnet_id = system_node_subnet_id + if node_subnet_id: + mc.hosted_system_profile.node_subnet_id = node_subnet_id # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: @@ -5315,7 +5315,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non Base behavior: if ``--vnet-subnet-id`` is provided, grant Network Contributor on that subnet to the cluster identity (SP or UAMI). BYO HOBO uses three separate - subnets (``--system-node-vnet-subnet-id``, ``--node-vnet-subnet-id``, + subnets (``--system-node-subnet-id``, ``--node-subnet-id``, ``--apiserver-subnet-id``) instead of ``--vnet-subnet-id``; without this override, cluster creation fails with ``ResourceMissingPermissionError`` on the BYO subnets. @@ -5325,7 +5325,7 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non for each. Skipping is honored via ``--skip-subnet-role-assignment``. """ # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO - # create (e.g. partial subnet trio, --system-node-vnet-subnet-id without + # create (e.g. partial subnet trio, --system-node-subnet-id without # --enable-hosted-system) cannot leave residual Network Contributor grants on # customer subnets. Trio validation is otherwise invoked later through # set_up_api_server_access_profile, which executes AFTER this method in the base @@ -5345,8 +5345,8 @@ def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> Non hobo_subnets = [] seen = set() for raw_key in ( - "system_node_vnet_subnet_id", - "node_vnet_subnet_id", + "system_node_subnet_id", + "node_subnet_id", "apiserver_subnet_id", ): subnet_id = self.context.raw_param.get(raw_key) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 2fb1cca1ebe..550c66e9d07 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6081,8 +6081,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, res "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--ssh-key-value={ssh_key_value}" ) @@ -6163,8 +6163,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resou "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--outbound-type loadBalancer " "--ssh-key-value={ssh_key_value}" @@ -6307,8 +6307,8 @@ def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group "aks create --resource-group={resource_group} --name={name} --location={location} " "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-vnet-subnet-id={system_node_subnet_id} " - "--node-vnet-subnet-id={node_subnet_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " "--apiserver-subnet-id={apiserver_subnet_id} " "--outbound-type userAssignedNATGateway " "--ssh-key-value={ssh_key_value}" From 5bea5f5f6ce6d2df273129158b028a8356241cf0 Mon Sep 17 00:00:00 2001 From: wenhug <50309350+wenhug@users.noreply.github.com> Date: Thu, 23 Apr 2026 00:07:29 +0000 Subject: [PATCH 10/10] test: drop BYO HOBO managedNATGateway live test RP rejects the combination of BYO VNet + managedNATGateway with "Outbound type is managedNATGateway but agent pool 'hostedpool' is using custom VNet, which is not allowed" (by design, enforced in natgatewayv2.go). For BYO VNet the RP auto-defaults outboundType to loadBalancer and only accepts loadBalancer or userAssignedNATGateway. The byovnet_slb and byovnet_user_natgw tests already cover the two supported BYO HOBO outbound modes, so this test was attempting an unsupported scenario and is removed. Signed-off-by: wenhug <50309350+wenhug@users.noreply.github.com> --- .../tests/latest/test_aks_commands.py | 82 ------------------- 1 file changed, 82 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 550c66e9d07..6c6460476af 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -6016,88 +6016,6 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso checks=[self.is_empty()], ) - @live_only() - @AllowLargeResponse() - @AKSCustomResourceGroupPreparer( - random_name_length=17, name_prefix="clitest", location="eastus2euap" - ) - def test_aks_automatic_sku_hosted_system_byovnet_natgw(self, resource_group, resource_group_location): - # reset the count so in replay mode the random names will start with 0 - self.test_resources_count = 0 - aks_name = self.create_random_name("cliakstest", 16) - vnet_name = self.create_random_name("clivnet", 16) - identity_name = self.create_random_name("cliakstest", 16) - self.kwargs.update( - { - "resource_group": resource_group, - "name": aks_name, - "location": resource_group_location, - "vnet_name": vnet_name, - "identity_name": identity_name, - "ssh_key_value": self.generate_ssh_keys(), - } - ) - - # user-assigned MSI is required for BYO VNet on automatic SKU - identity_id = self.cmd( - "identity create -g {resource_group} -n {identity_name} --query id -o tsv" - ).output.strip() - self.kwargs.update({"identity_id": identity_id}) - - # create a BYO VNet with 3 subnets: system-node, node, apiserver - self.cmd( - "network vnet create -g {resource_group} -n {vnet_name} " - "--address-prefix 10.0.0.0/8 " - "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" - ) - self.cmd( - "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " - "--name node --address-prefix 10.43.0.0/16" - ) - self.cmd( - "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " - "--name apiserver --address-prefix 10.44.0.0/28" - ) - system_node_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " - "--query id -o tsv" - ).output.strip() - node_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " - "--query id -o tsv" - ).output.strip() - apiserver_subnet_id = self.cmd( - "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " - "--query id -o tsv" - ).output.strip() - self.kwargs.update({ - "system_node_subnet_id": system_node_subnet_id, - "node_subnet_id": node_subnet_id, - "apiserver_subnet_id": apiserver_subnet_id, - }) - - # BYO VNet HOBO automatic cluster (default outbound = NAT gateway) - create_cmd = ( - "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " - "--enable-managed-identity --assign-identity {identity_id} " - "--system-node-subnet-id={system_node_subnet_id} " - "--node-subnet-id={node_subnet_id} " - "--apiserver-subnet-id={apiserver_subnet_id} " - "--ssh-key-value={ssh_key_value}" - ) - self.cmd( - create_cmd, - checks=[ - self.check("provisioningState", "Succeeded"), - self.check("sku.name", "Automatic"), - self.check("hostedSystemProfile.enabled", True), - self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), - self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), - self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), - ], - ) - @live_only() @AllowLargeResponse() @AKSCustomResourceGroupPreparer(