diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index 92bce607137..cdb72e0b9f9 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,6 +11,7 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ +* `az aks create`: Support BYO VNet for hosted-system automatic clusters via `--system-node-subnet-id` and `--node-subnet-id`; add `--disable-hosted-system` opt-out. 21.0.0b1 ++++++ diff --git a/src/aks-preview/azext_aks_preview/_help.py b/src/aks-preview/azext_aks_preview/_help.py index a06b6051ae8..486840c7f02 100644 --- a/src/aks-preview/azext_aks_preview/_help.py +++ b/src/aks-preview/azext_aks_preview/_help.py @@ -712,6 +712,27 @@ - name: --enable-hosted-system type: bool short-summary: Create a cluster with fully hosted system components. This applies only when creating a new automatic cluster. + long-summary: | + Deterministically opts the cluster into HOBO (Hosted Overlay System Pool). AKS hosts and manages the system node pool. + Cannot be used with `--disable-hosted-system`. + - name: --disable-hosted-system + type: bool + short-summary: Opt the automatic cluster out of hosted system components. + long-summary: | + Deterministically creates an automatic cluster WITHOUT HOBO, even in regions where HOBO is the default. + Only valid with `--sku automatic`. Mutually exclusive with `--enable-hosted-system`. + - name: --sys-node-subnet-id --system-node-subnet-id + type: string + short-summary: Resource ID of the subnet to be used by AKS-managed hosted system nodes (BYO VNet HOBO). + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--node-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. + - name: --node-subnet-id + type: string + short-summary: Resource ID of the subnet joined by tenant worker nodes in BYO VNet HOBO clusters. + long-summary: | + Only valid with `--enable-hosted-system`. Must be provided together with `--system-node-subnet-id` + and `--apiserver-subnet-id`, and all three subnets must belong to the same VNet. - name: --control-plane-scaling-size --cp-scaling-size type: string short-summary: (PREVIEW) The control plane scaling size for the cluster. @@ -814,6 +835,12 @@ text: az aks create -g MyResourceGroup -n MyManagedCluster --control-plane-scaling-size H4 - name: Create an automatic cluster with hosted system components enabled. text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system + - name: Create a hosted-system automatic cluster in a BYO VNet (NAT gateway outbound, the default). + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id + - name: Create a hosted-system automatic cluster in a BYO VNet with Load Balancer outbound. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --enable-hosted-system --system-node-subnet-id --node-subnet-id --apiserver-subnet-id --outbound-type loadBalancer + - name: Create an automatic cluster and opt out of hosted system components. + text: az aks create -g MyResourceGroup -n MyManagedCluster --sku automatic --disable-hosted-system """ diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index f3dfa8af56b..3f82f517451 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -235,6 +235,8 @@ validate_utc_offset, validate_vm_set_type, validate_vnet_subnet_id, + validate_system_node_subnet_id, + validate_node_subnet_id, validate_force_upgrade_disable_and_enable_parameters, validate_azure_service_mesh_revision, validate_artifact_streaming, @@ -1257,6 +1259,19 @@ def load_arguments(self, _): help="Enable Gateway API based ingress on App Routing via Istio" ) c.argument("enable_hosted_system", action="store_true", is_preview=True) + c.argument("disable_hosted_system", action="store_true", is_preview=True) + c.argument( + "system_node_subnet_id", + options_list=["--system-node-subnet-id", "--sys-node-subnet-id"], + validator=validate_system_node_subnet_id, + is_preview=True, + ) + c.argument( + "node_subnet_id", + options_list=["--node-subnet-id"], + validator=validate_node_subnet_id, + is_preview=True, + ) c.argument( "control_plane_scaling_size", options_list=["--control-plane-scaling-size", "--cp-scaling-size"], diff --git a/src/aks-preview/azext_aks_preview/_validators.py b/src/aks-preview/azext_aks_preview/_validators.py index aa43b9052a2..cdadab44b4c 100644 --- a/src/aks-preview/azext_aks_preview/_validators.py +++ b/src/aks-preview/azext_aks_preview/_validators.py @@ -355,6 +355,14 @@ def validate_apiserver_subnet_id(namespace): _validate_subnet_id(namespace.apiserver_subnet_id, "--apiserver-subnet-id") +def validate_system_node_subnet_id(namespace): + _validate_subnet_id(namespace.system_node_subnet_id, "--system-node-subnet-id") + + +def validate_node_subnet_id(namespace): + _validate_subnet_id(namespace.node_subnet_id, "--node-subnet-id") + + def _validate_subnet_id(subnet_id, name): if subnet_id is None or subnet_id == '': return diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index d73fd9b8b71..c57e41bccba 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -1181,6 +1181,9 @@ def aks_create( # app routing istio enable_app_routing_istio=False, enable_hosted_system=False, + disable_hosted_system=False, + system_node_subnet_id=None, + node_subnet_id=None, control_plane_scaling_size=None, # health monitor enable_continuous_control_plane_and_addon_monitor=False, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 60f22700aaa..4c98d442b12 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -545,6 +545,7 @@ def _get_disable_local_accounts(self, enable_validation: bool = False) -> bool: ) return disable_local_accounts + # pylint: disable=too-many-branches def _get_outbound_type( self, enable_validation: bool = False, @@ -602,12 +603,25 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_NONE, CONST_OUTBOUND_TYPE_BLOCK,] ): + # Preserve the user's explicit --outbound-type (if any) before default-completion + # overwrites it with CONST_OUTBOUND_TYPE_LOAD_BALANCER below. + user_supplied_outbound_type = self.raw_param.get("outbound_type") outbound_type = CONST_OUTBOUND_TYPE_LOAD_BALANCER - skuName = self.get_sku_name() - isVnetSubnetIdEmpty = self.get_vnet_subnet_id() in ["", None] - if skuName is not None and skuName == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and isVnetSubnetIdEmpty: - # outbound_type of Automatic SKU should be ManagedNATGateway if no subnet id provided. - outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY + sku_name = self.get_sku_name() + is_vnet_subnet_id_empty = self.get_vnet_subnet_id() in ["", None] + # BYO HOBO (hosted-system) scenarios provide a VNet via --system-node-subnet-id / + # --node-subnet-id instead of --vnet-subnet-id. + hobo_byo_subnets = self.has_byo_hobo_subnets() + if ( + sku_name is not None and sku_name == CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC and + is_vnet_subnet_id_empty + ): + # Default outbound for Automatic SKU without a VNet is managedNATGateway. + # For BYO HOBO, only apply this default when the user did NOT explicitly pass + # --outbound-type; otherwise we would silently overwrite a user's explicit + # --outbound-type loadBalancer (regression seen in BYO SLB scenario). + if not (hobo_byo_subnets and user_supplied_outbound_type): + outbound_type = CONST_OUTBOUND_TYPE_MANAGED_NAT_GATEWAY # validation # Note: The parameters involved in the validation are not verified in their own getters. @@ -628,7 +642,13 @@ def _get_outbound_type( CONST_OUTBOUND_TYPE_USER_DEFINED_ROUTING, CONST_OUTBOUND_TYPE_USER_ASSIGNED_NAT_GATEWAY, ]: - if self.get_vnet_subnet_id() in ["", None]: + # BYO HOBO scenarios satisfy the VNet requirement via + # --system-node-subnet-id / --node-subnet-id + # instead of --vnet-subnet-id. + if ( + self.get_vnet_subnet_id() in ["", None] and + not self.has_byo_hobo_subnets() + ): raise RequiredArgumentMissingError( "--vnet-subnet-id must be specified for userDefinedRouting and it must " "be pre-configured with a route table with egress rules" @@ -1925,8 +1945,18 @@ def _get_apiserver_subnet_id(self, enable_validation: bool = False) -> Union[str # validation if enable_validation: if self.decorator_mode == DecoratorMode.CREATE: + # Cross-validate the BYO VNet HOBO subnet trio on every CREATE so misuse + # (e.g. passing --system-node-subnet-id without --enable-hosted-system) + # is rejected up front rather than silently dropped. + self.validate_byo_hobo_subnet_trio() vnet_subnet_id = self.get_vnet_subnet_id() - if apiserver_subnet_id and vnet_subnet_id is None: + # For BYO VNet HOBO automatic clusters, --system-node-subnet-id and + # --node-subnet-id replace --vnet-subnet-id. + if ( + apiserver_subnet_id and + vnet_subnet_id is None and + not self.has_byo_hobo_subnets() + ): raise RequiredArgumentMissingError( '"--apiserver-subnet-id" requires "--vnet-subnet-id".') @@ -3951,10 +3981,105 @@ def get_enable_hosted_system(self) -> bool: :return: bool """ enable_hosted_system = self.raw_param.get("enable_hosted_system") + disable_hosted_system = self.raw_param.get("disable_hosted_system") + if enable_hosted_system and disable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) if enable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: raise RequiredArgumentMissingError('"--enable-hosted-system" requires "--sku automatic".') return enable_hosted_system + def get_disable_hosted_system(self) -> bool: + """Obtain the value of disable_hosted_system. + + :return: bool + """ + disable_hosted_system = self.raw_param.get("disable_hosted_system") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + if disable_hosted_system and enable_hosted_system: + raise MutuallyExclusiveArgumentError( + 'Cannot specify "--enable-hosted-system" and "--disable-hosted-system" at the same time.' + ) + if disable_hosted_system and self.get_sku_name() != CONST_MANAGED_CLUSTER_SKU_NAME_AUTOMATIC: + raise RequiredArgumentMissingError('"--disable-hosted-system" requires "--sku automatic".') + return disable_hosted_system + + def get_system_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of system_node_subnet_id. + + Validates that BYO VNet subnet flags are used together with + --enable-hosted-system and that the full triple + (system-node / node / apiserver) is provided. + + :return: str or None + """ + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") + self.validate_byo_hobo_subnet_trio() + return system_node_subnet_id + + def get_node_subnet_id(self) -> Union[str, None]: + """Obtain the value of node_subnet_id. + + :return: str or None + """ + node_subnet_id = self.raw_param.get("node_subnet_id") + self.validate_byo_hobo_subnet_trio() + return node_subnet_id + + def has_byo_hobo_subnets(self) -> bool: + """Return True when at least one of the BYO HOBO node subnet flags is set. + + The apiserver subnet flag is intentionally excluded here: it is a general + VNet-integration flag (not HOBO-specific) and including it would mis-classify + non-HOBO apiserver-vnet-integration clusters as BYO HOBO. + """ + return bool( + self.raw_param.get("system_node_subnet_id") or + self.raw_param.get("node_subnet_id") + ) + + def validate_byo_hobo_subnet_trio(self) -> None: + """Cross-validate the BYO VNet HOBO subnet flags. + + Rule: if any of --system-node-subnet-id, --node-subnet-id, or + --apiserver-subnet-id is set together with --enable-hosted-system, then + all three must be set (SDK requires all three subnets to share a VNet). + Any of these flags set without --enable-hosted-system is flagged so + the user doesn't silently get a non-HOBO cluster with the wrong shape. + """ + system_node_subnet_id = self.raw_param.get("system_node_subnet_id") + node_subnet_id = self.raw_param.get("node_subnet_id") + apiserver_subnet_id = self.raw_param.get("apiserver_subnet_id") + enable_hosted_system = self.raw_param.get("enable_hosted_system") + + # The HOBO-specific flags only make sense with --enable-hosted-system. + # --apiserver-subnet-id is a general VNet-integration flag, so only + # require HOBO on it when one of the other two is also set. + if self.has_byo_hobo_subnets() and not enable_hosted_system: + raise RequiredArgumentMissingError( + '"--system-node-subnet-id" and "--node-subnet-id" ' + 'require "--enable-hosted-system".' + ) + + if enable_hosted_system and ( + system_node_subnet_id or node_subnet_id or apiserver_subnet_id + ): + missing = [] + if not system_node_subnet_id: + missing.append("--system-node-subnet-id") + if not node_subnet_id: + missing.append("--node-subnet-id") + if not apiserver_subnet_id: + missing.append("--apiserver-subnet-id") + if missing: + raise RequiredArgumentMissingError( + "BYO VNet for hosted-system clusters requires all of " + "--system-node-subnet-id, --node-subnet-id, and " + "--apiserver-subnet-id to be provided together. " + f"Missing: {', '.join(missing)}." + ) + def get_control_plane_scaling_size(self) -> Union[str, None]: """Obtain the value of control_plane_scaling_size. @@ -4167,29 +4292,43 @@ def set_up_run_command(self, mc: ManagedCluster) -> ManagedCluster: def set_up_api_server_access_profile(self, mc: ManagedCluster) -> ManagedCluster: """Set up apiserverAccessProfile enableVnetIntegration and subnetId for the ManagedCluster object. - Note: Inherited and extended in aks-preview to set vnet integration configs. + Note: This is a full override (not calling super()) because the base acs module writes + `enableVnetIntegration` / `subnetId` via msrest-style `additional_properties`, which the + vendored 2026-02-02-preview SDK (azure.core Model) does not expose and would raise + AttributeError. The logic below mirrors the base implementation (authorized IP ranges, + private cluster, public FQDN, private DNS zone, fqdn subdomain) but writes + `enable_vnet_integration` and `subnet_id` as typed fields. :return: the ManagedCluster object """ - mc = super().set_up_api_server_access_profile(mc) + self._ensure_mc(mc) + + api_server_access_profile = None + api_server_authorized_ip_ranges = self.context.get_api_server_authorized_ip_ranges() + enable_private_cluster = self.context.get_enable_private_cluster() + disable_public_fqdn = self.context.get_disable_public_fqdn() + private_dns_zone = self.context.get_private_dns_zone() + if api_server_authorized_ip_ranges or enable_private_cluster: + # pylint: disable=no-member + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile( + authorized_ip_ranges=api_server_authorized_ip_ranges, + enable_private_cluster=True if enable_private_cluster else None, + enable_private_cluster_public_fqdn=False if disable_public_fqdn else None, + private_dns_zone=private_dns_zone, + ) if self.context.get_enable_apiserver_vnet_integration(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.enable_vnet_integration = True + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.enable_vnet_integration = True if self.context.get_apiserver_subnet_id(): - if mc.api_server_access_profile is None: + if api_server_access_profile is None: # pylint: disable=no-member - mc.api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() - mc.api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + api_server_access_profile = self.models.ManagedClusterAPIServerAccessProfile() + api_server_access_profile.subnet_id = self.context.get_apiserver_subnet_id() + mc.api_server_access_profile = api_server_access_profile - if ( - mc.api_server_access_profile is not None and - hasattr(mc.api_server_access_profile, 'additional_properties') and - mc.api_server_access_profile.additional_properties is not None - ): - # remove the additional properties that are set in official azure-cli/acs - mc.api_server_access_profile.additional_properties = {} + mc.fqdn_subdomain = self.context.get_fqdn_subdomain() return mc def build_gitops_addon_profile(self) -> ManagedClusterAddonProfile: @@ -5143,17 +5282,130 @@ def set_up_enable_hosted_components(self, mc: ManagedCluster) -> ManagedCluster: self._ensure_mc(mc) enable_hosted_components = self.context.get_enable_hosted_system() + disable_hosted_components = self.context.get_disable_hosted_system() if enable_hosted_components: if mc.hosted_system_profile is None: mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member mc.hosted_system_profile.enabled = True + # BYO VNet: plumb subnet IDs through to the SDK model. All three + # subnets (system-node / node / apiserver) must share a VNet, but + # the server enforces that check. + system_node_subnet_id = self.context.get_system_node_subnet_id() + node_subnet_id = self.context.get_node_subnet_id() + if system_node_subnet_id: + mc.hosted_system_profile.system_node_subnet_id = system_node_subnet_id + if node_subnet_id: + mc.hosted_system_profile.node_subnet_id = node_subnet_id + # Remove default agent pool profiles when hosted system profile is enabled if mc.agent_pool_profiles is not None: mc.agent_pool_profiles = None + elif disable_hosted_components: + # Deterministic opt-out: explicitly set enabled=False so customers + # don't get HOBO once the server flips the default in their region. + if mc.hosted_system_profile is None: + mc.hosted_system_profile = self.models.ManagedClusterHostedSystemProfile() # pylint: disable=no-member + mc.hosted_system_profile.enabled = False return mc + def process_add_role_assignment_for_vnet_subnet(self, mc: ManagedCluster) -> None: + """Extend base role assignment to also cover BYO VNet HOBO subnets. + + Base behavior: if ``--vnet-subnet-id`` is provided, grant Network Contributor on + that subnet to the cluster identity (SP or UAMI). BYO HOBO uses three separate + subnets (``--system-node-subnet-id``, ``--node-subnet-id``, + ``--apiserver-subnet-id``) instead of ``--vnet-subnet-id``; without this + override, cluster creation fails with ``ResourceMissingPermissionError`` on the + BYO subnets. + + Strategy: call super() so the original ``--vnet-subnet-id`` path still works, + then iterate over any HOBO BYO subnets and run the same role-assignment logic + for each. Skipping is honored via ``--skip-subnet-role-assignment``. + """ + # Fail-fast validation BEFORE any role assignment runs, so a malformed BYO HOBO + # create (e.g. partial subnet trio, --system-node-subnet-id without + # --enable-hosted-system) cannot leave residual Network Contributor grants on + # customer subnets. Trio validation is otherwise invoked later through + # set_up_api_server_access_profile, which executes AFTER this method in the base + # construct_mc_profile_default flow. + self.context.validate_byo_hobo_subnet_trio() + + # Preserve base behavior for the --vnet-subnet-id case. + super().process_add_role_assignment_for_vnet_subnet(mc) + + # Only extend for BYO VNet HOBO; outside that mode --apiserver-subnet-id keeps its + # generic apiserver-VNet-integration meaning and must NOT trigger an extra RBAC grant. + if not self.context.get_enable_hosted_system(): + return + + # By the time we reach here trio validation has already passed, so we have either + # all three HOBO subnets or none. Defend anyway — skip cleanly when absent. + hobo_subnets = [] + seen = set() + for raw_key in ( + "system_node_subnet_id", + "node_subnet_id", + "apiserver_subnet_id", + ): + subnet_id = self.context.raw_param.get(raw_key) + if subnet_id and subnet_id not in seen: + seen.add(subnet_id) + hobo_subnets.append(subnet_id) + + if not hobo_subnets: + return + + if self.context.get_skip_subnet_role_assignment(): + return + + service_principal_profile = mc.service_principal_profile + assign_identity = self.context.get_assign_identity() + + # For system-assigned identity clusters the SP does not exist yet and we can + # only grant after the cluster is created. Defer via the existing post-create + # flag AND stash the HOBO subnet list so the post-create handler can iterate it; + # base behavior only grants on --vnet-subnet-id, which is absent for BYO HOBO. + if service_principal_profile is None and not assign_identity: + self.context.set_intermediate( + "need_post_creation_vnet_permission_granting", + True, + overwrite_exists=True, + ) + self.context.set_intermediate( + "hobo_byo_subnets_pending_grant", + hobo_subnets, + overwrite_exists=True, + ) + return + + for subnet_id in hobo_subnets: + if self.context.external_functions.subnet_role_assignment_exists(self.cmd, subnet_id): + continue + if assign_identity: + identity_object_id = self.context.get_user_assigned_identity_object_id() + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + identity_object_id, + is_service_principal=False, + scope=subnet_id, + ) + else: + granted = self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + service_principal_profile.client_id, + scope=subnet_id, + ) + if not granted: + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + subnet_id, + ) + # pylint: disable=unused-argument def construct_mc_profile_preview(self, bypass_restore_defaults: bool = False) -> ManagedCluster: """The overall controller used to construct the default ManagedCluster profile. @@ -5322,16 +5574,32 @@ def immediate_processing_after_request(self, mc: ManagedCluster) -> None: # Grant vnet permission to system assigned identity RIGHT AFTER the cluster is put, this operation can # reduce latency for the role assignment take effect instant_cluster = self.client.get(self.context.get_resource_group_name(), self.context.get_name()) - if not self.context.external_functions.add_role_assignment( - self.cmd, - "Network Contributor", - instant_cluster.identity.principal_id, - scope=self.context.get_vnet_subnet_id(), - is_service_principal=False, - ): - logger.warning( - "Could not create a role assignment for subnet. Are you an Owner on this subscription?" - ) + # Determine the scopes to grant: base behavior uses --vnet-subnet-id only; BYO VNet HOBO + # uses the three HOBO subnets stashed by process_add_role_assignment_for_vnet_subnet. + # Iterate a list so both classic and HOBO cases share the same code path. + scopes = [] + vnet_subnet_id = self.context.get_vnet_subnet_id() + if vnet_subnet_id: + scopes.append(vnet_subnet_id) + hobo_subnets = self.context.get_intermediate( + "hobo_byo_subnets_pending_grant", default_value=[] + ) + for subnet in hobo_subnets or []: + if subnet and subnet not in scopes: + scopes.append(subnet) + for scope in scopes: + if not self.context.external_functions.add_role_assignment( + self.cmd, + "Network Contributor", + instant_cluster.identity.principal_id, + scope=scope, + is_service_principal=False, + ): + logger.warning( + "Could not create a role assignment for subnet %s. " + "Are you an Owner on this subscription?", + scope, + ) # pylint: disable=too-many-locals,too-many-branches def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 32cbffb7fa2..6c6460476af 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -5984,7 +5984,7 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso # so we only need to test for cluster creation here create_cmd = ( "aks create --resource-group={resource_group} --name={name} --location={location} " - "--sku automatic --enable-hosted-system " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " "--aks-custom-header AKSHTTPCustomFeatures=Microsoft.ContainerService/AutomaticSKUPreview," "AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-AutomaticHostedSystemProfilePreview " "--ssh-key-value={ssh_key_value}" @@ -6016,6 +6016,273 @@ def test_aks_automatic_sku_with_hosted_system_enabled(self, resource_group, reso checks=[self.is_empty()], ) + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_slb(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with loadBalancer outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type loadBalancer " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "loadBalancer"), + ], + ) + + # convert to Base SKU; expect the transition to succeed for the BYO VNet HOBO case. + # Wait for the cluster to become idle (no in-progress RP reconciliation) before starting + # the update; without this the RP returns 409 OperationNotAllowed for a few minutes after + # create completes while it finishes post-create reconciliation. + self.cmd( + "aks wait -g {resource_group} -n {name} --created --timeout 900", + checks=[self.is_empty()], + ) + # Poll with retries: `aks wait --created` returns once provisioningState hits Succeeded, + # but the RP can still have an internal in-progress reconciliation for several minutes + # afterwards which surfaces as 409 OperationNotAllowed on `aks update`. Back off and retry. + update_deadline = time.time() + 900 + last_error = None + while time.time() < update_deadline: + try: + self.cmd( + "aks update -g {resource_group} -n {name} --sku base", + checks=[self.check("sku.name", "Base")], + ) + last_error = None + break + except Exception as ex: # pylint: disable=broad-except + message = str(ex) + if "OperationNotAllowed" in message or "in progress" in message or "409" in message: + last_error = ex + time.sleep(60) + continue + raise + if last_error is not None: + raise last_error + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_hosted_system_byovnet_user_natgw(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + vnet_name = self.create_random_name("clivnet", 16) + identity_name = self.create_random_name("cliakstest", 16) + natgw_name = self.create_random_name("clinatgw", 16) + pip_name = self.create_random_name("clinatpip", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "vnet_name": vnet_name, + "identity_name": identity_name, + "natgw_name": natgw_name, + "pip_name": pip_name, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # user-assigned MSI is required for BYO VNet on automatic SKU + identity_id = self.cmd( + "identity create -g {resource_group} -n {identity_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"identity_id": identity_id}) + + # create a BYO VNet with 3 subnets: system-node, node, apiserver + self.cmd( + "network vnet create -g {resource_group} -n {vnet_name} " + "--address-prefix 10.0.0.0/8 " + "--subnet-name systemnode --subnet-prefix 10.42.0.0/20" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name node --address-prefix 10.43.0.0/16" + ) + self.cmd( + "network vnet subnet create -g {resource_group} --vnet-name {vnet_name} " + "--name apiserver --address-prefix 10.44.0.0/28" + ) + + # Create a user-assigned NAT gateway and attach it to the node subnet + self.cmd( + "network public-ip create -g {resource_group} -n {pip_name} " + "--sku Standard --allocation-method Static" + ) + self.cmd( + "network nat gateway create -g {resource_group} -n {natgw_name} " + "--public-ip-addresses {pip_name} --idle-timeout 10" + ) + natgw_id = self.cmd( + "network nat gateway show -g {resource_group} -n {natgw_name} --query id -o tsv" + ).output.strip() + self.kwargs.update({"natgw_id": natgw_id}) + + # Attach the NAT gateway to both the system-node and node subnets (egress paths) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name systemnode --nat-gateway {natgw_id}" + ) + self.cmd( + "network vnet subnet update -g {resource_group} --vnet-name {vnet_name} " + "--name node --nat-gateway {natgw_id}" + ) + + system_node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name systemnode " + "--query id -o tsv" + ).output.strip() + node_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name node " + "--query id -o tsv" + ).output.strip() + apiserver_subnet_id = self.cmd( + "network vnet subnet show -g {resource_group} --vnet-name {vnet_name} --name apiserver " + "--query id -o tsv" + ).output.strip() + self.kwargs.update({ + "system_node_subnet_id": system_node_subnet_id, + "node_subnet_id": node_subnet_id, + "apiserver_subnet_id": apiserver_subnet_id, + }) + + # BYO VNet HOBO automatic cluster with userAssignedNATGateway outbound + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --enable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--enable-managed-identity --assign-identity {identity_id} " + "--system-node-subnet-id={system_node_subnet_id} " + "--node-subnet-id={node_subnet_id} " + "--apiserver-subnet-id={apiserver_subnet_id} " + "--outbound-type userAssignedNATGateway " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", True), + self.check("hostedSystemProfile.systemNodeSubnetId", system_node_subnet_id), + self.check("hostedSystemProfile.nodeSubnetId", node_subnet_id), + self.check("apiServerAccessProfile.subnetId", apiserver_subnet_id), + self.check("networkProfile.outboundType", "userAssignedNATGateway"), + ], + ) + + @live_only() + @AllowLargeResponse() + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="eastus2euap" + ) + def test_aks_automatic_sku_with_hosted_system_disabled(self, resource_group, resource_group_location): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + aks_name = self.create_random_name("cliakstest", 16) + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "ssh_key_value": self.generate_ssh_keys(), + } + ) + + # opt out of HOBO even if the region defaults to it + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} " + "--sku automatic --disable-hosted-system --workspace-resource-id=/subscriptions/feb5b150-60fe-4441-be73-8c02a524f55a/resourceGroups/hobo-test-la-rg/providers/Microsoft.OperationalInsights/workspaces/hobo-test-la-ws " + "--ssh-key-value={ssh_key_value}" + ) + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + self.check("sku.name", "Automatic"), + self.check("hostedSystemProfile.enabled", False), + ], + ) + + # agent pool profiles should be present (non-HOBO automatic) + self.cmd( + "aks show --resource-group={resource_group} --name={name}", + checks=[self.greater_than("length(agentPoolProfiles)", 0)], + ) + @AllowLargeResponse() @AKSCustomResourceGroupPreparer( random_name_length=17, name_prefix="clitest", location="westus2"