Sync to 172.76.0

CHANGELOG.md

@@ -3,12 +3,18 @@
 Update this document for externally visible changes. Put most recent changes first.
 Once we push a new version to nuget.org add a double hash header for that version.
 
-# 172.64.0
+## 172.76.0
+
+- Fix scripting performance #165
+- Add new audit action type INFORMATION_PROTECTION_OPERATION_GROUP
+- Add new database level permissions ALTER ANY EXTERNAL MODEL, CREATE EXTERNAL MODEL and ALTER ANY INFORMATION PROTECTION
+
+## 172.64.0
 
 - Add DesignMode support to `QueryStoreOptions` class
 - Add Vector data type support
 
-# 172.61.0
+## 172.61.0
 
 - Remove major version restriction on Microsoft.Data.SqlClient dependency Fixes [Issue 188](https://github.com/microsoft/sqlmanagementobjects/issues/188)
 - Remove net6 binaries

Directory.Packages.props

@@ -2,26 +2,24 @@
      See https://github.com/Microsoft/MSBuildSdks/tree/main/src/CentralPackageVersions
      This file lists every nuget package dependency and its required version. 
      Do not put Version attributes on PackageReference tags in individual projects.
-     Add explicit references to transitive package dependencies if the msbuild binlog shows there
-     are double writes due to multiple versions of a package being imported.
-     It's common that we could import packages A and B that both depend on package C but 2 different versions.
-     Without a PackageReference to C, our build could pick up either version of C in a non-deterministic manner.
      The binlog viewer will show both versions being copied to the output folder as a double write.
 -->
 <Project>
   <PropertyGroup>
     <DotNetPackagesVersion>6.0.0</DotNetPackagesVersion>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+    <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
     <!--
         When adding new package dependencies here, make sure to also add those packages to
         Microsoft.SqlServer.Smo.TestUtils.nuspec if they affect any of the related test utilities.
         -->
     <PackageVersion Include="Azure.Core" Version="1.41.0" />
-    <PackageVersion Include="Azure.Identity" Version="1.11.4" />
+    <PackageVersion Include="Azure.Identity" Version="1.12.0" />
     <PackageVersion Include="Azure.ResourceManager" Version="1.8.0" />
     <PackageVersion Include="Azure.ResourceManager.Storage" Version="1.1.1" />
+    <PackageVersion Include="Azure.Storage.Blobs" Version="12.20.0" />
     <PackageVersion Include="Azure.Security.KeyVault.Secrets" Version="4.4.0" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="$(SqlClientPackageVersion)" />

global.json

@@ -1,6 +1,6 @@
 {
     "sdk": {
-        "version": "8.0.406",
+        "version": "8.0.409",
         "rollForward": "latestMinor"
     },
     "msbuild-sdks": {

src/Directory.Build.props

@@ -84,7 +84,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- these variables are referenced by packagebuild.proj for inclusion in nuspecs and used by packages.props -->
-    <SqlParserPackageVersion>172.18.0</SqlParserPackageVersion>
+    <SqlParserPackageVersion>172.20.0</SqlParserPackageVersion>
     <SqlClientPackage>Microsoft.Data.SqlClient</SqlClientPackage>
     <SqlClientPackageVersion>5.1.6</SqlClientPackageVersion>
   </PropertyGroup>

src/FunctionalTest/Framework/Helpers/AzureKeyVaultHelper.cs

@@ -3,27 +3,17 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Security;
-using System.Security.Cryptography.X509Certificates;
 using Azure.Identity;
-using Azure.ResourceManager;
-using Azure.ResourceManager.Storage;
 using Azure.Security.KeyVault.Secrets;
 using Microsoft.SqlServer.ADO.Identity;
 namespace Microsoft.SqlServer.Test.Manageability.Utils.Helpers
 {
     /// <summary>
     /// Retrieves a decrypted secret from Azure Key Vault or environment using certificate auth or client secret or managed identity
     /// </summary>
-    public class AzureKeyVaultHelper
+    public class AzureKeyVaultHelper : ICredential
     {
-        /// <summary>
-        /// The set of certificate thumbprints associated with the service principal.
-        /// If this collection is non-empty, AzureApplicationId and AzureTenantId must also be set to valid values.
-        /// Set these properties to use certificate-based authentication without relying on environment variables to specify the certificate.
-        /// </summary>
-        public IEnumerable<string> CertificateThumbprints { get; set; }
         /// <summary>
         /// The Azure application id associated with the service principal
         /// </summary>
@@ -39,14 +29,17 @@ public class AzureKeyVaultHelper
         /// <summary>
         /// The name of the Azure key vault where test secrets are stored.
         /// </summary>
-        public string KeyVaultName { get; private set; }
-
-        private static readonly IDictionary<string,SecureString> secretCache = new Dictionary<string, SecureString>();
+        public string KeyVaultName { get; set; }
+
+        /// <summary>
+        /// The AzureStorageHelper instance used to access the storage account
+        /// </summary>
+        public AzureStorageHelper StorageHelper { get; set; }
+        private static readonly IDictionary<string, SecureString> secretCache = new Dictionary<string, SecureString>();
         private static readonly object syncObj = new object();
         public static readonly string SSMS_TEST_SECRET_PREFIX = "SQLA-SSMS-Test-";
 
         private SecretClient secretClient = null;
-        private ArmClient armClient = null;
 
         /// <summary>
         /// Constructs a new AzureKeyVaultHelper that relies on an instance of Azure.Identity.DefaultAzureCredential to access the given vault.
@@ -56,7 +49,6 @@ public AzureKeyVaultHelper(string keyVaultName)
         {
 
             KeyVaultName = keyVaultName;
-            CertificateThumbprints = Enumerable.Empty<string>();
         }
 
         /// <summary>
@@ -101,7 +93,7 @@ public string GetDecryptedSecret(string secretName)
                     Console.WriteLine(@"Got Exception fetching secret. Type:{0}, Inner:{1}, Outer:{2}", e.GetType(), e.InnerException, e);
                     throw;
                 }
-                // Note we aren't bothering to cache secrets we found from GetEnvironmentVariable since that API is already fast                
+                // Note we aren't bothering to cache secrets we found from GetEnvironmentVariable since that API is already fast
                 lock (syncObj)
                 {
                     secretCache[secretName] = secret.StringToSecureString();
@@ -110,71 +102,38 @@ public string GetDecryptedSecret(string secretName)
             return secret;
         }
 
-        private Azure.Core.TokenCredential GetCredential()
+        /// <summary>
+        /// Returns a TokenCredential that implements Managed Identity, DefaultAzureCredential, and AzurePipelinesCredential in that order.
+        /// </summary>
+        /// <returns></returns>
+        public Azure.Core.TokenCredential GetCredential()
         {
+            TraceHelper.TraceInformation($"Getting credential for Azure in tenant {AzureTenantId}");
             // prefer managed identity then local user on dev machine over the certificate
             var credentials = new List<Azure.Core.TokenCredential>() { new ManagedIdentityCredential(AzureManagedIdentityClientId), new DefaultAzureCredential(new DefaultAzureCredentialOptions { ExcludeManagedIdentityCredential = true, TenantId = AzureTenantId }) };
-            foreach (var thumbprint in CertificateThumbprints ?? Enumerable.Empty<string>())
+            var options = new AzureDevOpsFederatedTokenCredentialOptions() { TenantId = AzureTenantId, ClientId = AzureApplicationId };
+            if (options.ServiceConnectionId != null)
             {
-                var certificate = FindCertificate(thumbprint);
-                if (certificate != null)
-                {
-                    credentials.Add(new ClientCertificateCredential(AzureTenantId, AzureApplicationId, certificate));
-                }
+                TraceHelper.TraceInformation($"Adding AzurePipelinesCredential for tenant id {options.TenantId} using service connection {options.ServiceConnectionId}");
+                credentials.Insert(0, new AzurePipelinesCredential(options.TenantId, options.ClientId, options.ServiceConnectionId, options.SystemAccessToken));
             }
-            credentials.Add(new AzureDevOpsFederatedTokenCredential(new AzureDevOpsFederatedTokenCredentialOptions() { TenantId = AzureTenantId, ClientId = AzureApplicationId }));
             return new ChainedTokenCredential(credentials.ToArray());
         }
 
+        /// <summary>
+        /// Returns the account access key for the given storage account resource id.
+        /// </summary>
+        /// <param name="storageAccountResourceId"></param>
+        /// <returns></returns>
         public string GetStorageAccountAccessKey(string storageAccountResourceId)
         {
             TraceHelper.TraceInformation($"Fetching storage access key for {storageAccountResourceId}");
-            if (armClient == null)
-            {
-                armClient = new ArmClient(GetCredential());
-            }
-            var storageAccount = armClient.GetStorageAccountResource(new Azure.Core.ResourceIdentifier(storageAccountResourceId));
-            return storageAccount.GetKeys().First().Value;
+            return new AzureStorageHelper(storageAccountResourceId, this).GetStorageAccountAccessKey(storageAccountResourceId);
         }
+    }
 
-        private static X509Certificate2 FindCertificate(string thumbprint)
-        {
-            X509Certificate2 certificate = null;
-            if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows))
-            {
-                var store = new X509Store(StoreName.My, StoreLocation.LocalMachine);
-                try
-                {
-                    store.Open(OpenFlags.ReadOnly);
-                    X509Certificate2Collection certificateCollection = store.Certificates.Find(X509FindType.FindByThumbprint, thumbprint, validOnly: false);
-                    if (certificateCollection.Count == 0)
-                    {
-                        TraceHelper.TraceInformation("Couldn't find Smo cert {0} in local machine. Looking in current user", thumbprint);
-                        var userStore = new X509Store(StoreName.My, StoreLocation.CurrentUser);
-                        userStore.Open(OpenFlags.ReadOnly);
-                        try
-                        {
-                            certificateCollection = userStore.Certificates.Find(X509FindType.FindByThumbprint,
-                                thumbprint, validOnly: false);
-                        }
-                        finally
-                        {
-                            userStore.Close();
-                        }
-                    }
-                    if (certificateCollection.Count != 0)
-                    {
-                        TraceHelper.TraceInformation("Found cert {0}", thumbprint);
-                        certificate = certificateCollection[0];
-                    }
-                }
-                finally
-                {
-                    store.Close();
-                }
-            }
-            return certificate;
-        }
-
+    public interface ICredential
+    {
+        Azure.Core.TokenCredential GetCredential();
     }
 }

src/FunctionalTest/Framework/Helpers/AzureStorageHelper.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+
+using System;
+using System.IO;
+using System.Linq;
+using Azure.ResourceManager;
+using Azure.ResourceManager.Storage;
+using Azure.Storage.Blobs;
+using Microsoft.SqlServer.Management.Smo;
+
+namespace Microsoft.SqlServer.Test.Manageability.Utils.Helpers
+{
+    /// <summary>
+    /// Contains methods for configuring Azure storage access by SQL instances
+    /// </summary>
+    public class AzureStorageHelper
+    {
+        private ArmClient ArmClient => new ArmClient(CredentialProvider.GetCredential());
+
+        public string Id { get; private set; }
+        public ICredential CredentialProvider { get; }
+
+        public AzureStorageHelper(string id, ICredential credentialProvider)
+        {
+            Id = id.Trim();
+            CredentialProvider = credentialProvider;
+        }
+
+        public string StorageAccountName => Id.Substring(Id.LastIndexOf('/') + 1);
+        public Credential EnsureCredential(Management.Smo.Server server)
+        {
+            var credential = new Credential(server, StorageAccountName + Guid.NewGuid());
+            credential.Create(StorageAccountName, GetStorageAccountAccessKey(Id));
+            return credential;
+        }
+
+        public string GetStorageAccountAccessKey(string storageAccountResourceId)
+        {
+            TraceHelper.TraceInformation($"Fetching storage access key for {storageAccountResourceId}");
+            var storageAccount = ArmClient.GetStorageAccountResource(new Azure.Core.ResourceIdentifier(storageAccountResourceId));
+            return storageAccount.GetKeys().First().Value;
+        }
+
+        /// <summary>
+        /// Downloads a blob from the given blob URL in Azure storage
+        /// </summary>
+        /// <param name="blobUrl"></param>
+        /// <returns></returns>
+        public string DownloadBlob(string blobUrl)
+        {
+            var ext = Path.GetExtension(blobUrl);
+            var path = Path.GetTempFileName();
+            var finalPath = Path.ChangeExtension(path, ext);
+            File.Move(path, finalPath);
+            var blobClient = new BlobClient(new Uri(blobUrl), CredentialProvider.GetCredential());
+            using (var rsp = blobClient.DownloadTo(finalPath))
+            {
+                if (rsp.IsError)
+                {
+                    throw new InvalidOperationException($"Unable to download blob '{blobUrl}'. Error: {rsp.ReasonPhrase}");
+                }
+            }
+            return finalPath;
+        }
+    }
+}