diff --git a/.gitignore b/.gitignore index b02321a54319..1e6128bc5658 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,7 @@ tests/bcl-test/SystemCoreXunit*.csproj tests/bcl-test/SystemXunit.csproj global.json .idea -device-tests-provisioning.csx -build-provisioning.csx -provision-xcode.csx +provision-shared.csx mono_crash.*.json *.binlog .vscode diff --git a/tools/devops/Makefile b/tools/devops/Makefile index 74771d5c78d6..873bc40bfc4b 100644 --- a/tools/devops/Makefile +++ b/tools/devops/Makefile @@ -1,44 +1,13 @@ TOP=../.. include $(TOP)/Make.config -device-tests-provisioning.csx: device-tests-provisioning.csx.in Makefile $(TOP)/Make.config - $(Q_GEN) sed \ - -e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \ - -e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \ - -e 's#@MIN_SHARPIE_URL@#$(MIN_SHARPIE_URL)#g' \ - -e 's#@INCLUDE_MAC@#$(INCLUDE_MAC)#g' \ - -e 's#@INCLUDE_IOS@#$(INCLUDE_IOS)#g' \ - -e 's#@INCLUDE_TVOS@#$(INCLUDE_TVOS)#g' \ - -e 's#@INCLUDE_WATCH@#$(INCLUDE_WATCH)#g' \ - $< > $@; - @echo "Generated $@:" - @cat $@ - -build-provisioning.csx: build-provisioning.csx.in Makefile $(TOP)/Make.config +provision-shared.csx: provision-shared.in.csx Makefile $(TOP)/Make.config $(Q_GEN) sed \ -e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \ + -e 's#@XCODE_ROOT_PATH@#$(XCODE_DEVELOPER_ROOT)#g' \ -e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \ -e 's#@MIN_SHARPIE_URL@#$(MIN_SHARPIE_URL)#g' \ - $< > $@; - @echo "Generated $@:" - @cat $@ - -mac-tests-provisioning.csx: mac-tests-provisioning.csx.in Makefile $(TOP)/Make.config - $(Q_GEN) sed \ - -e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \ - -e 's#@INCLUDE_MAC@#$(INCLUDE_MAC)#g' \ - -e 's#@INCLUDE_IOS@#$(INCLUDE_IOS)#g' \ - $< > $@; - @echo "Generated $@:" - @cat $@ - -provision-xcode.csx: provision-xcode.csx.in Makefile $(TOP)/Make.config - $(Q_GEN) sed \ - -e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \ - -e 's#@XCODE_ROOT_PATH@#$(XCODE_DEVELOPER_ROOT)#g' \ - $< > $@; - @echo "Generated $@:" - @cat $@ + $< > $@ LocProject.json: LocProject.json.in Makefile $(TOP)/Make.config $(Q_GEN) sed \ @@ -65,4 +34,4 @@ print-variable: print-variable-value-to-file: @echo $($(VARIABLE)) > "$(FILE)" -provisioning: build-provisioning.csx device-tests-provisioning.csx provision-xcode.csx +provisioning: provision-shared.csx diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index 7838afd4894c..4b284dfb5805 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -64,18 +64,8 @@ steps: MacDeveloper: $(mac-developer) HostedMacKeychainPassword: ${{ parameters.keyringPass }} - - task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provision Brew components' - inputs: - provisioning_script: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/provision-brew-packages.csx - provisioning_extra_args: '-vvvv' - github_token: ${{ parameters.gitHubToken }} - timeoutInMinutes: 30 - enabled: true - continueOnError: true # brew installation can be temperamental, and things usually work even if the installation fail. - - bash: | - make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops build-provisioning.csx + make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provisioning displayName: 'Generate provisionator files.' - task: xamops.azdevex.provisionator-task.provisionator@2 diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index dd8b8158bf5f..e8b12d92a345 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -220,7 +220,7 @@ steps: timeoutInMinutes: 5 - bash: | - make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ device-tests-provisioning.csx + make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ provisioning displayName: 'Generate Provisionator csx file' # Executed ONLY if we want to clear the provisionator cache. @@ -234,19 +234,7 @@ steps: - task: xamops.azdevex.provisionator-task.provisionator@2 displayName: 'Provision dependencies' inputs: - provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/device-tests-provisioning.csx - provisioning_extra_args: '-vvvv' - github_token: ${{ parameters.gitHubToken }} - timeoutInMinutes: 250 - -- bash: | - make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ mac-tests-provisioning.csx - displayName: 'Generate Provisionator csx file for macOS' - -- task: xamops.azdevex.provisionator-task.provisionator@2 - displayName: 'Provision Xamarin.Mac' - inputs: - provisioning_script: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/mac-tests-provisioning.csx + provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/build-provisioning.csx provisioning_extra_args: '-vvvv' github_token: ${{ parameters.gitHubToken }} timeoutInMinutes: 250 diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index afbd6e7dc216..9c8710d9b619 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -85,7 +85,7 @@ steps: $vsts.Agents.SetEnabled($pool, $agent, $False) displayName: "Disable macOS bot from pool" -- bash: make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provision-xcode.csx +- bash: make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provisioning displayName: 'Generate Xcode provisioning csx file' - task: xamops.azdevex.provisionator-task.provisionator@2 diff --git a/tools/devops/build-provisioning.csx b/tools/devops/build-provisioning.csx new file mode 100644 index 000000000000..de6a84226d26 --- /dev/null +++ b/tools/devops/build-provisioning.csx @@ -0,0 +1,8 @@ +#load "provision-shared.csx" + +if (!ProvisionXcode ()) + return 1; +ProvisionMono (); +ProvisionSharpie (); +ProvisionBrewPackages (); +SetDefaultXcodeInVSMac (); diff --git a/tools/devops/build-provisioning.csx.in b/tools/devops/build-provisioning.csx.in deleted file mode 100644 index 2fba863a6961..000000000000 --- a/tools/devops/build-provisioning.csx.in +++ /dev/null @@ -1,38 +0,0 @@ -#r "_provisionator/provisionator.dll" - -using System.IO; -using System.Reflection; -using System.Linq; - -using static Xamarin.Provisioning.ProvisioningScript; - -// Provision Xcode using the xip name declared in Make.config -Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true); - -// provisionator knows how to deal with this items -Item ("@MONO_PACKAGE@"); -Item ("@MIN_SHARPIE_URL@"); - -var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.Personal), "Library", "Preferences", "Xamarin", "Settings.plist"); -Item ("Override Apple SDK Settings") - .Action (item => { - DeleteSafe (appleSdkOverride); - CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ()); - Console.WriteLine ($"New iOS SDK Location: {GetSettingValue (appleSdkOverride, "AppleSdkRoot")}"); - }); - -void DeleteSafe (string file) -{ - if (File.Exists (file)) - File.Delete (file); -} - -void CreateSetting (string settingFile, string key, string value) -{ - Exec ("defaults", "write", settingFile, key, value); -} - -string GetSettingValue (string settingFile, string keyName) -{ - return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault (); -} diff --git a/tools/devops/device-tests-provisioning.csx.in b/tools/devops/device-tests-provisioning.csx.in deleted file mode 100644 index 1834b99bbe52..000000000000 --- a/tools/devops/device-tests-provisioning.csx.in +++ /dev/null @@ -1,15 +0,0 @@ -#r "_provisionator/provisionator.dll" - -using System.IO; -using System.Reflection; -using System.Linq; - -using static Xamarin.Provisioning.ProvisioningScript; - -// Provision Xcode using the xip name declared in Make.config -Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true); - -Item ("@MONO_PACKAGE@"); -Item ("@MIN_SHARPIE_URL@"); - -BrewPackages ("p7zip"); diff --git a/tools/devops/mac-tests-provisioning.csx.in b/tools/devops/mac-tests-provisioning.csx.in deleted file mode 100644 index 7bbea84ce87e..000000000000 --- a/tools/devops/mac-tests-provisioning.csx.in +++ /dev/null @@ -1,34 +0,0 @@ -#r "_provisionator/provisionator.dll" - -using System.IO; -using System.Reflection; -using System.Linq; - -using static Xamarin.Provisioning.ProvisioningScript; - -BrewPackages ("p7zip"); - -var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library", "Preferences", "Xamarin", "Settings.plist"); -Item ("Override Apple SDK Settings") - .Condition (item => !File.Exists (appleSdkOverride) || GetSettingValue (appleSdkOverride, "AppleSdkRoot") != GetSelectedXcodePath ()) - .Action (item => { - DeleteSafe (appleSdkOverride); - CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ()); - Console.WriteLine ($"New VSMac iOS SDK Location: {GetSelectedXcodePath ()}"); - }); - -void DeleteSafe (string file) -{ - if (File.Exists (file)) - File.Delete (file); -} - -void CreateSetting (string settingFile, string key, string value) -{ - Exec ("defaults", "write", settingFile, key, value); -} - -string GetSettingValue (string settingFile, string keyName) -{ - return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault (); -} diff --git a/tools/devops/provision-brew-packages.csx b/tools/devops/provision-brew-packages.csx index 291ec5e294cd..1b0789b311f6 100644 --- a/tools/devops/provision-brew-packages.csx +++ b/tools/devops/provision-brew-packages.csx @@ -1,9 +1,3 @@ -BrewPackages ( - "shellcheck", - "yamllint", - "cmake", - "p7zip", - "msitools", - "wget", - "azure-cli" - ); +#load "provision-shared.csx" + +ProvisionBrewPackages (); diff --git a/tools/devops/provision-shared.csx b/tools/devops/provision-shared.csx deleted file mode 100644 index f01699237aee..000000000000 --- a/tools/devops/provision-shared.csx +++ /dev/null @@ -1,103 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -using Newtonsoft.Json.Linq; - -using Xamarin.Provisioning; -using Xamarin.Provisioning.Model; -using Xamarin.Provisioning.IO; - -var commit = Environment.GetEnvironmentVariable ("BUILD_SOURCEVERSION"); -var provision_from_commit = Environment.GetEnvironmentVariable ("PROVISION_FROM_COMMIT") ?? commit; - -// Looks for a variable either in the environment, or in current repo's Make.config. -// Returns null if the variable couldn't be found. -IEnumerable make_config = null; -string FindConfigurationVariable (string variable, string hash = "HEAD") -{ - var value = Environment.GetEnvironmentVariable (variable); - if (!string.IsNullOrEmpty (value)) - return value; - - if (make_config == null) { - try { - make_config = Exec ("git", "show", $"{hash}:Make.config"); - } catch { - Console.WriteLine ("Could not find a Make.config"); - return null; - } - } - foreach (var line in make_config) { - if (line.StartsWith (variable + "=", StringComparison.Ordinal)) - return line.Substring (variable.Length + 1); - } - - return null; -} - -string FindVariable (string variable) -{ - var value = FindConfigurationVariable (variable, provision_from_commit); - if (!string.IsNullOrEmpty (value)) - return value; - - throw new Exception ($"Could not find {variable} in environment nor in the commit's ({commit}) manifest."); -} - -void ExecVerbose (string filename, params string[] args) -{ - Console.WriteLine ($"{filename} {string.Join (" ", args)}"); - Exec (filename, args); -} - -bool IsAtLeastVersion(string actualVer, string minVer) -{ - if (actualVer.Equals(minVer, StringComparison.OrdinalIgnoreCase)) - { - return true; - } - - var actualVerChars = actualVer.ToCharArray(); - var minVerChars = minVer.ToCharArray(); - - var length = Math.Min (minVerChars.Length, actualVerChars.Length); - - var i = 0; - while (i < length) - { - if (actualVerChars[i] > minVerChars[i]) - { - return true; - } - else if (minVerChars[i] > actualVerChars[i]) - { - return false; - } - i++; - } - - if (actualVerChars.Length == minVerChars.Length) - { - return true; - } - - return actualVerChars.Length > minVerChars.Length; -} - -void RemoveXcodeSymlinks (string xcodePath) -{ - Console.WriteLine ($"Checkig if '{xcodePath}' is a symlink..."); - var resolvedPath = Symlink.Resolve (xcodePath); - Console.WriteLine ($"Path resolved: '{resolvedPath}'"); - if (resolvedPath is string) { - Console.WriteLine ($"Removing '{xcodePath}' symlink."); - Symlink.Delete (xcodePath); - Console.WriteLine ($"Renaming '{resolvedPath}' into '{xcodePath}'"); - ElevatedExec ("/bin/mv", resolvedPath, xcodePath); - } else - Console.WriteLine ($"'{xcodePath}' is not a symlink."); -} diff --git a/tools/devops/provision-shared.in.csx b/tools/devops/provision-shared.in.csx new file mode 100644 index 000000000000..53816241322a --- /dev/null +++ b/tools/devops/provision-shared.in.csx @@ -0,0 +1,118 @@ +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using Newtonsoft.Json.Linq; + +using Xamarin.Provisioning; +using Xamarin.Provisioning.Model; +using Xamarin.Provisioning.IO; + +var commit = Environment.GetEnvironmentVariable ("BUILD_SOURCEVERSION"); +var provision_from_commit = Environment.GetEnvironmentVariable ("PROVISION_FROM_COMMIT") ?? commit; + +void RemoveXcodeSymlinks (string xcodePath) +{ + Console.WriteLine ($"Checking if '{xcodePath}' is a symlink..."); + var resolvedPath = Symlink.Resolve (xcodePath); + Console.WriteLine ($"Path resolved: '{resolvedPath}'"); + if (resolvedPath is string) { + Console.WriteLine ($"Removing '{xcodePath}' symlink."); + Symlink.Delete (xcodePath); + Console.WriteLine ($"Renaming '{resolvedPath}' into '{xcodePath}'"); + ElevatedExec ("/bin/mv", resolvedPath, xcodePath); + } else + Console.WriteLine ($"'{xcodePath}' is not a symlink."); +} + +void ListXcodes () +{ + Console.WriteLine ($"Xcodes:"); + var lines = Exec ("bash", "-c", "ls -lad /Applications/Xcode*"); + foreach (var line in lines) + Console.WriteLine ($"\t{line}"); +} + +// Provision Xcode using the xip name declared in Make.config +// +// Overrides: +// * The current commit can be overridden by setting the PROVISION_FROM_COMMIT variable. +bool ProvisionXcode () +{ + + if (string.IsNullOrEmpty (provision_from_commit)) { + Console.Error.WriteLine ($"Either BUILD_SOURCEVERSION or PROVISION_FROM_COMMIT must be set."); + Environment.Exit (1); + return false; + } + + ListXcodes (); + + // Provision Xcode + Console.WriteLine ($"Provisioning Xcode from {provision_from_commit}..."); + + // Let's turn symlink into an actual path + var reqXcode = Path.GetDirectoryName (Path.GetDirectoryName ("@XCODE_ROOT_PATH@")); + RemoveXcodeSymlinks (reqXcode); + + // Provision Xcode using the xip name declared in Make.config + Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true); + + LogInstalledXcodes (); + return true; +} + +void ProvisionBrewPackages () +{ + BrewPackages ( + "shellcheck", + "yamllint", + "cmake", + "p7zip", + "msitools", + "wget", + "azure-cli" + ); +} + +void SetDefaultXcodeInVSMac () +{ + var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library", "Preferences", "Xamarin", "Settings.plist"); + Item ("Override Apple SDK Settings") + .Condition (item => !File.Exists (appleSdkOverride) || GetSettingValue (appleSdkOverride, "AppleSdkRoot") != GetSelectedXcodePath ()) + .Action (item => { + DeleteSafe (appleSdkOverride); + CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ()); + Console.WriteLine ($"New VSMac iOS SDK Location: {GetSelectedXcodePath ()}"); + }); + +} + +void DeleteSafe (string file) +{ + if (File.Exists (file)) + File.Delete (file); +} + +void CreateSetting (string settingFile, string key, string value) +{ + Exec ("defaults", "write", settingFile, key, value); +} + +string GetSettingValue (string settingFile, string keyName) +{ + return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault (); +} + +void ProvisionMono () +{ + Item ("@MONO_PACKAGE@"); +} + +void ProvisionSharpie () +{ + Item ("@MIN_SHARPIE_URL@"); +} diff --git a/tools/devops/provision-xcode.csx b/tools/devops/provision-xcode.csx new file mode 100644 index 000000000000..4a4785ea7462 --- /dev/null +++ b/tools/devops/provision-xcode.csx @@ -0,0 +1,4 @@ +#load "provision-shared.csx" + +if (ProvisionXcode ()) + return 1; diff --git a/tools/devops/provision-xcode.csx.in b/tools/devops/provision-xcode.csx.in deleted file mode 100644 index ea30ad8f6ec8..000000000000 --- a/tools/devops/provision-xcode.csx.in +++ /dev/null @@ -1,34 +0,0 @@ -#load "provision-shared.csx" - -// Provision Xcode -// -// Overrides: -// * The current commit can be overridden by setting the PROVISION_FROM_COMMIT variable. - -void ListXcodes () -{ - Console.WriteLine ($"Xcodes:"); - var lines = Exec ("bash", "-c", "ls -lad /Applications/Xcode*"); - foreach (var line in lines) - Console.WriteLine ($"\t{line}"); -} - -if (string.IsNullOrEmpty (provision_from_commit)) { - Console.Error.WriteLine ($"Either BUILD_SOURCEVERSION or PROVISION_FROM_COMMIT must be set."); - Environment.Exit (1); - return 1; -} - -ListXcodes (); - -// Provision Xcode -Console.WriteLine ($"Provisioning Xcode from {provision_from_commit}..."); - -// Let's turn symlink into an actual path -var reqXcode = Path.GetDirectoryName (Path.GetDirectoryName ("@XCODE_ROOT_PATH@")); -RemoveXcodeSymlinks (reqXcode); - -// Provision Xcode using the xip name declared in Make.config -Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true); - -LogInstalledXcodes ();