From 39f47b25b732c27ed704d122d4758366d0f950b6 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Wed, 25 Oct 2017 18:24:40 -0400 Subject: [PATCH] [build] Add `make prepare-image-dependencies` target. We are investigating building Xamarin.Android atop Visual Studio Team System (VSTS) in addition to our current Jenkins setup, and one of the issues we're running into is "bootstrapping": VSTS is configured to create a "fresh" VM for each build. **Pro**: It should allow builds to be more reliable, as previous build artifacts won't be present, and thus won't cause/hide errors. **Con**: *Previous build artifacts are not present*. Previous build artifacts such as *downloading and extracting* the Android NDK & SDK, using `brew` to install dependencies, building MXE... Ensuring that the dependencies are installed through `make prepare` can be quite time consuming. What we want is a way to ensure that the "build image" -- *what's already installed* when the VM boots -- contains all of our desired dependencies. Furthermore, we *don't* want to have the responsible parties checkout and build xamarin-android in order to determine what the dependencies should be. Attempt to square this circle by adding a new `make prepare-image-dependencies` target, which processes `@(AndroidSdkItem)`, `@(AndroidNdkItem)`, `@(AntItem)`, and `@(RequiredProgram)` to create a `prepare-image-dependencies.sh` script which will download and install the required dependencies. The generated shell script does *not* take the state of the machine running `make prepare-image-dependencies` into consideration. This allows the target to be executed on one machine, and the output run on another. $ make prepare-image-dependencies # creates `prepare-image-dependencies.sh` *Note*: `make prepare-image-dependencies` does not currently deal with MXE. (Building MXE on the VSTS VM is *very* time consuming, so it's something we need to take care of. It is not *yet* dealt with.) --- Makefile | 4 ++ .../scripts/PrepareImageDependencies.targets | 49 +++++++++++++ .../scripts/prepare-image-dependencies.sh.in | 72 +++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 build-tools/scripts/PrepareImageDependencies.targets create mode 100755 build-tools/scripts/prepare-image-dependencies.sh.in diff --git a/Makefile b/Makefile index 80cca6ecaab..3bc2ae9477b 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,10 @@ ifeq ($(USE_MSBUILD),1) done endif # msbuild +prepare-image-dependencies: + $(MSBUILD) $(MSBUILD_FLAGS) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \ + /p:AndroidSupportedHostJitAbis=mxe-Win32:mxe-Win64 + include build-tools/scripts/BuildEverything.mk include tests/api-compatibility/api-compatibility.mk diff --git a/build-tools/scripts/PrepareImageDependencies.targets b/build-tools/scripts/PrepareImageDependencies.targets new file mode 100644 index 00000000000..551a57ee7dc --- /dev/null +++ b/build-tools/scripts/PrepareImageDependencies.targets @@ -0,0 +1,49 @@ + + + Debug + + + + + + + + <_Dir Include="sdk" /> + <_Dir Include="ndk" /> + <_Dir Include="ant" /> + + + <_Package + Condition=" '%(HostOS)' == '$(HostOS)' Or '%(HostOS)' == '' " + Include="@(AndroidSdkItem->'$(AndroidUri)/%(RelUrl)%(Identity) sdk/%(DestDir)')" + /> + <_Package + Condition=" '%(HostOS)' == '$(HostOS)' Or '%(HostOS)' == '' " + Include="@(AndroidNdkItem->'$(AndroidUri)/%(RelUrl)%(Identity) ndk/%(DestDir)')" + /> + <_Package + Condition=" '%(HostOS)' == '$(HostOS)' Or '%(HostOS)' == '' " + Include="@(AntItem->'$(AntUri)/%(RelUrl)%(Identity) ant/%(DestDir)')" + /> + + + <_Brew + Condition=" '$(HostOS)' == 'Darwin' And '%(RequiredProgram.Homebrew)' != '' " + Include="@(RequiredProgram->'%(Homebrew)')" + /> + + + <_Packages>@(_Package->'%(Identity)', ' +') + <_Dirs>@(_Dir->'%(Identity)', ' +') + <_Brews>@(_Brew->'%(Identity)', ' +') + + + + + \ No newline at end of file diff --git a/build-tools/scripts/prepare-image-dependencies.sh.in b/build-tools/scripts/prepare-image-dependencies.sh.in new file mode 100755 index 00000000000..13f672ac46f --- /dev/null +++ b/build-tools/scripts/prepare-image-dependencies.sh.in @@ -0,0 +1,72 @@ +#!/bin/bash -e +set -o xtrace + +ARCHIVE_PATH="$HOME/android-archives" +TOOLCHAIN_PATH="$HOME/android-toolchain" + +TOOLCHAIN_DIRS=" +@TOOLCHAIN_DIRS@ +" + +# format: URL path/under/$TOOLCHAIN_PATH +PACKAGES=" +@PACKAGES@ +" + +BREW_PACKAGES=" +@BREWS@ +" + +function Download () +{ + local url="$1" + local archive="$ARCHIVE_PATH/`basename "$url"`" + if [ -f "$archive" ]; then + return + fi + curl -o "$archive" "$url" +} + +function Install () +{ + local url="$1" + local subdir="$2" + + local file=`basename "$url"` + local archive="$ARCHIVE_PATH/$file" + local targetdir="$TOOLCHAIN_PATH/$subdir" + + local flagdir="$TOOLCHAIN_PATH/$(echo $subdir | sed 's#/.*$##g')" + local flagfile="$flagdir/.$file" + + mkdir -p "$targetdir" + if [ ! -f "$flagfile" ]; then + local extractdir=`mktemp -d` + unzip -d "$extractdir" "$ARCHIVE_PATH/$file" + mv "$extractdir"/*/* "$targetdir" + touch "$flagfile" + rm -Rf "$extractdir" + fi +} + +echo "$TOOLCHAIN_DIRS" | while read line ; do + if [ -z "$line" ]; then + continue + fi + rm -Rf "$TOOLCHAIN_PATH/$line" +done + +echo "$PACKAGES" | while read line ; do + if [ -z "$line" ]; then + continue + fi + Download $line + Install $line +done + +echo "$BREW_PACKAGES" | while read line ; do + if [ -z "$line" ]; then + continue + fi + brew install $line +done