diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 00f7bdb772..2421ed65f7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,31 +1,31 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader - 8a3aaf73bc04edffd49429f44e35a70d2ba9e9c9 + d8b40e59161956e59d8f707e45ce684e7f40bc27 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + 34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + 34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + 34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + 34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + 34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5 https://github.com/dotnet/arcade-services @@ -35,9 +35,9 @@ https://github.com/dotnet/arcade-services cd705029f2675970b42f9273ae359d0926c5e815 - + https://github.com/dotnet/xharness - 89cb4b1d368e0f15b4df8e02a176dd1f1c33958b + 6f4215e4ef72ecf808621f15ab92b2ccefc270c4 https://github.com/dotnet/roslyn @@ -59,9 +59,9 @@ https://github.com/dotnet/symreader-converter c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0 - + https://github.com/dotnet/xliff-tasks - 397ff033b467003d51619f9ac3928e02a4d4178f + 9eccf0f490397043a39454d810b82a0474687226 diff --git a/eng/Versions.props b/eng/Versions.props index ee6e711e95..f2759a7a54 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -113,8 +113,8 @@ 2.4.0 2.0.3 2.4.0 - 6.0.0-beta.21253.2 - 6.0.0-beta.21253.2 + 7.0.0-beta.22056.6 + 7.0.0-beta.22056.6 1.22.0 1.1.2 2.0.0 @@ -127,10 +127,10 @@ 1.1.0-beta.20074.1 1.1.0-beta-20464-02 1.1.0-beta-20464-02 - 6.0.0-beta.21253.2 - 1.0.0-beta.21376.1 + 7.0.0-beta.22056.6 + 1.0.0-beta.22057.1 1.1.0-beta.20074.1 - 1.0.0-prerelease.21254.2 - 1.1.225302 + 1.0.0-prerelease.22060.1 + 2.0.0-preview.1.22060.3 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a0b5fc37f4..18823840b1 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } +$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") +if ($dotnet6Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 2734601c13..ad3fb74fd2 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet5-internal-transport') fi +# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet6-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal-transport') +fi + # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c0817..8943da242f 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/arm/sources.list.trusty b/eng/common/cross/arm/sources.list.trusty deleted file mode 100644 index 07d8f88d82..0000000000 --- a/eng/common/cross/arm/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/tizen-build-rootfs.sh b/eng/common/cross/arm/tizen-build-rootfs.sh new file mode 100644 index 0000000000..9fdb32e920 --- /dev/null +++ b/eng/common/cross/arm/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 0000000000..fb12ade725 --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/arm/trusty-lttng-2.4.patch b/eng/common/cross/arm/trusty-lttng-2.4.patch deleted file mode 100644 index 8e4dd7ae71..0000000000 --- a/eng/common/cross/arm/trusty-lttng-2.4.patch +++ /dev/null @@ -1,71 +0,0 @@ -From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001 -From: Mathieu Desnoyers -Date: Thu, 7 May 2015 13:25:04 -0400 -Subject: [PATCH] Fix: building probe providers with C++ compiler - -Robert Daniels wrote: -> > I'm attempting to use lttng userspace tracing with a C++ application -> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6 -> > release of lttng. I've compiled lttng-modules, lttng-ust, and -> > lttng-tools and have been able to get a simple test working with C -> > code. When I attempt to run the hello.cxx test on my target it will -> > segfault. -> -> -> I spent a little time digging into this issue and finally discovered the -> cause of my segfault with ARM C++ tracepoints. -> -> There is a struct called 'lttng_event' in ust-events.h which contains an -> empty union 'u'. This was the cause of my issue. Under C, this empty union -> compiles to a zero byte member while under C++ it compiles to a one byte -> member, and in my case was four-byte aligned which caused my C++ code to -> have the 'cds_list_head node' offset incorrectly by four bytes. This lead -> to an incorrect linked list structure which caused my issue. -> -> Since this union is empty, I simply removed it from the struct and everything -> worked correctly. -> -> I don't know the history or purpose behind this empty union so I'd like to -> know if this is a safe fix. If it is I can submit a patch with the union -> removed. - -That's a very nice catch! - -We do not support building tracepoint probe provider with -g++ yet, as stated in lttng-ust(3): - -"- Note for C++ support: although an application instrumented with - tracepoints can be compiled with g++, tracepoint probes should be - compiled with gcc (only tested with gcc so far)." - -However, if it works fine with this fix, then I'm tempted to take it, -especially because removing the empty union does not appear to affect -the layout of struct lttng_event as seen from liblttng-ust, which must -be compiled with a C compiler, and from probe providers compiled with -a C compiler. So all we are changing is the layout of a probe provider -compiled with a C++ compiler, which is anyway buggy at the moment, -because it is not compatible with the layout expected by liblttng-ust -compiled with a C compiler. - -Reported-by: Robert Daniels -Signed-off-by: Mathieu Desnoyers ---- - include/lttng/ust-events.h | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h -index 328a875..3d7a274 100644 ---- a/usr/include/lttng/ust-events.h -+++ b/usr/include/lttng/ust-events.h -@@ -407,8 +407,6 @@ struct lttng_event { - void *_deprecated1; - struct lttng_ctx *ctx; - enum lttng_ust_instrumentation instrumentation; -- union { -- } u; - struct cds_list_head node; /* Event list in session */ - struct cds_list_head _deprecated2; - void *_deprecated3; --- -2.7.4 - diff --git a/eng/common/cross/arm/trusty.patch b/eng/common/cross/arm/trusty.patch deleted file mode 100644 index 2f2972f8eb..0000000000 --- a/eng/common/cross/arm/trusty.patch +++ /dev/null @@ -1,97 +0,0 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900 -+++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900 -@@ -65,17 +65,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_val_compare_and_swap_1(addr, old, _new); -+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_val_compare_and_swap_8(addr, old, _new); -+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new); - #endif - } - _uatomic_link_error(); -@@ -100,20 +100,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_and_and_fetch_1(addr, val); -+ __sync_and_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_and_and_fetch_2(addr, val); -+ __sync_and_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_and_and_fetch_8(addr, val); -+ __sync_and_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -139,20 +139,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_or_and_fetch_1(addr, val); -+ __sync_or_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_or_and_fetch_2(addr, val); -+ __sync_or_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_or_and_fetch_8(addr, val); -+ __sync_or_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -180,17 +180,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_add_and_fetch_1(addr, val); -+ return __sync_add_and_fetch_1((uint8_t *) addr, val); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_add_and_fetch_2(addr, val); -+ return __sync_add_and_fetch_2((uint16_t *) addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t *) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_add_and_fetch_8(addr, val); -+ return __sync_add_and_fetch_8((uint64_t *) addr, val); - #endif - } - _uatomic_link_error(); diff --git a/eng/common/cross/arm64/sources.list.trusty b/eng/common/cross/arm64/sources.list.trusty deleted file mode 100644 index 07d8f88d82..0000000000 --- a/eng/common/cross/arm64/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 0000000000..f27fc4fb34 --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index c29c8267e7..42516bbeeb 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -106,7 +106,6 @@ __AndroidPackages+=" libandroid-glob" __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -__AndroidPackages+=" openldap" for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6969834f85..e94d13d62e 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,10 +6,10 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo " for FreeBSD can be: freebsd11 or freebsd12." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" echo " for illumos can be: illumos." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 @@ -32,10 +32,9 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeCommunity=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm10-libs" -__AlpinePackagesEdgeMain+=" python3" -__AlpinePackagesEdgeMain+=" libedit" +__AlpinePackages+=" lldb-dev" +__AlpinePackages+=" python3" +__AlpinePackages+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -55,32 +54,30 @@ __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" zlib1g-dev" -__UbuntuPackages+=" libldap2-dev" - -# ML.NET dependencies -__UbuntuPackages+=" libomp5" -__UbuntuPackages+=" libomp-dev" __AlpinePackages+=" curl-dev" __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__AlpinePackages+=" openldap-dev" -__FreeBSDBase="12.1-RELEASE" +__FreeBSDBase="12.2-RELEASE" __FreeBSDPkg="1.12.0" +__FreeBSDABI="12" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" -__FreeBSDPackages+=" libslapi-2.4" +__FreeBSDPackages+=" terminfo-db" __IllumosPackages="icu-64.2nb2" __IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" zlib-1.2.11" -__IllumosPackages+=" openldap-client-2.4.49" + +# ML.NET dependencies +__UbuntuPackages+=" libomp5" +__UbuntuPackages+=" libomp-dev" __UseMirror=0 @@ -102,6 +99,15 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -119,6 +125,8 @@ while :; do __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') unset __LLDB_Package ;; x86) @@ -147,11 +155,6 @@ while :; do no-lldb) unset __LLDB_Package ;; - trusty) # Ubuntu 14.04 - if [ "$__CodeName" != "jessie" ]; then - __CodeName=trusty - fi - ;; xenial) # Ubuntu 16.04 if [ "$__CodeName" != "jessie" ]; then __CodeName=xenial @@ -182,8 +185,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for armel and arm64." + if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for arm, armel and arm64." usage; exit 1; fi @@ -191,29 +194,34 @@ while :; do __UbuntuRepo= __Tizen=tizen ;; - alpine|alpine3.9) + alpine|alpine3.13) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.9 + __AlpineVersion=3.13 + __AlpinePackages+=" llvm10-libs" ;; - alpine3.13) + alpine3.14) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.13 - # Alpine 3.13 has all the packages we need in the 3.13 repository - __AlpinePackages+=$__AlpinePackagesEdgeCommunity - __AlpinePackagesEdgeCommunity= - __AlpinePackages+=$__AlpinePackagesEdgeMain - __AlpinePackagesEdgeMain= + __AlpineVersion=3.14 + __AlpinePackages+=" llvm11-libs" ;; freebsd11) __FreeBSDBase="11.3-RELEASE" + __FreeBSDABI="11" ;& freebsd12) __CodeName=freebsd __BuildArch=x64 __SkipUnmount=1 ;; + freebsd13) + __CodeName=freebsd + __FreeBSDBase="13.0-RELEASE" + __FreeBSDABI="13" + __BuildArch=x64 + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __BuildArch=x64 @@ -237,6 +245,12 @@ while :; do shift done +if [ -e "$__Keyring" ]; then + __Keyring="--keyring=$__Keyring" +else + __Keyring="" +fi + if [ "$__BuildArch" == "armel" ]; then __LLDB_Package="lldb-3.5-dev" fi @@ -274,26 +288,12 @@ if [[ "$__CodeName" == "alpine" ]]; then -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages - if [[ -n "$__AlpinePackagesEdgeMain" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeMain - fi - - if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeCommunity - fi - rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p $__RootfsDir/usr/local/etc + JOBS="$(getconf _NPROCESSORS_ONLN)" wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - # For now, ask for 11 ABI even on 12. This can be revisited later. - echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf mkdir -p $__RootfsDir/tmp # get and build package manager @@ -301,7 +301,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # needed for install to succeed mkdir -p $__RootfsDir/host/etc - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # install packages we need. INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update @@ -352,7 +352,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install @@ -364,13 +364,6 @@ elif [[ -n $__CodeName ]]; then umount $__RootfsDir/* || true fi - if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/trusty.patch - patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch - popd - fi - if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index fc11001aa7..f7878dddd3 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,18 +3,26 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) + set(FREEBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) else() set(CMAKE_SYSTEM_NAME Linux) + set(LINUX 1) endif() set(CMAKE_SYSTEM_VERSION 1) +if(EXISTS ${CROSS_ROOTFS}/etc/tizen-release) + set(TIZEN 1) +elseif(EXISTS ${CROSS_ROOTFS}/android_platform) + set(ANDROID 1) +endif() + if(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") - if("$ENV{__DistroRid}" MATCHES "tizen.*") + if(TIZEN) set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm") @@ -26,6 +34,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) @@ -33,7 +44,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") else() set(TOOLCHAIN "aarch64-linux-gnu") endif() - if("$ENV{__DistroRid}" MATCHES "tizen.*") + if(TIZEN) set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") @@ -42,9 +53,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "s390x") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") -elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif (FREEBSD) set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(triple "x86_64-unknown-freebsd11") + set(triple "x86_64-unknown-freebsd12") elseif (ILLUMOS) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(TOOLCHAIN "x86_64-illumos") @@ -57,7 +68,11 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(DEFINED TIZEN_TOOLCHAIN) +if(TIZEN) + if(TARGET_ARCH_NAME STREQUAL "arm") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) + endif() if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) @@ -68,7 +83,7 @@ if(DEFINED TIZEN_TOOLCHAIN) endif() endif() -if("$ENV{__DistroRid}" MATCHES "android.*") +if(ANDROID) if(TARGET_ARCH_NAME STREQUAL "arm") set(ANDROID_ABI armeabi-v7a) elseif(TARGET_ARCH_NAME STREQUAL "arm64") @@ -76,7 +91,9 @@ if("$ENV{__DistroRid}" MATCHES "android.*") endif() # extract platform number required by the NDK's toolchain - string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + file(READ "${CROSS_ROOTFS}/android_platform" RID_FILE_CONTENTS) + string(REPLACE "RID=" "" ANDROID_RID "${RID_FILE_CONTENTS}") + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "${ANDROID_RID}") set(ANDROID_TOOLCHAIN clang) set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository @@ -85,12 +102,15 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif(FREEBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -138,24 +158,24 @@ function(add_toolchain_linker_flag Flag) if (NOT Config STREQUAL "") set(CONFIG_SUFFIX "_${Config}") endif() - set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) - set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) + set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) + set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) endfunction() -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +if(LINUX) add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") endif() -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") @@ -174,7 +194,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) +if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT ANDROID) OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -201,8 +221,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-Wno-error=unused-command-line-argument) endif() -if(DEFINED TIZEN_TOOLCHAIN) - if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") +if(TIZEN) + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 39abdbecdc..84c1d0cc2e 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -53,7 +53,7 @@ fi function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - InitializeDotNetCli + InitializeDotNetCli true local dotnet_root=$_InitializeDotNetCli if [ -z "$toolpath" ]; then diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index d6efeb4434..5c94e98632 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -55,6 +55,9 @@ case $cpuname in aarch64) buildarch=arm64 ;; + loongarch64) + buildarch=loongarch64 + ;; amd64|x86_64) buildarch=x64 ;; @@ -70,7 +73,7 @@ case $cpuname in ;; esac -dotnetRoot="$repo_root/.dotnet" +dotnetRoot="${repo_root}.dotnet" if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 deleted file mode 100644 index 0728b1a8b5..0000000000 --- a/eng/common/generate-graph-files.ps1 +++ /dev/null @@ -1,86 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens - [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) - [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) - [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion, # darc's version - [string] $graphvizVersion = '2.38', # GraphViz version - [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about - # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies -) - -function CheckExitCode ([string]$stage) -{ - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - $ErrorActionPreference = 'Stop' - . $PSScriptRoot\tools.ps1 - - Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - - Push-Location $PSScriptRoot - - Write-Host 'Installing darc...' - . .\darc-init.ps1 -darcVersion $darcVersion - CheckExitCode 'Running darc-init' - - $engCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory - $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external' - $installBin = Join-Path $graphvizInstallDir 'bin' - - Write-Host 'Installing dot...' - .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose - - $darcExe = "$env:USERPROFILE\.dotnet\tools" - $darcExe = Resolve-Path "$darcExe\darc.exe" - - Create-Directory $outputFolder - - # Generate 3 graph descriptions: - # 1. Flat with coherency information - # 2. Graphviz (dot) file - # 3. Standard dependency graph - $graphVizFilePath = "$outputFolder\graphviz.txt" - $graphVizImageFilePath = "$outputFolder\graph.png" - $normalGraphFilePath = "$outputFolder\graph-full.txt" - $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" ) - - if ($includeToolset) { - Write-Host 'Toolsets will be included in the graph...' - $baseOptions += @( '--include-toolset' ) - } - - Write-Host 'Generating standard dependency graph...' - & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath - CheckExitCode 'Generating normal dependency graph' - - Write-Host 'Generating flat dependency graph and graphviz file...' - & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath - CheckExitCode 'Generating flat and graphviz dependency graph' - - Write-Host "Generating graph image $graphVizFilePath" - $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" - CheckExitCode 'Generating graphviz image' - - Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" -} -catch { - if (!$includeToolset) { - Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow - Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow - } - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} finally { - Pop-Location -} \ No newline at end of file diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 24c00b5be9..25e97ac007 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop" Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') -$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } if (Test-Path -Path $exclusionsFilePath) { @@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to # Template files $jsonFiles = @() -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $xlfFiles = @() @@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object { $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } -$locFiles = $jsonFiles + $xlfFiles +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles $locJson = @{ Projects = @( @@ -92,14 +99,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n" Pop-Location if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json } else { - New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." exit 1 diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 5bd205b5da..3e6a8d6acf 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -10,7 +10,7 @@ force=false download_retries=5 retry_wait_time_seconds=30 global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -A native_assets +declare -a native_assets . $scriptroot/pipeline-logging-functions.sh . $scriptroot/native/common-library.sh diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 418c09930c..92b77347d9 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -45,11 +45,11 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - $nugetConfigPath = "$RepoRoot\NuGet.config" + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" if (-Not (Test-Path -Path $nugetConfigPath)) { Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 + ExitWithExitCode 1 } $endpoints = New-Object System.Collections.ArrayList @@ -85,7 +85,7 @@ function SetupCredProvider { #Workaround for https://github.com/microsoft/msbuild/issues/4430 function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetTempDir = Join-Path $RepoRoot "dotnet" $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) $dotnet = "$dotnetTempDir\dotnet.exe" $restoreProjPath = "$PSScriptRoot\restore.proj" diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index e2233e7812..9378223ba0 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -39,7 +39,7 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - local nugetConfigPath="$repo_root/NuGet.config" + local nugetConfigPath="{$repo_root}NuGet.config" if [ ! "$nugetConfigPath" ]; then Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index f46d5efe2e..beb9c4648e 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,5 +1,4 @@ - - + net472 diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index c640123000..f041e5ddd9 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -5,6 +5,8 @@ Param( [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, + [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index adf707c8fe..ca38268c44 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -276,7 +276,8 @@ function Get-MachineArchitecture { } if (($ProcessorArchitecture -Eq "AMD64") -Or ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64")) { + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64")) { return "x64" } return "x86" diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index bf272dcf55..080c2c283a 100755 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh @@ -148,8 +148,12 @@ function NewScriptShim { fi if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi fi local shim_contents=$'#!/usr/bin/env bash\n' diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh deleted file mode 100755 index aed19d07d5..0000000000 --- a/eng/common/native/find-native-compiler.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -# This file locates the native compiler with the given name and version and sets the environment variables to locate it. -# - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -if [ $# -lt 0 ] -then - echo "Usage..." - echo "find-native-compiler.sh " - echo "Specify the name of compiler (clang or gcc)." - echo "Specify the major version of compiler." - echo "Specify the minor version of compiler." - exit 1 -fi - -. $scriptroot/../pipeline-logging-functions.sh - -compiler="$1" -cxxCompiler="$compiler++" -majorVersion="$2" -minorVersion="$3" - -if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi - -check_version_exists() { - desired_version=-1 - - # Set up the environment to be used for building with the desired compiler. - if command -v "$compiler-$1.$2" > /dev/null; then - desired_version="-$1.$2" - elif command -v "$compiler$1$2" > /dev/null; then - desired_version="$1$2" - elif command -v "$compiler-$1$2" > /dev/null; then - desired_version="-$1$2" - fi - - echo "$desired_version" -} - -if [ -z "$CLR_CC" ]; then - - # Set default versions - if [ -z "$majorVersion" ]; then - # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) - elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi - - for version in "${versions[@]}"; do - parts=(${version//./ }) - desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" - if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi - done - - if [ -z "$majorVersion" ]; then - if command -v "$compiler" > /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." - fi - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" - else - Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." - exit 1 - fi - else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" - else - Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi - fi - fi - fi - else - desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" - if [ "$desired_version" = "-1" ]; then - Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." - exit 1 - fi - fi - - if [ -z "$CC" ]; then - export CC="$(command -v "$compiler$desired_version")" - export CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi - fi -else - if [ ! -f "$CLR_CC" ]; then - Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" - exit 1 - fi - export CC="$CLR_CC" - export CXX="$CLR_CXX" -fi - -if [ -z "$CC" ]; then - Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." - exit 1 -fi - -export CCC_CC="$CC" -export CCC_CXX="$CXX" -export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh new file mode 100644 index 0000000000..6d7ba15e5f --- /dev/null +++ b/eng/common/native/init-compiler.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables +# +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! + +if [[ "$#" -lt 3 ]]; then + echo "Usage..." + echo "init-compiler.sh