dlang · thewilsonator · Sep 17, 2019
diff --git a/.azure-pipelines/lib.sh b/.azure-pipelines/lib.sh
@@ -0,0 +1,70 @@
+# Shared functions between individual scripts
+
+CURL_USER_AGENT="DMD-CI $(curl --version | head -n 1)"
+DMD_DIR="$PWD"
+N=$(($(nproc)+1))
+
+clone() {
+    local url="$1"
+    local path="$2"
+    local branch="$3"
+    for i in {0..4}; do
+        if git clone --depth 1 --branch "$branch" "$url" "$path" "${@:4}" --quiet; then
+            break
+        elif [ $i -lt 4 ]; then
+            sleep $((1 << $i))
+        else
+            echo "Failed to clone: ${url}"
+            exit 1
+        fi
+    done
+}
+
+download() {
+    local url="$1"
+    local path="$2"
+    curl -fsSL -A "$CURL_USER_AGENT" --connect-timeout 5 --speed-time 30 --speed-limit 1024 --retry 5 --retry-delay 5 "$url" -o "$path"
+}
+
+install_grep() {
+    local tools_dir="${DMD_DIR}/tools"
+    mkdir -p "$tools_dir"
+    cd "$tools_dir"
+    download "http://downloads.dlang.org/other/grep-3.1.zip" "grep-3.1.zip"
+    unzip "grep-3.1.zip" # contains grep.exe
+    export PATH="${tools_dir}:$PATH"
+}
+
+################################################################################
+# Download Grep
+################################################################################
+
+install_grep() {
+    local tools_dir="${DMD_DIR}/tools"
+    mkdir -p "$tools_dir"
+    cd "$tools_dir"
+    download "http://downloads.dlang.org/other/grep-3.1.zip" "grep-3.1.zip"
+    unzip "grep-3.1.zip" # contains grep.exe
+    export PATH="${tools_dir}:$PATH"
+}
+
+################################################################################
+# Checkout other repositories
+################################################################################
+
+clone_repos() {
+    local REPO_BRANCH="$SYSTEM_PULLREQUEST_TARGETBRANCH"
+
+    for proj in druntime phobos; do
+        if [ "$REPO_BRANCH" != master ] && [ "$REPO_BRANCH" != stable ] &&
+                ! git ls-remote --exit-code --heads "https://github.com/dlang/$proj.git" "$REPO_BRANCH" > /dev/null; then
+            # use master as fallback for other repos to test feature branches
+            clone "https://github.com/dlang/$proj.git" "${DMD_DIR}/../$proj" master
+            echo "[GIT_CLONE] Switched $proj to branch master \$(REPO_BRANCH=$REPO_BRANCH)"
+        else
+            clone "https://github.com/dlang/$proj.git" "${DMD_DIR}/../$proj" "$REPO_BRANCH"
+            echo "[GIT_CLONE] Switched $proj to branch $REPO_BRANCH"
+        fi
+    done
+}
+
diff --git a/.azure-pipelines/windows-visual-studio.sh b/.azure-pipelines/windows-visual-studio.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$DIR"/lib.sh
+
+CURL_USER_AGENT="DMD-CI $(curl --version | head -n 1)"
+DMD_DIR="$PWD"
+
+################################################################################
+# Download LDC
+################################################################################
+
+ldc() {
+    LDC_DIR="ldc2-${LDC_VERSION}-windows-multilib"
+    LDC_INSTALLER="${LDC_DIR}.7z"
+    LDC_URL="https://github.com/ldc-developers/ldc/releases/download/v${LDC_VERSION}/${LDC_INSTALLER}"
+
+    if [ ! -e "$LDC_INSTALLER" ] ; then
+        download "$LDC_URL" "$LDC_INSTALLER"
+    fi
+
+    7z x "$LDC_INSTALLER" > /dev/null
+    if [ ! -e "$LDC_DIR/bin/ldmd2.exe" ] ; then
+      echo "Unexpected LDC installation, $LDC_INSTALLER/bin/ldmd2.exe missing"
+      exit 1
+    fi
+}
+
+################################################################################
+# Download VisualD
+################################################################################
+
+visuald() {
+    local VISUALD_INSTALLER="VisualD-${VISUALD_VER}.exe"
+    local VISUALD_URL="https://github.com/dlang/visuald/releases/download/${VISUALD_VER}/${VISUALD_INSTALLER}"
+    if [ ! -e "$VISUALD_INSTALLER" ] ; then
+        download "$VISUALD_URL" "$VISUALD_INSTALLER"
+    fi
+}
+
+################################################################################
+# Download DigitalMars Make
+################################################################################
+
+dm_make() {
+    download "http://downloads.dlang.org/other/dm857c.zip" dmc.zip
+    unzip dmc.zip > /dev/null
+    export DMC="$PWD/dm/bin/dmc.exe"
+    export DM_MAKE="$PWD/dm/bin/make.exe"
+    mkdir -p dm/path
+    cp "$DMC" "$DM_MAKE" "dm/path"
+}
+
+echo "[STEP]: Downloading LDC"
+ldc
+
+echo "[STEP]: Downloading VisualD"
+visuald
+
+echo "[STEP]: Downloading DigitalMars make"
+dm_make
+
+echo "[STEP]: Downloading grep"
+install_grep
+
+echo "[STEP]: Cloning repositories"
+clone_repos
diff --git a/.azure-pipelines/windows-visual-studio.yml b/.azure-pipelines/windows-visual-studio.yml
@@ -0,0 +1,84 @@
+steps:
+  - script: |
+      :: Use Windows CRLF line endings for checked-out text files
+      git config --global core.autocrlf true
+      set
+    displayName: Print environment variables
+  - checkout: self
+    fetchDepth: 1
+  - script: |
+      bash --version
+      sh --login .azure-pipelines/windows-visual-studio.sh
+    displayName: Download requires binaries
+  - script: |
+      @echo on
+      call "%VSINSTALLDIR%\VC\Auxiliary\Build\vcvarsall.bat" %ARCH%
+      @echo on
+      set VISUALD_INSTALLER=VisualD-%VISUALD_VER%.exe
+      set DMD_DIR=%cd%
+      set DMD=%DMD_DIR%\generated\Windows\Release\Win32\dmd.exe
+      set DMD_TESTSUITE_MAKE_ARGS=-j3
+      set DM_MAKE=%DMD_DIR%\dm\path\make.exe
+      set LDC_DIR=%DMD_DIR%\ldc2-%LDC_VERSION%-windows-multilib
+
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where cl.exe`) DO (SET MSVC_CC=%%~fsF)
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where lib.exe`) DO (SET MSVC_AR=%%~fsF)
+      REM this returns two lines (GNU's link.exe is on the second line)
+      REM Just take the first one
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where link.exe`) DO (SET MSVC_LD=%%~fsF
+        GOTO :Next)
+      :Next
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where make.exe`) DO (SET GNU_MAKE=%%~fsF)
+
+      REM WORKAROUND: move files to a directory without spaces
+      copy "%MSVC_AR%" "%DMD_DIR%\dm\path\lib.exe"
+      copy "%MSVC_LD%" "%DMD_DIR%\dm\path\link.exe"
+      copy "%MSVC_CC%" "%DMD_DIR%\dm\path\cl.exe"
+      set MSVC_AR=%DMD_DIR%\dm\path\lib.exe
+      set MSVC_CC=%DMD_DIR%\dm\path\cl.exe
+      set MSVC_LD=%DMD_DIR%\dm\path\link.exe
+
+      REM expose dm_make as default make
+      set PATH=%DMD_DIR%\dm\path;%DMD_DIR%\tools;%PATH%
+      dir "%DMD_DIR%\tools"
+      "%DMD_DIR%\tools\grep.exe" --version
+      echo %PATH%
+      grep --version
+
+      .\%VISUALD_INSTALLER% /S
+      REM configure DMD path
+      REM reg add "HKLM\SOFTWARE\DMD" /v InstallationFolder /t REG_SZ /d "%DMD_DIR%\dmd2" /reg:32 /f
+      REM configure LDC path
+      reg add "HKLM\SOFTWARE\LDC" /v InstallationFolder /t REG_SZ /d "%LDC_DIR%" /reg:32 /f
+
+      REM build via VS projects with LDC
+      cd src
+      set LDC_ARGS=%LDC_ARGS% /p:DCompiler=LDC
+      msbuild /target:dmd /p:Configuration=Release /p:Platform=Win32 %LDC_ARGS% vcbuild\dmd.sln || exit /B 1
+
+      %DMD% --version
+      grep --version
+
+      REM Check: run druntime unittests
+      cd "%DMD_DIR%\..\druntime"
+      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" target || exit /B 2
+      echo "[DRUNTIME] running tests..."
+      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest32mscoff || exit /B 3
+      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" test_all || exit /B 4
+
+      REM Check: build phobos
+      cd "%DMD_DIR%\..\phobos"
+      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "AR=%MSVC_AR%" "MAKE=%DM_MAKE%" || exit /B 5
+
+      REM Build DMD VERSION + string imports (not built by VisualD)
+      copy "%DMD_DIR%\VERSION" "%DMD_DIR%\generated\Windows\Release\Win32\VERSION"
+
+      REM Run DMD testsuite
+      cd "%DMD_DIR%\test"
+      cp %DMD_DIR%\..\phobos\phobos32mscoff.lib .
+      "%GNU_MAKE%" -j%NUMBER_OF_PROCESSORS% all MODEL=32mscoff ARGS="-O -inline -g" OS=windows DMD="%DMD%" "CC=%MSVC_CC%" DMD_MODEL=win32 || exit /B 6
+
+      cd "%DMD_DIR%\..\phobos"
+      REM Check: build phobos unittests
+      cp %LDC_DIR%\lib32\libcurl.dll .
+      "%DM_MAKE%" -f win64.mak unittest MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%"
diff --git a/.azure-pipelines/windows.sh b/.azure-pipelines/windows.sh
@@ -0,0 +1,118 @@
+#!/bin/sh
+
+set -eux -o pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+. "$DIR"/lib.sh
+
+GNU_MAKE="$(which make)"
+
+install_host_dmd() {
+    download "http://downloads.dlang.org/releases/2.x/${HOST_DMD_VERSION}/dmd.${HOST_DMD_VERSION}.windows.7z" dmd2.7z
+    7z x dmd2.7z > /dev/null
+    export PATH="$PWD/dmd2/windows/bin/:$PATH"
+    export HOST_DC="$PWD/dmd2/windows/bin/dmd.exe"
+    export DM_MAKE="$PWD/dmd2/windows/bin/make.exe"
+    dmd --version
+}
+
+################################################################################
+# Setup required tools
+################################################################################
+
+# WORKAROUND: install a newer grep version
+# REASON: the preinstalled version is buggy (see also: https://github.com/dlang/dmd/pull/9398#issuecomment-468773638)
+install_grep
+
+echo "D_VERSION: $HOST_DMD_VERSION"
+echo "VSINSTALLDIR: $VSINSTALLDIR"
+echo "GNU_MAKE: $("${GNU_MAKE}" --version)"
+echo "GREP_VERSION: $(grep --version)"
+
+################################################################################
+# Install the host compiler
+################################################################################
+
+if [ "$D_COMPILER" == "dmd" ]; then
+    install_host_dmd
+else
+    echo 'Invalid $D_COMPILER provided'.
+    exit 1
+fi
+
+################################################################################
+# Checkout other repositories
+################################################################################
+
+clone_repos
+
+################################################################################
+# Prepare build flags
+################################################################################
+
+export CC="${MSVC_CC}"
+export AR="${MSVC_AR}"
+
+if [ "$MODEL" == "64" ] ; then
+    export MODEL_FLAG="-m64"
+    MAKE_FILE="win64.mak"
+else
+    export MODEL_FLAG="-m32"
+    MAKE_FILE="win32.mak"
+fi
+
+################################################################################
+# Build DMD
+################################################################################
+
+DMD_BIN_PATH="$DMD_DIR/generated/windows/release/${MODEL}/dmd"
+
+cd "${DMD_DIR}/src"
+"${DM_MAKE}" -f "${MAKE_FILE}" reldmd DMD="$DMD_BIN_PATH"
+
+################################################################################
+# WORKAROUND: Make the paths to CC and AR whitespace free
+# REASON: Druntime & Phobos Makefiles as the variables don't use quotation
+################################################################################
+
+ln -s "$(dirname "$CC")" "${DMD_DIR}/../ccdir"
+export CC="$DMD_DIR/../ccdir/cl.exe"
+export AR="$DMD_DIR/../ccdir/lib.exe"
+
+################################################################################
+# WORKAROUND: Build zlib separately with DigitalMars make
+# REASON: whitespace path variables in DigitalMars make from indirect invocation from Phobos
+################################################################################
+
+cd "${DMD_DIR}/../phobos/etc/c/zlib"
+${DM_MAKE} -f win64.mak "MODEL=${MODEL}" "zlib${MODEL}.lib" CC="${CC}" LIB="${AR}" VCDIR="${VCINSTALLDIR}"
+
+################################################################################
+# Build Druntime and Phobos
+################################################################################
+
+for proj in druntime phobos; do
+    cd "${DMD_DIR}/../${proj}"
+    "${DM_MAKE}" -f "${MAKE_FILE}" DMD="$DMD_BIN_PATH" CC="$CC" AR="$MSVC_AR" VCDIR="${VCINSTALLDIR}" CFLAGS="/C7"
+done
+
+################################################################################
+# Run DMD testsuite
+################################################################################
+cd "${DMD_DIR}/test"
+
+# WORKAROUND: Copy the built Phobos library in the path
+# REASON: LIB argument doesn't seem to work
+cp "${DMD_DIR}/../phobos/phobos64.lib" .
+
+DMD_TESTSUITE_MAKE_ARGS="-j$N" "${GNU_MAKE}" -j1 start_all_tests ARGS="-O -inline -g" MODEL="$MODEL"  MODEL_FLAG="$MODEL_FLAG"
+
+################################################################################
+# Prepare artifacts
+################################################################################
+
+mkdir -p "${DMD_DIR}/artifacts"
+cd "${DMD_DIR}/artifacts"
+cp "${DMD_DIR}/../phobos/phobos64.lib" .
+cp "${DMD_BIN_PATH}" .
diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml
@@ -0,0 +1,21 @@
+steps:
+  - script: |
+      :: Use Windows CRLF line endings for checked-out text files
+      git config --global core.autocrlf true
+      set
+    displayName: Print environment variables
+  - checkout: self
+    fetchDepth: 1
+  - script: |
+      @echo on
+      call "%VSINSTALLDIR%\VC\Auxiliary\Build\vcvarsall.bat" %ARCH%
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where cl.exe`) DO (SET MSVC_CC=%%~fsF)
+      FOR /F "tokens=* USEBACKQ" %%F IN (`where lib.exe`) DO (SET MSVC_AR=%%~fsF)
+      bash --version
+      sh --login .azure-pipelines/windows.sh
+    displayName: Build DMD
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: windows-$(ARCH)
+      targetPath: artifacts
+    displayName: Publish artifacts