From 8a42d14c6625b6f635a9a2176847aafda2fe9b0c Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Wed, 26 Jan 2022 09:03:30 +0100 Subject: [PATCH 1/6] Autobuild: Use bash consistently The reason for using 'sh' instead of invoking the scripts directly seems to have been that the exec bit on the *prepare*.sh files had been missing. 408b985b7808135a505295bdcf05b6bef25f52ad fixed this already. Using bash consistently avoids unexpected issues due to usage of bash best-practices in non-bash environments. All those scripts run on Ubuntu or Mac where bash is available, so there's no need to artificially limit the usable language features. --- .github/workflows/autobuild.yml | 4 ++-- autobuild/android/autobuild_apk_1_prepare.sh | 2 +- autobuild/mac/artifacts/autobuild_mac_1_prepare.sh | 2 +- autobuild/mac/codeQL/autobuild_mac_1_prepare.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml index 197faea7e1..4785ad73c7 100644 --- a/.github/workflows/autobuild.yml +++ b/.github/workflows/autobuild.yml @@ -95,7 +95,7 @@ jobs: - config_name: AndroidAPK (artifact+codeQL) target_os: android building_on_os: ubuntu-20.04 - cmd1_prebuild: "sh ./autobuild/android/autobuild_apk_1_prepare.sh" # this step needs sh instead of bash for permissions + cmd1_prebuild: "./autobuild/android/autobuild_apk_1_prepare.sh" cmd2_build: "./autobuild/android/autobuild_apk_2_build.sh" cmd3_postbuild: "./autobuild/android/autobuild_apk_3_copy_files.sh" uses_codeql: true @@ -103,7 +103,7 @@ jobs: - config_name: Linux (artifacts+codeQL) target_os: linux building_on_os: ubuntu-18.04 - cmd1_prebuild: "sh ./autobuild/linux/autobuild_deb_1_prepare.sh" # this step needs sh instead of bash for permissions + cmd1_prebuild: "./autobuild/linux/autobuild_deb_1_prepare.sh" cmd2_build: "./autobuild/linux/autobuild_deb_2_build.sh" cmd3_postbuild: "./autobuild/linux/autobuild_deb_3_copy_files.sh" uses_codeql: true diff --git a/autobuild/android/autobuild_apk_1_prepare.sh b/autobuild/android/autobuild_apk_1_prepare.sh index e2970c7ce6..2ad25b6a37 100755 --- a/autobuild/android/autobuild_apk_1_prepare.sh +++ b/autobuild/android/autobuild_apk_1_prepare.sh @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/bin/bash -e # autobuild_1_prepare: set up environment, install Qt & dependencies diff --git a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh index c9206b5cb2..f54ce2f206 100755 --- a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/bin/bash -e # autobuild_1_prepare: set up environment, install Qt & dependencies diff --git a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh index 51d7f25c6e..f54ce2f206 100755 --- a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # autobuild_1_prepare: set up environment, install Qt & dependencies From 7aa64a73f4235bf09108c7a5ae5b3d8f0b0d4104 Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Wed, 26 Jan 2022 09:11:19 +0100 Subject: [PATCH 2/6] Autobuild: Clean up Android Docker leftovers The Android autobuild scripts contained a whole directory of archived code. The name suggests that it is unused. The repository does not contain any references to this directory either. Therefore remove it. Furthermore, the actively used scripts contained commented out code from the apparent conversion from a Dockerfile to bash scripts. It also contained references to older versions. This commit removes those instances of disabled code. Should we ever need it again, git has all the history. --- .../archive___android_build_docker/Dockerfile | 82 ---- .../__example__jamulusbuild.sh | 16 - .../archive___android_build_docker/action.yml | 7 - .../build_qt_project.sh | 6 - .../install-qt.sh | 355 ------------------ autobuild/android/autobuild_apk_1_prepare.sh | 18 +- autobuild/android/autobuild_apk_2_build.sh | 1 - 7 files changed, 1 insertion(+), 484 deletions(-) delete mode 100644 autobuild/android/archive___android_build_docker/Dockerfile delete mode 100755 autobuild/android/archive___android_build_docker/__example__jamulusbuild.sh delete mode 100644 autobuild/android/archive___android_build_docker/action.yml delete mode 100755 autobuild/android/archive___android_build_docker/build_qt_project.sh delete mode 100755 autobuild/android/archive___android_build_docker/install-qt.sh diff --git a/autobuild/android/archive___android_build_docker/Dockerfile b/autobuild/android/archive___android_build_docker/Dockerfile deleted file mode 100644 index 478ac5222d..0000000000 --- a/autobuild/android/archive___android_build_docker/Dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -FROM ubuntu:18.04 - -#based on https://gitlab.com/vikingsoftware/qt5.12.4androiddocker -#LABEL "Maintainer"="Guenter Schwann" -#LABEL "version"="0.3" - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y build-essential git zip unzip bzip2 p7zip-full wget curl chrpath libxkbcommon-x11-0 && \ -# Dependencies to create Android pkg - apt-get install -y openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle && \ -# Clean apt cache - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Add Android tools and platform tools to PATH -ENV ANDROID_HOME /opt/android/android-sdk -ENV ANDROID_SDK_ROOT /opt/android/android-sdk -ENV ANDROID_NDK_ROOT /opt/android/android-ndk - -# paths for Android SDK -RUN mkdir -p ${ANDROID_SDK_ROOT}/cmdline-tools/latest/ -RUN mkdir -p ${ANDROID_SDK_ROOT}/build-tools/latest/ - -# Install Android sdk -#https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -#https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -RUN downloadfile="downloadfile" && \ - wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip && \ - unzip -q downloadfile && \ - rm downloadfile && \ - mv cmdline-tools/* /opt/android/android-sdk/cmdline-tools/latest/ && \ - rm -r cmdline-tools - -# Install Android ndk -#https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip -#https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip -RUN downloadfile="downloadfile" && \ - wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip && \ - unzip -q downloadfile && \ - rm downloadfile && \ - mv android-ndk-r21d /opt/android/android-ndk - -# Add Android tools and platform tools to PATH -ENV ANDROID_HOME /opt/android/android-sdk -ENV ANDROID_SDK_ROOT /opt/android/android-sdk -ENV ANDROID_NDK_ROOT /opt/android/android-ndk -ENV PATH $PATH:$ANDROID_HOME/tools -ENV PATH $PATH:$ANDROID_HOME/platform-tools -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ - -ENV ANDROID_NDK_HOST linux-x86_64 - -ENV ANDROID_SDKMANAGER $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager - -# Install Android SDK -RUN yes | $ANDROID_SDKMANAGER --licenses && $ANDROID_SDKMANAGER --update -RUN $ANDROID_SDKMANAGER "platforms;android-17" -RUN $ANDROID_SDKMANAGER "platforms;android-28" -RUN $ANDROID_SDKMANAGER "platforms;android-30" -RUN $ANDROID_SDKMANAGER "build-tools;28.0.3" -RUN $ANDROID_SDKMANAGER "build-tools;30.0.2" - - -#5.15.2 -ENV MY_QT_VERSION 5.15.2 - -# Download / install Qt -####ADD https://code.qt.io/cgit/qbs/qbs.git/plain/scripts/install-qt.sh ./ -COPY install-qt.sh /install-qt.sh -RUN bash install-qt.sh --version $MY_QT_VERSION --target android --toolchain android qtbase qt3d qtdeclarative qtandroidextras qtconnectivity qtgamepad qtlocation qtmultimedia qtquickcontrols2 qtremoteobjects qtscxml qtsensors qtserialport qtsvg qtimageformats qttools qtspeech qtwebchannel qtwebsockets qtwebview qtxmlpatterns qttranslations - -# Set the QTDIR environment variable -ENV QTDIR="/opt/Qt/$MY_QT_VERSION/android" - - -COPY build_qt_project.sh /build_qt_project.sh -RUN chmod +x /build_qt_project.sh - -ENTRYPOINT /bin/bash /build_qt_project.sh - - diff --git a/autobuild/android/archive___android_build_docker/__example__jamulusbuild.sh b/autobuild/android/archive___android_build_docker/__example__jamulusbuild.sh deleted file mode 100755 index 82152e5d2e..0000000000 --- a/autobuild/android/archive___android_build_docker/__example__jamulusbuild.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# execute this file in an empty folder - -#clean & download code & change into folder -rm -r jamulus -git clone --recurse-submodules https://github.com/corrados/jamulus.git -cd jamulus - -#################################################### -### these two lines run in toplevel folder of repo## -#################################################### - -#build & run docker -docker build -t myjambuild ./android/automated_build -docker run --rm -v `pwd`:/jamulus -w /jamulus -e CONFIG=release myjambuild diff --git a/autobuild/android/archive___android_build_docker/action.yml b/autobuild/android/archive___android_build_docker/action.yml deleted file mode 100644 index 64cb1789f1..0000000000 --- a/autobuild/android/archive___android_build_docker/action.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: 'Run Android-Build-Docker' -description: 'Run Android-Build-Docker' -runs: - using: 'docker' - image: ./Dockerfile - args: - - CONFIG=release diff --git a/autobuild/android/archive___android_build_docker/build_qt_project.sh b/autobuild/android/archive___android_build_docker/build_qt_project.sh deleted file mode 100755 index aab6b6d545..0000000000 --- a/autobuild/android/archive___android_build_docker/build_qt_project.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -"${QTDIR}"/bin/qmake -spec android-clang CONFIG+="${CONFIG}" -/opt/android/android-ndk/prebuilt/linux-x86_64/bin/make -"${ANDROID_NDK_ROOT}"/prebuilt/"${ANDROID_NDK_HOST}"/bin/make INSTALL_ROOT=android-build -f Makefile install -"${QTDIR}"/bin/androiddeployqt --input $(ls *.json) --output android-build --android-platform android-30 --jdk "${JAVA_HOME}" --gradle diff --git a/autobuild/android/archive___android_build_docker/install-qt.sh b/autobuild/android/archive___android_build_docker/install-qt.sh deleted file mode 100755 index 2bae78a963..0000000000 --- a/autobuild/android/archive___android_build_docker/install-qt.sh +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env bash -############################################################################# -## -## Copyright (C) 2019 Richard Weickelt. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qbs. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -## https://code.qt.io/cgit/qbs/qbs.git/tree/scripts/install-qt.sh - -set -eu - -function help() { - cat < - Root directory where to install the components. - Maps to C:/Qt on Windows, /opt/Qt on Linux, /usr/local/Qt on Mac - by default. - - -f, --force - Force download and do not attempt to re-use an existing installation. - - --host - The host operating system. Can be one of linux_x64, mac_x64, - windows_x86. Auto-detected by default. - - --target - The desired target platform. Can be one of desktop, android, ios. - The default value is desktop. - - --toolchain - The toolchain that has been used to build the binaries. - Possible values depend on --host and --target, respectively: - - linux_x64 - android - any, android_armv7, android_arm64_v8a - desktop - gcc_64 (default) - - mac_x64 - android - any, android_armv7, android_arm64_v8a - desktop - clang_64 (default), - ios - ios - - windows_x86 - android - any, android_armv7, android_arm64_v8a - desktop - win64_mingw73, win64_msvc2017_64 (default) - - --version - The desired Qt version. Currently supported are all versions - above 5.9.0. - -EOF -} - -TARGET_PLATFORM=desktop -COMPONENTS= -VERSION= -FORCE_DOWNLOAD=false -MD5_TOOL=md5sum - -case "$OSTYPE" in - *linux*) - HOST_OS=linux_x64 - INSTALL_DIR=/opt/Qt - TOOLCHAIN=gcc_64 - ;; - *darwin*) - HOST_OS=mac_x64 - INSTALL_DIR=/usr/local/Qt - TOOLCHAIN=clang_64 - MD5_TOOL="md5 -r" - ;; - msys) - HOST_OS=windows_x86 - INSTALL_DIR=/c/Qt - TOOLCHAIN=win64_msvc2015_64 - ;; - *) - HOST_OS= - INSTALL_DIR= - ;; -esac - -while [ $# -gt 0 ]; do - case "$1" in - --directory|-d) - INSTALL_DIR="$2" - shift - ;; - --force|-f) - FORCE_DOWNLOAD=true - ;; - --host) - HOST_OS="$2" - shift - ;; - --target) - TARGET_PLATFORM="$2" - shift - ;; - --toolchain) - TOOLCHAIN=$(echo $2 | tr '[A-Z]' '[a-z]') - shift - ;; - --version) - VERSION="$2" - shift - ;; - --help|-h) - help - exit 0 - ;; - *) - COMPONENTS="${COMPONENTS} $1" - ;; - esac - shift -done - -if [ -z "${HOST_OS}" ]; then - echo "No --host specified or auto-detection failed." >&2 - exit 1 -fi - -if [ -z "${INSTALL_DIR}" ]; then - echo "No --directory specified or auto-detection failed." >&2 - exit 1 -fi - -if [ -z "${VERSION}" ]; then - echo "No --version specified." >&2 - exit 1 -fi - -if [ -z "${COMPONENTS}" ]; then - echo "No components specified." >&2 - exit 1 -fi - -case "$TARGET_PLATFORM" in - android) - ;; - ios) - ;; - desktop) - ;; - *) - echo "Error: TARGET_PLATFORM=${TARGET_PLATFORM} is not valid." >&2 - exit 1 - ;; -esac - -HASH=$(echo "${OSTYPE} ${TARGET_PLATFORM} ${TOOLCHAIN} ${VERSION} ${INSTALL_DIR}" | ${MD5_TOOL} | head -c 16) -HASH_FILEPATH="${INSTALL_DIR}/${HASH}.manifest" -INSTALLATION_IS_VALID=false -if ! ${FORCE_DOWNLOAD} && [ -f "${HASH_FILEPATH}" ]; then - INSTALLATION_IS_VALID=true - while read filepath; do - if [ ! -e "${filepath}" ]; then - INSTALLATION_IS_VALID=false - break - fi - done <"${HASH_FILEPATH}" -fi - -if ${INSTALLATION_IS_VALID}; then - echo "Already installed. Skipping download." >&2 - exit 0 -fi - -MIRRORS="\ - http://download.qt.io \ - http://ftp.acc.umu.se/mirror/qt.io/qtproject \ - http://qt.mirrors.tds.net/qt \ - http://ftp.fau.de/qtproject \ -" - -for MIRROR in ${MIRRORS}; do - if curl "${MIRROR}/online" -s -f -o /dev/null; then - break; - else - echo "Server ${MIRROR} not availabe. Trying next alternative..." >&2 - MIRROR="" - fi -done - -DOWNLOAD_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'install-qt'` - -# -# The repository structure is a mess. Try different URL variants -# -function compute_url(){ - local COMPONENT=$1 - local CURL="curl -s -L" - local BASE_URL="${MIRROR}/online/qtsdkrepository/${HOST_OS}/${TARGET_PLATFORM}" - local ANDROID_ARCH=$(echo ${TOOLCHAIN##android_}) - - if [[ "${COMPONENT}" =~ "qtcreator" ]]; then - - SHORT_VERSION=${VERSION%??} - BASE_URL="${MIRROR}/official_releases/qtcreator" - REMOTE_PATH="${SHORT_VERSION}/${VERSION}/installer_source/${HOST_OS}/qtcreator.7z" - echo "${BASE_URL}/${REMOTE_PATH}" - return 0 - - else - REMOTE_BASES=( - # New repository format (>=6.0.0) - "qt6_${VERSION//./}/qt.qt6.${VERSION//./}.${TOOLCHAIN}" - "qt6_${VERSION//./}_${ANDROID_ARCH}/qt.qt6.${VERSION//./}.${TOOLCHAIN}" - "qt6_${VERSION//./}_${ANDROID_ARCH}/qt.qt6.${VERSION//./}.${COMPONENT}.${TOOLCHAIN}" - # New repository format (>=5.9.6) - "qt5_${VERSION//./}/qt.qt5.${VERSION//./}.${TOOLCHAIN}" - "qt5_${VERSION//./}/qt.qt5.${VERSION//./}.${COMPONENT}.${TOOLCHAIN}" - # Multi-abi Android since 5.14 - "qt5_${VERSION//./}/qt.qt5.${VERSION//./}.${TARGET_PLATFORM}" - "qt5_${VERSION//./}/qt.qt5.${VERSION//./}.${COMPONENT}.${TARGET_PLATFORM}" - # Older repository format (<5.9.0) - "qt5_${VERSION//./}/qt.${VERSION//./}.${TOOLCHAIN}" - "qt5_${VERSION//./}/qt.${VERSION//./}.${COMPONENT}.${TOOLCHAIN}" - ) - - for REMOTE_BASE in ${REMOTE_BASES[*]}; do - REMOTE_PATH="$(${CURL} ${BASE_URL}/${REMOTE_BASE}/ | grep -o -E "[[:alnum:]_.\-]*7z" | grep "${COMPONENT}" | tail -1)" - if [ ! -z "${REMOTE_PATH}" ]; then - echo "${BASE_URL}/${REMOTE_BASE}/${REMOTE_PATH}" - return 0 - fi - done - fi - - echo "Could not determine a remote URL for ${COMPONENT} with version ${VERSION}">&2 - exit 1 -} - -mkdir -p ${INSTALL_DIR} -rm -f "${HASH_FILEPATH}" - -for COMPONENT in ${COMPONENTS}; do - - URL="$(compute_url ${COMPONENT})" - echo "Downloading ${COMPONENT} ${URL}..." >&2 - curl --progress-bar -L -o ${DOWNLOAD_DIR}/package.7z ${URL} >&2 - 7z x -y -o${INSTALL_DIR} ${DOWNLOAD_DIR}/package.7z >/dev/null 2>&1 - 7z l -ba -slt -y ${DOWNLOAD_DIR}/package.7z | tr '\\' '/' | sed -n -e "s|^Path\ =\ |${INSTALL_DIR}/|p" >> "${HASH_FILEPATH}" 2>/dev/null - rm -f ${DOWNLOAD_DIR}/package.7z - - # - # conf file is needed for qmake - # - if [ "${COMPONENT}" == "qtbase" ]; then - if [[ "${TOOLCHAIN}" =~ "win64_mingw" ]]; then - SUBDIR="${TOOLCHAIN/win64_/}_64" - elif [[ "${TOOLCHAIN}" =~ "win32_mingw" ]]; then - SUBDIR="${TOOLCHAIN/win32_/}_32" - elif [[ "${TOOLCHAIN}" =~ "win64_msvc" ]]; then - SUBDIR="${TOOLCHAIN/win64_/}" - elif [[ "${TOOLCHAIN}" =~ "win32_msvc" ]]; then - SUBDIR="${TOOLCHAIN/win32_/}" - elif [[ "${TOOLCHAIN}" =~ "any" ]] && [[ "${TARGET_PLATFORM}" == "android" ]]; then - SUBDIR="android" - else - SUBDIR="${TOOLCHAIN}" - fi - - if [ "${TARGET_PLATFORM}" == "android" ] && [ ! "${VERSION}" \< "6.0.0" ]; then - CONF_FILE="${INSTALL_DIR}/${VERSION}/${SUBDIR}/bin/target_qt.conf" - sed -i "s|target|../$TOOLCHAIN|g" "${CONF_FILE}" - sed -i "/HostPrefix/ s|$|gcc_64|g" "${CONF_FILE}" - ANDROID_QMAKE_FILE="${INSTALL_DIR}/${VERSION}/${SUBDIR}/bin/qmake" - QMAKE_FILE="${INSTALL_DIR}/${VERSION}/gcc_64/bin/qmake" - sed -i "s|\/home\/qt\/work\/install\/bin\/qmake|$QMAKE_FILE|g" "${ANDROID_QMAKE_FILE}" - else - CONF_FILE="${INSTALL_DIR}/${VERSION}/${SUBDIR}/bin/qt.conf" - echo "[Paths]" > ${CONF_FILE} - echo "Prefix = .." >> ${CONF_FILE} - fi - - # Adjust the license to be able to run qmake - # sed with -i requires intermediate file on Mac OS - PRI_FILE="${INSTALL_DIR}/${VERSION}/${SUBDIR}/mkspecs/qconfig.pri" - sed -i.bak 's/Enterprise/OpenSource/g' "${PRI_FILE}" - sed -i.bak 's/licheck.*//g' "${PRI_FILE}" - rm "${PRI_FILE}.bak" - - # Print the directory so that the caller can - # adjust the PATH variable. - echo $(dirname "${CONF_FILE}") - elif [[ "${COMPONENT}" =~ "qtcreator" ]]; then - if [ "${HOST_OS}" == "mac_x64" ]; then - echo "${INSTALL_DIR}/Qt Creator.app/Contents/MacOS" - else - echo "${INSTALL_DIR}/bin" - fi - fi - -done - diff --git a/autobuild/android/autobuild_apk_1_prepare.sh b/autobuild/android/autobuild_apk_1_prepare.sh index 2ad25b6a37..91a22ddf4c 100755 --- a/autobuild/android/autobuild_apk_1_prepare.sh +++ b/autobuild/android/autobuild_apk_1_prepare.sh @@ -7,10 +7,6 @@ ### PROCEDURE ### ################### -#FROM ubuntu:18.04 -#based on https://gitlab.com/vikingsoftware/qt5.12.4androiddocker -#LABEL "Maintainer"="Guenter Schwann" -#LABEL "version"="0.3" export DEBIAN_FRONTEND="noninteractive" echo "::set-env name=DEBIAN_FRONTEND::${DEBIAN_FRONTEND}" @@ -42,8 +38,6 @@ mkdir -p "${ANDROID_SDK_ROOT}"/cmdline-tools/latest/ mkdir -p "${ANDROID_SDK_ROOT}"/build-tools/latest/ # Install Android sdk -#https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -#https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip downloadfile="downloadfile" wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip unzip -q downloadfile @@ -52,8 +46,6 @@ mv cmdline-tools/* /opt/android/android-sdk/cmdline-tools/latest/ rm -r cmdline-tools # Install Android ndk -#https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip -#https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip downloadfile="downloadfile" wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip unzip -q downloadfile @@ -63,19 +55,12 @@ mv android-ndk-r21d /opt/android/android-ndk # Install Android SDK yes | "${ANDROID_SDKMANAGER}" --licenses -#echo yes | $ANDROID_SDKMANAGER --licenses "${ANDROID_SDKMANAGER}" --update -#$ANDROID_SDKMANAGER "platforms;android-17" -#$ANDROID_SDKMANAGER "platforms;android-28" "${ANDROID_SDKMANAGER}" "platforms;android-30" -#$ANDROID_SDKMANAGER "build-tools;28.0.3" "${ANDROID_SDKMANAGER}" "build-tools;30.0.2" - # Download / install Qt -####ADD https://code.qt.io/cgit/qbs/qbs.git/plain/scripts/install-qt.sh ./ -#COPY install-qt.sh /install-qt.sh THIS_SCRIPT=$(readlink -f "${0}") # Absolute path this script is in, thus /home/user/bin THIS_SCRIPT_PATH=$(dirname "${THIS_SCRIPT}") @@ -84,14 +69,13 @@ bash "${THIS_SCRIPT_PATH}"/install-qt.sh --version "${MY_QT_VERSION}" --target a # Set the QTDIR environment variable export QTDIR="/opt/Qt/${MY_QT_VERSION}/android" - #necessary echo "::set-env name=QTDIR::${QTDIR}" echo "::set-env name=ANDROID_NDK_ROOT::${ANDROID_NDK_ROOT}" echo "::set-env name=ANDROID_NDK_HOST::${ANDROID_NDK_HOST}" echo "::set-env name=JAVA_HOME::${JAVA_HOME}" -#nce to have +#nice to have echo "::set-env name=ANDROID_HOME::${ANDROID_HOME}" echo "::set-env name=ANDROID_SDK_ROOT::${ANDROID_SDK_ROOT}" echo "::set-env name=PATH::${PATH}" diff --git a/autobuild/android/autobuild_apk_2_build.sh b/autobuild/android/autobuild_apk_2_build.sh index 3a60b2e41b..e7d532edff 100755 --- a/autobuild/android/autobuild_apk_2_build.sh +++ b/autobuild/android/autobuild_apk_2_build.sh @@ -15,7 +15,6 @@ source $(dirname $(readlink -f "${BASH_SOURCE[0]}"))/../ensure_THIS_JAMULUS_PROJ cd "${THIS_JAMULUS_PROJECT_PATH}" -#$QTDIR/bin/qmake -spec android-clang CONFIG+=$CONFIG "${QTDIR}"/bin/qmake -spec android-clang CONFIG+=release echo . echo . From 901b3d1eb139b41e0b5e65b41d986a36d8fef53b Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Wed, 26 Jan 2022 09:23:30 +0100 Subject: [PATCH 3/6] Autobuild Android: Avoid cleanups, condense calls Cleaning up unnecessary files such as the apt cache is important when building docker images. When running on Github Actions we shouldn't care about it as the whole runtime environment is destroyed more efficiently after each run. `apt` usage can also be condensed into a single call to reduce the (likely very minimal) startup overhead. --- autobuild/android/autobuild_apk_1_prepare.sh | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/autobuild/android/autobuild_apk_1_prepare.sh b/autobuild/android/autobuild_apk_1_prepare.sh index 91a22ddf4c..63f54dfff2 100755 --- a/autobuild/android/autobuild_apk_1_prepare.sh +++ b/autobuild/android/autobuild_apk_1_prepare.sh @@ -11,13 +11,10 @@ export DEBIAN_FRONTEND="noninteractive" echo "::set-env name=DEBIAN_FRONTEND::${DEBIAN_FRONTEND}" -sudo apt-get update -sudo apt-get -qq -y install build-essential git zip unzip bzip2 p7zip-full wget curl chrpath libxkbcommon-x11-0 +sudo apt-get -qq update # Dependencies to create Android pkg -sudo apt-get -qq -y install openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle -# Clean apt cache -sudo apt-get clean -rm -rf /var/lib/apt/lists/* +sudo apt-get -qq -y install build-essential git zip unzip bzip2 p7zip-full wget curl chrpath libxkbcommon-x11-0 \ + openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle # Add Android tools and platform tools to PATH export ANDROID_HOME="/opt/android/android-sdk" @@ -38,21 +35,15 @@ mkdir -p "${ANDROID_SDK_ROOT}"/cmdline-tools/latest/ mkdir -p "${ANDROID_SDK_ROOT}"/build-tools/latest/ # Install Android sdk -downloadfile="downloadfile" wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip unzip -q downloadfile -rm downloadfile mv cmdline-tools/* /opt/android/android-sdk/cmdline-tools/latest/ -rm -r cmdline-tools # Install Android ndk -downloadfile="downloadfile" wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip unzip -q downloadfile -rm downloadfile mv android-ndk-r21d /opt/android/android-ndk - # Install Android SDK yes | "${ANDROID_SDKMANAGER}" --licenses "${ANDROID_SDKMANAGER}" --update From 8f208f55d915a78d97196ff4975759880210da9e Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Wed, 26 Jan 2022 09:38:28 +0100 Subject: [PATCH 4/6] Autobuild: Define versions and paths as variables This refactors several autobuild scripts to avoid repeated hardcoding of versions or paths. --- autobuild/android/autobuild_apk_1_prepare.sh | 17 ++++++++++------- .../mac/artifacts/autobuild_mac_1_prepare.sh | 8 ++++---- autobuild/mac/codeQL/autobuild_mac_1_prepare.sh | 8 ++++---- .../autobuild_windowsinstaller_1_prepare.ps1 | 8 ++++++-- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/autobuild/android/autobuild_apk_1_prepare.sh b/autobuild/android/autobuild_apk_1_prepare.sh index 63f54dfff2..2e709e674c 100755 --- a/autobuild/android/autobuild_apk_1_prepare.sh +++ b/autobuild/android/autobuild_apk_1_prepare.sh @@ -7,6 +7,8 @@ ### PROCEDURE ### ################### +COMMANDLINETOOLS_VERSION=6858069 +ANDROID_NDK_VERSION=r21d export DEBIAN_FRONTEND="noninteractive" echo "::set-env name=DEBIAN_FRONTEND::${DEBIAN_FRONTEND}" @@ -18,8 +20,9 @@ sudo apt-get -qq -y install build-essential git zip unzip bzip2 p7zip-full wget # Add Android tools and platform tools to PATH export ANDROID_HOME="/opt/android/android-sdk" -export ANDROID_SDK_ROOT="/opt/android/android-sdk" +export ANDROID_SDK_ROOT="${ANDROID_HOME}" export ANDROID_NDK_ROOT="/opt/android/android-ndk" +COMMANDLINETOOLS_DIR="${ANDROID_SDK_ROOT}"/cmdline-tools/latest/ export PATH="${PATH}:${ANDROID_HOME}/tools" export PATH="${PATH}:${ANDROID_HOME}/platform-tools" @@ -28,21 +31,21 @@ export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" # other variables export MY_QT_VERSION="5.15.2" export ANDROID_NDK_HOST="linux-x86_64" -export ANDROID_SDKMANAGER="${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" +export ANDROID_SDKMANAGER="${COMMANDLINETOOLS_DIR}/bin/sdkmanager" # paths for Android SDK -mkdir -p "${ANDROID_SDK_ROOT}"/cmdline-tools/latest/ +mkdir -p "${COMMANDLINETOOLS_DIR}" mkdir -p "${ANDROID_SDK_ROOT}"/build-tools/latest/ # Install Android sdk -wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip +wget -q -O downloadfile https://dl.google.com/android/repository/commandlinetools-linux-${COMMANDLINETOOLS_VERSION}_latest.zip unzip -q downloadfile -mv cmdline-tools/* /opt/android/android-sdk/cmdline-tools/latest/ +mv cmdline-tools/* "${COMMANDLINETOOLS_DIR}" # Install Android ndk -wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip +wget -q -O downloadfile https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip unzip -q downloadfile -mv android-ndk-r21d /opt/android/android-ndk +mv android-ndk-${ANDROID_NDK_VERSION} "${ANDROID_NDK_ROOT}" # Install Android SDK yes | "${ANDROID_SDKMANAGER}" --licenses diff --git a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh index f54ce2f206..48e7e01cee 100755 --- a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh @@ -7,6 +7,7 @@ if [ "$#" -ne "1" ]; then exit 1 fi +QT_DIR=/usr/local/opt/qt QT_VER=$1 ################### @@ -15,11 +16,11 @@ QT_VER=$1 echo "Install dependencies..." python3 -m pip install aqtinstall -python3 -m aqt install-qt --outputdir /usr/local/opt/qt mac desktop ${QT_VER} +python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} -# add the qt binaries to the path +# Add the qt binaries to the PATH. # The clang_64 entry can be dropped when Qt <6.2 compatibility is no longer needed. -for qt_path in /usr/local/opt/qt/${QT_VER}/macos/bin /usr/local/opt/qt/${QT_VER}/clang_64/bin; do +for qt_path in "${QT_DIR}"/${QT_VER}/macos/bin "${QT_DIR}"/${QT_VER}/clang_64/bin; do if [[ -d $qt_path ]]; then export -p PATH="${qt_path}:${PATH}" break @@ -27,4 +28,3 @@ for qt_path in /usr/local/opt/qt/${QT_VER}/macos/bin /usr/local/opt/qt/${QT_VER} done echo "::set-env name=PATH::${PATH}" echo "the path is ${PATH}" - diff --git a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh index f54ce2f206..48e7e01cee 100755 --- a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh @@ -7,6 +7,7 @@ if [ "$#" -ne "1" ]; then exit 1 fi +QT_DIR=/usr/local/opt/qt QT_VER=$1 ################### @@ -15,11 +16,11 @@ QT_VER=$1 echo "Install dependencies..." python3 -m pip install aqtinstall -python3 -m aqt install-qt --outputdir /usr/local/opt/qt mac desktop ${QT_VER} +python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} -# add the qt binaries to the path +# Add the qt binaries to the PATH. # The clang_64 entry can be dropped when Qt <6.2 compatibility is no longer needed. -for qt_path in /usr/local/opt/qt/${QT_VER}/macos/bin /usr/local/opt/qt/${QT_VER}/clang_64/bin; do +for qt_path in "${QT_DIR}"/${QT_VER}/macos/bin "${QT_DIR}"/${QT_VER}/clang_64/bin; do if [[ -d $qt_path ]]; then export -p PATH="${qt_path}:${PATH}" break @@ -27,4 +28,3 @@ for qt_path in /usr/local/opt/qt/${QT_VER}/macos/bin /usr/local/opt/qt/${QT_VER} done echo "::set-env name=PATH::${PATH}" echo "the path is ${PATH}" - diff --git a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 index 094720e413..50a0fbe8e4 100644 --- a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 +++ b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 @@ -19,6 +19,10 @@ $ErrorActionPreference = "Stop" ### PROCEDURE ### ################### +$QtDir = 'C:\Qt' +$Qt32Version = "5.15.2" +$Qt64Version = "5.15.2" + echo "Install Qt..." # Install Qt pip install aqtinstall @@ -29,7 +33,7 @@ if ( !$? ) echo "Get Qt 64 bit..." # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir C:\Qt windows desktop 5.15.2 win64_msvc2019_64 +aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt64Version}" win64_msvc2019_64 if ( !$? ) { throw "64bit Qt installation failed with exit code $LastExitCode" @@ -37,7 +41,7 @@ if ( !$? ) echo "Get Qt 32 bit..." # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir C:\Qt windows desktop 5.15.2 win32_msvc2019 +aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt32Version}" win32_msvc2019 if ( !$? ) { throw "32bit Qt installation failed with exit code $LastExitCode" From b27414f8edefd42f831d12fcf4b73404c4742454 Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Wed, 26 Jan 2022 09:44:27 +0100 Subject: [PATCH 5/6] Autobuild: Pin aqt and JACK versions This ensures that we always install specific versions of aqt and JACK. This helps with build reproducibility. It's also a requirement to be able to cache parts of the build environment. --- autobuild/mac/artifacts/autobuild_mac_1_prepare.sh | 3 ++- autobuild/mac/codeQL/autobuild_mac_1_prepare.sh | 3 ++- .../autobuild_windowsinstaller_1_prepare.ps1 | 13 ++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh index 48e7e01cee..6b496575bd 100755 --- a/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/artifacts/autobuild_mac_1_prepare.sh @@ -9,13 +9,14 @@ fi QT_DIR=/usr/local/opt/qt QT_VER=$1 +AQTINSTALL_VERSION=2.0.5 ################### ### PROCEDURE ### ################### echo "Install dependencies..." -python3 -m pip install aqtinstall +python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} # Add the qt binaries to the PATH. diff --git a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh index 48e7e01cee..6b496575bd 100755 --- a/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh +++ b/autobuild/mac/codeQL/autobuild_mac_1_prepare.sh @@ -9,13 +9,14 @@ fi QT_DIR=/usr/local/opt/qt QT_VER=$1 +AQTINSTALL_VERSION=2.0.5 ################### ### PROCEDURE ### ################### echo "Install dependencies..." -python3 -m pip install aqtinstall +python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}" python3 -m aqt install-qt --outputdir "${QT_DIR}" mac desktop ${QT_VER} # Add the qt binaries to the PATH. diff --git a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 index 50a0fbe8e4..19d4ca1fc3 100644 --- a/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 +++ b/autobuild/windows/autobuild_windowsinstaller_1_prepare.ps1 @@ -22,10 +22,13 @@ $ErrorActionPreference = "Stop" $QtDir = 'C:\Qt' $Qt32Version = "5.15.2" $Qt64Version = "5.15.2" +$AqtinstallVersion = "2.0.5" +$JackVersion = "1.9.17" +$MsvcVersion = "2019" echo "Install Qt..." # Install Qt -pip install aqtinstall +pip install "aqtinstall==$AqtinstallVersion" if ( !$? ) { throw "pip install aqtinstall failed with exit code $LastExitCode" @@ -33,7 +36,7 @@ if ( !$? ) echo "Get Qt 64 bit..." # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt64Version}" win64_msvc2019_64 +aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt64Version}" "win64_msvc${MsvcVersion}_64" if ( !$? ) { throw "64bit Qt installation failed with exit code $LastExitCode" @@ -41,7 +44,7 @@ if ( !$? ) echo "Get Qt 32 bit..." # intermediate solution if the main server is down: append e.g. " -b https://mirrors.ocf.berkeley.edu/qt/" to the "aqt"-line below -aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt32Version}" win32_msvc2019 +aqt install-qt --outputdir "${QtDir}" windows desktop "${Qt32Version}" "win32_msvc${MsvcVersion}" if ( !$? ) { throw "32bit Qt installation failed with exit code $LastExitCode" @@ -56,7 +59,7 @@ if ($BuildOption -Eq "jackonwindows") { echo "Install JACK2 64-bit..." # Install JACK2 64-bit - choco install --no-progress -y jack + choco install --no-progress -y jack --version "${JackVersion}" if ( !$? ) { throw "64bit jack installation failed with exit code $LastExitCode" @@ -64,7 +67,7 @@ if ($BuildOption -Eq "jackonwindows") echo "Install JACK2 32-bit..." # Install JACK2 32-bit (need to force choco install as it detects 64 bits as installed) - choco install --no-progress -y -f --forcex86 jack + choco install --no-progress -y -f --forcex86 jack --version "${JackVersion}" if ( !$? ) { throw "32bit jack installation failed with exit code $LastExitCode" From 111321311f8bb2c7131f30b58bccd66e2e533eb5 Mon Sep 17 00:00:00 2001 From: Christian Hoffmann Date: Sat, 5 Feb 2022 15:41:20 +0100 Subject: [PATCH 6/6] ChangeLog: Add #2345 (Autobuild refactor/version pinning) --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index db8ab4a9b0..9945da35d4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -112,6 +112,9 @@ - Internal: Prepared autobuild logic for upcoming Qt6 compatibility (#2328). (contributed by @hoffie, @softins) +- Internal: Release build process has been refactored and uses pinned dependencies for better reproducibility now. (#2345) + (contributed by @hoffie) + ### 3.8.1 (2021-10-23) ###