From 2ddf898427e3518857beba09b465a0d300c112cf Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sat, 23 May 2020 22:48:40 -0600 Subject: [PATCH 01/33] - Bourne Shell shebang - Almquist shellcheck - Error trap - Strict shell options - Return 1 from functions on error instead of exit 1 since we have an error trap and the errexit option set --- build/build.sh | 8 +++--- build/functions.sh | 24 ++++++++-------- grove/build/build_rpm.sh | 25 +++++++++-------- grove/grovetccfg/build/build_rpm.sh | 25 +++++++++-------- infrastructure/docker/build/clean_build.sh | 9 +++--- traffic_monitor/build/build_rpm.sh | 28 +++++++++---------- traffic_ops/build/build_rpm.sh | 30 ++++++++++---------- traffic_ops_ort/build/build_rpm.sh | 12 ++++---- traffic_portal/build/build_rpm.sh | 22 +++++++-------- traffic_router/build/build_rpm.sh | 30 ++++++++++---------- traffic_router/tomcat-rpm/build_rpm.sh | 8 +++--- traffic_stats/build/build_rpm.sh | 32 +++++++++++----------- 12 files changed, 128 insertions(+), 125 deletions(-) diff --git a/build/build.sh b/build/build.sh index 1f252111e6..38a66e4486 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}"; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; # By default all sub-projects are built. Supply a list of projects to build if # only a subset is wanted. diff --git a/build/functions.sh b/build/functions.sh index 90628b2a77..bbad54d8c2 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,13 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash # --------------------------------------- # versionOk checks version number against required version. # ``versionOk 1.2.3 2.0.4.7'' returns false value indicating # version you have is not at least version you need # if versionOk $haveversion $needversion; then -# echo "Need at least version $needversion"; exit 1 +# echo "Need at least version $needversion"; return 1 # fi function versionOk() { local h=$1 n=$2 @@ -65,7 +63,7 @@ function getBuildNumber() { # Expect it's from the released tarball -- if BUILD_NUMBER file is not present, abort if [[ ! -f $TC_DIR/BUILD_NUMBER ]]; then echo "Not in git repository and no BUILD_NUMBER present -- aborting!" - exit 1 + return 1 fi grep -v '^#' $TC_DIR/BUILD_NUMBER fi @@ -75,7 +73,7 @@ function getBuildNumber() { function getVersion() { local d="$1" local vf="$d/VERSION" - [ -r $vf ] || { echo "Could not read $vf: $!"; exit 1; } + [ -r $vf ] || { echo "Could not read $vf: $!"; return 1; } cat "$vf" } @@ -99,7 +97,7 @@ function checkEnvironment { export RPMBUILD="$WORKSPACE/rpmbuild" export DIST="$WORKSPACE/dist" - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } + mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; return 1; } # verify required tools available in path -- extra tools required by subsystem are passed in for pgm in git rpmbuild "$@"; do @@ -109,7 +107,7 @@ function checkEnvironment { requiredGitVersion=1.7.12 if ! versionOk $(git --version | tr -dc 0-9. ) "$requiredGitVersion"; then echo "$(git --version) must be at least $requiredGitVersion" - exit 1 + return 1 fi echo "Build environment has been verified." @@ -125,7 +123,7 @@ function checkEnvironment { function createSourceDir() { local target="$1-$TC_VERSION" local srcpath="$RPMBUILD/SOURCES/$target" - mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; exit 1; } + mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; return 1; } echo "$srcpath" } @@ -147,7 +145,7 @@ function buildRpm () { --define "commit $(getCommit)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ -ba SPECS/$package.spec || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } + { echo "RPM BUILD FAILED: $?"; return 1; } echo echo "========================================================================================" @@ -155,8 +153,8 @@ function buildRpm () { echo "========================================================================================" echo - cp "$RPMBUILD/RPMS/$(uname -m)/$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } - cp "$RPMBUILD/SRPMS/$srpm" "$DIST/." || { echo "Could not copy $srpm to $DIST: $?"; exit 1; } + cp "$RPMBUILD/RPMS/$(uname -m)/$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; return 1; } + cp "$RPMBUILD/SRPMS/$srpm" "$DIST/." || { echo "Could not copy $srpm to $DIST: $?"; return 1; } done } diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index e02a533a90..eb818b080b 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,15 +10,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } + [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -42,7 +45,7 @@ function checkGroveEnvironment() { # grove needs to be built with go 1.14 or greater verify_and_set_go_version if [[ $? -ne 1 ]]; then - exit 0 + return 0 fi echo "==================================================" @@ -64,18 +67,18 @@ function initBuildArea() { # prep build environment [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; exit 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; exit 1; } + [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- function buildRpmGrove() { # build - $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; exit 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; exit 1; } + $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } + $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/grove-${GROVE_VERSION}.tgz grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; exit 1; } + tar -cvzf $RPMBUILD/SOURCES/grove-${GROVE_VERSION}.tgz grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -93,7 +96,7 @@ function buildRpmGrove() { fi # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/grove.spec || { echo "rpmbuild failed: $?" >&2; exit 1; } + rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/grove.spec || { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . [ -e $DIST ] || mkdir -p $DIST diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 16d0b6a540..ee5e284268 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,15 +10,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } + [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -42,7 +45,7 @@ function checkGroveEnvironment() { # grovetccfg needs to be built with go 1.14 or greater verify_and_set_go_version if [[ $? -ne 1 ]]; then - exit 0 + return 0 fi echo "==================================================" @@ -65,18 +68,18 @@ function initBuildArea() { # prep build environment [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; exit 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; exit 1; } + [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- function buildRpmGrove() { # build - $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; exit 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; exit 1; } + $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } + $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; exit 1; } + tar -cvzf $RPMBUILD/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -94,7 +97,7 @@ function buildRpmGrove() { fi # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/${PACKAGE}.spec || { echo "rpmbuild failed: $?" >&2; exit 1; } + rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/${PACKAGE}.spec || { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . [ -e $DIST ] || mkdir -p $DIST diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index 65729d17f5..86222454d9 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,6 +10,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}"; cleanup; exit $exit_code' EXIT; +set -o errexit -o nounset; # Fix ownership of output files @@ -26,8 +29,6 @@ cleanup() { setowner /trafficcontrol /trafficcontrol/dist } -trap cleanup EXIT - set -o xtrace; # set owner of dist dir -- cleans up existing dist permissions... diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 35b438dd6e..772872060f 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; function importFunctions() { local script=$(readlink -f "$0") @@ -23,7 +23,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -31,12 +31,12 @@ function importFunctions() { #---------------------------------------- function initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local tm_dest=$(createSourceDir traffic_monitor) cd "$TM_DIR" || \ - { echo "Could not cd to $TM_DIR: $?"; exit 1; } + { echo "Could not cd to $TM_DIR: $?"; return 1; } echo "PATH: $PATH" echo "GOPATH: $GOPATH" @@ -45,7 +45,7 @@ function initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/net/publicsuffix golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; exit 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_monitor go_build=(go build -v) @@ -54,15 +54,15 @@ function initBuildArea() { go_build+=(-gcflags 'all=-N -l'); fi; "${go_build[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" || \ - { echo "Could not build traffic_monitor binary"; exit 1; } + { echo "Could not build traffic_monitor binary"; return 1; } rsync -av ./ "$tm_dest"/ || \ - { echo "Could not copy to $tm_dest: $?"; exit 1; } + { echo "Could not copy to $tm_dest: $?"; return 1; } cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $tm_dest) || { echo "Could not create tar archive $tm_dest.tgz: $?"; exit 1; } - cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $tm_dest) || { echo "Could not create tar archive $tm_dest.tgz: $?"; return 1; } + cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } @@ -70,7 +70,7 @@ function initBuildArea() { function preBuildChecks() { if [[ -e "$TM_DIR"/traffic_monitor ]]; then echo "Found $TM_DIR/traffic_monitor, please remove before retrying to build" - exit 1 + return 1 fi } diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index a2e4cbcce9..f8ab778521 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function importFunctions() { @@ -24,7 +24,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -32,11 +32,11 @@ function importFunctions() { # --------------------------------------- function initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } local dest=$(createSourceDir traffic_ops) cd "$TO_DIR" || \ - { echo "Could not cd to $TO_DIR: $?"; exit 1; } + { echo "Could not cd to $TO_DIR: $?"; return 1; } echo "PATH: $PATH" echo "GOPATH: $GOPATH" @@ -45,7 +45,7 @@ function initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; exit 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_ops_golang pushd traffic_ops_golang @@ -55,29 +55,29 @@ function initBuildArea() { go_build+=(-gcflags 'all=-N -l'); fi; "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ - { echo "Could not build traffic_ops_golang binary"; exit 1; } + { echo "Could not build traffic_ops_golang binary"; return 1; } popd # compile db/admin pushd app/db "${go_build[@]}" -o admin || \ - { echo "Could not build db/admin binary"; exit 1; } + { echo "Could not build db/admin binary"; return 1; } popd # compile TO profile converter pushd install/bin/convert_profile "${go_build[@]}" || \ - { echo "Could not build convert_profile binary"; exit 1; } + { echo "Could not build convert_profile binary"; return 1; } popd rsync -av etc install "$dest"/ || \ - { echo "Could not copy to $dest: $?"; exit 1; } + { echo "Could not copy to $dest: $?"; return 1; } rsync -av app/{bin,conf,cpanfile,db,lib,public,script,templates} "$dest"/app/ || \ - { echo "Could not copy to $dest/app: $?"; exit 1; } + { echo "Could not copy to $dest/app: $?"; return 1; } tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES $(basename "$dest") || \ - { echo "Could not create tar archive $dest.tgz: $?"; exit 1; } + { echo "Could not create tar archive $dest.tgz: $?"; return 1; } cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + { echo "Could not copy spec files: $?"; return 1; } export PLUGINS=$(grep -l -P '(?/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail -o xtrace; #---------------------------------------- function importFunctions() { @@ -26,7 +24,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh"; if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" >&2; - exit 1; + return 1; fi . "$functions_sh"; } diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index a30f6f016e..058ee49064 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash -# -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function importFunctions() { @@ -24,7 +24,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -33,18 +33,18 @@ function importFunctions() { # --------------------------------------- function initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local ts_dest=$(createSourceDir traffic_portal) cd "$TP_DIR" || \ - { echo "Could not cd to $TP_DIR: $?"; exit 1; } + { echo "Could not cd to $TP_DIR: $?"; return 1; } rsync -av ./ "$ts_dest"/ || \ - { echo "Could not copy to $to_dest: $?"; exit 1; } - cp -r "$TP_DIR"/ "$ts_dest" || { echo "Could not copy $TP_DIR to $ts_dest: $?"; exit 1; } + { echo "Could not copy to $to_dest: $?"; return 1; } + cp -r "$TP_DIR"/ "$ts_dest" || { echo "Could not copy $TP_DIR to $ts_dest: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; exit 1; } - cp "$TP_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } + cp "$TP_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 16be21eb3b..ade564224b 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function importFunctions() { @@ -25,7 +25,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -37,22 +37,22 @@ function buildRpmTrafficRouter () { export STARTUP_SCRIPT_DIR="/lib/systemd/system" export STARTUP_SCRIPT_LOC="../core/src/main/lib/systemd/system" - cd "$TR_DIR" || { echo "Could not cd to $TR_DIR: $?"; exit 1; } + cd "$TR_DIR" || { echo "Could not cd to $TR_DIR: $?"; return 1; } mvn -P rpm-build -Dmaven.test.skip=true -DminimumTPS=1 clean package || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } + { echo "RPM BUILD FAILED: $?"; return 1; } local rpm=$(find -name \*.rpm) if [[ -z $rpm ]]; then echo "Could not find rpm file $RPM in $(pwd)" - exit 1; + return 1; fi echo "========================================================================================" echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." echo "========================================================================================" echo - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } + mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; return 1; } - cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } + cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; return 1; } } @@ -86,18 +86,18 @@ function adaptEnvironment() { # --------------------------------------- function initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } tr_dest=$(createSourceDir traffic_router) export MVN_CMD="mvn versions:set -DnewVersion=$TC_VERSION" echo $MVN_CMD (cd $TR_DIR; $MVN_CMD) - cp -r "$TR_DIR"/{build,connector,core} "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; exit 1; } - cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; exit 1; } + cp -r "$TR_DIR"/{build,connector,core} "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; return 1; } + cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; return 1; } # tar/gzip the source - tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" $(basename $tr_dest) || { echo "Could not create tar archive $tr_dest: $?"; exit 1; } + tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" $(basename $tr_dest) || { echo "Could not create tar archive $tr_dest: $?"; return 1; } echo "The build area has been initialized." } @@ -106,7 +106,7 @@ function initBuildArea() { function buildRpmTomcat () { echo "Building the rpm for Tomcat." - cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; exit 1; } + cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; return 1; } ./build_rpm.sh } diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 7a4a6ef17d..5795aafc68 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash -# -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- function checkEnvironment() { diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 416b080fdf..f363e65837 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; function importFunctions() { local script=$(readlink -f "$0") @@ -23,7 +23,7 @@ function importFunctions() { functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } @@ -31,12 +31,12 @@ function importFunctions() { #---------------------------------------- function initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local ts_dest=$(createSourceDir traffic_stats) cd "$TS_DIR" || \ - { echo "Could not cd to $TS_DIR: $?"; exit 1; } + { echo "Could not cd to $TS_DIR: $?"; return 1; } echo "PATH: $PATH" echo "GOPATH: $GOPATH" @@ -45,27 +45,27 @@ function initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/net/publicsuffix || \ - { echo "Could not get go package dependencies"; exit 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_stats go build -v || \ - { echo "Could not build traffic_stats binary"; exit 1; } + { echo "Could not build traffic_stats binary"; return 1; } # compile influx_db_tools pushd influxdb_tools go build -v sync/sync_ts_databases.go || \ - { echo "Could not build sync_ts_databases binary"; exit 1; } + { echo "Could not build sync_ts_databases binary"; return 1; } go build -v create/create_ts_databases.go || \ - { echo "Could not build create_ts_databases binary"; exit 1; } + { echo "Could not build create_ts_databases binary"; return 1; } popd rsync -aLv ./ "$ts_dest"/ || \ - { echo "Could not copy to $ts_dest: $?"; exit 1; } + { echo "Could not copy to $ts_dest: $?"; return 1; } cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; exit 1; } - cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } + cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } @@ -73,7 +73,7 @@ function initBuildArea() { function preBuildChecks() { if [[ -e "$TS_DIR"/traffic_stats ]]; then echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" - exit 1 + return 1 fi } From a47eb4119272c57a91de6ec67b8dde8c990f6e5c Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sat, 23 May 2020 23:09:52 -0600 Subject: [PATCH 02/33] Remove non-standard "function" keyword (SC2112) --- build/functions.sh | 26 +++++++++++++------------- grove/build/build_rpm.sh | 10 +++++----- grove/grovetccfg/build/build_rpm.sh | 10 +++++----- traffic_monitor/build/build_rpm.sh | 7 ++++--- traffic_ops/build/build_rpm.sh | 4 ++-- traffic_ops_ort/build/build_rpm.sh | 4 ++-- traffic_portal/build/build_rpm.sh | 4 ++-- traffic_router/build/build_rpm.sh | 10 +++++----- traffic_router/tomcat-rpm/build_rpm.sh | 8 ++++---- traffic_stats/build/build_rpm.sh | 7 ++++--- 10 files changed, 46 insertions(+), 44 deletions(-) diff --git a/build/functions.sh b/build/functions.sh index bbad54d8c2..0af3049c22 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -20,7 +20,7 @@ # if versionOk $haveversion $needversion; then # echo "Need at least version $needversion"; return 1 # fi -function versionOk() { +versionOk() { local h=$1 n=$2 # string compare -- no need to do more if the same [[ $h == $n ]] && return 0 @@ -41,19 +41,19 @@ function versionOk() { } # --------------------------------------- -function getRevCount() { +getRevCount() { local buildNum=$(getBuildNumber) echo ${buildNum%.*} } # --------------------------------------- -function isInGitTree() { +isInGitTree() { # ignore output -- use exit status git rev-parse --is-inside-work-tree >/dev/null 2>&1 } # --------------------------------------- -function getBuildNumber() { +getBuildNumber() { local in_git=$() if isInGitTree; then local commits=$(git rev-list HEAD 2>/dev/null | wc -l) @@ -70,7 +70,7 @@ function getBuildNumber() { } # --------------------------------------- -function getVersion() { +getVersion() { local d="$1" local vf="$d/VERSION" [ -r $vf ] || { echo "Could not read $vf: $!"; return 1; } @@ -78,18 +78,18 @@ function getVersion() { } # --------------------------------------- -function getRhelVersion { +getRhelVersion() { echo el$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)) } # --------------------------------------- -function getCommit() { +getCommit() { local buildNum=$(getBuildNumber) echo ${buildNum%.*} } # --------------------------------------- -function checkEnvironment { +checkEnvironment() { export TC_VERSION=$(getVersion "$TC_DIR") export BUILD_NUMBER=$(getBuildNumber) export RHEL_VERSION=$(getRhelVersion) @@ -120,7 +120,7 @@ function checkEnvironment { } # --------------------------------------- -function createSourceDir() { +createSourceDir() { local target="$1-$TC_VERSION" local srcpath="$RPMBUILD/SOURCES/$target" mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; return 1; } @@ -128,7 +128,7 @@ function createSourceDir() { } # --------------------------------------- -function buildRpm () { +buildRpm() { for package in "$@"; do local pre="${package}-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION}" local rpm="${pre}.$(uname -m).rpm" @@ -159,7 +159,7 @@ function buildRpm () { } # --------------------------------------- -function createTarball() { +createTarball() { local projDir=$(cd "$1"; pwd) local projName=trafficcontrol local version=$(getVersion "$TC_DIR") @@ -177,7 +177,7 @@ function createTarball() { } # --------------------------------------- -function createDocsTarball() { +createDocsTarball() { local projDir=$(cd "$1"; pwd) local projName=trafficcontrol local version=$(getVersion "$TC_DIR") @@ -197,7 +197,7 @@ function createDocsTarball() { # ---------------------------------------- # verify if the go compiler is version 1.14 or higher, returns 0 if if not. returns 1 if it is. # -function verify_and_set_go_version () { +verify_and_set_go_version() { GO_VERSION="none" GO="none" go_in_path=`type -p go` diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index eb818b080b..8db3f8056c 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" @@ -28,7 +28,7 @@ function importFunctions() { } #---------------------------------------- -function checkGroveEnvironment() { +checkGroveEnvironment() { echo "Verifying the build configuration environment." local script=$(readlink -f "$0") @@ -62,7 +62,7 @@ function checkGroveEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { cd "$GROVE_DIR" # prep build environment @@ -72,7 +72,7 @@ function initBuildArea() { } # --------------------------------------- -function buildRpmGrove() { +buildRpmGrove() { # build $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; return 1; } @@ -89,7 +89,7 @@ function buildRpmGrove() { if ! id $spec_owner >/dev/null 2>&1; then chown $(id -u):$(id -g) build/grove.spec - function give_spec_back { + give_spec_back() { chown ${spec_owner}:${spec_group} build/grove.spec } trap give_spec_back EXIT diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index ee5e284268..2d319e507c 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" @@ -28,7 +28,7 @@ function importFunctions() { } #---------------------------------------- -function checkGroveEnvironment() { +checkGroveEnvironment() { echo "Verifying the build configuration environment." local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") @@ -63,7 +63,7 @@ function checkGroveEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { cd "$GROVETC_DIR" # prep build environment @@ -73,7 +73,7 @@ function initBuildArea() { } # --------------------------------------- -function buildRpmGrove() { +buildRpmGrove() { # build $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } @@ -90,7 +90,7 @@ function buildRpmGrove() { if ! id $spec_owner >/dev/null 2>&1; then chown $(id -u):$(id -g) build/${PACKAGE}.spec - function give_spec_back { + give_spec_back() { chown ${spec_owner}:${spec_group} build/${PACKAGE}.spec } trap give_spec_back EXIT diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 772872060f..13d6e1bd8e 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -15,7 +15,8 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; set -o errexit -o nounset -o pipefail; -function importFunctions() { +#---------------------------------------- +importFunctions() { local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") export TM_DIR=$(dirname "$scriptdir") @@ -29,7 +30,7 @@ function importFunctions() { } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } @@ -67,7 +68,7 @@ function initBuildArea() { echo "The build area has been initialized." } -function preBuildChecks() { +preBuildChecks() { if [[ -e "$TM_DIR"/traffic_monitor ]]; then echo "Found $TM_DIR/traffic_monitor, please remove before retrying to build" return 1 diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index f8ab778521..ee1b775bf9 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") export TO_DIR=$(dirname "$scriptdir") @@ -30,7 +30,7 @@ function importFunctions() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index 9779c8f540..ec82821722 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail -o xtrace; #---------------------------------------- -function importFunctions() { +importFunctions() { local script=$(readlink -f "$0"); local scriptdir=$(dirname "$script"); export ORT_DIR=$(dirname "$scriptdir"); @@ -30,7 +30,7 @@ function importFunctions() { } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area for Traffic Ops ORT"; mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index 058ee49064..0db9da520d 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") export TP_DIR=$(dirname "$scriptdir") @@ -31,7 +31,7 @@ function importFunctions() { # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index ade564224b..7821433f18 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { echo "Verifying the build configuration environment." local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") @@ -31,7 +31,7 @@ function importFunctions() { } #---------------------------------------- -function buildRpmTrafficRouter () { +buildRpmTrafficRouter () { echo "Building the rpm." export STARTUP_SCRIPT_DIR="/lib/systemd/system" @@ -58,7 +58,7 @@ function buildRpmTrafficRouter () { #---------------------------------------- -function adaptEnvironment() { +adaptEnvironment() { echo "Verifying the build configuration environment." # get traffic_control src path -- relative to build_rpm.sh script export PACKAGE="traffic_router" @@ -84,7 +84,7 @@ function adaptEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } @@ -103,7 +103,7 @@ function initBuildArea() { } #---------------------------------------- -function buildRpmTomcat () { +buildRpmTomcat () { echo "Building the rpm for Tomcat." cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; return 1; } diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 5795aafc68..1d10bea537 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -16,7 +16,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0 set -o errexit -o nounset -o pipefail; #---------------------------------------- -function checkEnvironment() { +checkEnvironment() { echo "Verifying the build configuration environment." local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") @@ -48,7 +48,7 @@ function checkEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } export VERSION=$TOMCAT_VERSION @@ -65,12 +65,12 @@ function initBuildArea() { } #---------------------------------------- -function buildRpmTomcat () { +buildRpmTomcat () { export SPEC_FILE_NAME=tomcat.spec buildRpmForEl 7 } -function buildRpmForEl () { +buildRpmForEl () { echo "Building the rpm for "$RHEL_VERSION"." cd $RPMBUILD diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index f363e65837..324cd93f7c 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -15,7 +15,8 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; set -o errexit -o nounset -o pipefail; -function importFunctions() { +#---------------------------------------- +importFunctions() { local script=$(readlink -f "$0") local scriptdir=$(dirname "$script") export TS_DIR=$(dirname "$scriptdir") @@ -29,7 +30,7 @@ function importFunctions() { } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } @@ -70,7 +71,7 @@ function initBuildArea() { echo "The build area has been initialized." } -function preBuildChecks() { +preBuildChecks() { if [[ -e "$TS_DIR"/traffic_stats ]]; then echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" return 1 From 117491de5d7a5a3845df93c7da79c1f3c649afd4 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sat, 23 May 2020 23:33:19 -0600 Subject: [PATCH 03/33] Declare variables before setting to avoid masking return values --- build/build.sh | 6 ++-- build/functions.sh | 47 ++++++++++++++++---------- grove/build/build_rpm.sh | 23 +++++++------ grove/grovetccfg/build/build_rpm.sh | 23 +++++++------ infrastructure/cdn-in-a-box/Makefile | 4 +-- traffic_monitor/build/build_rpm.sh | 14 +++++--- traffic_ops/build/build_rpm.sh | 16 +++++---- traffic_ops_ort/build/build_rpm.sh | 13 ++++--- traffic_portal/build/build_rpm.sh | 14 +++++--- traffic_router/build/build_rpm.sh | 36 ++++++++++++-------- traffic_router/tomcat-rpm/build_rpm.sh | 14 +++++--- traffic_stats/build/build_rpm.sh | 14 +++++--- 12 files changed, 136 insertions(+), 88 deletions(-) diff --git a/build/build.sh b/build/build.sh index 38a66e4486..8b55818a14 100755 --- a/build/build.sh +++ b/build/build.sh @@ -19,8 +19,10 @@ set -o errexit -o nounset -o pipefail; # only a subset is wanted. # make sure we start out in traffic_control dir -topscript=$(readlink -f $0) -export TC_DIR=$(dirname $(dirname "$topscript")) +topscript='' TC_DIR='' +topscript="$(readlink -f "$0")" +TC_DIR="$(dirname "$(dirname "$topscript")")" +export TC_DIR [[ -n $TC_DIR ]] && cd "$TC_DIR" || { echo "Could not cd $TC_DIR"; exit 1; } . build/functions.sh diff --git a/build/functions.sh b/build/functions.sh index 0af3049c22..5bb5312e6d 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -42,7 +42,8 @@ versionOk() { # --------------------------------------- getRevCount() { - local buildNum=$(getBuildNumber) + local buildNum + buildNum=$(getBuildNumber) echo ${buildNum%.*} } @@ -54,10 +55,11 @@ isInGitTree() { # --------------------------------------- getBuildNumber() { - local in_git=$() + local in_git='' if isInGitTree; then - local commits=$(git rev-list HEAD 2>/dev/null | wc -l) - local sha=$(git rev-parse --short=8 HEAD) + local commits sha + commits="$(git rev-list HEAD 2>/dev/null | wc -l)" + sha="$(git rev-parse --short=8 HEAD)" echo "$commits.$sha" else # Expect it's from the released tarball -- if BUILD_NUMBER file is not present, abort @@ -84,18 +86,21 @@ getRhelVersion() { # --------------------------------------- getCommit() { - local buildNum=$(getBuildNumber) + local buildNum + buildNum="$(getBuildNumber)" echo ${buildNum%.*} } # --------------------------------------- checkEnvironment() { - export TC_VERSION=$(getVersion "$TC_DIR") - export BUILD_NUMBER=$(getBuildNumber) - export RHEL_VERSION=$(getRhelVersion) - export WORKSPACE=${WORKSPACE:-$TC_DIR} - export RPMBUILD="$WORKSPACE/rpmbuild" - export DIST="$WORKSPACE/dist" + TC_VERSION='' BUILD_NUMBER='' RHEL_VERSION='' RPMBUILD='' DIST='' + TC_VERSION="$(getVersion "$TC_DIR")" + BUILD_NUMBER="$(getBuildNumber)" + RHEL_VERSION="$(getRhelVersion)" + WORKSPACE="${WORKSPACE:-$TC_DIR}" + RPMBUILD="$WORKSPACE/rpmbuild" + DIST="$WORKSPACE/dist" + export TC_VERSION BUILD_NUMBER RHEL_VERSION WORKSPACE RPMBUILD DIST mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; return 1; } @@ -131,7 +136,8 @@ createSourceDir() { buildRpm() { for package in "$@"; do local pre="${package}-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION}" - local rpm="${pre}.$(uname -m).rpm" + local rpm + rpm="${pre}.$(uname -m).rpm" local srpm="${pre}.src.rpm" echo "Building the rpm." if [[ "$DEBUG_BUILD" == true ]]; then @@ -160,11 +166,14 @@ buildRpm() { # --------------------------------------- createTarball() { - local projDir=$(cd "$1"; pwd) + local projDir + projDir="$(cd "$1"; pwd)" local projName=trafficcontrol - local version=$(getVersion "$TC_DIR") - local tarball="dist/apache-$projName-$version.tar.gz" - local tardir=$(basename $tarball .tar.gz) + local version + version="$(getVersion "$TC_DIR")" + local tarball="dist/apache-${projName}-${version}.tar.gz" + local tardir + tardir="$(basename "$tarball" .tar.gz)" # Create a BULDNUMBER file and add to tarball local bndir=$(mktemp -d) @@ -178,9 +187,11 @@ createTarball() { # --------------------------------------- createDocsTarball() { - local projDir=$(cd "$1"; pwd) + local projDir + projDir="$(cd "$1"; pwd)" local projName=trafficcontrol - local version=$(getVersion "$TC_DIR") + local version + version="$(getVersion "$TC_DIR")" local tarball="dist/apache-$projName-$version-docs.tar.gz" local tardir="${projDir}/docs/build/" diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 8db3f8056c..48af76302d 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -31,16 +31,19 @@ importFunctions() { checkGroveEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - - export GROVE_DIR=$(dirname "$scriptdir") - export GROVE_VERSION=`cat ${GROVE_DIR}/VERSION` - export PACKAGE="grove" - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export RPMBUILD="${GROVE_DIR}/rpmbuild" - export DIST="${TC_DIR}/dist" - export RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + local script scriptdir + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + + GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' + GROVE_DIR=$(dirname "$scriptdir") + GROVE_VERSION="$(cat "${GROVE_DIR}/VERSION")" + PACKAGE="grove" + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + RPMBUILD="${GROVE_DIR}/rpmbuild" + DIST="${TC_DIR}/dist" + RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + export GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM # grove needs to be built with go 1.14 or greater verify_and_set_go_version diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 2d319e507c..a067de339a 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -30,17 +30,20 @@ importFunctions() { #---------------------------------------- checkGroveEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") + local script scriptdir + script=$(readlink -f "$0") + scriptdir=$(dirname "$script") - export GROVETC_DIR=$(dirname "$scriptdir") - export GROVE_DIR=$(dirname "$GROVETC_DIR") - export GROVE_VERSION=`cat ${GROVE_DIR}/VERSION` - export PACKAGE="grovetccfg" - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export RPMBUILD="${GROVE_DIR}/rpmbuild" - export DIST="${TC_DIR}/dist" - export RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + GROVETC_DIR='' GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' + GROVETC_DIR=$(dirname "$scriptdir") + GROVE_DIR=$(dirname "$GROVETC_DIR") + GROVE_VERSION="$(cat "${GROVE_DIR}/VERSION")" + PACKAGE="grovetccfg" + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + RPMBUILD="${GROVE_DIR}/rpmbuild" + DIST="${TC_DIR}/dist" + RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + export GROVETC_DIR GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM # grovetccfg needs to be built with go 1.14 or greater verify_and_set_go_version diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index c3e2acc28a..3edbeb46e7 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -32,8 +32,8 @@ endif PKG_FLAGS := -v BUILD_NUMBER := $(shell git rev-list HEAD 2>/dev/null | wc -l | tr -d '[[:space:]]').$(shell git rev-parse --short=8 HEAD) TC_VERSION := $(shell cat "../../VERSION") -TOMCAT_VERSION := $(shell grep 'export TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d '=' -f 2) -TOMCAT_RELEASE := $(shell grep 'export TOMCAT_RELEASE=' ../../traffic_router/build/build_rpm.sh | cut -d '=' -f 2) +TOMCAT_VERSION := $(shell grep '^\s*TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d= -f2) +TOMCAT_RELEASE := $(shell grep '^\s*TOMCAT_RELEASE=' ../../traffic_router/build/build_rpm.sh | cut -d= -f2) RHEL_VERSION := el7 diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 13d6e1bd8e..e41402ec9e 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -17,10 +17,13 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TM_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TM_DIR") + local script scriptdir + TM_DIR='' TC_DIR='' + script=$(readlink -f "$0") + scriptdir=$(dirname "$script") + TM_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TM_DIR")" + export TM_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" @@ -35,7 +38,8 @@ initBuildArea() { mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local tm_dest=$(createSourceDir traffic_monitor) + local tm_dest + tm_dest="$(createSourceDir traffic_monitor)" cd "$TM_DIR" || \ { echo "Could not cd to $TM_DIR: $?"; return 1; } diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index ee1b775bf9..ad69a2b692 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -17,10 +17,12 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TO_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TO_DIR") + local script scriptdir + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + TO_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TO_DIR")" + export TO_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" @@ -34,7 +36,8 @@ initBuildArea() { echo "Initializing the build area." mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } - local dest=$(createSourceDir traffic_ops) + local dest + dest="$(createSourceDir traffic_ops)" cd "$TO_DIR" || \ { echo "Could not cd to $TO_DIR: $?"; return 1; } @@ -79,7 +82,8 @@ initBuildArea() { cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ { echo "Could not copy spec files: $?"; return 1; } - export PLUGINS=$(grep -l -P '(?&2; @@ -34,7 +36,8 @@ initBuildArea() { echo "Initializing the build area for Traffic Ops ORT"; mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} - local dest=$(createSourceDir traffic_ops_ort); + local dest; + dest=$(createSourceDir traffic_ops_ort); cd "$ORT_DIR"; echo "PATH: $PATH"; diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index 0db9da520d..e0994c8b9d 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -17,10 +17,13 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TP_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TP_DIR") + local script scriptdir + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + TP_DIR='' TC_DIR='' + TP_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TP_DIR")" + export TP_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" @@ -36,7 +39,8 @@ initBuildArea() { mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local ts_dest=$(createSourceDir traffic_portal) + local tp_dest + tp_dest="$(createSourceDir traffic_portal)" cd "$TP_DIR" || \ { echo "Could not cd to $TP_DIR: $?"; return 1; } rsync -av ./ "$ts_dest"/ || \ diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 7821433f18..20ce3a510e 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -18,10 +18,13 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TR_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TR_DIR") + local script scriptdir + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + TR_DIR='' TC_DIR='' + TR_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TR_DIR")" + export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "Error: Can't find $functions_sh" @@ -41,7 +44,8 @@ buildRpmTrafficRouter () { mvn -P rpm-build -Dmaven.test.skip=true -DminimumTPS=1 clean package || \ { echo "RPM BUILD FAILED: $?"; return 1; } - local rpm=$(find -name \*.rpm) + local rpm + rpm="$(find . -name \*.rpm | head -n1)" if [[ -z $rpm ]]; then echo "Could not find rpm file $RPM in $(pwd)" return 1; @@ -61,16 +65,18 @@ buildRpmTrafficRouter () { adaptEnvironment() { echo "Verifying the build configuration environment." # get traffic_control src path -- relative to build_rpm.sh script - export PACKAGE="traffic_router" - export TC_VERSION=$(getVersion "$TC_DIR") - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export BUILD_LOCK=$(getBuildNumber).${RHEL_VERSION} - export WORKSPACE=${WORKSPACE:-$TC_DIR} - export RPMBUILD="$WORKSPACE/rpmbuild" - export DIST="$WORKSPACE/dist" - export RPM="${PACKAGE}-${TC_VERSION}-${BUILD_NUMBER}.x86_64.rpm" - export TOMCAT_VERSION=8.5 - export TOMCAT_RELEASE=32 + PACKAGE='' TC_VERSION='' BUILD_LOCK='' RPMBUILD='' DIST='' RPM='' TOMCAT_VERSION='' TOMCAT_RELEASE='' + PACKAGE="traffic_router" + TC_VERSION=$(getVersion "$TC_DIR") + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + BUILD_LOCK=$(getBuildNumber).${RHEL_VERSION} + WORKSPACE=${WORKSPACE:-$TC_DIR} + RPMBUILD="$WORKSPACE/rpmbuild" + DIST="$WORKSPACE/dist" + RPM="${PACKAGE}-${TC_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + TOMCAT_VERSION=8.5 + TOMCAT_RELEASE=32 + export PACKAGE TC_VERSION BUILD_NUMBER BUILD_LOCK WORKSPACE RPMBUILD DIST RPM TOMCAT_VERSION TOMCAT_RELEASE echo "==================================================" echo "WORKSPACE: $WORKSPACE" diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 1d10bea537..910d5243ee 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -18,10 +18,13 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- checkEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TR_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TR_DIR") + local script scriptdir + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + TR_DIR='' TC_DIR='' + TR_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TR_DIR")" + export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "Error: Can't find $functions_sh" @@ -79,7 +82,8 @@ buildRpmForEl () { --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ -ba SPECS/$SPEC_FILE_NAME || \ { echo "RPM BUILD FAILED: $?"; exit 1; } - local rpm=$(find ./RPMS -name $RPM) + local rpm + rpm="$(find ./RPMS -name "$RPM")" if [[ -z $rpm ]]; then echo "Could not find rpm file $RPM in $(pwd)" exit 1; diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 324cd93f7c..1d4615f112 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -17,10 +17,13 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TS_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TS_DIR") + local script scriptdir + TS_DIR='' TC_DIR='' + script="$(readlink -f "$0")" + scriptdir="$(dirname "$script")" + TS_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TS_DIR")" + export TS_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" if [[ ! -r $functions_sh ]]; then echo "error: can't find $functions_sh" @@ -35,7 +38,8 @@ initBuildArea() { mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local ts_dest=$(createSourceDir traffic_stats) + local ts_dest + ts_dest="$(createSourceDir traffic_stats)" cd "$TS_DIR" || \ { echo "Could not cd to $TS_DIR: $?"; return 1; } From 4cfc828d3d078c72639277325d9a94363920bb50 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 14:15:09 -0600 Subject: [PATCH 04/33] Reformat RPM scripts and spec files --- build/build.sh | 2 +- build/functions.sh | 84 +++---- grove/build/build_rpm.sh | 6 +- grove/build/grove.spec | 22 +- grove/grovetccfg/build/build_rpm.sh | 6 +- traffic_monitor/build/traffic_monitor.spec | 37 ++- traffic_ops/build/build_rpm.sh | 8 +- traffic_ops/build/traffic_ops.spec | 224 +++++++++--------- traffic_ops_ort/build/traffic_ops_ort.spec | 24 +- traffic_portal/build/traffic_portal.spec | 62 ++--- traffic_router/build/build_rpm.sh | 2 +- traffic_router/tomcat-rpm/build_rpm.sh | 66 +++--- traffic_router/tomcat-rpm/tomcat.spec | 12 +- .../astats_over_http/astats-git-build.spec | 24 +- .../astats_over_http/astats_over_http.spec | 24 +- traffic_server/spec/trafficserver.spec | 24 +- traffic_stats/build/build_rpm.sh | 32 +-- traffic_stats/build/traffic_stats.spec | 34 +-- 18 files changed, 345 insertions(+), 348 deletions(-) diff --git a/build/build.sh b/build/build.sh index 8b55818a14..29cfddc348 100755 --- a/build/build.sh +++ b/build/build.sh @@ -38,7 +38,7 @@ else # Always build tarball when building everything.. projects=(tarball) for p in "${projects_to_build[@]}"; do - p=${p%%/*} + p=${p%%/*} if [[ $p != "traffic_monitor_golang" ]]; then projects+=($p) fi diff --git a/build/functions.sh b/build/functions.sh index 5bb5312e6d..a3dbc12b36 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -81,7 +81,7 @@ getVersion() { # --------------------------------------- getRhelVersion() { - echo el$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)) + echo el$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)) } # --------------------------------------- @@ -177,12 +177,12 @@ createTarball() { # Create a BULDNUMBER file and add to tarball local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" + getBuildNumber >"$bndir/BUILD_NUMBER" - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$projDir" --exclude-vcs --transform "s@^@$tardir/@S" $(git ls-files) - rm -r "$bndir" - echo "$tarball" + # create the tarball only from files in repo and BUILD_NUMBER + tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$projDir" --exclude-vcs --transform "s@^@$tardir/@S" $(git ls-files) + rm -r "$bndir" + echo "$tarball" } # --------------------------------------- @@ -197,47 +197,47 @@ createDocsTarball() { # Create a BULDNUMBER file and add to tarball local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" + getBuildNumber >"$bndir/BUILD_NUMBER" - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$tardir" . --exclude-vcs - rm -r "$bndir" - echo "$tarball" + # create the tarball only from files in repo and BUILD_NUMBER + tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$tardir" . --exclude-vcs + rm -r "$bndir" + echo "$tarball" } # ---------------------------------------- # verify if the go compiler is version 1.14 or higher, returns 0 if if not. returns 1 if it is. -# +# verify_and_set_go_version() { - GO_VERSION="none" - GO="none" - go_in_path=`type -p go` - for g in $go_in_path /usr/bin/go /usr/local/go/bin/go; do - if [[ -z $g ]] || [[ ! -x $g ]]; then - continue - fi - - go_version=`$g version | awk '{print $3}'` - - if [[ $go_version =~ go([1-9])\.([1-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 1 ]] && [[ ${BASH_REMATCH[2]} -ge 14 ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - GO=$g; export GO - PATH=`dirname $g`:$PATH; export PATH - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - echo "will use $g" - return 1 - else - if [[ $go_version =~ go([1-9])\.([1-9]+) ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - continue - fi - fi - done - - if [[ $GO == none ]]; then - echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" - return 0 - fi + GO_VERSION="none" + GO="none" + go_in_path=`type -p go` + for g in $go_in_path /usr/bin/go /usr/local/go/bin/go; do + if [[ -z $g ]] || [[ ! -x $g ]]; then + continue + fi + + go_version=`$g version | awk '{print $3}'` + + if [[ $go_version =~ go([1-9])\.([1-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 1 ]] && [[ ${BASH_REMATCH[2]} -ge 14 ]]; then + GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION + GO=$g; export GO + PATH=`dirname $g`:$PATH; export PATH + echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" + echo "will use $g" + return 1 + else + if [[ $go_version =~ go([1-9])\.([1-9]+) ]]; then + GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION + echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" + continue + fi + fi + done + + if [[ $GO == none ]]; then + echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" + return 0 + fi } diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 48af76302d..f5a4b01999 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -90,12 +90,12 @@ buildRpmGrove() { spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/grove.spec + chown $(id -u):$(id -g) build/grove.spec give_spec_back() { chown ${spec_owner}:${spec_group} build/grove.spec - } - trap give_spec_back EXIT + } + trap give_spec_back EXIT fi # build RPM diff --git a/grove/build/grove.spec b/grove/build/grove.spec index 2f0f345aa4..31f24a46c3 100644 --- a/grove/build/grove.spec +++ b/grove/build/grove.spec @@ -10,18 +10,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -Summary: Grove HTTP Caching Proxy -Name: grove -Version: %{version} -Release: %{build_number} -License: Apache License, Version 2.0 -Group: Base System/System Tools -Prefix: /usr/sbin/%{name} -Source: %{_sourcedir}/%{name}-%{version}.tgz -URL: https://github.com/apache/trafficcontrol/%{name} +Summary: Grove HTTP Caching Proxy +Name: grove +Version: %{version} +Release: %{build_number} +License: Apache License, Version 2.0 +Group: Base System/System Tools +Prefix: /usr/sbin/%{name} +Source: %{_sourcedir}/%{name}-%{version}.tgz +URL: https://github.com/apache/trafficcontrol/%{name} Distribution: CentOS Linux -Vendor: Apache Software Foundation -BuildRoot: %{buildroot} +Vendor: Apache Software Foundation +BuildRoot: %{buildroot} # %define PACKAGEDIR %{prefix} diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index a067de339a..f787cc5afc 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -91,12 +91,12 @@ buildRpmGrove() { spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/${PACKAGE}.spec + chown $(id -u):$(id -g) build/${PACKAGE}.spec give_spec_back() { chown ${spec_owner}:${spec_group} build/${PACKAGE}.spec - } - trap give_spec_back EXIT + } + trap give_spec_back EXIT fi # build RPM diff --git a/traffic_monitor/build/traffic_monitor.spec b/traffic_monitor/build/traffic_monitor.spec index abf8cfe0bf..571893c9ab 100644 --- a/traffic_monitor/build/traffic_monitor.spec +++ b/traffic_monitor/build/traffic_monitor.spec @@ -15,17 +15,17 @@ # # RPM spec file for the Go version of Traffic Monitor (tm). # -%define debug_package %{nil} -Name: traffic_monitor -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Monitor the caches -Packager: david_neuman2 at Cable dot Comcast dot com -Vendor: Apache Software Foundation -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol -Source: %{_sourcedir}/traffic_monitor-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_monitor +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Monitor the caches +Packager: david_neuman2 at Cable dot Comcast dot com +Vendor: Apache Software Foundation +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol +Source: %{_sourcedir}/traffic_monitor-%{traffic_control_version}.tgz %description Installs traffic_monitor @@ -38,8 +38,8 @@ Installs traffic_monitor # copy traffic_monitor binary godir=src/github.com/apache/trafficcontrol/traffic_monitor ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -r "$TC_DIR"/traffic_monitor/* . + cd "$godir" && \ + cp -r "$TC_DIR"/traffic_monitor/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } %install @@ -67,16 +67,13 @@ cp "$src"/build/traffic_monitor.logrotate "${RPM_BUILD_ROOT}"/etc/logrotate.d/t /usr/bin/getent group traffic_monitor >/dev/null if [ $? -ne 0 ]; then - /usr/sbin/groupadd -g 423 traffic_monitor fi /usr/bin/getent passwd traffic_monitor >/dev/null if [ $? -ne 0 ]; then - /usr/sbin/useradd -g traffic_monitor -u 423 -d /opt/traffic_monitor -M traffic_monitor - fi /usr/bin/passwd -l traffic_monitor >/dev/null @@ -89,10 +86,10 @@ fi #don't install over the top of java TM. This is a workaround since yum doesn't respect the Conflicts tag. if [[ $(rpm -q traffic_monitor --qf "%{VERSION}-%{RELEASE}") < 1.9.0 ]] then - echo -e "\n****************\n" - echo "A java version of traffic_monitor is installed. Please backup/remove that version before installing the golang version of traffic_monitor." - echo -e "\n****************\n" - exit 1 + echo -e "\n****************\n" + echo "A java version of traffic_monitor is installed. Please backup/remove that version before installing the golang version of traffic_monitor." + echo -e "\n****************\n" + exit 1 fi %post diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index ad69a2b692..6b10e0e770 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -48,7 +48,7 @@ initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; return 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_ops_golang pushd traffic_ops_golang @@ -58,19 +58,19 @@ initBuildArea() { go_build+=(-gcflags 'all=-N -l'); fi; "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ - { echo "Could not build traffic_ops_golang binary"; return 1; } + { echo "Could not build traffic_ops_golang binary"; return 1; } popd # compile db/admin pushd app/db "${go_build[@]}" -o admin || \ - { echo "Could not build db/admin binary"; return 1; } + { echo "Could not build db/admin binary"; return 1; } popd # compile TO profile converter pushd install/bin/convert_profile "${go_build[@]}" || \ - { echo "Could not build convert_profile binary"; return 1; } + { echo "Could not build convert_profile binary"; return 1; } popd rsync -av etc install "$dest"/ || \ diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index c2e682b1cd..d658746acd 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -56,139 +56,139 @@ Built: %(date) by %{getenv: USER} %setup %build - # update version referenced in the source - perl -pi.bak -e 's/__VERSION__/%{version}-%{release}/' app/lib/UI/Utils.pm - - # copy traffic_ops_golang binary - godir=src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang - ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp "$TC_DIR"/traffic_ops/traffic_ops_golang/traffic_ops_golang . - ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } - - # copy TO DB admin - db_admin_dir=src/github.com/apache/trafficcontrol/traffic_ops/app/db - ( mkdir -p "$db_admin_dir" && \ - cd "$db_admin_dir" && \ - cp "$TC_DIR"/traffic_ops/app/db/admin . - ) || { echo "Could not copy go db admin at $(pwd): $!"; exit 1; }; - - # copy TO profile converter - convert_dir=src/github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile - ( mkdir -p "$convert_dir" && \ - cd "$convert_dir" && \ - cp "$TC_DIR"/traffic_ops/install/bin/convert_profile/convert_profile . - ) || { echo "Could not copy go profile converter at $(pwd): $!"; exit 1; }; + # update version referenced in the source + perl -pi.bak -e 's/__VERSION__/%{version}-%{release}/' app/lib/UI/Utils.pm + + # copy traffic_ops_golang binary + godir=src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang + ( mkdir -p "$godir" && \ + cd "$godir" && \ + cp "$TC_DIR"/traffic_ops/traffic_ops_golang/traffic_ops_golang . + ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } + + # copy TO DB admin + db_admin_dir=src/github.com/apache/trafficcontrol/traffic_ops/app/db + ( mkdir -p "$db_admin_dir" && \ + cd "$db_admin_dir" && \ + cp "$TC_DIR"/traffic_ops/app/db/admin . + ) || { echo "Could not copy go db admin at $(pwd): $!"; exit 1; }; + + # copy TO profile converter + convert_dir=src/github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile + ( mkdir -p "$convert_dir" && \ + cd "$convert_dir" && \ + cp "$TC_DIR"/traffic_ops/install/bin/convert_profile/convert_profile . + ) || { echo "Could not copy go profile converter at $(pwd): $!"; exit 1; }; %install - if [ -d $RPM_BUILD_ROOT ]; then + if [ -d $RPM_BUILD_ROOT ]; then %__rm -rf $RPM_BUILD_ROOT - fi + fi - if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR} ]; then + if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR} ]; then %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR} - fi + fi - %__cp -R $RPM_BUILD_DIR/traffic_ops-%{version}/* $RPM_BUILD_ROOT/%{PACKAGEDIR} - echo "go rming $RPM_BUILD_ROOT/%{PACKAGEDIR}/{pkg,src,bin}" - %__rm -rf $RPM_BUILD_ROOT/%{PACKAGEDIR}/{pkg,src,bin} + %__cp -R $RPM_BUILD_DIR/traffic_ops-%{version}/* $RPM_BUILD_ROOT/%{PACKAGEDIR} + echo "go rming $RPM_BUILD_ROOT/%{PACKAGEDIR}/{pkg,src,bin}" + %__rm -rf $RPM_BUILD_ROOT/%{PACKAGEDIR}/{pkg,src,bin} - %__mkdir -p $RPM_BUILD_ROOT/var/www/files - %__cp install/data/perl/osversions.cfg $RPM_BUILD_ROOT/var/www/files/. - %__cp install/data/json/osversions.json $RPM_BUILD_ROOT/var/www/files/. + %__mkdir -p $RPM_BUILD_ROOT/var/www/files + %__cp install/data/perl/osversions.cfg $RPM_BUILD_ROOT/var/www/files/. + %__cp install/data/json/osversions.json $RPM_BUILD_ROOT/var/www/files/. - if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing ]; then - %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing - fi + if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing ]; then + %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing + fi - # install traffic_ops_golang binary - if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/bin ]; then - %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/bin - fi + # install traffic_ops_golang binary + if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/bin ]; then + %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/bin + fi - src=src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang - %__cp -p "$src"/traffic_ops_golang "${RPM_BUILD_ROOT}"/opt/traffic_ops/app/bin/traffic_ops_golang + src=src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang + %__cp -p "$src"/traffic_ops_golang "${RPM_BUILD_ROOT}"/opt/traffic_ops/app/bin/traffic_ops_golang - db_admin_src=src/github.com/apache/trafficcontrol/traffic_ops/app/db - %__cp -p "$db_admin_src"/admin "${RPM_BUILD_ROOT}"/opt/traffic_ops/app/db/admin - %__rm $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/db/*.go + db_admin_src=src/github.com/apache/trafficcontrol/traffic_ops/app/db + %__cp -p "$db_admin_src"/admin "${RPM_BUILD_ROOT}"/opt/traffic_ops/app/db/admin + %__rm $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/db/*.go - convert_profile_src=src/github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile - %__cp -p "$convert_profile_src"/convert_profile "${RPM_BUILD_ROOT}"/opt/traffic_ops/install/bin/convert_profile - %__rm $RPM_BUILD_ROOT/%{PACKAGEDIR}/install/bin/convert_profile/*.go + convert_profile_src=src/github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile + %__cp -p "$convert_profile_src"/convert_profile "${RPM_BUILD_ROOT}"/opt/traffic_ops/install/bin/convert_profile + %__rm $RPM_BUILD_ROOT/%{PACKAGEDIR}/install/bin/convert_profile/*.go %pre - /usr/bin/getent group %{TRAFFIC_OPS_GROUP} || /usr/sbin/groupadd -r %{TRAFFIC_OPS_GROUP} - /usr/bin/getent passwd %{TRAFFIC_OPS_USER} || /usr/sbin/useradd -r -d %{PACKAGEDIR} -s /sbin/nologin %{TRAFFIC_OPS_USER} -g %{TRAFFIC_OPS_GROUP} - if [ -d %{PACKAGEDIR}/app/conf ]; then - echo -e "\nBacking up config files.\n" - if [ -f /var/tmp/traffic_ops-backup.tar ]; then - %__rm /var/tmp/traffic_ops-backup.tar - fi - cd %{PACKAGEDIR} && tar cf /var/tmp/traffic_ops-backup.tar app/public/routing app/conf app/db/dbconf.yml app/local app/cpanfile.snapshot - fi - - # upgrade - if [ "$1" == "2" ]; then + /usr/bin/getent group %{TRAFFIC_OPS_GROUP} || /usr/sbin/groupadd -r %{TRAFFIC_OPS_GROUP} + /usr/bin/getent passwd %{TRAFFIC_OPS_USER} || /usr/sbin/useradd -r -d %{PACKAGEDIR} -s /sbin/nologin %{TRAFFIC_OPS_USER} -g %{TRAFFIC_OPS_GROUP} + if [ -d %{PACKAGEDIR}/app/conf ]; then + echo -e "\nBacking up config files.\n" + if [ -f /var/tmp/traffic_ops-backup.tar ]; then + %__rm /var/tmp/traffic_ops-backup.tar + fi + cd %{PACKAGEDIR} && tar cf /var/tmp/traffic_ops-backup.tar app/public/routing app/conf app/db/dbconf.yml app/local app/cpanfile.snapshot + fi + + # upgrade + if [ "$1" == "2" ]; then systemctl stop traffic_ops - fi + fi %post - %__cp %{PACKAGEDIR}/etc/init.d/traffic_ops /etc/init.d/traffic_ops - %__mkdir -p /var/www/files - %__mkdir -p /etc/cron.d/ - %__cp %{PACKAGEDIR}/etc/cron.d/trafops_dnssec_refresh /etc/cron.d/trafops_dnssec_refresh - %__cp %{PACKAGEDIR}/etc/cron.d/trafops_clean_isos /etc/cron.d/trafops_clean_isos - %__cp %{PACKAGEDIR}/etc/cron.d/autorenew_certs /etc/cron.d/autorenew_certs - %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops /etc/logrotate.d/traffic_ops - %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_golang /etc/logrotate.d/traffic_ops_golang - %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_access /etc/logrotate.d/traffic_ops_access - %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_perl_access /etc/logrotate.d/traffic_ops_perl_access - %__cp %{PACKAGEDIR}/etc/profile.d/traffic_ops.sh /etc/profile.d/traffic_ops.sh - %__chown root:root /etc/init.d/traffic_ops - %__chown root:root /etc/cron.d/trafops_dnssec_refresh - %__chown root:root /etc/cron.d/autorenew_certs - %__chown root:root /etc/cron.d/trafops_clean_isos - %__chown root:root /etc/logrotate.d/traffic_ops - %__chown root:root /etc/logrotate.d/traffic_ops_golang - %__chown root:root /etc/logrotate.d/traffic_ops_access - %__chown root:root /etc/logrotate.d/traffic_ops_perl_access - %__chmod +x /etc/init.d/traffic_ops - %__chmod +x %{PACKAGEDIR}/install/bin/* - /sbin/chkconfig --add traffic_ops - - %__mkdir -p %{TRAFFIC_OPS_LOG_DIR} - - if [ -f /var/tmp/traffic_ops-backup.tar ]; then - echo -e "\nRestoring config files.\n" + %__cp %{PACKAGEDIR}/etc/init.d/traffic_ops /etc/init.d/traffic_ops + %__mkdir -p /var/www/files + %__mkdir -p /etc/cron.d/ + %__cp %{PACKAGEDIR}/etc/cron.d/trafops_dnssec_refresh /etc/cron.d/trafops_dnssec_refresh + %__cp %{PACKAGEDIR}/etc/cron.d/trafops_clean_isos /etc/cron.d/trafops_clean_isos + %__cp %{PACKAGEDIR}/etc/cron.d/autorenew_certs /etc/cron.d/autorenew_certs + %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops /etc/logrotate.d/traffic_ops + %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_golang /etc/logrotate.d/traffic_ops_golang + %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_access /etc/logrotate.d/traffic_ops_access + %__cp %{PACKAGEDIR}/etc/logrotate.d/traffic_ops_perl_access /etc/logrotate.d/traffic_ops_perl_access + %__cp %{PACKAGEDIR}/etc/profile.d/traffic_ops.sh /etc/profile.d/traffic_ops.sh + %__chown root:root /etc/init.d/traffic_ops + %__chown root:root /etc/cron.d/trafops_dnssec_refresh + %__chown root:root /etc/cron.d/autorenew_certs + %__chown root:root /etc/cron.d/trafops_clean_isos + %__chown root:root /etc/logrotate.d/traffic_ops + %__chown root:root /etc/logrotate.d/traffic_ops_golang + %__chown root:root /etc/logrotate.d/traffic_ops_access + %__chown root:root /etc/logrotate.d/traffic_ops_perl_access + %__chmod +x /etc/init.d/traffic_ops + %__chmod +x %{PACKAGEDIR}/install/bin/* + /sbin/chkconfig --add traffic_ops + + %__mkdir -p %{TRAFFIC_OPS_LOG_DIR} + + if [ -f /var/tmp/traffic_ops-backup.tar ]; then + echo -e "\nRestoring config files.\n" cd %{PACKAGEDIR} && tar xf /var/tmp/traffic_ops-backup.tar - fi - - # install - if [ "$1" = "1" ]; then - # see postinstall, the .reconfigure file triggers init(). - echo -e "\nRun /opt/traffic_ops/install/bin/postinstall from the root home directory to complete the install.\n" - fi - - # upgrade - if [ "$1" == "2" ]; then - echo -e "\n\nTo complete the update, perform the following steps:\n" - echo -e "1. If any *.rpmnew files are in /opt/traffic_ops/..., reconcile with any local changes\n" - echo -e "2. Run './db/admin --env production upgrade'\n" - echo -e " from the /opt/traffic_ops/app directory.\n" - echo -e "To start Traffic Ops: systemctl start traffic_ops\n"; - echo -e "To stop Traffic Ops: systemctl stop traffic_ops\n\n"; - fi - /bin/chown -R %{TRAFFIC_OPS_USER}:%{TRAFFIC_OPS_GROUP} %{PACKAGEDIR} - /bin/chown -R %{TRAFFIC_OPS_USER}:%{TRAFFIC_OPS_GROUP} %{TRAFFIC_OPS_LOG_DIR} - setcap cap_net_bind_service=+ep %{PACKAGEDIR}/app/bin/traffic_ops_golang + fi + + # install + if [ "$1" = "1" ]; then + # see postinstall, the .reconfigure file triggers init(). + echo -e "\nRun /opt/traffic_ops/install/bin/postinstall from the root home directory to complete the install.\n" + fi + + # upgrade + if [ "$1" == "2" ]; then + echo -e "\n\nTo complete the update, perform the following steps:\n" + echo -e "1. If any *.rpmnew files are in /opt/traffic_ops/..., reconcile with any local changes\n" + echo -e "2. Run './db/admin --env production upgrade'\n" + echo -e " from the /opt/traffic_ops/app directory.\n" + echo -e "To start Traffic Ops: systemctl start traffic_ops\n"; + echo -e "To stop Traffic Ops: systemctl stop traffic_ops\n\n"; + fi + /bin/chown -R %{TRAFFIC_OPS_USER}:%{TRAFFIC_OPS_GROUP} %{PACKAGEDIR} + /bin/chown -R %{TRAFFIC_OPS_USER}:%{TRAFFIC_OPS_GROUP} %{TRAFFIC_OPS_LOG_DIR} + setcap cap_net_bind_service=+ep %{PACKAGEDIR}/app/bin/traffic_ops_golang %preun if [ "$1" = "0" ]; then - # stop service before starting the uninstall - systemctl stop traffic_ops + # stop service before starting the uninstall + systemctl stop traffic_ops fi %postun @@ -197,8 +197,8 @@ if [ "$1" = "0" ]; then # this is an uninstall %__rm -rf %{PACKAGEDIR} %__rm /etc/init.d/traffic_ops - /usr/bin/getent passwd %{TRAFFIC_OPS_USER} || /usr/sbin/userdel %{TRAFFIC_OPS_USER} - /usr/bin/getent group %{TRAFFIC_OPS_GROUP} || /usr/sbin/groupdel %{TRAFFIC_OPS_GROUP} + /usr/bin/getent passwd %{TRAFFIC_OPS_USER} || /usr/sbin/userdel %{TRAFFIC_OPS_USER} + /usr/bin/getent group %{TRAFFIC_OPS_GROUP} || /usr/sbin/groupdel %{TRAFFIC_OPS_GROUP} fi %files diff --git a/traffic_ops_ort/build/traffic_ops_ort.spec b/traffic_ops_ort/build/traffic_ops_ort.spec index 1e14f6b900..cbbd99febe 100644 --- a/traffic_ops_ort/build/traffic_ops_ort.spec +++ b/traffic_ops_ort/build/traffic_ops_ort.spec @@ -16,16 +16,16 @@ # RPM spec file for Traffic Stats (tm). # %define debug_package %{nil} -Name: traffic_ops_ort -Summary: Installs ORT script for Traffic Control caches -Version: %{traffic_control_version} -Release: %{build_number} -License: Apache License, Version 2.0 -Group: Applications/Communications -Source0: traffic_ops_ort-%{version}.tgz -URL: https://github.com/apache/trafficcontrol/ -Vendor: Apache Software Foundation -Packager: dev at trafficcontrol dot Apache dot org +Name: traffic_ops_ort +Summary: Installs ORT script for Traffic Control caches +Version: %{traffic_control_version} +Release: %{build_number} +License: Apache License, Version 2.0 +Group: Applications/Communications +Source0: traffic_ops_ort-%{version}.tgz +URL: https://github.com/apache/trafficcontrol/ +Vendor: Apache Software Foundation +Packager: dev at trafficcontrol dot Apache dot org %{?el6:Requires: perl-JSON, perl-libwww-perl, perl-Crypt-SSLeay, perl-Digest-SHA} %{?el7:Requires: perl-JSON, perl-libwww-perl, perl-Crypt-SSLeay, perl-LWP-Protocol-https, perl-Digest-SHA} @@ -41,8 +41,8 @@ tar xvf %{SOURCE0} -C $RPM_SOURCE_DIR # copy atstccfg binary godir=src/github.com/apache/trafficcontrol/traffic_ops_ort/atstccfg ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp "$TC_DIR"/traffic_ops_ort/atstccfg/atstccfg . + cd "$godir" && \ + cp "$TC_DIR"/traffic_ops_ort/atstccfg/atstccfg . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } diff --git a/traffic_portal/build/traffic_portal.spec b/traffic_portal/build/traffic_portal.spec index f5e992252e..bc39d44ebb 100644 --- a/traffic_portal/build/traffic_portal.spec +++ b/traffic_portal/build/traffic_portal.spec @@ -15,15 +15,15 @@ # # RPM spec file for the Traffic Portal # -%define debug_package %{nil} -Name: traffic_portal -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Traffic Portal -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/ -Source: %{_sourcedir}/traffic_portal-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_portal +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Traffic Portal +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/ +Source: %{_sourcedir}/traffic_portal-%{traffic_control_version}.tgz AutoReqProv: no Requires: nodejs @@ -40,24 +40,24 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz %setup %build - /usr/bin/npm install - /usr/bin/bower install - /usr/bin/grunt dist + /usr/bin/npm install + /usr/bin/bower install + /usr/bin/grunt dist %install - %__mkdir -p ${RPM_BUILD_ROOT}/etc/init.d - %__mkdir -p ${RPM_BUILD_ROOT}/etc/logrotate.d - %__mkdir -p ${RPM_BUILD_ROOT}/etc/traffic_portal - %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/public - %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/server - %__mkdir -p ${RPM_BUILD_ROOT}/var/log/traffic_portal + %__mkdir -p ${RPM_BUILD_ROOT}/etc/init.d + %__mkdir -p ${RPM_BUILD_ROOT}/etc/logrotate.d + %__mkdir -p ${RPM_BUILD_ROOT}/etc/traffic_portal + %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/public + %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/server + %__mkdir -p ${RPM_BUILD_ROOT}/var/log/traffic_portal - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/server.js ${RPM_BUILD_ROOT}%{traffic_portal_home}/. - %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/conf ${RPM_BUILD_ROOT}/etc/traffic_portal/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/init.d/traffic_portal ${RPM_BUILD_ROOT}/etc/init.d/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal ${RPM_BUILD_ROOT}/etc/logrotate.d/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal-access ${RPM_BUILD_ROOT}/etc/logrotate.d/. - %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/* ${RPM_BUILD_ROOT}%{traffic_portal_home}/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/server.js ${RPM_BUILD_ROOT}%{traffic_portal_home}/. + %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/conf ${RPM_BUILD_ROOT}/etc/traffic_portal/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/init.d/traffic_portal ${RPM_BUILD_ROOT}/etc/init.d/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal ${RPM_BUILD_ROOT}/etc/logrotate.d/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal-access ${RPM_BUILD_ROOT}/etc/logrotate.d/. + %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/* ${RPM_BUILD_ROOT}%{traffic_portal_home}/. # creates dynamic json file needed at runtime for traffic portal to display release info VERSION=%{version}-%{build_number} @@ -68,13 +68,13 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz echo -e $JSON_VERSION > ${RPM_BUILD_ROOT}%{traffic_portal_home}/public/traffic_portal_release.json %post - echo "Successfully installed the traffic_portal assets to " %{traffic_portal_home} - %__chmod +x %{traffic_portal_home}/node_modules/forever/bin/forever - %__chmod +x /etc/init.d/traffic_portal - echo "Successfully installed the 'traffic_portal' service" - /sbin/chkconfig traffic_portal on - echo "" - echo "Start with 'service traffic_portal start'" + echo "Successfully installed the traffic_portal assets to " %{traffic_portal_home} + %__chmod +x %{traffic_portal_home}/node_modules/forever/bin/forever + %__chmod +x /etc/init.d/traffic_portal + echo "Successfully installed the 'traffic_portal' service" + /sbin/chkconfig traffic_portal on + echo "" + echo "Start with 'service traffic_portal start'" %files %defattr(644,root,root,755) diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 20ce3a510e..16bdcaf4f0 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -113,7 +113,7 @@ buildRpmTomcat () { echo "Building the rpm for Tomcat." cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; return 1; } - ./build_rpm.sh + ./build_rpm.sh } # --------------------------------------- diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 910d5243ee..f84977ae7b 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -42,29 +42,29 @@ checkEnvironment() { echo "==================================================" echo "WORKSPACE: $WORKSPACE" - echo "TOMCAT_RELEASE: $TOMCAT_RELEASE" #defined in traffic_router - echo "TOMCAT_VERSION: $TOMCAT_VERSION" #defined in traffic_router - echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router - echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router + echo "TOMCAT_RELEASE: $TOMCAT_RELEASE" #defined in traffic_router + echo "TOMCAT_VERSION: $TOMCAT_VERSION" #defined in traffic_router + echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router + echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router echo "RPM: $RPM" echo "--------------------------------------------------" } # --------------------------------------- initBuildArea() { - echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } - export VERSION=$TOMCAT_VERSION - export RELEASE=$TOMCAT_RELEASE + echo "Initializing the build area." + mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + export VERSION=$TOMCAT_VERSION + export RELEASE=$TOMCAT_RELEASE - echo "Downloading Tomcat $VERSION.$RELEASE..." - curl -fo "$RPMBUILD"/SOURCES/apache-tomcat-$VERSION.$RELEASE.tar.gz https://archive.apache.org/dist/tomcat/tomcat-8/v$VERSION.$RELEASE/bin/apache-tomcat-$VERSION.$RELEASE.tar.gz || \ - { echo "Could not download Tomcat $VERSION.$RELEASE: $?"; exit 1; } + echo "Downloading Tomcat $VERSION.$RELEASE..." + curl -fo "$RPMBUILD"/SOURCES/apache-tomcat-$VERSION.$RELEASE.tar.gz https://archive.apache.org/dist/tomcat/tomcat-8/v$VERSION.$RELEASE/bin/apache-tomcat-$VERSION.$RELEASE.tar.gz || \ + { echo "Could not download Tomcat $VERSION.$RELEASE: $?"; exit 1; } - cp "$TR_DIR/tomcat-rpm/tomcat.service" "$RPMBUILD/SOURCES/" || { echo "Could not copy source files: $?"; exit 1; } - cp "$TR_DIR/tomcat-rpm/tomcat.spec" "$RPMBUILD/SPECS/" || { echo "Could not copy spec files: $?"; exit 1; } + cp "$TR_DIR/tomcat-rpm/tomcat.service" "$RPMBUILD/SOURCES/" || { echo "Could not copy source files: $?"; exit 1; } + cp "$TR_DIR/tomcat-rpm/tomcat.spec" "$RPMBUILD/SPECS/" || { echo "Could not copy spec files: $?"; exit 1; } - echo "The build area has been initialized." + echo "The build area has been initialized." } #---------------------------------------- @@ -74,27 +74,27 @@ buildRpmTomcat () { } buildRpmForEl () { - echo "Building the rpm for "$RHEL_VERSION"." + echo "Building the rpm for "$RHEL_VERSION"." - cd $RPMBUILD - rpmbuild --define "_topdir $(pwd)" \ - --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ - --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ - -ba SPECS/$SPEC_FILE_NAME || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } - local rpm - rpm="$(find ./RPMS -name "$RPM")" - if [[ -z $rpm ]]; then - echo "Could not find rpm file $RPM in $(pwd)" - exit 1; - fi - echo "========================================================================================" - echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." - echo "========================================================================================" - echo - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } + cd $RPMBUILD + rpmbuild --define "_topdir $(pwd)" \ + --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ + --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ + -ba SPECS/$SPEC_FILE_NAME || \ + { echo "RPM BUILD FAILED: $?"; exit 1; } + local rpm + rpm="$(find ./RPMS -name "$RPM")" + if [[ -z $rpm ]]; then + echo "Could not find rpm file $RPM in $(pwd)" + exit 1; + fi + echo "========================================================================================" + echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." + echo "========================================================================================" + echo + mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } - cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } + cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } } checkEnvironment diff --git a/traffic_router/tomcat-rpm/tomcat.spec b/traffic_router/tomcat-rpm/tomcat.spec index 119e89d6cf..a196fe230c 100644 --- a/traffic_router/tomcat-rpm/tomcat.spec +++ b/traffic_router/tomcat-rpm/tomcat.spec @@ -54,15 +54,15 @@ rm -rf ${RPM_BUILD_ROOT} # This here takes care of stopping and removing tomcat before installing new files %pretrans if [[ -e "/etc/init.d/tomcat" ]]; then - echo "Disabling and stopping SysV tomcat service..." - chkconfig tomcat off - service stop tomcat + echo "Disabling and stopping SysV tomcat service..." + chkconfig tomcat off + service stop tomcat fi if [ -d /opt/apache-tomcat-* ]; then - echo "Deleting unmanaged Tomcat install from < 2.3 version of Traffic Router" - rm -rf /opt/apache-tomcat-* - rm -rf /opt/tomcat + echo "Deleting unmanaged Tomcat install from < 2.3 version of Traffic Router" + rm -rf /opt/apache-tomcat-* + rm -rf /opt/tomcat fi %pre diff --git a/traffic_server/plugins/astats_over_http/astats-git-build.spec b/traffic_server/plugins/astats_over_http/astats-git-build.spec index 4e73069204..ae4f3466dc 100644 --- a/traffic_server/plugins/astats_over_http/astats-git-build.spec +++ b/traffic_server/plugins/astats_over_http/astats-git-build.spec @@ -21,18 +21,18 @@ %global no_commits %(git log astats_over_http.c | grep '^commit' | wc -l) %global _find_debuginfo_dwz_opts %{nil} -Name: astats_over_http -Version: 1.3.0 -Release: %{no_commits}.%{commit}%{?dist} -Epoch: 434 -Summary: Apache Traffic Server %{name} plugin -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: trafficserver >= 6011 -BuildRequires: trafficserver >= 6011 +Name: astats_over_http +Version: 1.3.0 +Release: %{no_commits}.%{commit}%{?dist} +Epoch: 434 +Summary: Apache Traffic Server %{name} plugin +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: trafficserver >= 6011 +BuildRequires: trafficserver >= 6011 %description Apache Traffic Server plugin diff --git a/traffic_server/plugins/astats_over_http/astats_over_http.spec b/traffic_server/plugins/astats_over_http/astats_over_http.spec index 0caaad09ae..8a26f97961 100644 --- a/traffic_server/plugins/astats_over_http/astats_over_http.spec +++ b/traffic_server/plugins/astats_over_http/astats_over_http.spec @@ -18,18 +18,18 @@ %global install_prefix "/opt" -Name: astats_over_http -Version: 1.3 -Release: 1%{?dist} -Summary: Apache Traffic Server %{name} plugin -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http -Source0: %{name}.tar.gz -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: trafficserver = 5.2.0 -BuildRequires: trafficserver = 5.2.0 +Name: astats_over_http +Version: 1.3 +Release: 1%{?dist} +Summary: Apache Traffic Server %{name} plugin +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http +Source0: %{name}.tar.gz +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: trafficserver = 5.2.0 +BuildRequires: trafficserver = 5.2.0 %description Apache Traffic Server plugin diff --git a/traffic_server/spec/trafficserver.spec b/traffic_server/spec/trafficserver.spec index 4c4afb2788..f23c4af262 100644 --- a/traffic_server/spec/trafficserver.spec +++ b/traffic_server/spec/trafficserver.spec @@ -18,19 +18,19 @@ %global install_prefix "/opt" -Name: trafficserver -Version: 5.2.0 -Release: 1%{?dist} -Summary: Apache Traffic Server -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://trafficserver.apache.org/ +Name: trafficserver +Version: 5.2.0 +Release: 1%{?dist} +Summary: Apache Traffic Server +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://trafficserver.apache.org/ Source0: %{name}-%{version}.tar.bz2 -Patch: %{name}-%{version}-791ddc4.patch -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: tcl, hwloc, pcre, openssl, libcap -BuildRequires: autoconf, automake, libtool, gcc-c++, glibc-devel, openssl-devel, expat-devel, pcre, libcap-devel, pcre-devel, perl-ExtUtils-MakeMaker, tcl-devel, hwloc-devel +Patch: %{name}-%{version}-791ddc4.patch +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: tcl, hwloc, pcre, openssl, libcap +BuildRequires: autoconf, automake, libtool, gcc-c++, glibc-devel, openssl-devel, expat-devel, pcre, libcap-devel, pcre-devel, perl-ExtUtils-MakeMaker, tcl-devel, hwloc-devel %description Apache Traffic Server with Comcast modifications and environment specific modifications diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 1d4615f112..242053f1a5 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -43,25 +43,25 @@ initBuildArea() { cd "$TS_DIR" || \ { echo "Could not cd to $TS_DIR: $?"; return 1; } - echo "PATH: $PATH" - echo "GOPATH: $GOPATH" - go version - go env + echo "PATH: $PATH" + echo "GOPATH: $GOPATH" + go version + go env - # get x/* packages (everything else should be properly vendored) - go get -v golang.org/x/net/publicsuffix || \ - { echo "Could not get go package dependencies"; return 1; } + # get x/* packages (everything else should be properly vendored) + go get -v golang.org/x/net/publicsuffix || \ + { echo "Could not get go package dependencies"; return 1; } - # compile traffic_stats - go build -v || \ - { echo "Could not build traffic_stats binary"; return 1; } + # compile traffic_stats + go build -v || \ + { echo "Could not build traffic_stats binary"; return 1; } # compile influx_db_tools pushd influxdb_tools go build -v sync/sync_ts_databases.go || \ - { echo "Could not build sync_ts_databases binary"; return 1; } + { echo "Could not build sync_ts_databases binary"; return 1; } go build -v create/create_ts_databases.go || \ - { echo "Could not build create_ts_databases binary"; return 1; } + { echo "Could not build create_ts_databases binary"; return 1; } popd rsync -aLv ./ "$ts_dest"/ || \ @@ -76,10 +76,10 @@ initBuildArea() { } preBuildChecks() { - if [[ -e "$TS_DIR"/traffic_stats ]]; then - echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" - return 1 - fi + if [[ -e "$TS_DIR"/traffic_stats ]]; then + echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" + return 1 + fi } # --------------------------------------- diff --git a/traffic_stats/build/traffic_stats.spec b/traffic_stats/build/traffic_stats.spec index 52425eface..8c651bcf36 100644 --- a/traffic_stats/build/traffic_stats.spec +++ b/traffic_stats/build/traffic_stats.spec @@ -15,17 +15,17 @@ # # RPM spec file for Traffic Stats (tm). # -%define debug_package %{nil} -Name: traffic_stats -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Tool to pull data from traffic monitor and store in Influxdb -Packager: david_neuman2 at Cable dot Comcast dot com -Vendor: Apache Software Foundation -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol -Source: %{_sourcedir}/traffic_stats-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_stats +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Tool to pull data from traffic monitor and store in Influxdb +Packager: david_neuman2 at Cable dot Comcast dot com +Vendor: Apache Software Foundation +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol +Source: %{_sourcedir}/traffic_stats-%{traffic_control_version}.tgz %description Installs traffic_stats which performs the follwing functions: @@ -40,15 +40,15 @@ Installs traffic_stats which performs the follwing functions: # copy traffic_stats client godir=src/github.com/apache/trafficcontrol/traffic_stats ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -L -r "$TC_DIR"/traffic_stats/* . + cd "$godir" && \ + cp -L -r "$TC_DIR"/traffic_stats/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } # copy influxdb_tools godir=src/github.com/apache/trafficcontrol/traffic_stats/influxdb_tools ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -r "$TC_DIR"/traffic_stats/influxdb_tools/* . + cd "$godir" && \ + cp -r "$TC_DIR"/traffic_stats/influxdb_tools/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } %install @@ -70,8 +70,8 @@ cp "$src"/traffic_stats_seelog.xml "${RPM_BUILD_ROOT}"/opt/traffic_stats/conf/tr cp "$src"/traffic_stats.init "${RPM_BUILD_ROOT}"/etc/init.d/traffic_stats cp "$src"/traffic_stats.logrotate "${RPM_BUILD_ROOT}"/etc/logrotate.d/traffic_stats cp "$src"/grafana/*.js "${RPM_BUILD_ROOT}"/usr/share/grafana/public/dashboards/ -cp "$src"/influxdb_tools/sync_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ -cp "$src"/influxdb_tools/create_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ +cp "$src"/influxdb_tools/sync_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ +cp "$src"/influxdb_tools/create_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ %pre From 87e3e472a88bdf4b774b0124448fb7d06511a013 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sat, 23 May 2020 23:58:19 -0600 Subject: [PATCH 05/33] - Quote variables - Single brackets for Bourne shell, not double - Use -n instead of ! -z - Use braces for variables inside strings --- build/build.sh | 22 ++++++------ build/functions.sh | 41 +++++++++++----------- grove/build/build_rpm.sh | 21 ++++++----- grove/grovetccfg/build/build_rpm.sh | 20 +++++------ infrastructure/docker/build/clean_build.sh | 14 ++++---- traffic_monitor/build/build_rpm.sh | 23 +++++++----- traffic_ops/build/build_rpm.sh | 6 ++-- traffic_ops_ort/build/build_rpm.sh | 16 ++++----- traffic_portal/build/build_rpm.sh | 8 ++--- traffic_router/build/build_rpm.sh | 10 +++--- traffic_router/tomcat-rpm/build_rpm.sh | 12 +++---- traffic_server/spec/trafficserver.spec | 22 ++++++------ traffic_stats/build/build_rpm.sh | 6 ++-- 13 files changed, 113 insertions(+), 108 deletions(-) diff --git a/build/build.sh b/build/build.sh index 29cfddc348..5484b9ee25 100755 --- a/build/build.sh +++ b/build/build.sh @@ -23,14 +23,14 @@ topscript='' TC_DIR='' topscript="$(readlink -f "$0")" TC_DIR="$(dirname "$(dirname "$topscript")")" export TC_DIR -[[ -n $TC_DIR ]] && cd "$TC_DIR" || { echo "Could not cd $TC_DIR"; exit 1; } +[ -n "$TC_DIR" ] && cd "$TC_DIR" || { echo "Could not cd $TC_DIR"; exit 1; } . build/functions.sh checkEnvironment -if [[ $# -gt 0 ]]; then +if [ $# -gt 0 ]; then projects=( "$*" ) else # get all subdirs containing build/build_rpm.sh @@ -49,33 +49,33 @@ fi declare -a badproj declare -a goodproj for p in "${projects[@]}"; do - if [[ $p == tarball ]]; then + if [ "$p" = tarball ]; then if isInGitTree; then echo "----- Building tarball ..." - tarball=$(createTarball "$TC_DIR") - ls -l $tarball + tarball="$(createTarball "$TC_DIR")" + ls -l "$tarball" else echo "---- Skipping tarball creation" fi continue fi - if [[ $p == docs ]]; then + if [ "$p" = docs ]; then if isInGitTree; then echo "----- Building docs ..." pushd docs make html popd tarball=$(createDocsTarball "${TC_DIR}") - ls -l $tarball + ls -l "$tarball" else echo "---- Skipping docs creation" fi continue fi # strip trailing / - p=${p%/} - bldscript="$p/build/build_rpm.sh" - if [[ ! -x $bldscript ]]; then + p="${p%/}" + bldscript="${p}/build/build_rpm.sh" + if [ ! -x "$bldscript" ]; then echo "$bldscript not found" badproj+=($p) continue @@ -85,7 +85,7 @@ for p in "${projects[@]}"; do if $bldscript; then goodproj+=($p) else - echo "$p failed: $!" + echo "${p} failed: ${bldscript}" badproj+=($p) fi done diff --git a/build/functions.sh b/build/functions.sh index a3dbc12b36..de8aa87082 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -21,20 +21,20 @@ # echo "Need at least version $needversion"; return 1 # fi versionOk() { - local h=$1 n=$2 + local h="$1" n="$2" # string compare -- no need to do more if the same - [[ $h == $n ]] && return 0 + [ "$h" = "$n" ] && return 0 # split into fields - local have=(${h//\./ }) - local need=(${n//\./ }) + local have="${h//\./ }" + local need="${n//\./ }" # cmp first entry of each array. Bail when unequal. - while [[ -n $have && $have -eq $need ]]; do + while [ -n "$have" ] && [ "$have" = "$need" ]; do # pop 1st entry from each have=("${have[@]:1}") need=("${need[@]:1}") done - if [[ ${have:-0} -lt ${need:-0} ]]; then + if [ "${have:-0}" -lt "${need:-0}" ]; then return 1 fi return 0 @@ -44,7 +44,7 @@ versionOk() { getRevCount() { local buildNum buildNum=$(getBuildNumber) - echo ${buildNum%.*} + echo "${buildNum%.*}" } # --------------------------------------- @@ -63,11 +63,11 @@ getBuildNumber() { echo "$commits.$sha" else # Expect it's from the released tarball -- if BUILD_NUMBER file is not present, abort - if [[ ! -f $TC_DIR/BUILD_NUMBER ]]; then + if [ ! -f "${TC_DIR}/BUILD_NUMBER" ]; then echo "Not in git repository and no BUILD_NUMBER present -- aborting!" return 1 fi - grep -v '^#' $TC_DIR/BUILD_NUMBER + grep -v '^#' "${TC_DIR}/BUILD_NUMBER" fi } @@ -75,7 +75,7 @@ getBuildNumber() { getVersion() { local d="$1" local vf="$d/VERSION" - [ -r $vf ] || { echo "Could not read $vf: $!"; return 1; } + [ -r "$vf" ] || { echo "Could not read $vf: $!"; return 1; } cat "$vf" } @@ -88,7 +88,7 @@ getRhelVersion() { getCommit() { local buildNum buildNum="$(getBuildNumber)" - echo ${buildNum%.*} + echo "${buildNum%.*}" } # --------------------------------------- @@ -102,15 +102,15 @@ checkEnvironment() { DIST="$WORKSPACE/dist" export TC_VERSION BUILD_NUMBER RHEL_VERSION WORKSPACE RPMBUILD DIST - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; return 1; } + mkdir -p "$DIST" || { echo "Could not create ${DIST}: ${?}"; return 1; } # verify required tools available in path -- extra tools required by subsystem are passed in for pgm in git rpmbuild "$@"; do - type $pgm 2>/dev/null || { echo "$pgm not found in PATH"; } + type "$pgm" 2>/dev/null || { echo "$pgm not found in PATH"; } done # verify git version requiredGitVersion=1.7.12 - if ! versionOk $(git --version | tr -dc 0-9. ) "$requiredGitVersion"; then + if ! versionOk "$(git --version | tr -dc 0-9. )" "$requiredGitVersion"; then echo "$(git --version) must be at least $requiredGitVersion" return 1 fi @@ -126,7 +126,7 @@ checkEnvironment() { # --------------------------------------- createSourceDir() { - local target="$1-$TC_VERSION" + local target="${1}-${TC_VERSION}" local srcpath="$RPMBUILD/SOURCES/$target" mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; return 1; } echo "$srcpath" @@ -211,13 +211,14 @@ createDocsTarball() { verify_and_set_go_version() { GO_VERSION="none" GO="none" - go_in_path=`type -p go` - for g in $go_in_path /usr/bin/go /usr/local/go/bin/go; do - if [[ -z $g ]] || [[ ! -x $g ]]; then + go_in_path="$(type -p go)" + local group_1='' group_2='' + for g in "$go_in_path" /usr/bin/go /usr/local/go/bin/go; do + if [ -z "$g" ] || [ ! -x "$g" ]; then continue fi - go_version=`$g version | awk '{print $3}'` + go_version="$($g version | awk '{print $3}')" if [[ $go_version =~ go([1-9])\.([1-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 1 ]] && [[ ${BASH_REMATCH[2]} -ge 14 ]]; then GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION @@ -235,7 +236,7 @@ verify_and_set_go_version() { fi done - if [[ $GO == none ]]; then + if [ "$GO" = none ]; then echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" return 0 fi diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index f5a4b01999..27e23ca89f 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -17,10 +17,10 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } + [ -n "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" return 1 fi @@ -69,8 +69,8 @@ initBuildArea() { cd "$GROVE_DIR" # prep build environment - [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" + [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } @@ -81,7 +81,7 @@ buildRpmGrove() { $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/grove-${GROVE_VERSION}.tgz grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } + tar -cvzf "${RPMBUILD}/SOURCES/grove-${GROVE_VERSION}.tgz" grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -89,11 +89,11 @@ buildRpmGrove() { spec=build/grove.spec spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) - if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/grove.spec + if ! id "$spec_owner" >/dev/null 2>&1; then + chown "$(id -u):$(id -g)" build/grove.spec give_spec_back() { - chown ${spec_owner}:${spec_group} build/grove.spec + chown "${spec_owner}:${spec_group}" build/grove.spec } trap give_spec_back EXIT fi @@ -102,12 +102,11 @@ buildRpmGrove() { rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/grove.spec || { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . - [ -e $DIST ] || mkdir -p $DIST - cp $RPMBUILD/RPMS/x86_64/${RPM} $DIST + [ -e "$DIST" ] || mkdir -p "$DIST" + cp "${RPMBUILD}/RPMS/x86_64/${RPM}" "$DIST" } importFunctions checkGroveEnvironment initBuildArea buildRpmGrove - diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index f787cc5afc..da84fe3166 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -17,10 +17,10 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } + [ -n "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" return 1 fi @@ -70,8 +70,8 @@ initBuildArea() { cd "$GROVETC_DIR" # prep build environment - [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" + [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } @@ -82,7 +82,7 @@ buildRpmGrove() { $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } + tar -cvzf "${RPMBUILD}/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz" ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -90,11 +90,11 @@ buildRpmGrove() { spec=build/${PACKAGE}.spec spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) - if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/${PACKAGE}.spec + if ! id "$spec_owner" >/dev/null 2>&1; then + chown "$(id -u):$(id -g)" build/${PACKAGE}.spec give_spec_back() { - chown ${spec_owner}:${spec_group} build/${PACKAGE}.spec + chown "${spec_owner}:${spec_group}" build/${PACKAGE}.spec } trap give_spec_back EXIT fi @@ -103,8 +103,8 @@ buildRpmGrove() { rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/${PACKAGE}.spec || { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . - [ -e $DIST ] || mkdir -p $DIST - cp $RPMBUILD/RPMS/x86_64/${RPM} $DIST + [ -e "$DIST" ] || mkdir -p "$DIST" + cp "${RPMBUILD}/RPMS/x86_64/${RPM}" "$DIST" } importFunctions diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index 86222454d9..556a7ab577 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -22,7 +22,7 @@ set -o errexit -o nounset; setowner() { own=$(stat -c '%u:%g' "$1") shift - [[ -n $* ]] && chown -R "${own}" "$@" + [ -n "$*" ] && chown -R "${own}" "$@" } cleanup() { @@ -34,14 +34,14 @@ set -o xtrace; # set owner of dist dir -- cleans up existing dist permissions... export GOPATH=/tmp/go; tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; -mkdir -p ${GOPATH}/{src,pkg,bin} $tc_dir; +mkdir -p ${GOPATH}/{src,pkg,bin} "$tc_dir"; ( set -o errexit; - rsync -a /trafficcontrol/ $tc_dir; - if ! [[ -d ${tc_dir}/.git ]]; then + rsync -a /trafficcontrol/ "$tc_dir"; + if ! [ -d ${tc_dir}/.git ]; then rsync -a /trafficcontrol/.git $tc_dir; # Docker for Windows compatibility fi; - rm -rf ${tc_dir}/dist; + rm -rf "${tc_dir}/dist"; mkdir -p /trafficcontrol/dist; - ln -s /trafficcontrol/dist ${tc_dir}/dist; ) && \ - cd $tc_dir && + ln -s /trafficcontrol/dist "${tc_dir}/dist") && \ + cd "$tc_dir" && ( ( ( (./build/build.sh "$1" 2>&1; echo $? >&3) | tee ./dist/build-"$1".log >&4) 3>&1) | (read -r x; exit "$x"); ) 4>&1 diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index e41402ec9e..23556b20ce 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -25,7 +25,7 @@ importFunctions() { TC_DIR="$(dirname "$TM_DIR")" export TM_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" return 1 fi @@ -53,27 +53,32 @@ initBuildArea() { { echo "Could not get go package dependencies"; return 1; } # compile traffic_monitor - go_build=(go build -v) - if [[ "$DEBUG_BUILD" == true ]]; then + gcflags='' + ldflags="-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; - go_build+=(-gcflags 'all=-N -l'); + gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; #strip binary fi; - "${go_build[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" || \ + set -o nounset; } + go build -v -gcflags "$gcflags" -ldflags "$ldflags" || \ { echo "Could not build traffic_monitor binary"; return 1; } - rsync -av ./ "$tm_dest"/ || \ + cp -av ./ "$tm_dest"/ || \ { echo "Could not copy to $tm_dest: $?"; return 1; } - cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ + cp -av "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $tm_dest) || { echo "Could not create tar archive $tm_dest.tgz: $?"; return 1; } + tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$tm_dest")" || { echo "Could not create tar archive $tm_dest.tgz: $?"; return 1; } cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } preBuildChecks() { - if [[ -e "$TM_DIR"/traffic_monitor ]]; then + if [ -e "$TM_DIR"/traffic_monitor ]; then echo "Found $TM_DIR/traffic_monitor, please remove before retrying to build" return 1 fi diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 6b10e0e770..838f834458 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -24,7 +24,7 @@ importFunctions() { TC_DIR="$(dirname "$TO_DIR")" export TO_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" return 1 fi @@ -48,7 +48,7 @@ initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; return 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_ops_golang pushd traffic_ops_golang @@ -77,7 +77,7 @@ initBuildArea() { { echo "Could not copy to $dest: $?"; return 1; } rsync -av app/{bin,conf,cpanfile,db,lib,public,script,templates} "$dest"/app/ || \ { echo "Could not copy to $dest/app: $?"; return 1; } - tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES $(basename "$dest") || \ + tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$dest")" || \ { echo "Could not create tar archive $dest.tgz: $?"; return 1; } cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ { echo "Could not copy spec files: $?"; return 1; } diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index 1d53f83f77..649496fa68 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -18,13 +18,13 @@ set -o errexit -o nounset -o pipefail -o xtrace; #---------------------------------------- importFunctions() { local script scriptdir; - script=$(readlink -f "$0"); - scriptdir=$(dirname "$script"); - ORT_DIR=$(dirname "$scriptdir"); - TC_DIR=$(dirname "$ORT_DIR"); + script="$(readlink -f "$0")"; + scriptdir="$(dirname "$script")"; + ORT_DIR="$(dirname "$scriptdir")"; + TC_DIR="$(dirname "$ORT_DIR")"; export ORT_DIR TC_DIR; functions_sh="$TC_DIR/build/functions.sh"; - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" >&2; return 1; fi @@ -49,20 +49,20 @@ initBuildArea() { GC=(go build) GFLAGS=(-v) - if [[ "$DEBUG_BUILD" == true ]]; then + if [ "$DEBUG_BUILD" == true ]; then echo "DEBUG_BUILD is enabled, building without optimization or inlining..."; GFLAGS+=(--gcflags 'all=-N -l'); fi; pushd atstccfg; - "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=`git rev-parse HEAD` -X main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%s'` -X main.Version=${TC_VERSION}"; + "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}"; popd; cp -p traffic_ops_ort.pl "$dest"; cp -p supermicro_udev_mapper.pl "$dest"; mkdir -p "${dest}/atstccfg"; cp -a atstccfg/* "${dest}/atstccfg"; - tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES $(basename "$dest"); + tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$dest")"; cp build/traffic_ops_ort.spec "$RPMBUILD"/SPECS/.; echo "The build area has been initialized."; diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index e0994c8b9d..d7bb63b997 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -25,7 +25,7 @@ importFunctions() { TC_DIR="$(dirname "$TP_DIR")" export TP_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" return 1 fi @@ -43,11 +43,11 @@ initBuildArea() { tp_dest="$(createSourceDir traffic_portal)" cd "$TP_DIR" || \ { echo "Could not cd to $TP_DIR: $?"; return 1; } - rsync -av ./ "$ts_dest"/ || \ + rsync -av ./ "$tp_dest"/ || \ { echo "Could not copy to $to_dest: $?"; return 1; } - cp -r "$TP_DIR"/ "$ts_dest" || { echo "Could not copy $TP_DIR to $ts_dest: $?"; return 1; } + cp -r "$TP_DIR"/ "$tp_dest" || { echo "Could not copy $TP_DIR to $tp_dest: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } + tar -czvf "$tp_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$tp_dest")" || { echo "Could not create tar archive ${tp_dest}.tgz: $?"; return 1; } cp "$TP_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 16bdcaf4f0..10437c3001 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -26,7 +26,7 @@ importFunctions() { TC_DIR="$(dirname "$TR_DIR")" export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" return 1 fi @@ -46,7 +46,7 @@ buildRpmTrafficRouter () { local rpm rpm="$(find . -name \*.rpm | head -n1)" - if [[ -z $rpm ]]; then + if [ -z "$rpm" ]; then echo "Could not find rpm file $RPM in $(pwd)" return 1; fi @@ -97,13 +97,13 @@ initBuildArea() { tr_dest=$(createSourceDir traffic_router) export MVN_CMD="mvn versions:set -DnewVersion=$TC_VERSION" - echo $MVN_CMD - (cd $TR_DIR; $MVN_CMD) + echo "$MVN_CMD" + (cd "$TR_DIR"; $MVN_CMD) cp -r "$TR_DIR"/{build,connector,core} "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; return 1; } cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; return 1; } # tar/gzip the source - tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" $(basename $tr_dest) || { echo "Could not create tar archive $tr_dest: $?"; return 1; } + tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" "$(basename "$tr_dest")" || { echo "Could not create tar archive $tr_dest: $?"; return 1; } echo "The build area has been initialized." } diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index f84977ae7b..ed0f3e5f34 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -26,7 +26,7 @@ checkEnvironment() { TC_DIR="$(dirname "$TR_DIR")" export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" exit 1 fi @@ -44,8 +44,8 @@ checkEnvironment() { echo "WORKSPACE: $WORKSPACE" echo "TOMCAT_RELEASE: $TOMCAT_RELEASE" #defined in traffic_router echo "TOMCAT_VERSION: $TOMCAT_VERSION" #defined in traffic_router - echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router - echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router + echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router + echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router echo "RPM: $RPM" echo "--------------------------------------------------" } @@ -74,9 +74,9 @@ buildRpmTomcat () { } buildRpmForEl () { - echo "Building the rpm for "$RHEL_VERSION"." + echo "Building the rpm for ${RHEL_VERSION}." - cd $RPMBUILD + cd "$RPMBUILD" rpmbuild --define "_topdir $(pwd)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ @@ -84,7 +84,7 @@ buildRpmForEl () { { echo "RPM BUILD FAILED: $?"; exit 1; } local rpm rpm="$(find ./RPMS -name "$RPM")" - if [[ -z $rpm ]]; then + if [ -z "$rpm" ]; then echo "Could not find rpm file $RPM in $(pwd)" exit 1; fi diff --git a/traffic_server/spec/trafficserver.spec b/traffic_server/spec/trafficserver.spec index f23c4af262..9090c86998 100644 --- a/traffic_server/spec/trafficserver.spec +++ b/traffic_server/spec/trafficserver.spec @@ -18,18 +18,18 @@ %global install_prefix "/opt" -Name: trafficserver -Version: 5.2.0 -Release: 1%{?dist} -Summary: Apache Traffic Server -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://trafficserver.apache.org/ +Name: trafficserver +Version: 5.2.0 +Release: 1%{?dist} +Summary: Apache Traffic Server +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://trafficserver.apache.org/ Source0: %{name}-%{version}.tar.bz2 -Patch: %{name}-%{version}-791ddc4.patch -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: tcl, hwloc, pcre, openssl, libcap +Patch: %{name}-%{version}-791ddc4.patch +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: tcl, hwloc, pcre, openssl, libcap BuildRequires: autoconf, automake, libtool, gcc-c++, glibc-devel, openssl-devel, expat-devel, pcre, libcap-devel, pcre-devel, perl-ExtUtils-MakeMaker, tcl-devel, hwloc-devel %description diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 242053f1a5..1ae49f1401 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -25,7 +25,7 @@ importFunctions() { TC_DIR="$(dirname "$TS_DIR")" export TS_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" return 1 fi @@ -69,14 +69,14 @@ initBuildArea() { cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } + tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$ts_dest")" || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } preBuildChecks() { - if [[ -e "$TS_DIR"/traffic_stats ]]; then + if [ -e "${TS_DIR}/traffic_stats" ]; then echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" return 1 fi From f72c9b2d505d8593da59540b6224b07f0a396b3a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 00:31:39 -0600 Subject: [PATCH 06/33] Rewrite verify_and_set_go_version to use sed instead of BASH_REMATCH and treat 0 as a successful exit code instead of 1 --- build/functions.sh | 26 ++++++++++++++------------ grove/build/build_rpm.sh | 7 +++---- grove/grovetccfg/build/build_rpm.sh | 7 +++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/build/functions.sh b/build/functions.sh index de8aa87082..59e424826c 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -220,25 +220,27 @@ verify_and_set_go_version() { go_version="$($g version | awk '{print $3}')" - if [[ $go_version =~ go([1-9])\.([1-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 1 ]] && [[ ${BASH_REMATCH[2]} -ge 14 ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - GO=$g; export GO - PATH=`dirname $g`:$PATH; export PATH - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - echo "will use $g" - return 1 - else - if [[ $go_version =~ go([1-9])\.([1-9]+) ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" + version_pattern='.*go([1-9]+)\.([1-9]+).*' + if echo "$go_version" | grep -E "$version_pattern"; then + group_1="$(echo "$go_version" | sed -E "s/.*${version_pattern}/\1/")" + group_2="$(echo "$go_version" | sed -E "s/${version_pattern}/\2/")" + if [ ! "$group_1" -ge 1 ] || [ ! "$group_2" -ge 14 ]; then + GO_VERSION="${group_1}.${group_2}"; export GO_VERSION + echo "go version for $g is ${group_1}.${group_2}" continue fi + GO_VERSION="${group_1}.${group_2}"; export GO_VERSION + GO=$g; export GO + PATH="$(dirname "$g"):${PATH}"; export PATH + echo "go version for $g is ${group_1}.${group_2}" + echo "will use $g" + return 0 fi done if [ "$GO" = none ]; then echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" - return 0 + return 1 fi } diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 27e23ca89f..2684ecfa9d 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -46,10 +46,9 @@ checkGroveEnvironment() { export GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM # grove needs to be built with go 1.14 or greater - verify_and_set_go_version - if [[ $? -ne 1 ]]; then - return 0 - fi + if ! verify_and_set_go_version; then + return $?; + fi; echo "==================================================" echo "GO_VERSION: $GO_VERSION" diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index da84fe3166..c933f25e87 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -46,10 +46,9 @@ checkGroveEnvironment() { export GROVETC_DIR GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM # grovetccfg needs to be built with go 1.14 or greater - verify_and_set_go_version - if [[ $? -ne 1 ]]; then - return 0 - fi + if ! verify_and_set_go_version; then + return $?; + fi; echo "==================================================" echo "GO_VERSION: $GO_VERSION" From 60c7f48ac32167b4d62a75f7446f6036f1b8c413 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 00:33:58 -0600 Subject: [PATCH 07/33] Eliminate Bash-only brace expansions --- grove/build/build_rpm.sh | 4 +++- grove/grovetccfg/build/build_rpm.sh | 4 +++- infrastructure/docker/build/clean_build.sh | 5 ++++- traffic_monitor/build/build_rpm.sh | 4 +++- traffic_ops/build/build_rpm.sh | 10 +++++++--- traffic_ops_ort/build/build_rpm.sh | 4 +++- traffic_portal/build/build_rpm.sh | 4 +++- traffic_router/build/build_rpm.sh | 6 ++++-- traffic_router/tomcat-rpm/build_rpm.sh | 4 +++- traffic_stats/build/build_rpm.sh | 4 +++- 10 files changed, 36 insertions(+), 13 deletions(-) diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 2684ecfa9d..be074b0e0e 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -70,7 +70,9 @@ initBuildArea() { # prep build environment [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p BUILD RPMS SOURCES) || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index c933f25e87..ace9e414cc 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -71,7 +71,9 @@ initBuildArea() { # prep build environment [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p BUILD RPMS SOURCES) || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index 556a7ab577..bf6402fae0 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -34,7 +34,10 @@ set -o xtrace; # set owner of dist dir -- cleans up existing dist permissions... export GOPATH=/tmp/go; tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; -mkdir -p ${GOPATH}/{src,pkg,bin} "$tc_dir"; +(mkdir -p "$GOPATH" + cd "$GOPATH" + mkdir -p src pkg bin "$(dirname "$tc_dir")" +) ( set -o errexit; rsync -a /trafficcontrol/ "$tc_dir"; if ! [ -d ${tc_dir}/.git ]; then diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 23556b20ce..4f3ba7d549 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -35,7 +35,9 @@ importFunctions() { #---------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local tm_dest diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 838f834458..2465ab1e8d 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -34,7 +34,9 @@ importFunctions() { # --------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } local dest dest="$(createSourceDir traffic_ops)" @@ -75,8 +77,10 @@ initBuildArea() { rsync -av etc install "$dest"/ || \ { echo "Could not copy to $dest: $?"; return 1; } - rsync -av app/{bin,conf,cpanfile,db,lib,public,script,templates} "$dest"/app/ || \ - { echo "Could not copy to $dest/app: $?"; return 1; } + if ! (cd app; rsync -av bin conf cpanfile db lib public script templates "${dest}/app"); then + echo "Could not copy to $dest/app" + return 1 + fi tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$dest")" || \ { echo "Could not create tar archive $dest.tgz: $?"; return 1; } cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index 649496fa68..dfa9ccd732 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -34,7 +34,9 @@ importFunctions() { #---------------------------------------- initBuildArea() { echo "Initializing the build area for Traffic Ops ORT"; - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } local dest; dest=$(createSourceDir traffic_ops_ort); diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index d7bb63b997..ded816da98 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -36,7 +36,9 @@ importFunctions() { # --------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local tp_dest diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 10437c3001..c0a8b10c55 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -92,14 +92,16 @@ adaptEnvironment() { # --------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } tr_dest=$(createSourceDir traffic_router) export MVN_CMD="mvn versions:set -DnewVersion=$TC_VERSION" echo "$MVN_CMD" (cd "$TR_DIR"; $MVN_CMD) - cp -r "$TR_DIR"/{build,connector,core} "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; return 1; } + cp -r "$TR_DIR"/build "$TR_DIR"/connector "$TR_DIR"/core "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; return 1; } cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; return 1; } # tar/gzip the source diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index ed0f3e5f34..d78a204945 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -53,7 +53,9 @@ checkEnvironment() { # --------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } export VERSION=$TOMCAT_VERSION export RELEASE=$TOMCAT_RELEASE diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 1ae49f1401..f5879bfbe9 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -35,7 +35,9 @@ importFunctions() { #---------------------------------------- initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source local ts_dest From 9da95b335adf0ae1f3afd2c7ab3e1f4b70ba00b7 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 00:38:57 -0600 Subject: [PATCH 08/33] Use subshells instead of pushd/popd --- build/build.sh | 4 ++-- traffic_ops/build/build_rpm.sh | 14 ++++++-------- traffic_ops_ort/build/build_rpm.sh | 5 ++--- traffic_stats/build/build_rpm.sh | 5 ++--- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/build/build.sh b/build/build.sh index 5484b9ee25..294b8ab44f 100755 --- a/build/build.sh +++ b/build/build.sh @@ -62,9 +62,9 @@ for p in "${projects[@]}"; do if [ "$p" = docs ]; then if isInGitTree; then echo "----- Building docs ..." - pushd docs + ( cd docs make html - popd + ) tarball=$(createDocsTarball "${TC_DIR}") ls -l "$tarball" else diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 2465ab1e8d..affb0fda3f 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -53,7 +53,7 @@ initBuildArea() { { echo "Could not get go package dependencies"; return 1; } # compile traffic_ops_golang - pushd traffic_ops_golang + cd traffic_ops_golang go_build=(go build -v); if [[ "$DEBUG_BUILD" == true ]]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; @@ -61,19 +61,17 @@ initBuildArea() { fi; "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ { echo "Could not build traffic_ops_golang binary"; return 1; } - popd + cd - # compile db/admin - pushd app/db + (cd app/db "${go_build[@]}" -o admin || \ - { echo "Could not build db/admin binary"; return 1; } - popd + { echo "Could not build db/admin binary"; return 1;}) # compile TO profile converter - pushd install/bin/convert_profile + (cd install/bin/convert_profile "${go_build[@]}" || \ - { echo "Could not build convert_profile binary"; return 1; } - popd + { echo "Could not build convert_profile binary"; return 1; }) rsync -av etc install "$dest"/ || \ { echo "Could not copy to $dest: $?"; return 1; } diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index dfa9ccd732..3037ead185 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -56,9 +56,8 @@ initBuildArea() { GFLAGS+=(--gcflags 'all=-N -l'); fi; - pushd atstccfg; - "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}"; - popd; + (cd atstccfg; + "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}") cp -p traffic_ops_ort.pl "$dest"; cp -p supermicro_udev_mapper.pl "$dest"; diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index f5879bfbe9..05f9903191 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -59,12 +59,11 @@ initBuildArea() { { echo "Could not build traffic_stats binary"; return 1; } # compile influx_db_tools - pushd influxdb_tools + (cd influxdb_tools go build -v sync/sync_ts_databases.go || \ { echo "Could not build sync_ts_databases binary"; return 1; } go build -v create/create_ts_databases.go || \ - { echo "Could not build create_ts_databases binary"; return 1; } - popd + { echo "Could not build create_ts_databases binary"; return 1; }) rsync -aLv ./ "$ts_dest"/ || \ { echo "Could not copy to $ts_dest: $?"; return 1; } From 2fe6ff45b05bc464c511079f449cd358aba3866c Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 00:50:50 -0600 Subject: [PATCH 09/33] Do not use Bash arrays --- build/build.sh | 34 +++++++++++++++--------------- build/functions.sh | 11 ++++++++-- traffic_ops/build/build_rpm.sh | 15 +++++++------ traffic_ops_ort/build/build_rpm.sh | 14 ++++++------ 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/build/build.sh b/build/build.sh index 294b8ab44f..2023202548 100755 --- a/build/build.sh +++ b/build/build.sh @@ -31,24 +31,24 @@ checkEnvironment if [ $# -gt 0 ]; then - projects=( "$*" ) + projects="$*" else # get all subdirs containing build/build_rpm.sh - projects_to_build=( */build/build_rpm.sh ) + projects_to_build='*/build/build_rpm.sh' # Always build tarball when building everything.. - projects=(tarball) - for p in "${projects_to_build[@]}"; do - p=${p%%/*} - if [[ $p != "traffic_monitor_golang" ]]; then - projects+=($p) + projects=tarball + for p in ${projects_to_build}; do + p="${p%%/*}" + if [ "$p" != "traffic_monitor_golang" ]; then + projects="${projects} ${p}" fi done fi -declare -a badproj -declare -a goodproj -for p in "${projects[@]}"; do +badproj='' +goodproj='' +for p in ${projects}; do if [ "$p" = tarball ]; then if isInGitTree; then echo "----- Building tarball ..." @@ -77,30 +77,30 @@ for p in "${projects[@]}"; do bldscript="${p}/build/build_rpm.sh" if [ ! -x "$bldscript" ]; then echo "$bldscript not found" - badproj+=($p) + badproj="${badproj} ${p}" continue fi echo "----- Building $p ..." if $bldscript; then - goodproj+=($p) + goodproj="${goodproj} ${p}" else echo "${p} failed: ${bldscript}" - badproj+=($p) + badproj="${badproj} ${p}" fi done -if [[ ${#goodproj[@]} -ne 0 ]]; then +if [ "$(echo "${goodproj}" | wc -w)" -ne 0 ]; then echo "The following subdirectories built successfully: " - for p in "${goodproj[@]}"; do + for p in ${goodproj}; do echo " $p" done echo "See $(pwd)/dist for newly built rpms." fi -if [[ ${#badproj[@]} -ne 0 ]]; then +if [ "$(echo "${badproj}" | wc -w)" -ne 0 ]; then echo "The following subdirectories had errors: " - for p in "${badproj[@]}"; do + for p in ${badproj}; do echo " $p" done exit 1 diff --git a/build/functions.sh b/build/functions.sh index 59e424826c..00cc6d2f0a 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -13,6 +13,11 @@ # # shellcheck shell=ash +removeFirstArg() { + shift + echo "$@" +} + # --------------------------------------- # versionOk checks version number against required version. # ``versionOk 1.2.3 2.0.4.7'' returns false value indicating @@ -31,9 +36,11 @@ versionOk() { # cmp first entry of each array. Bail when unequal. while [ -n "$have" ] && [ "$have" = "$need" ]; do # pop 1st entry from each - have=("${have[@]:1}") - need=("${need[@]:1}") + have="$(removeFirstArg ${have})" + need="$(removeFirstArg ${need})" done + have="${have%% *}"; + need="${need%% *}"; if [ "${have:-0}" -lt "${need:-0}" ]; then return 1 fi diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index affb0fda3f..7d16ab3ef2 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -54,23 +54,26 @@ initBuildArea() { # compile traffic_ops_golang cd traffic_ops_golang - go_build=(go build -v); - if [[ "$DEBUG_BUILD" == true ]]; then + gcflags='' + ldflags='' + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; - go_build+=(-gcflags 'all=-N -l'); + gcflags="${gcflags} all=-N -l"; fi; - "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ + set -o nounset; } + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ { echo "Could not build traffic_ops_golang binary"; return 1; } cd - # compile db/admin (cd app/db - "${go_build[@]}" -o admin || \ + go build -v -o admin -gcflags "$gcflags" -ldflags "$ldflags" || \ { echo "Could not build db/admin binary"; return 1;}) # compile TO profile converter (cd install/bin/convert_profile - "${go_build[@]}" || \ + go build -v -gcflags "$gcflags" -ldflags "$ldflags" || \ { echo "Could not build convert_profile binary"; return 1; }) rsync -av etc install "$dest"/ || \ diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index 3037ead185..a6b53e5082 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -49,15 +49,17 @@ initBuildArea() { go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix; - GC=(go build) - GFLAGS=(-v) - if [ "$DEBUG_BUILD" == true ]; then - echo "DEBUG_BUILD is enabled, building without optimization or inlining..."; - GFLAGS+=(--gcflags 'all=-N -l'); + gcflags='' + ldflags='' + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then + echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; + gcflags="${gcflags} all=-N -l"; fi; + set -o nounset; } (cd atstccfg; - "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}") + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}") cp -p traffic_ops_ort.pl "$dest"; cp -p supermicro_udev_mapper.pl "$dest"; From 587f1f6f589d55bdef5012bcc769deafd46de0eb Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:32:31 -0600 Subject: [PATCH 10/33] Avoid using non-standard tar options --exclude-cvs and --transform and support an arbitrary number of files --- build/functions.sh | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/build/functions.sh b/build/functions.sh index 00cc6d2f0a..d623cf0066 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -179,17 +179,23 @@ createTarball() { local version version="$(getVersion "$TC_DIR")" local tarball="dist/apache-${projName}-${version}.tar.gz" - local tardir - tardir="$(basename "$tarball" .tar.gz)" + local tarDir + tarDir="$(basename "$tarball" .tar.gz)" + local tarLink + ( trap 'rm -f "${projDir}/BUILD_NUMBER" "$projLink"' EXIT + projLink="$(dirname "$projDir")/${tarDir}" + ln -sf "$projDir" "$projLink" - # Create a BULDNUMBER file and add to tarball - local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" + # Create a BUILDNUMBER file and add to tarball + getBuildNumber >"${projDir}/BUILD_NUMBER" - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$projDir" --exclude-vcs --transform "s@^@$tardir/@S" $(git ls-files) - rm -r "$bndir" - echo "$tarball" + # create the tarball from BUILD_NUMBER and files tracked in git + git ls-files | + sed "s|^|${tarDir}/|g" | + tar -czf "$tarball" -C "${projLink}/.." -T - "${tarDir}/BUILD_NUMBER" + # ^ read files from stdin to avoid tar argument limit + ) + echo "$tarball" } # --------------------------------------- @@ -200,16 +206,15 @@ createDocsTarball() { local version version="$(getVersion "$TC_DIR")" local tarball="dist/apache-$projName-$version-docs.tar.gz" - local tardir="${projDir}/docs/build/" + local tarDir="${projDir}/docs/build/" - # Create a BULDNUMBER file and add to tarball - local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" + # Create a BUILDNUMBER file and add to tarball + getBuildNumber >"${tarDir}/BUILD_NUMBER" - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$tardir" . --exclude-vcs - rm -r "$bndir" - echo "$tarball" + # create the tarball only from files in repo and BUILD_NUMBER + tar -czf "$tarball" -C "$tarDir" . + rm "${tarDir}/BUILD_NUMBER" + echo "$tarball" } # ---------------------------------------- From 8c543e612eabab5acfea0e4584639eb58a5dd841 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 14:01:30 -0600 Subject: [PATCH 11/33] Group commands to use intended control flow --- build/build.sh | 5 ++++- build/functions.sh | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build/build.sh b/build/build.sh index 2023202548..4ce14b9774 100755 --- a/build/build.sh +++ b/build/build.sh @@ -23,7 +23,10 @@ topscript='' TC_DIR='' topscript="$(readlink -f "$0")" TC_DIR="$(dirname "$(dirname "$topscript")")" export TC_DIR -[ -n "$TC_DIR" ] && cd "$TC_DIR" || { echo "Could not cd $TC_DIR"; exit 1; } +if [ -z "$TC_DIR" ] || ! cd "$TC_DIR"; then + echo "Could not cd TC_DIR ${TC_DIR}"; + exit 1; +fi; . build/functions.sh diff --git a/build/functions.sh b/build/functions.sh index d623cf0066..85b2e4ca8a 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -152,12 +152,13 @@ buildRpm() { echo '%__os_install_post %{nil}' >> /etc/rpm/macros; # Do not strip binaries before packaging fi; - cd "$RPMBUILD" && \ + (cd "$RPMBUILD" && \ rpmbuild --define "_topdir $(pwd)" \ --define "traffic_control_version $TC_VERSION" \ --define "commit $(getCommit)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ - -ba SPECS/$package.spec || \ + -ba SPECS/$package.spec \ + ) || \ { echo "RPM BUILD FAILED: $?"; return 1; } echo From 98a1db2e832bfadeb11e6f94b01f0f1a058d60f2 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:41:17 -0600 Subject: [PATCH 12/33] Optionally skip stripping binaries as a rpmbuild option instead of editing RPM definitions --- build/functions.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/build/functions.sh b/build/functions.sh index 85b2e4ca8a..2e74929f71 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -147,17 +147,21 @@ buildRpm() { rpm="${pre}.$(uname -m).rpm" local srpm="${pre}.src.rpm" echo "Building the rpm." - if [[ "$DEBUG_BUILD" == true ]]; then + { set +o nounset + set -- # Clear arguments for reuse + if [ "$DEBUG_BUILD" = true ]; then echo 'RPM will not strip binaries before packaging.'; - echo '%__os_install_post %{nil}' >> /etc/rpm/macros; # Do not strip binaries before packaging + set -- "$@" --define '%__os_install_post %{nil}' # Do not strip binaries before packaging fi; + set -o nounset; } (cd "$RPMBUILD" && \ rpmbuild --define "_topdir $(pwd)" \ - --define "traffic_control_version $TC_VERSION" \ - --define "commit $(getCommit)" \ - --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ + --define "traffic_control_version $TC_VERSION" \ + --define "commit $(getCommit)" \ + --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ -ba SPECS/$package.spec \ + "$@" # variable number of arguments ) || \ { echo "RPM BUILD FAILED: $?"; return 1; } From dc44a1db35f57bc54ea9d8580d705cc8459052b6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:41:55 -0600 Subject: [PATCH 13/33] Compress RPMs with xz level 2 compression --- build/functions.sh | 2 ++ grove/build/build_rpm.sh | 11 +++++++++-- grove/grovetccfg/build/build_rpm.sh | 11 +++++++++-- traffic_router/build/pom.xml | 4 ++++ traffic_router/tomcat-rpm/build_rpm.sh | 3 +++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/build/functions.sh b/build/functions.sh index 2e74929f71..32164c9364 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -153,6 +153,8 @@ buildRpm() { echo 'RPM will not strip binaries before packaging.'; set -- "$@" --define '%__os_install_post %{nil}' # Do not strip binaries before packaging fi; + set -- "$@" --define '%_source_payload w2.xzdio' # xz level 2 compression for text files + set -- "$@" --define '%_binary_payload w2.xzdio' # xz level 2 compression for binary files set -o nounset; } (cd "$RPMBUILD" && \ diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index be074b0e0e..76da35abb6 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -99,8 +99,15 @@ buildRpmGrove() { trap give_spec_back EXIT fi - # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/grove.spec || { echo "rpmbuild failed: $?" >&2; return 1; } + # build RPM with xz level 2 compression + rpmbuild \ + --define "_topdir $RPMBUILD" \ + --define "version ${GROVE_VERSION}" \ + --define "build_number ${BUILD_NUMBER}" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ + -ba build/grove.spec || + { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . [ -e "$DIST" ] || mkdir -p "$DIST" diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index ace9e414cc..8fe5516be0 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -100,8 +100,15 @@ buildRpmGrove() { trap give_spec_back EXIT fi - # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/${PACKAGE}.spec || { echo "rpmbuild failed: $?" >&2; return 1; } + # build RPM with xz level 2 compression + rpmbuild \ + --define "_topdir $RPMBUILD" \ + --define "version ${GROVE_VERSION}" \ + --define "build_number ${BUILD_NUMBER}" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ + -ba build/${PACKAGE}.spec || + { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . [ -e "$DIST" ] || mkdir -p "$DIST" diff --git a/traffic_router/build/pom.xml b/traffic_router/build/pom.xml index 49caa35931..bab5b3426e 100644 --- a/traffic_router/build/pom.xml +++ b/traffic_router/build/pom.xml @@ -133,6 +133,10 @@ ${project.parent.artifactId} ${env.BUILD_NUMBER}.${env.RHEL_VERSION} x86_64 + + _source_payload w2.xzdio + _binary_payload w2.xzdio + ${deploy.dir} diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index d78a204945..768ad3525b 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -79,9 +79,12 @@ buildRpmForEl () { echo "Building the rpm for ${RHEL_VERSION}." cd "$RPMBUILD" + # build RPM with xz level 2 compression rpmbuild --define "_topdir $(pwd)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ -ba SPECS/$SPEC_FILE_NAME || \ { echo "RPM BUILD FAILED: $?"; exit 1; } local rpm From d487ccf690e81401ec0c9a7decbe9649bdb6ec3c Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:42:26 -0600 Subject: [PATCH 14/33] Strip the go binaries from `go build` in case RPM does not strip them later --- traffic_ops/build/build_rpm.sh | 2 ++ traffic_ops_ort/build/build_rpm.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 7d16ab3ef2..3aff345064 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -60,6 +60,8 @@ initBuildArea() { if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; # strip binary fi; set -o nounset; } go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index a6b53e5082..ee13c2fc8c 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -55,6 +55,8 @@ initBuildArea() { if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; # strip binary fi; set -o nounset; } From 5e7492f68d9abe1c38e3c3d01f5e9a6cc5b4083a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:42:22 -0600 Subject: [PATCH 15/33] Statically link go binaries --- grove/build/build_rpm.sh | 4 +++- grove/grovetccfg/build/build_rpm.sh | 4 +++- traffic_monitor/build/build_rpm.sh | 3 ++- traffic_ops/build/build_rpm.sh | 7 ++++--- traffic_ops_ort/build/build_rpm.sh | 3 ++- traffic_stats/build/build_rpm.sh | 8 +++++--- 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 76da35abb6..accc92623b 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -78,8 +78,10 @@ initBuildArea() { # --------------------------------------- buildRpmGrove() { # build + ldflags='-s -w' + tags='osusergo netgo' $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; return 1; } + $GO build -v -ldflags "${ldflags} -X main.Version=$GROVE_VERSION" -tags "$tags" || { echo "Failed to build grove: $?" >&2; return 1; } # tar tar -cvzf "${RPMBUILD}/SOURCES/grove-${GROVE_VERSION}.tgz" grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 8fe5516be0..6e96138749 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -79,8 +79,10 @@ initBuildArea() { # --------------------------------------- buildRpmGrove() { # build + ldflags='-s -w' + tags='osusergo netgo' $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } + $GO build -v -ldflags "${ldflags} -X main.Version=$GROVE_VERSION" -tags "$tags" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } # tar tar -cvzf "${RPMBUILD}/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz" ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 4f3ba7d549..a56f4dc27a 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -57,6 +57,7 @@ initBuildArea() { # compile traffic_monitor gcflags='' ldflags="-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" + tags='osusergo netgo' { set +o nounset; if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; @@ -65,7 +66,7 @@ initBuildArea() { ldflags="${ldflags} -s -w"; #strip binary fi; set -o nounset; } - go build -v -gcflags "$gcflags" -ldflags "$ldflags" || \ + go build -v -gcflags "$gcflags" -ldflags "$ldflags" -tags "$tags" || \ { echo "Could not build traffic_monitor binary"; return 1; } cp -av ./ "$tm_dest"/ || \ diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 3aff345064..abff541a90 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -56,6 +56,7 @@ initBuildArea() { cd traffic_ops_golang gcflags='' ldflags='' + tags='osusergo netgo' { set +o nounset; if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; @@ -64,18 +65,18 @@ initBuildArea() { ldflags="${ldflags} -s -w"; # strip binary fi; set -o nounset; } - go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" -tags "$tags" || \ { echo "Could not build traffic_ops_golang binary"; return 1; } cd - # compile db/admin (cd app/db - go build -v -o admin -gcflags "$gcflags" -ldflags "$ldflags" || \ + go build -v -o admin -gcflags "$gcflags" -ldflags "$ldflags" -tags "$tags" || \ { echo "Could not build db/admin binary"; return 1;}) # compile TO profile converter (cd install/bin/convert_profile - go build -v -gcflags "$gcflags" -ldflags "$ldflags" || \ + go build -v -gcflags "$gcflags" -ldflags "$ldflags" -tags="$tags" || \ { echo "Could not build convert_profile binary"; return 1; }) rsync -av etc install "$dest"/ || \ diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index ee13c2fc8c..d0263d5ca4 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -51,6 +51,7 @@ initBuildArea() { gcflags='' ldflags='' + tags='osusergo netgo' { set +o nounset; if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; @@ -61,7 +62,7 @@ initBuildArea() { set -o nounset; } (cd atstccfg; - go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}") + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" -tags "$tags") cp -p traffic_ops_ort.pl "$dest"; cp -p supermicro_udev_mapper.pl "$dest"; diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 05f9903191..34ba8514b7 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -49,20 +49,22 @@ initBuildArea() { echo "GOPATH: $GOPATH" go version go env + ldflags='-s -w' + tags='osusergo netgo' # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/net/publicsuffix || \ { echo "Could not get go package dependencies"; return 1; } # compile traffic_stats - go build -v || \ + go build -v -ldflags "$ldflags" -tags "$tags" || \ { echo "Could not build traffic_stats binary"; return 1; } # compile influx_db_tools (cd influxdb_tools - go build -v sync/sync_ts_databases.go || \ + go build -v -ldflags "$ldflags" -tags="$tags" sync/sync_ts_databases.go || \ { echo "Could not build sync_ts_databases binary"; return 1; } - go build -v create/create_ts_databases.go || \ + go build -v -ldflags "$ldflags" -tags="$tags" create/create_ts_databases.go || \ { echo "Could not build create_ts_databases binary"; return 1; }) rsync -aLv ./ "$ts_dest"/ || \ From 76dbe912b22d6a93719a3f6da46c1d6095640068 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:44:25 -0600 Subject: [PATCH 16/33] Use a more reliable way to check the OS version and add a default version --- build/functions.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/build/functions.sh b/build/functions.sh index 32164c9364..68d017c9c3 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -88,7 +88,17 @@ getVersion() { # --------------------------------------- getRhelVersion() { - echo el$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)) + local releasever + local redhat_release=/etc/redhat-release + local default_version=7 + if [ -e $redhat_release ]; then + releasever="$(rpm -q --qf '%{version}' -f $redhat_release)" + else + echo "${redhat_release} not found, defaulting to major release ${default_version}" >/dev/stderr + releasever=${default_version} + fi; + + echo "el${releasever}" } # --------------------------------------- From 0d2020dd08e3926ab0d70b62dfe718f00f0701b6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:49:57 -0600 Subject: [PATCH 17/33] Use sed for pattern replacements, not perl --- traffic_ops/build/traffic_ops.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index d658746acd..3a105e2df0 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -57,7 +57,7 @@ Built: %(date) by %{getenv: USER} %build # update version referenced in the source - perl -pi.bak -e 's/__VERSION__/%{version}-%{release}/' app/lib/UI/Utils.pm + sed -i.bak 's/__VERSION__/%{version}-%{release}/g' app/lib/UI/Utils.pm # copy traffic_ops_golang binary godir=src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang From 89db273de7dbf55281a1f4476b391cb2e48406cb Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 01:51:04 -0600 Subject: [PATCH 18/33] Unglob files list in RPM spec file --- traffic_monitor/build/traffic_monitor.spec | 11 ++++++----- traffic_ops/build/traffic_ops.spec | 22 ++++++++++++++++------ traffic_portal/build/traffic_portal.spec | 11 ++++++++--- traffic_stats/build/traffic_stats.spec | 15 +++++++++------ 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/traffic_monitor/build/traffic_monitor.spec b/traffic_monitor/build/traffic_monitor.spec index 571893c9ab..4d89e54bb5 100644 --- a/traffic_monitor/build/traffic_monitor.spec +++ b/traffic_monitor/build/traffic_monitor.spec @@ -99,8 +99,8 @@ fi %files %defattr(644, traffic_monitor, traffic_monitor, 755) -%config(noreplace) /opt/traffic_monitor/conf/traffic_monitor.cfg -%config(noreplace) /opt/traffic_monitor/conf/traffic_ops.cfg +%config(noreplace) %attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/traffic_monitor.cfg +%config(noreplace) %attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/traffic_ops.cfg %config(noreplace) /etc/logrotate.d/traffic_monitor %dir /opt/traffic_monitor @@ -112,9 +112,10 @@ fi %dir /opt/traffic_monitor/var/log %dir /opt/traffic_monitor/var/run -%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/* -%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/* -%attr(755, traffic_monitor, traffic_monitor) /opt/traffic_monitor/bin/* +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/index.html +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/script.js +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/style.css +%attr(755, traffic_monitor, traffic_monitor) /opt/traffic_monitor/bin/traffic_monitor %attr(755, traffic_monitor, traffic_monitor) /etc/init.d/traffic_monitor %preun diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index 3a105e2df0..dcb8bdf984 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -203,19 +203,29 @@ fi %files %defattr(644,root,root,755) -%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/* -%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/script/* -%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/db/*.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/config2json +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/extensions +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/osversions-convert.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/routes.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/start.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/bin/traffic_ops_golang +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/script/cdn +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/script/detect10ginterfaces.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/script/generate_raid0_files.pl +%attr(755,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/db/reverse_schema.pl %config(noreplace) %attr(750,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) /opt/traffic_ops/app/conf %config(noreplace) %attr(750,%{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) /opt/traffic_ops/app/db/dbconf.yml %config(noreplace)/var/www/files/osversions.cfg %config(noreplace)/var/www/files/osversions.json +%attr(755, %{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/db/admin +%attr(755, %{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/install/bin/convert_profile/convert_profile +%{PACKAGEDIR}/etc +%{PACKAGEDIR}/app/bin/checks +%{PACKAGEDIR}/app/bin/db +%{PACKAGEDIR}/app/bin/tests %{PACKAGEDIR}/app/cpanfile %{PACKAGEDIR}/app/db %{PACKAGEDIR}/app/lib %{PACKAGEDIR}/app/public %{PACKAGEDIR}/app/templates %{PACKAGEDIR}/install -%attr(755, %{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/app/db/admin -%attr(755, %{TRAFFIC_OPS_USER},%{TRAFFIC_OPS_GROUP}) %{PACKAGEDIR}/install/bin/convert_profile/convert_profile -%{PACKAGEDIR}/etc diff --git a/traffic_portal/build/traffic_portal.spec b/traffic_portal/build/traffic_portal.spec index bc39d44ebb..ce572aa02e 100644 --- a/traffic_portal/build/traffic_portal.spec +++ b/traffic_portal/build/traffic_portal.spec @@ -57,6 +57,7 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/init.d/traffic_portal ${RPM_BUILD_ROOT}/etc/init.d/. %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal ${RPM_BUILD_ROOT}/etc/logrotate.d/. %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal-access ${RPM_BUILD_ROOT}/etc/logrotate.d/. + %__rm -f ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/package-lock.json %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/* ${RPM_BUILD_ROOT}%{traffic_portal_home}/. # creates dynamic json file needed at runtime for traffic portal to display release info @@ -79,11 +80,15 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz %files %defattr(644,root,root,755) %attr(755,root,root) /etc/init.d/traffic_portal -%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/* +%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/forever +%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/monitor %config(noreplace)/etc/traffic_portal/conf/config.js %config(noreplace)%{traffic_portal_home}/public/traffic_portal_properties.json %dir /var/log/traffic_portal -%{traffic_portal_home}/* +%{traffic_portal_home}/node_modules +%{traffic_portal_home}/package.json +%{traffic_portal_home}/public +%{traffic_portal_home}/server +%{traffic_portal_home}/server.js /etc/logrotate.d/traffic_portal /etc/logrotate.d/traffic_portal-access -/etc/init.d/traffic_portal diff --git a/traffic_stats/build/traffic_stats.spec b/traffic_stats/build/traffic_stats.spec index 8c651bcf36..3bd7d7a7b7 100644 --- a/traffic_stats/build/traffic_stats.spec +++ b/traffic_stats/build/traffic_stats.spec @@ -113,8 +113,8 @@ fi %files %defattr(644, traffic_stats, traffic_stats, 755) -%config(noreplace) /opt/traffic_stats/conf/traffic_stats.cfg -%config(noreplace) /opt/traffic_stats/conf/traffic_stats_seelog.xml +%config(noreplace) %attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/traffic_stats.cfg +%config(noreplace) %attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/traffic_stats_seelog.xml %config(noreplace) /etc/logrotate.d/traffic_stats %dir /opt/traffic_stats @@ -128,11 +128,14 @@ fi %dir /usr/share/grafana/public/dashboards %dir /opt/traffic_stats/influxdb_tools -%attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/* -%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/bin/* +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/bin/traffic_stats %attr(755, traffic_stats, traffic_stats) /etc/init.d/traffic_stats -%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/* -%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/* +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_cachegroup.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_deliveryservice.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_scripted.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_server.js +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/create_ts_databases +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/sync_ts_databases %preun # args for hooks: https://www.ibm.com/developerworks/library/l-rpm2/ From 83e421367bc66f106c855576eac2a7eb2cfdf7c6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 24 May 2020 02:16:37 -0600 Subject: [PATCH 19/33] Rewrite build command to not use 4 nested subshells and 4 file descriptors --- infrastructure/docker/build/clean_build.sh | 30 ++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index bf6402fae0..627a8304b0 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -13,20 +13,20 @@ # # shellcheck shell=ash trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}"; cleanup; exit $exit_code' EXIT; -set -o errexit -o nounset; +set -o errexit -o nounset -o pipefail; # Fix ownership of output files # $1 is file or dir with correct ownership # remaining args are files/dirs to be fixed, recursively setowner() { - own=$(stat -c '%u:%g' "$1") + own=$(stat -c%u:%g "$1" 2>/dev/null || stat -f%u:%g "$1") shift [ -n "$*" ] && chown -R "${own}" "$@" } cleanup() { - setowner /trafficcontrol /trafficcontrol/dist + setowner "$tc_volume" "${tc_volume}/dist" } set -o xtrace; @@ -34,17 +34,21 @@ set -o xtrace; # set owner of dist dir -- cleans up existing dist permissions... export GOPATH=/tmp/go; tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; +tc_volume='/trafficcontrol' (mkdir -p "$GOPATH" cd "$GOPATH" mkdir -p src pkg bin "$(dirname "$tc_dir")" ) -( set -o errexit; - rsync -a /trafficcontrol/ "$tc_dir"; - if ! [ -d ${tc_dir}/.git ]; then - rsync -a /trafficcontrol/.git $tc_dir; # Docker for Windows compatibility - fi; - rm -rf "${tc_dir}/dist"; - mkdir -p /trafficcontrol/dist; - ln -s /trafficcontrol/dist "${tc_dir}/dist") && \ - cd "$tc_dir" && - ( ( ( (./build/build.sh "$1" 2>&1; echo $? >&3) | tee ./dist/build-"$1".log >&4) 3>&1) | (read -r x; exit "$x"); ) 4>&1 +rsync -a "${tc_volume}/" "$tc_dir"; +if ! [ -d ${tc_dir}/.git ]; then + rsync -a "${tc_volume}/.git" $tc_dir; # Docker for Windows compatibility +fi + +cd "$tc_dir" +rm -rf "dist" +mkdir -p "${tc_volume}/dist" +ln -sf "${tc_volume}/dist" "dist" + +for project in "$@"; do + ./build/build.sh "${project}" 2>&1 | tee "dist/build-${project//\//-}.log" +done From 8cd5df7e378c6b1b22ae293bab7e532c7264111a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 12:20:36 -0600 Subject: [PATCH 20/33] - Remove gitignored files and dist symlink in case clean_build.sh was already run - Do not copy dist directory - Gitignore grove and grovetccfg binaries --- .gitignore | 2 ++ infrastructure/docker/build/clean_build.sh | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c415772126..9f72bff5b8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ cpanfile.snapshot rpm/build.number rpmbuild/ docs/build/* +/grove/grove +/grove/grovetccfg/grovetccfg local traffic_ops_extensions/log/* traffic_ops_extensions/lib/Extensions/InfluxDB/log/* diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index 627a8304b0..c70bcda10c 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -39,12 +39,15 @@ tc_volume='/trafficcontrol' cd "$GOPATH" mkdir -p src pkg bin "$(dirname "$tc_dir")" ) -rsync -a "${tc_volume}/" "$tc_dir"; +rsync -a --exclude=dist "${tc_volume}/" "$tc_dir"; if ! [ -d ${tc_dir}/.git ]; then rsync -a "${tc_volume}/.git" $tc_dir; # Docker for Windows compatibility fi cd "$tc_dir" +# In case the mirrored repo already exists, remove gitignored files +git clean -fX + rm -rf "dist" mkdir -p "${tc_volume}/dist" ln -sf "${tc_volume}/dist" "dist" From 87c25e78df1dad01fce9269eb8e1f43e5cc62410 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 00:13:55 -0600 Subject: [PATCH 21/33] Use realpath instead of readlink -f --- build/build.sh | 6 +++--- build/functions.sh | 11 +++++++++++ grove/build/build_rpm.sh | 2 +- grove/grovetccfg/build/build_rpm.sh | 2 +- traffic_monitor/build/build_rpm.sh | 2 +- traffic_ops/build/build_rpm.sh | 3 ++- traffic_ops_ort/build/build_rpm.sh | 2 +- traffic_portal/build/build_rpm.sh | 2 +- traffic_router/build/build_rpm.sh | 2 +- traffic_router/core/src/main/scripts/postinstall.sh | 6 +++--- traffic_router/tomcat-rpm/build_rpm.sh | 2 +- traffic_stats/build/build_rpm.sh | 2 +- 12 files changed, 27 insertions(+), 15 deletions(-) diff --git a/build/build.sh b/build/build.sh index 4ce14b9774..1110bed5cd 100755 --- a/build/build.sh +++ b/build/build.sh @@ -18,9 +18,11 @@ set -o errexit -o nounset -o pipefail; # By default all sub-projects are built. Supply a list of projects to build if # only a subset is wanted. +. build/functions.sh + # make sure we start out in traffic_control dir topscript='' TC_DIR='' -topscript="$(readlink -f "$0")" +topscript="$(realpath "$0")" TC_DIR="$(dirname "$(dirname "$topscript")")" export TC_DIR if [ -z "$TC_DIR" ] || ! cd "$TC_DIR"; then @@ -28,8 +30,6 @@ if [ -z "$TC_DIR" ] || ! cd "$TC_DIR"; then exit 1; fi; -. build/functions.sh - checkEnvironment diff --git a/build/functions.sh b/build/functions.sh index 68d017c9c3..dc76ea4ea9 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -13,6 +13,17 @@ # # shellcheck shell=ash +if ! type -p realpath; then + # by default, macOS does not have realpath + realpath() { + ls "$( + cd "$(dirname "$0")" + pwd -P # -P resolves symlinks + )/$(basename "$0")" + } + export -f realpath +fi; + removeFirstArg() { shift echo "$@" diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index accc92623b..599d61ad3f 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -32,7 +32,7 @@ checkGroveEnvironment() { echo "Verifying the build configuration environment." local script scriptdir - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 6e96138749..069458d824 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -31,7 +31,7 @@ importFunctions() { checkGroveEnvironment() { echo "Verifying the build configuration environment." local script scriptdir - script=$(readlink -f "$0") + script=$(realpath "$0") scriptdir=$(dirname "$script") GROVETC_DIR='' GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index a56f4dc27a..b3bab56c98 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -19,7 +19,7 @@ set -o errexit -o nounset -o pipefail; importFunctions() { local script scriptdir TM_DIR='' TC_DIR='' - script=$(readlink -f "$0") + script=$(realpath "$0") scriptdir=$(dirname "$script") TM_DIR="$(dirname "$scriptdir")" TC_DIR="$(dirname "$TM_DIR")" diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index abff541a90..b348d79944 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -14,11 +14,12 @@ # shellcheck shell=ash trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; set -o errexit -o nounset -o pipefail; +set -o xtrace #---------------------------------------- importFunctions() { local script scriptdir - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" TO_DIR="$(dirname "$scriptdir")" TC_DIR="$(dirname "$TO_DIR")" diff --git a/traffic_ops_ort/build/build_rpm.sh b/traffic_ops_ort/build/build_rpm.sh index d0263d5ca4..09d2c4aafd 100755 --- a/traffic_ops_ort/build/build_rpm.sh +++ b/traffic_ops_ort/build/build_rpm.sh @@ -18,7 +18,7 @@ set -o errexit -o nounset -o pipefail -o xtrace; #---------------------------------------- importFunctions() { local script scriptdir; - script="$(readlink -f "$0")"; + script="$(realpath "$0")"; scriptdir="$(dirname "$script")"; ORT_DIR="$(dirname "$scriptdir")"; TC_DIR="$(dirname "$ORT_DIR")"; diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index ded816da98..9e371f0a62 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -18,7 +18,7 @@ set -o errexit -o nounset -o pipefail; #---------------------------------------- importFunctions() { local script scriptdir - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" TP_DIR='' TC_DIR='' TP_DIR="$(dirname "$scriptdir")" diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index c0a8b10c55..ae7423deb1 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -19,7 +19,7 @@ set -o errexit -o nounset -o pipefail; importFunctions() { echo "Verifying the build configuration environment." local script scriptdir - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" TR_DIR='' TC_DIR='' TR_DIR="$(dirname "$scriptdir")" diff --git a/traffic_router/core/src/main/scripts/postinstall.sh b/traffic_router/core/src/main/scripts/postinstall.sh index 1c017ec200..f50299778d 100644 --- a/traffic_router/core/src/main/scripts/postinstall.sh +++ b/traffic_router/core/src/main/scripts/postinstall.sh @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -keytool=$(dirname $(readlink -f $(which java)))/keytool +keytool=$(dirname $(realpath "$(which java)"))/keytool cd /opt/traffic_router/conf if [ -f /opt/traffic_router/conf/*.crt ]; then for file in *.crt; do alias=$(echo $file |sed -e 's/.crt//g' |tr [:upper:] [:lower:]) - cacerts=$(/bin/find $(dirname $(readlink -f $(which java)))/.. -name cacerts) + cacerts=$(/bin/find $(dirname $(realpath "$(which java)"))/.. -name cacerts) $keytool -list -alias $alias -keystore $cacerts -storepass changeit -noprompt > /dev/null if [ $? -ne 0 ]; then @@ -31,7 +31,7 @@ fi echo -e " cd /opt/traffic_router/conf -keytool=\$(dirname \$(readlink -f \$(which java)))/keytool +keytool=\$(dirname \$(realpath \$(which java)))/keytool if [ ! -f /opt/traffic_router/conf/keyStore.jks ]; then \n \$keytool -genkeypair -v -alias \$(hostname -f) -dname \"CN=\$(hostname -f), OU=APIDefault, O=Apache Traffic Control, L=Denver, ST=Colorado, C=US\" -keystore \$(pwd)/keyStore.jks -storepass changeit -keyalg RSA -ext KeyUsage=\"digitalSignature,keyEncipherment,keyCertSign\" -ext BasicConstraints:\"critical=ca:true\" -storetype JKS -validity 3650 diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 768ad3525b..118b17880b 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -19,7 +19,7 @@ set -o errexit -o nounset -o pipefail; checkEnvironment() { echo "Verifying the build configuration environment." local script scriptdir - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" TR_DIR='' TC_DIR='' TR_DIR="$(dirname "$scriptdir")" diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 34ba8514b7..96ae8c2aab 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -19,7 +19,7 @@ set -o errexit -o nounset -o pipefail; importFunctions() { local script scriptdir TS_DIR='' TC_DIR='' - script="$(readlink -f "$0")" + script="$(realpath "$0")" scriptdir="$(dirname "$script")" TS_DIR="$(dirname "$scriptdir")" TC_DIR="$(dirname "$TS_DIR")" From 98e0c8addab6b2cc9ba00a8117d58d75d8732a7b Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 02:16:36 -0600 Subject: [PATCH 22/33] Various cross-platform scripting improvements - Specify that we want to build for Linux, regardless of host OS - Use awk to print the first field of wc, since BSD wc starts with a tab - Support stat -c on BSD systems - npm, bower, and grunt are already in the path, do not prefix with /usr/bin/ - Specify recursive copy with -R, not -r - Use ${file.separator} in Traffic Router POMs - If the build environment is Cygwin, use a Windows GOPATH --- CHANGELOG.md | 1 + build/functions.sh | 24 ++++++++++++++++++++-- grove/build/build_rpm.sh | 5 ++++- grove/grovetccfg/build/build_rpm.sh | 5 ++++- infrastructure/docker/build/clean_build.sh | 5 ++++- traffic_portal/build/traffic_portal.spec | 6 +++--- traffic_router/build/build_rpm.sh | 3 ++- traffic_router/build/pom.xml | 2 ++ traffic_router/connector/pom.xml | 2 +- traffic_router/core/pom.xml | 2 +- traffic_router/shared/pom.xml | 2 +- traffic_router/tomcat-rpm/build_rpm.sh | 4 +++- traffic_stats/build/traffic_stats.spec | 4 ++-- 13 files changed, 50 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8ec762d1..8969ae5a20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Router: Added support for topology-based delivery services - Updated /servers/details to use multiple interfaces in API v3 - Added [Edge Traffic Routing](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_router.html#edge-traffic-routing) feature which allows Traffic Router to localize more DNS record types than just the routing name for DNS delivery services +- Added the ability to speedily build development RPMs from any OS without needing Docker - Astats csv support - astats will now respond to `Accept: text/csv` and return a csv formatted stats list - Updated /deliveryservices/{{ID}}/servers to use multiple interfaces in API v3 - Updated /deliveryservices/{{ID}}/servers/eligible to use multiple interfaces in API v3 diff --git a/build/functions.sh b/build/functions.sh index dc76ea4ea9..4a444f8358 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -24,6 +24,23 @@ if ! type -p realpath; then export -f realpath fi; +if { ! stat -c%u . >/dev/null && stat -f%u .; } >/dev/null 2>&1; then + #BSD stat uses -f as its formatting flag instead of -c + stat() { + local format='' + while getopts c: option; do + case "$option" in + c) format="$OPTARG";; + *) return 1 + esac + done + shift $(( OPTIND - 1 )) + unset OPTIND + $(which stat) "-f${format}" "$@" + } + export -f stat +fi + removeFirstArg() { shift echo "$@" @@ -76,7 +93,7 @@ getBuildNumber() { local in_git='' if isInGitTree; then local commits sha - commits="$(git rev-list HEAD 2>/dev/null | wc -l)" + commits="$(git rev-list HEAD 2>/dev/null | wc -l | awk '{print $1}')" # awk is for BSD compatibility sha="$(git rev-parse --short=8 HEAD)" echo "$commits.$sha" else @@ -127,8 +144,10 @@ checkEnvironment() { RHEL_VERSION="$(getRhelVersion)" WORKSPACE="${WORKSPACE:-$TC_DIR}" RPMBUILD="$WORKSPACE/rpmbuild" + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" DIST="$WORKSPACE/dist" - export TC_VERSION BUILD_NUMBER RHEL_VERSION WORKSPACE RPMBUILD DIST + export TC_VERSION BUILD_NUMBER RHEL_VERSION WORKSPACE RPMBUILD GOOS RPM_TARGET_OS DIST mkdir -p "$DIST" || { echo "Could not create ${DIST}: ${?}"; return 1; } @@ -183,6 +202,7 @@ buildRpm() { --define "traffic_control_version $TC_VERSION" \ --define "commit $(getCommit)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ + --define "_target_os $RPM_TARGET_OS" \ -ba SPECS/$package.spec \ "$@" # variable number of arguments ) || \ diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 599d61ad3f..86d5098e0c 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -43,7 +43,9 @@ checkGroveEnvironment() { RPMBUILD="${GROVE_DIR}/rpmbuild" DIST="${TC_DIR}/dist" RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" - export GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" + export GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM GOOS RPM_TARGET_OS # grove needs to be built with go 1.14 or greater if ! verify_and_set_go_version; then @@ -106,6 +108,7 @@ buildRpmGrove() { --define "_topdir $RPMBUILD" \ --define "version ${GROVE_VERSION}" \ --define "build_number ${BUILD_NUMBER}" \ + --define "_target_os ${RPM_TARGET_OS}" \ --define '%_source_payload w2.xzdio' \ --define '%_binary_payload w2.xzdio' \ -ba build/grove.spec || diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 069458d824..6179b8829f 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -43,7 +43,9 @@ checkGroveEnvironment() { RPMBUILD="${GROVE_DIR}/rpmbuild" DIST="${TC_DIR}/dist" RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" - export GROVETC_DIR GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" + export GROVETC_DIR GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM GOOS RPM_TARGET_OS # grovetccfg needs to be built with go 1.14 or greater if ! verify_and_set_go_version; then @@ -107,6 +109,7 @@ buildRpmGrove() { --define "_topdir $RPMBUILD" \ --define "version ${GROVE_VERSION}" \ --define "build_number ${BUILD_NUMBER}" \ + --define "_target_os ${RPM_TARGET_OS}" \ --define '%_source_payload w2.xzdio' \ --define '%_binary_payload w2.xzdio' \ -ba build/${PACKAGE}.spec || diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh index c70bcda10c..427a0bd123 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/infrastructure/docker/build/clean_build.sh @@ -32,8 +32,11 @@ cleanup() { set -o xtrace; # set owner of dist dir -- cleans up existing dist permissions... -export GOPATH=/tmp/go; +export GOPATH=/tmp/go GOOS="${GOOS:-linux}"; tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; +if which cygpath 2>/dev/null; then + GOPATH="$(cygpath -w "$GOPATH")" # cygwin compatibility +fi tc_volume='/trafficcontrol' (mkdir -p "$GOPATH" cd "$GOPATH" diff --git a/traffic_portal/build/traffic_portal.spec b/traffic_portal/build/traffic_portal.spec index ce572aa02e..00635c0d78 100644 --- a/traffic_portal/build/traffic_portal.spec +++ b/traffic_portal/build/traffic_portal.spec @@ -40,9 +40,9 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz %setup %build - /usr/bin/npm install - /usr/bin/bower install - /usr/bin/grunt dist + npm install + bower install + grunt dist %install %__mkdir -p ${RPM_BUILD_ROOT}/etc/init.d diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index ae7423deb1..d4082fd198 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -74,9 +74,10 @@ adaptEnvironment() { RPMBUILD="$WORKSPACE/rpmbuild" DIST="$WORKSPACE/dist" RPM="${PACKAGE}-${TC_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + RPM_TARGET_OS="${RPM_TARGET_OS:-linux}" TOMCAT_VERSION=8.5 TOMCAT_RELEASE=32 - export PACKAGE TC_VERSION BUILD_NUMBER BUILD_LOCK WORKSPACE RPMBUILD DIST RPM TOMCAT_VERSION TOMCAT_RELEASE + export PACKAGE TC_VERSION BUILD_NUMBER BUILD_LOCK WORKSPACE RPMBUILD DIST RPM RPM_TARGET_OS TOMCAT_VERSION TOMCAT_RELEASE echo "==================================================" echo "WORKSPACE: $WORKSPACE" diff --git a/traffic_router/build/pom.xml b/traffic_router/build/pom.xml index bab5b3426e..d7c5a080a2 100644 --- a/traffic_router/build/pom.xml +++ b/traffic_router/build/pom.xml @@ -92,6 +92,7 @@ TOMCAT_VERSION TOMCAT_RELEASE RHEL_VERSION + RPM_TARGET_OS STARTUP_SCRIPT_DIR STARTUP_SCRIPT_LOC @@ -133,6 +134,7 @@ ${project.parent.artifactId} ${env.BUILD_NUMBER}.${env.RHEL_VERSION} x86_64 + ${env.RPM_TARGET_OS} _source_payload w2.xzdio _binary_payload w2.xzdio diff --git a/traffic_router/connector/pom.xml b/traffic_router/connector/pom.xml index 3a5f5a67b3..1f85a06883 100644 --- a/traffic_router/connector/pom.xml +++ b/traffic_router/connector/pom.xml @@ -43,7 +43,7 @@ true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/core/pom.xml b/traffic_router/core/pom.xml index 0cf0e096db..f39b743bc7 100644 --- a/traffic_router/core/pom.xml +++ b/traffic_router/core/pom.xml @@ -101,7 +101,7 @@ true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/shared/pom.xml b/traffic_router/shared/pom.xml index 6938dc2086..d7e86902ff 100644 --- a/traffic_router/shared/pom.xml +++ b/traffic_router/shared/pom.xml @@ -50,7 +50,7 @@ under the License. true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 118b17880b..bb07a24a20 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -37,6 +37,7 @@ checkEnvironment() { export PACKAGE="tomcat" export WORKSPACE=${WORKSPACE:-$TC_DIR} export RPMBUILD="$WORKSPACE/rpmbuild" + export RPM_TARGET_OS="${RPM_TARGET_OS:-linux}" export DIST="$WORKSPACE/dist" export RPM="${PACKAGE}-${TOMCAT_VERSION}.${TOMCAT_RELEASE}-${BUILD_NUMBER}.${RHEL_VERSION}.x86_64.rpm" @@ -83,9 +84,10 @@ buildRpmForEl () { rpmbuild --define "_topdir $(pwd)" \ --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ + --define "_target_os ${RPM_TARGET_OS}" \ --define '%_source_payload w2.xzdio' \ --define '%_binary_payload w2.xzdio' \ - -ba SPECS/$SPEC_FILE_NAME || \ + -ba SPECS/$SPEC_FILE_NAME || { echo "RPM BUILD FAILED: $?"; exit 1; } local rpm rpm="$(find ./RPMS -name "$RPM")" diff --git a/traffic_stats/build/traffic_stats.spec b/traffic_stats/build/traffic_stats.spec index 3bd7d7a7b7..680224a4ab 100644 --- a/traffic_stats/build/traffic_stats.spec +++ b/traffic_stats/build/traffic_stats.spec @@ -41,14 +41,14 @@ Installs traffic_stats which performs the follwing functions: godir=src/github.com/apache/trafficcontrol/traffic_stats ( mkdir -p "$godir" && \ cd "$godir" && \ - cp -L -r "$TC_DIR"/traffic_stats/* . + cp -LR "$TC_DIR"/traffic_stats/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } # copy influxdb_tools godir=src/github.com/apache/trafficcontrol/traffic_stats/influxdb_tools ( mkdir -p "$godir" && \ cd "$godir" && \ - cp -r "$TC_DIR"/traffic_stats/influxdb_tools/* . + cp -R "$TC_DIR"/traffic_stats/influxdb_tools/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } %install From 907b2d373467f0f415aa1b06068e3b02a504f540 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 21:04:46 -0600 Subject: [PATCH 23/33] - Fix bug from 4cf95296ed where failing the environment check did not stop the build script - Specify environment requirements by component --- build/build.sh | 5 ++-- build/functions.sh | 40 +++++++++++++++++++------- grove/build/build_rpm.sh | 1 + grove/grovetccfg/build/build_rpm.sh | 1 + traffic_monitor/build/build_rpm.sh | 2 +- traffic_ops/build/build_rpm.sh | 2 +- traffic_portal/build/build_rpm.sh | 2 +- traffic_router/build/build_rpm.sh | 2 +- traffic_router/tomcat-rpm/build_rpm.sh | 2 +- traffic_stats/build/build_rpm.sh | 2 +- 10 files changed, 40 insertions(+), 19 deletions(-) diff --git a/build/build.sh b/build/build.sh index 1110bed5cd..7b1afacbea 100755 --- a/build/build.sh +++ b/build/build.sh @@ -30,9 +30,6 @@ if [ -z "$TC_DIR" ] || ! cd "$TC_DIR"; then exit 1; fi; -checkEnvironment - - if [ $# -gt 0 ]; then projects="$*" else @@ -55,6 +52,7 @@ for p in ${projects}; do if [ "$p" = tarball ]; then if isInGitTree; then echo "----- Building tarball ..." + checkEnvironment -e rpmbuild tarball="$(createTarball "$TC_DIR")" ls -l "$tarball" else @@ -65,6 +63,7 @@ for p in ${projects}; do if [ "$p" = docs ]; then if isInGitTree; then echo "----- Building docs ..." + checkEnvironment -i python,sphinx-build,make, -e rpmbuild ( cd docs make html ) diff --git a/build/functions.sh b/build/functions.sh index 4a444f8358..a60f66a70e 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -138,6 +138,36 @@ getCommit() { # --------------------------------------- checkEnvironment() { + include_programs='' exclude_programs='' + while getopts i:e: option; do + case "$option" in + i) include_programs="$OPTARG";; + e) exclude_programs="$OPTARG";; + *) return 1 + esac + done + unset OPTIND + include_file="$(mktemp)" + exclude_file="$(mktemp)" + printf '%s\n' git rpmbuild ${include_programs//,/ } | sort >"$include_file" + printf '%s\n' ${exclude_programs//,/ } | sort >"$exclude_file" + programs="$(comm -23 "$include_file" "$exclude_file")" + rm "$include_file" "$exclude_file" + + # verify required tools available in path -- extra tools required by subsystem are passed in + for program in $programs; do + if ! type "$program"; then + echo "$program not found in PATH" + return 1 + fi + done + # verify git version + requiredGitVersion=1.7.12 + if ! versionOk "$(git --version | tr -dc 0-9. )" "$requiredGitVersion"; then + echo "$(git --version) must be at least $requiredGitVersion" + return 1 + fi + TC_VERSION='' BUILD_NUMBER='' RHEL_VERSION='' RPMBUILD='' DIST='' TC_VERSION="$(getVersion "$TC_DIR")" BUILD_NUMBER="$(getBuildNumber)" @@ -151,16 +181,6 @@ checkEnvironment() { mkdir -p "$DIST" || { echo "Could not create ${DIST}: ${?}"; return 1; } - # verify required tools available in path -- extra tools required by subsystem are passed in - for pgm in git rpmbuild "$@"; do - type "$pgm" 2>/dev/null || { echo "$pgm not found in PATH"; } - done - # verify git version - requiredGitVersion=1.7.12 - if ! versionOk "$(git --version | tr -dc 0-9. )" "$requiredGitVersion"; then - echo "$(git --version) must be at least $requiredGitVersion" - return 1 - fi echo "Build environment has been verified." echo "==================================================" diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index 86d5098e0c..7e49e0e915 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -120,6 +120,7 @@ buildRpmGrove() { } importFunctions +checkEnvironment -i go checkGroveEnvironment initBuildArea buildRpmGrove diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 6179b8829f..4eb57b552b 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -121,6 +121,7 @@ buildRpmGrove() { } importFunctions +checkEnvironment -i go checkGroveEnvironment initBuildArea buildRpmGrove diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index b3bab56c98..fede55b6f9 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -91,6 +91,6 @@ preBuildChecks() { importFunctions preBuildChecks -checkEnvironment go +checkEnvironment -i go,rsync initBuildArea buildRpm traffic_monitor diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index b348d79944..3fa1e42e7b 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -99,6 +99,6 @@ initBuildArea() { # --------------------------------------- importFunctions -checkEnvironment go +checkEnvironment -i go,rsync initBuildArea buildRpm traffic_ops diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index 9e371f0a62..7533e0c3aa 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -58,6 +58,6 @@ initBuildArea() { # --------------------------------------- importFunctions -checkEnvironment npm node +checkEnvironment -i npm,bower,grunt,compass,rsync initBuildArea buildRpm traffic_portal diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index d4082fd198..a6908e9be4 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -122,7 +122,7 @@ buildRpmTomcat () { # --------------------------------------- importFunctions -checkEnvironment +checkEnvironment -i mvn adaptEnvironment initBuildArea buildRpmTrafficRouter diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index bb07a24a20..352fa99bd4 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -104,6 +104,6 @@ buildRpmForEl () { cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } } -checkEnvironment +checkEnvironment -i curl initBuildArea buildRpmTomcat diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 96ae8c2aab..26bc56e5a6 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -89,6 +89,6 @@ preBuildChecks() { importFunctions preBuildChecks -checkEnvironment go +checkEnvironment -i go,rsync initBuildArea buildRpm traffic_stats From f071e7eb656da6f72f682b3837f388b016869067 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 2 Jun 2020 23:46:51 -0600 Subject: [PATCH 24/33] Move the note about specifying the Traffic Ops host to the "Installing The Traffic Portal Developer Environment" section --- docs/source/development/traffic_portal.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/development/traffic_portal.rst b/docs/source/development/traffic_portal.rst index 5f3e5a3d09..16f12758f2 100644 --- a/docs/source/development/traffic_portal.rst +++ b/docs/source/development/traffic_portal.rst @@ -32,7 +32,6 @@ To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly us * `Grunt CLI 1.2.0 or above `_ * Access to a working instance of Traffic Ops -.. note:: The Traffic Portal consumes the Traffic Ops API. Modify traffic_portal/conf/config.js to specify the location of Traffic Ops. Traffic Portal Project Tree Overview ===================================== @@ -90,3 +89,5 @@ Installing The Traffic Portal Developer Environment #. Modify ``log.stream`` to be ``./server/log/access.log``. #. Navigate to http(s)://localhost:[port|sslPort defined in ``traffic_portal/conf/config.js``] + +.. note:: The Traffic Portal consumes the Traffic Ops API. Modify traffic_portal/conf/config.js to specify the location of Traffic Ops. From 16963ed57ed6da9e5003e56095a064f2b218f0ed Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 2 Jun 2020 23:58:26 -0600 Subject: [PATCH 25/33] - Add subsection for installing the global NPM packages - Move Compass instructions to "Software Requirements" section - Add reference labels --- docs/source/development/traffic_portal.rst | 83 +++++++++++++--------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/docs/source/development/traffic_portal.rst b/docs/source/development/traffic_portal.rst index 16f12758f2..26bcd6632d 100644 --- a/docs/source/development/traffic_portal.rst +++ b/docs/source/development/traffic_portal.rst @@ -21,6 +21,8 @@ Introduction ============ Traffic Portal is an `AngularJS `_ client served from a `Node.js `_ web server designed to consume the :ref:`to-api`. Traffic Portal is the official replacement for the legacy Traffic Ops UI. +.. _dev-tp-software-requirements: + Software Requirements ===================== To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly used) environment that has the following installed: @@ -32,53 +34,70 @@ To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly us * `Grunt CLI 1.2.0 or above `_ * Access to a working instance of Traffic Ops +Install Global NPM Packages +--------------------------- -Traffic Portal Project Tree Overview -===================================== -* **traffic_control/traffic_portal/app/src** - contains HTML, JavaScript and :abbr:`SCSS (Sassy CSS)` source files. +Bower and the Grunt CLI can be installed using NPM. -Installing The Traffic Portal Developer Environment -=================================================== -#. Clone the `Traffic Control Repository `_ -#. Navigate to the ``traffic_portal`` subdirectory of your cloned repository. -#. Run ``npm install`` to install application dependencies into ``traffic_portal/node_modules``. Only needs to be done the first time unless ``traffic_portal/package.json`` changes. -#. Run ``bower install`` to install client-side dependencies into ``traffic_portal/app/bower_components``. Only needs to be done the first time unless ``traffic_portal/bower.json`` changes. -#. Make sure that compass is installed and functioning correctly by running ``compass version``. If compass is not available, then it can be installed using ``gem``'s manually, or using ``bundle`` +.. code-block:: shell + :caption: Install Bower and Grunt CLI + + npm -g install bower grunt-cli + +.. _dev-tp-compass: + +Install Compass +--------------- - .. tip:: Bundle will automitcally install the correct version of the gems +Compass can be installed using ``gem`` manually, or by using ``bundle`` - #. ``brew install ruby``/``apt-get install ruby``/``yum install ruby`` +.. tip:: Bundle will automatically install the correct version of the gems. - #. ``gem update --system`` +#. ``brew install ruby``/``apt-get install ruby``/``yum install ruby`` - #. At this point, you can either manually install the gems or use bundler +#. ``gem update --system`` - #. For manually: ``gem install sass compass`` +#. At this point, you can either manually install the gems or use bundler - #. For automatically: ``gem install bundle && bundle install`` + #. For manually: ``gem install sass compass`` - .. note:: Bundle requires ruby versions > 2.3.0, so if you're using a version of ruby < 2.3.0 then this will not work. + #. For automatically: ``gem install bundle && bundle install`` - #. Make sure that ``compass`` and ``sass`` are part of your ``PATH`` environment variable. + .. note:: Bundle requires ruby versions > 2.3.0, so if you're using a version of ruby < 2.3.0 then this will not work. - #. If not, you can see where gem installs ``compass`` and ``sass`` by running: - ``gem environment`` +#. Make sure that ``compass`` and ``sass`` are part of your ``PATH`` environment variable. - #. In there, you can see where ruby is installing all the gems. Add that path to your ``PATH`` environment variable. - For example, it is ``/usr/local/lib/ruby/gems/2.7.0/gems/compass-1.0.3/bin/`` for this test setup. +#. If not, you can see where gem installs ``compass`` and ``sass`` by running: + ``gem environment`` - #. Once you have installed ``compass`` successfully, make sure you can reach it by typing: - ``compass version`` - This should give a valid output. For example, for the test setup, the output is: +#. In there, you can see where ruby is installing all the gems. Add that path to your ``PATH`` environment variable. + For example, it is ``/usr/local/lib/ruby/gems/2.7.0/gems/compass-1.0.3/bin/`` for this test setup. - .. code-block:: text - :caption: Compass version output +#. Once you have installed ``compass`` successfully, make sure you can reach it by typing: + ``compass version`` + This should give a valid output. For example, for the test setup, the output is: - Compass 1.0.3 (Polaris) - Copyright (c) 2008-2020 Chris Eppstein - Released under the MIT License. - Compass is charityware. - Please make a tax deductable donation for a worthy cause: http://umdf.org/compass +.. code-block:: text + :caption: Compass version output + + Compass 1.0.3 (Polaris) + Copyright (c) 2008-2020 Chris Eppstein + Released under the MIT License. + Compass is charityware. + Please make a tax deductable donation for a worthy cause: http://umdf.org/compass + + +Traffic Portal Project Tree Overview +===================================== +* **traffic_control/traffic_portal/app/src** - contains HTML, JavaScript and :abbr:`SCSS (Sassy CSS)` source files. + +Installing The Traffic Portal Developer Environment +=================================================== +#. Clone the `Traffic Control Repository `_ +#. Navigate to the ``traffic_portal`` subdirectory of your cloned repository. +#. Run ``npm install`` to install application dependencies into ``traffic_portal/node_modules``. Only needs to be done the first time unless ``traffic_portal/package.json`` changes. +#. Run ``bower install`` to install client-side dependencies into ``traffic_portal/app/bower_components``. Only needs to be done the first time unless ``traffic_portal/bower.json`` changes. +#. Make sure that compass is installed and functioning correctly by running ``compass version``. If compass is not available, then it can be installed following the instructions under :ref:`dev-tp-compass`. #. Run ``grunt`` to package the application into ``traffic_portal/app/dist``, start a local HTTPS server (Express), and start a file watcher. #. Modify ``traffic_portal/conf/config.js``: From 1db3ce41579a73e61c4207bb334a927b84b9f70a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 3 Jun 2020 17:44:24 -0600 Subject: [PATCH 26/33] Add instructions for getting OpenJDK 8 on macOS --- docs/source/development/traffic_router.rst | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/source/development/traffic_router.rst b/docs/source/development/traffic_router.rst index 2a7970cc3b..c9fe941a0b 100644 --- a/docs/source/development/traffic_router.rst +++ b/docs/source/development/traffic_router.rst @@ -40,6 +40,30 @@ To work on Traffic Router you need a \*nix (MacOS and Linux are most commonly us * Tomcat Native >= 1.2.16 * Not Tomcat - You do not need a Tomcat installation for development. An embedded version is launched for development testing instead. +.. _dev-tr-mac-jdk: + +Get OpenJDK 8 on macOS +-------------------------- +If you are on macOS, OpenJDK 8 is not available from Homebrew by default, but it can still be installed using Homebrew with little effort. + +Using Homebrew, |AdoptOpenJDK instructions|_ + +.. code-block:: shell + :caption: Install OpenJDK 8 on macOS + + brew tap AdoptOpenJDK/openjdk + brew cask install adoptopenjdk8 + +Next, set the JAVA_HOME environment variable. Add this line to your ``~/.bash_profile``: + +.. code-block:: shell + :caption: Set JAVA_HOME environment variable + + export JAVA_HOME=$(/usr/libexec/java_home -v1.8) + +.. |AdoptOpenJDK instructions| replace:: add the AdoptOpenJDK tap and install the ``adoptopenjdk8`` cask +.. _AdoptOpenJDK instructions: https://github.com/AdoptOpenJDK/homebrew-openjdk#other-versions + Traffic Router Project Tree Overview ==================================== * ``traffic_control/traffic_traffic_router/`` - base directory for Traffic Router From 26873a094052a9c3f6415db75d1a1311ea32b5f9 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 3 Jun 2020 00:02:19 -0600 Subject: [PATCH 27/33] Move instructions for building docs to documentation guidelines page --- docs/source/development/building.rst | 20 ++++++++----------- .../development/documentation_guidelines.rst | 13 +++++++++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/source/development/building.rst b/docs/source/development/building.rst index f00601fbfa..eb34f8e47a 100644 --- a/docs/source/development/building.rst +++ b/docs/source/development/building.rst @@ -74,28 +74,24 @@ Output :file:`{component}-{version}.rpm` files, build logs and source tarballs w .. _build-with-dc: Build Using ``docker-compose`` -============================== If the ``pkg`` script fails, ``docker-compose`` can still be used to build the projects directly. The compose file can be found at ``infrastructure/docker/build/docker-compose.yml`` under the repository's root directory. It can be passed directly to ``docker-compose``, either from the ``infrastructure/docker/build/`` directory or by explicitly passing a path to the ``infrastructure/docker/build/docker-compose.yml`` file via ``-f``. It is recommended that between builds ``docker-compose down -v`` is run to prevent caching of old build steps. The service names are the same as the project names described above in `Usage`_, and similar to the ``pkg`` script, the build results, logs and source tarballs may all be found in the ``dist`` directory after completion. .. note:: Calling ``docker-compose`` in the way described above will build _all_ projects, not just the default projects. .. seealso:: `The Docker Compose command line reference `_ -Building Individual Components -============================== -Each Traffic Control component can be individually built, and the instructions for doing so may be found in their respective component's development documentation. -.. _docs-build: -Building This Documentation ---------------------------- -This documentation uses the `Sphinx documentation build system `_, and as such requires a Python3 version that is at least 3.4.1. It also has dependency on Sphinx, and Sphinx extensions and themes. All of these can be easily installed using `pip` by referencing the requirements file like so: .. code-block:: shell - :caption: Run from the Repository's Root Directory - python3 -m pip install --user -r docs/source/requirements.txt -Once all dependencies have been satisfied, build using the Makefile at ``docs/Makefile``. -Alternatively, it is also possible to :ref:`pkg` or to :ref:`build-with-dc`, both of which will output a documentation "tarball" to ``dist/``. + +Building Individual Components +============================== +Each Traffic Control component can be individually built, and the instructions for doing so may be found in their respective component's development documentation. + +Building This Documentation +--------------------------- +To build the documentation, see :ref:`docs-build`. diff --git a/docs/source/development/documentation_guidelines.rst b/docs/source/development/documentation_guidelines.rst index e7319acc13..06c4558d4d 100644 --- a/docs/source/development/documentation_guidelines.rst +++ b/docs/source/development/documentation_guidelines.rst @@ -22,9 +22,20 @@ The Apache Traffic Control documentation is written in :abbr:`RST (reStructuredT .. seealso:: `The docutils RST reference `_. +.. _docs-build: + Building ======== -To build the documentation, see :ref:`docs-build`. +This documentation uses the `Sphinx documentation build system `_, and as such requires a Python3 version that is at least 3.4.1. It also has dependency on Sphinx, and Sphinx extensions and themes. All of these can be easily installed using `pip` by referencing the requirements file like so: + +.. code-block:: shell + :caption: Run from the Repository's Root Directory + + python3 -m pip install --user -r docs/source/requirements.txt + +Once all dependencies have been satisfied, build using the Makefile at ``docs/Makefile``. + +Alternatively, it is also possible to :ref:`pkg` or to :ref:`build-with-dc`, both of which will output a documentation "tarball" to ``dist/``. Writing ======= From 019d9512b8e68a0d7f8fcf78df0177422e5f44c5 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 25 May 2020 02:52:48 -0600 Subject: [PATCH 28/33] - Move clean_build.sh to the main build directory for easy access - Change directories to the repo before running if we are not already there - Clean build the tarball to exclude gitignored files - If no projects are provided to clean_build.sh, build them all --- .../docker/build => build}/clean_build.sh | 13 ++- docs/source/development/building.rst | 80 +++++++++++++++++++ docs/source/development/traffic_monitor.rst | 2 + docs/source/development/traffic_ops.rst | 2 + docs/source/development/traffic_router.rst | 2 + docs/source/development/traffic_stats.rst | 2 + infrastructure/docker/build/Dockerfile-docs | 2 +- infrastructure/docker/build/Dockerfile-grove | 2 +- .../docker/build/Dockerfile-grovetccfg | 2 +- infrastructure/docker/build/Dockerfile-source | 3 +- .../docker/build/Dockerfile-traffic_monitor | 2 +- .../docker/build/Dockerfile-traffic_ops | 2 +- .../docker/build/Dockerfile-traffic_ops_ort | 2 +- .../docker/build/Dockerfile-traffic_portal | 2 +- .../docker/build/Dockerfile-traffic_router | 2 +- .../docker/build/Dockerfile-traffic_stats | 2 +- 16 files changed, 111 insertions(+), 11 deletions(-) rename {infrastructure/docker/build => build}/clean_build.sh (81%) diff --git a/infrastructure/docker/build/clean_build.sh b/build/clean_build.sh similarity index 81% rename from infrastructure/docker/build/clean_build.sh rename to build/clean_build.sh index 427a0bd123..5e82629821 100755 --- a/infrastructure/docker/build/clean_build.sh +++ b/build/clean_build.sh @@ -31,13 +31,20 @@ cleanup() { set -o xtrace; +if ! script_path="$(readlink "$0")"; then + script_path="$0"; +fi; +cd "$(dirname "$script_path")"; +if ! tc_volume="$(git rev-parse --show-toplevel 2>/dev/null)"; then + tc_volume='/trafficcontrol'; # Default repo location for ATC builder Docker images +fi; + # set owner of dist dir -- cleans up existing dist permissions... export GOPATH=/tmp/go GOOS="${GOOS:-linux}"; tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; if which cygpath 2>/dev/null; then GOPATH="$(cygpath -w "$GOPATH")" # cygwin compatibility fi -tc_volume='/trafficcontrol' (mkdir -p "$GOPATH" cd "$GOPATH" mkdir -p src pkg bin "$(dirname "$tc_dir")" @@ -55,6 +62,10 @@ rm -rf "dist" mkdir -p "${tc_volume}/dist" ln -sf "${tc_volume}/dist" "dist" +if [ $# -eq 0 ]; then + set -- tarball traffic_monitor traffic_ops traffic_ops_ort traffic_portal traffic_router traffic_stats grove grove/grovetccfg docs +fi + for project in "$@"; do ./build/build.sh "${project}" 2>&1 | tee "dist/build-${project//\//-}.log" done diff --git a/docs/source/development/building.rst b/docs/source/development/building.rst index eb34f8e47a..fb3d183466 100644 --- a/docs/source/development/building.rst +++ b/docs/source/development/building.rst @@ -74,19 +74,99 @@ Output :file:`{component}-{version}.rpm` files, build logs and source tarballs w .. _build-with-dc: Build Using ``docker-compose`` +------------------------------ If the ``pkg`` script fails, ``docker-compose`` can still be used to build the projects directly. The compose file can be found at ``infrastructure/docker/build/docker-compose.yml`` under the repository's root directory. It can be passed directly to ``docker-compose``, either from the ``infrastructure/docker/build/`` directory or by explicitly passing a path to the ``infrastructure/docker/build/docker-compose.yml`` file via ``-f``. It is recommended that between builds ``docker-compose down -v`` is run to prevent caching of old build steps. The service names are the same as the project names described above in `Usage`_, and similar to the ``pkg`` script, the build results, logs and source tarballs may all be found in the ``dist`` directory after completion. .. note:: Calling ``docker-compose`` in the way described above will build _all_ projects, not just the default projects. .. seealso:: `The Docker Compose command line reference `_ +.. _dev-building-natively: +Build the RPMs Natively +======================= +A developer may end up building the RPMs several times to test or :ref:`debug ` code changes, so it can be desirable to build the RPMs quickly for this purpose. Natively building the RPMs has the lowest build time of any method. +Install the Dependencies +------------------------ + +.. table:: Build dependencies for Traffic Control + + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | | Common dependencies | :ref:`dev-traffic-monitor` | :ref:`dev-traffic-ops` | :ref:`dev-traffic-portal` | :ref:`dev-traffic-router` | :ref:`dev-traffic-stats` | Grove | Grove TC Config (grovetccfg) | :ref:`Docs ` | + +====================================+=====================+============================+========================+===========================+===========================+==========================+==========+==============================+==========================+ + | macOS (homebrew_)\ [3]_,\ [4]_ | - rpm | - go | - go | - npm | - maven | - go | - go | - go | - python3 | + | | | | | - bower | | | | | - sphinx-doc | + | | | | | - grunt-cli | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | CentOS/Red Hat/Fedora (yum_)\ [5]_ | - git | | | - epel-release | - java-1.8.0-openjdk | | | | - python3-devel | + | | - rpm-build | | | - npm | - maven | | | | - gcc | + | | - rsync | | | - nodejs-grunt-cli | | | | | - make | + | | | | | - ruby-devel | | | | | | + | | | | | - gcc | | | | | | + | | | | | - make | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Arch Linux (pacman_) | - git | - go | - go | - npm | - jdk8-openjdk | - go | - go | - go | - python-pip | + | | - rpm-tools | | | - bower | - maven | | | | - python-sphinx | + | | - diff | | | - grunt-cli | | | | | - make | + | | - rsync | | | - ruby | | | | | | + | | | | | - gcc | | | | | | + | | | | | - make | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Windows (cygwin_)\ [6]_ | - git | | | - ruby-devel | - curl | | | | | + | | - rpm-build | | | - make | | | | | | + | | - rsync | | | - gcc-g++ | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Windows (chocolatey_) | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | + | | | | | | - maven | | | | - pip | + | | | | | | | | | | - make | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + +.. _homebrew: https://brew.sh/ +.. _yum: https://wiki.centos.org/PackageManagement/Yum +.. _pacman: https://www.archlinux.org/pacman/ +.. _cygwin: https://cygwin.com/ +.. _chocolatey: https://chocolatey.org/ + +.. [3] If you are on macOS, you additionally need to :ref:`dev-tr-mac-jdk`. + +.. [4] On macOS, in order to add the ``sphinx-build`` binary to your ``PATH`` after installing the ``sphinx-doc`` package, run ``brew link --force sphinx-doc`` + +.. [5] If you are on CentOS, you need to `download Go directly `_ instead of using a package manager in order to get the latest Go version. For most users, the desired architecture is AMD64/x86_64. + +.. [6] If you are on Windows, you need to install both the Cygwin packages and the Chocolatey packages in order to build the Apache Traffic Control RPMs natively. + +.. |AdoptOpenJDK instructions| replace:: add the AdoptOpenJDK tap and install the ``adoptopenjdk8`` cask +.. _AdoptOpenJDK instructions: https://github.com/AdoptOpenJDK/homebrew-openjdk#other-versions + +After installing the packages using your platform's package manager, + + - Install the :ref:`global NPM packages ` and :ref:`Compass `.. to build Traffic Portal. + + - Install the Python 3 modules used to :ref:`build the documentation `. + +Run ``build/clean_build.sh`` directly +------------------------------------- + +In a terminal, nativate to the root directory of the repository. You can run ``build/clean_build.sh`` with no arguments to build all components. .. code-block:: shell + :caption: ``build/clean_build.sh`` with no arguments + + build/clean_build.sh + +This is the equivalent of running + +.. code-block:: shell + :caption: ``build/clean_build.sh`` with all components + + build/clean_build.sh tarball traffic_monitor traffic_ops traffic_portal traffic_router traffic_stats grove grove/grovetccfg docs +If any component fails to build, no further component builds will be attempted. +Regardless of which OS the RPMs were built on, they are meant to be installed on CentOS 7. The exception is if the RPMs were built on CentOS 8, in which case the target OS for the RPMs is CentOS 8. +.. warning:: Although there are no known issues with natively-built RPMs, the official, supported method of building the RPMs is by using :ref:`pkg ` or :ref:`docker-compose `. Use natively-built RPMs at your own risk. Building Individual Components ============================== diff --git a/docs/source/development/traffic_monitor.rst b/docs/source/development/traffic_monitor.rst index f917c02bec..e4be3f9120 100644 --- a/docs/source/development/traffic_monitor.rst +++ b/docs/source/development/traffic_monitor.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-monitor: + *************** Traffic Monitor *************** diff --git a/docs/source/development/traffic_ops.rst b/docs/source/development/traffic_ops.rst index 95d6744511..9f6f97b1e4 100644 --- a/docs/source/development/traffic_ops.rst +++ b/docs/source/development/traffic_ops.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-ops: + *********** Traffic Ops *********** diff --git a/docs/source/development/traffic_router.rst b/docs/source/development/traffic_router.rst index c9fe941a0b..321d874d97 100644 --- a/docs/source/development/traffic_router.rst +++ b/docs/source/development/traffic_router.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-router: + ************** Traffic Router ************** diff --git a/docs/source/development/traffic_stats.rst b/docs/source/development/traffic_stats.rst index d2663e1a1c..ea83360919 100644 --- a/docs/source/development/traffic_stats.rst +++ b/docs/source/development/traffic_stats.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-stats: + ************* Traffic Stats ************* diff --git a/infrastructure/docker/build/Dockerfile-docs b/infrastructure/docker/build/Dockerfile-docs index 325e0bef06..9b9dae6bad 100644 --- a/infrastructure/docker/build/Dockerfile-docs +++ b/infrastructure/docker/build/Dockerfile-docs @@ -44,7 +44,7 @@ RUN python3 -m pip install --upgrade setuptools && \ python3 -m pip install -r /docs.requirements.txt ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh docs # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-grove b/infrastructure/docker/build/Dockerfile-grove index 878fc0a73c..4473934aae 100644 --- a/infrastructure/docker/build/Dockerfile-grove +++ b/infrastructure/docker/build/Dockerfile-grove @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / ENV GOPATH=/go RUN mkdir -p /go/src/github.com/apache && ln -s /tmp/trafficcontrol /go/src/github.com/apache/trafficcontrol diff --git a/infrastructure/docker/build/Dockerfile-grovetccfg b/infrastructure/docker/build/Dockerfile-grovetccfg index 6a3ffbc459..3e06710927 100644 --- a/infrastructure/docker/build/Dockerfile-grovetccfg +++ b/infrastructure/docker/build/Dockerfile-grovetccfg @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / ENV GOPATH=/go RUN mkdir -p /go/src/github.com/apache && ln -s /tmp/trafficcontrol /go/src/github.com/apache/trafficcontrol diff --git a/infrastructure/docker/build/Dockerfile-source b/infrastructure/docker/build/Dockerfile-source index 8d8e756d4f..e4e9d17f74 100644 --- a/infrastructure/docker/build/Dockerfile-source +++ b/infrastructure/docker/build/Dockerfile-source @@ -37,6 +37,7 @@ RUN yum -y install \ yum -y clean all WORKDIR /trafficcontrol -CMD ./build/build.sh tarball +ADD build/clean_build.sh / +CMD /clean_build.sh tarball # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_monitor b/infrastructure/docker/build/Dockerfile-traffic_monitor index b8e854fcf8..f2a67bc3cf 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_monitor +++ b/infrastructure/docker/build/Dockerfile-traffic_monitor @@ -45,7 +45,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_monitor # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_ops b/infrastructure/docker/build/Dockerfile-traffic_ops index f4acf8398e..99632f2407 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_ops +++ b/infrastructure/docker/build/Dockerfile-traffic_ops @@ -50,7 +50,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_ops # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_ops_ort b/infrastructure/docker/build/Dockerfile-traffic_ops_ort index 5909d6f775..77ebdf207f 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_ops_ort +++ b/infrastructure/docker/build/Dockerfile-traffic_ops_ort @@ -31,7 +31,7 @@ RUN curl -Lo go.tgz https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ln -s /usr/local/go/bin/go /usr/bin/go && \ rm go.tgz -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_ops_ort #vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_portal b/infrastructure/docker/build/Dockerfile-traffic_portal index 4049b4c9fa..86043e485d 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_portal +++ b/infrastructure/docker/build/Dockerfile-traffic_portal @@ -54,7 +54,7 @@ RUN echo '{ "allow_root": true }' > /root/.bowerrc ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_portal # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_router b/infrastructure/docker/build/Dockerfile-traffic_router index af7c7b83ec..b167371ff5 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_router +++ b/infrastructure/docker/build/Dockerfile-traffic_router @@ -47,7 +47,7 @@ RUN curl -L https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3 ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_router # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_stats b/infrastructure/docker/build/Dockerfile-traffic_stats index 631960dc3e..aa1ddab67b 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_stats +++ b/infrastructure/docker/build/Dockerfile-traffic_stats @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_stats # vi:syntax=Dockerfile From 037fe84a7999236e18521b3bb6e4d1347ffc6c5f Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 29 May 2020 21:06:17 -0600 Subject: [PATCH 29/33] Support native building in CDN-in-a-Box makefile --- docs/source/admin/quick_howto/ciab.rst | 9 ++++--- docs/source/development/debugging.rst | 4 +++ infrastructure/cdn-in-a-box/Makefile | 36 ++++++++++++++++++++------ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/docs/source/admin/quick_howto/ciab.rst b/docs/source/admin/quick_howto/ciab.rst index 81693e8b68..9725101ac8 100644 --- a/docs/source/admin/quick_howto/ciab.rst +++ b/docs/source/admin/quick_howto/ciab.rst @@ -38,7 +38,9 @@ The CDN in a Box directory is found within the Traffic Control repository at :fi .. note:: These can also be specified via the ``RPM`` variable to a direct Docker build of the component - with the exception of Traffic Router, which instead accepts ``JDK8_RPM`` to specify a Java Development Kit RPM, ``TRAFFIC_ROUTER_RPM`` to specify a Traffic Router RPM, and ``TOMCAT_RPM`` to specify an Apache Tomcat RPM. -These can all be supplied manually via the steps in :ref:`dev-building` (for Traffic Control component RPMs) or via some external source. Alternatively, the :file:`infrastructure/cdn-in-a-box/Makefile` file contains recipes to build all of these - simply run :manpage:`make(1)`\ [2]_ from the :file:`infrastructure/cdn-in-a-box/` directory. Once all RPM dependencies have been satisfied, run ``docker-compose build`` from the :file:`infrastructure/cdn-in-a-box/` directory to construct the images needed to run CDN in a Box. +These can all be supplied manually via the steps in :ref:`dev-building` (for Traffic Control component RPMs) or via some external source. Alternatively, the :file:`infrastructure/cdn-in-a-box/Makefile` file contains recipes to build all of these - simply run :manpage:`make(1)` from the :file:`infrastructure/cdn-in-a-box/` directory. Once all RPM dependencies have been satisfied, run ``docker-compose build`` from the :file:`infrastructure/cdn-in-a-box/` directory to construct the images needed to run CDN in a Box. + +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make native`` instead of ``make`` to build the RPMs quickly. Another option is running ``make -j4`` to build 4 components at once, if your computer can handle it. .. tip:: When updating CDN-in-a-Box, there is no need to remove old images before building new ones. Docker detects which files are updated and only reuses cached layers that have not changed. @@ -66,7 +68,7 @@ In a typical scenario, if the steps in `Building`_ have been followed, all that' | Traffic Monitor | Web interface and API on port 80 | N/A | N/A | +---------------------------------+----------------------------------------------------------------+---------------------------------------+-------------------------------------------+ | Traffic Ops | Main API endpoints on port 6443, with a direct route to the | ``TO_ADMIN_USER`` in `variables.env`_ | ``TO_ADMIN_PASSWORD`` in `variables.env`_ | - | | Perl API on port 60443\ [3]_ | | | + | | Perl API on port 60443\ [2]_ | | | +---------------------------------+----------------------------------------------------------------+---------------------------------------+-------------------------------------------+ | Traffic Ops PostgresQL Database | PostgresQL connections accepted on port 5432 (database name: | ``DB_USER`` in `variables.env`_ | ``DB_USER_PASS`` in `variables.env`_ | | | ``DB_NAME`` in `variables.env`_) | | | @@ -131,8 +133,7 @@ variables.env .. note:: While these port settings can be changed without hampering the function of the CDN in a Box system, note that changing a port without also changing the matching port-mapping in :file:`infrastructure/cdn-in-a-box/docker-compose.yml` for the affected service *will* make it unreachable from the host. .. [1] It is perfectly possible to build and run all containers without Docker Compose, but it's not recommended and not covered in this guide. -.. [2] Consider ``make -j`` to build quickly, if your computer can handle multiple builds at once. -.. [3] Please do NOT use the Perl endpoints directly. The CDN will only work properly if everything hits the Go API, which will proxy to the Perl endpoints as needed. +.. [2] Please do NOT use the Perl endpoints directly. The CDN will only work properly if everything hits the Go API, which will proxy to the Perl endpoints as needed. X.509 SSL/TLS Certificates ========================== diff --git a/docs/source/development/debugging.rst b/docs/source/development/debugging.rst index 953e1ea9b6..23c8628b71 100644 --- a/docs/source/development/debugging.rst +++ b/docs/source/development/debugging.rst @@ -40,6 +40,8 @@ Traffic Monitor make very-clean make debug +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make debug native`` instead of ``make debug`` to build the RPMs quickly. + * Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TM_DEBUG_ENABLE`` to ``true``. * Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficmonitor`` image without reusing any cached layers to make sure it uses our fresh ``traffic_monitor.rpm``. Then, start CDN-in-a-Box. @@ -87,6 +89,8 @@ Traffic Ops (Go) make very-clean make debug +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make debug native`` instead of ``make debug`` to build the RPMs quickly. + * Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TO_DEBUG_ENABLE`` to ``true``. * Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficops-go`` image without reusing any cached layers to make sure it uses our fresh ``traffic_ops.rpm``. Then, start CDN-in-a-Box. diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index 3edbeb46e7..8ab0d444f6 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -29,7 +29,9 @@ ifneq ($(PWD),$(makefile_dir)) $(error This makefile MUST be run from within its directory) endif +PKG_COMMAND := ../../pkg PKG_FLAGS := -v +BUILD_SUFFIX := _build BUILD_NUMBER := $(shell git rev-list HEAD 2>/dev/null | wc -l | tr -d '[[:space:]]').$(shell git rev-parse --short=8 HEAD) TC_VERSION := $(shell cat "../../VERSION") TOMCAT_VERSION := $(shell grep '^\s*TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d= -f2) @@ -54,12 +56,30 @@ TS_SOURCE := $(wildcard ../../traffic_stats/**/*) # Default target; builds all pre-requisite rpms from source trees all: cache/traffic_ops_ort.rpm traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm -debug: PKG_FLAGS += -d -ifneq ($(MAKECMDGOALS), debug) +ifneq ($(filter debug,$(MAKECMDGOALS)),) +PKG_FLAGS += -d +export DEBUG_BUILD = true +ifneq ($(MAKECMDGOALS),debug) +MAKECMDGOALS := $(filter-out debug,$(MAKECMDGOALS)) debug: $(MAKECMDGOALS) else debug: all endif +endif + +ifneq ($(filter native,$(MAKECMDGOALS)),) +PKG_COMMAND := ../../build/clean_build.sh +PKG_FLAGS := +BUILD_SUFFIX := +ifneq ($(MAKECMDGOALS),native) +MAKECMDGOALS := $(filter-out native,$(MAKECMDGOALS)) +native: $(MAKECMDGOALS) +else +native: all +endif +endif + +#.PHONY: native # Actual output rpm recipies traffic_monitor/traffic_monitor.rpm: ../../dist/traffic_monitor-$(SPECIAL_SAUCE) @@ -79,22 +99,22 @@ cache/traffic_ops_ort.rpm: ../../dist/traffic_ops_ort-$(SPECIAL_SAUCE) # Dist rpms ../../dist/traffic_monitor-$(SPECIAL_SAUCE): $(TM_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_monitor_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_monitor$(BUILD_SUFFIX) ../../dist/traffic_ops-$(SPECIAL_SAUCE): $(TO_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_ops_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_ops$(BUILD_SUFFIX) ../../dist/traffic_portal-$(SPECIAL_SAUCE): $(TP_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_portal_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_portal$(BUILD_SUFFIX) ../../dist/traffic_rou%er-$(SPECIAL_SAUCE) ../../dist/tomca%-$(SPECIAL_SEASONING): $(TR_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_router_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_router$(BUILD_SUFFIX) ../../dist/traffic_stats-$(SPECIAL_SAUCE): $(TS_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_stats_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_stats$(BUILD_SUFFIX) ../../dist/traffic_ops_ort-$(SPECIAL_SAUCE): $(ORT_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_ops_ort_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_ops_ort$(BUILD_SUFFIX) clean: $(RM) traffic_monitor/traffic_monitor.rpm traffic_ops/traffic_ops.rpm traffic_portal/traffic_portal.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm cache/traffic_ops_ort.rpm traffic_stats/traffic_stats.rpm From 7a2245775f4b7ec7e4ee868d7c9ed5170860c914 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 8 Jun 2020 23:01:00 -0600 Subject: [PATCH 30/33] Docs touch-ups - Fix label references - Fix some typos - Link to the Windows footnote from the Chocolatey package row --- docs/source/development/building.rst | 10 +++++----- docs/source/development/traffic_portal.rst | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/source/development/building.rst b/docs/source/development/building.rst index fb3d183466..478cfd40d4 100644 --- a/docs/source/development/building.rst +++ b/docs/source/development/building.rst @@ -85,7 +85,7 @@ If the ``pkg`` script fails, ``docker-compose`` can still be used to build the p Build the RPMs Natively ======================= -A developer may end up building the RPMs several times to test or :ref:`debug ` code changes, so it can be desirable to build the RPMs quickly for this purpose. Natively building the RPMs has the lowest build time of any method. +A developer may end up building the RPMs several times to test or :ref:`debug ` code changes, so it can be desirable to build the RPMs quickly for this purpose. Natively building the RPMs has the lowest build time of any building method. Install the Dependencies ------------------------ @@ -117,7 +117,7 @@ Install the Dependencies | | - rpm-build | | | - make | | | | | | | | - rsync | | | - gcc-g++ | | | | | | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ - | Windows (chocolatey_) | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | + | Windows (chocolatey_)\ [6]_ | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | | | | | | | - maven | | | | - pip | | | | | | | | | | | - make | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ @@ -141,14 +141,14 @@ Install the Dependencies After installing the packages using your platform's package manager, - - Install the :ref:`global NPM packages ` and :ref:`Compass `.. to build Traffic Portal. + - Install the :ref:`global NPM packages ` and :ref:`Compass ` to build Traffic Portal. - Install the Python 3 modules used to :ref:`build the documentation `. Run ``build/clean_build.sh`` directly ------------------------------------- -In a terminal, nativate to the root directory of the repository. You can run ``build/clean_build.sh`` with no arguments to build all components. +In a terminal, navigate to the root directory of the repository. You can run ``build/clean_build.sh`` with no arguments to build all components. .. code-block:: shell :caption: ``build/clean_build.sh`` with no arguments @@ -174,4 +174,4 @@ Each Traffic Control component can be individually built, and the instructions f Building This Documentation --------------------------- -To build the documentation, see :ref:`docs-build`. +See instructions for :ref:`building the documentation `. diff --git a/docs/source/development/traffic_portal.rst b/docs/source/development/traffic_portal.rst index 26bcd6632d..c6f8191d34 100644 --- a/docs/source/development/traffic_portal.rst +++ b/docs/source/development/traffic_portal.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-portal: + ************** Traffic Portal ************** @@ -34,6 +36,8 @@ To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly us * `Grunt CLI 1.2.0 or above `_ * Access to a working instance of Traffic Ops +.. _dev-tp-global-npm: + Install Global NPM Packages --------------------------- From 71784b822ea9922b24008ad4d666b74563ff8152 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 16 Jun 2020 08:48:58 -0600 Subject: [PATCH 31/33] Recursively remove untracked files in clean_build.sh, not just the top level --- build/clean_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/clean_build.sh b/build/clean_build.sh index 5e82629821..8e36e04f2b 100755 --- a/build/clean_build.sh +++ b/build/clean_build.sh @@ -56,7 +56,7 @@ fi cd "$tc_dir" # In case the mirrored repo already exists, remove gitignored files -git clean -fX +git clean -fdX rm -rf "dist" mkdir -p "${tc_volume}/dist" From 6596da539337fe6a3b3bdb356a1b7536e8aab511 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 18 Jun 2020 15:27:39 -0600 Subject: [PATCH 32/33] Add native phony make target --- infrastructure/cdn-in-a-box/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index 8ab0d444f6..176aa87e41 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -51,7 +51,7 @@ TP_SOURCE := $(wildcard ../../traffic_portal/**/*) TR_SOURCE := $(wildcard ../../traffic_router/**/*) TS_SOURCE := $(wildcard ../../traffic_stats/**/*) -.PHONY: clean very-clean all nearly-all debug +.PHONY: clean very-clean all nearly-all debug native # Default target; builds all pre-requisite rpms from source trees all: cache/traffic_ops_ort.rpm traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm From 65ae41215d1928896fabb5cf1d18d957d691099e Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 24 Jun 2020 14:26:55 -0600 Subject: [PATCH 33/33] Docs building changes - Use the sphinx module to build the docs, not the sphinx-build executable - check for python3, not python --- build/build.sh | 2 +- docs/Makefile | 2 +- docs/source/development/building.rst | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/build/build.sh b/build/build.sh index 7b1afacbea..ebb442a3f0 100755 --- a/build/build.sh +++ b/build/build.sh @@ -63,7 +63,7 @@ for p in ${projects}; do if [ "$p" = docs ]; then if isInGitTree; then echo "----- Building docs ..." - checkEnvironment -i python,sphinx-build,make, -e rpmbuild + checkEnvironment -i python3,make, -e rpmbuild ( cd docs make html ) diff --git a/docs/Makefile b/docs/Makefile index f48302f1da..958775ca91 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,7 +20,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD = python3 -m sphinx PAPER = letter BUILDDIR = build diff --git a/docs/source/development/building.rst b/docs/source/development/building.rst index 478cfd40d4..58d11ad2ca 100644 --- a/docs/source/development/building.rst +++ b/docs/source/development/building.rst @@ -95,11 +95,11 @@ Install the Dependencies +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ | | Common dependencies | :ref:`dev-traffic-monitor` | :ref:`dev-traffic-ops` | :ref:`dev-traffic-portal` | :ref:`dev-traffic-router` | :ref:`dev-traffic-stats` | Grove | Grove TC Config (grovetccfg) | :ref:`Docs ` | +====================================+=====================+============================+========================+===========================+===========================+==========================+==========+==============================+==========================+ - | macOS (homebrew_)\ [3]_,\ [4]_ | - rpm | - go | - go | - npm | - maven | - go | - go | - go | - python3 | - | | | | | - bower | | | | | - sphinx-doc | + | macOS (homebrew_)\ [3]_ | - rpm | - go | - go | - npm | - maven | - go | - go | - go | - python3 | + | | | | | - bower | | | | | | | | | | | - grunt-cli | | | | | | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ - | CentOS/Red Hat/Fedora (yum_)\ [5]_ | - git | | | - epel-release | - java-1.8.0-openjdk | | | | - python3-devel | + | CentOS/Red Hat/Fedora (yum_)\ [4]_ | - git | | | - epel-release | - java-1.8.0-openjdk | | | | - python3-devel | | | - rpm-build | | | - npm | - maven | | | | - gcc | | | - rsync | | | - nodejs-grunt-cli | | | | | - make | | | | | | - ruby-devel | | | | | | @@ -113,11 +113,11 @@ Install the Dependencies | | | | | - gcc | | | | | | | | | | | - make | | | | | | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ - | Windows (cygwin_)\ [6]_ | - git | | | - ruby-devel | - curl | | | | | + | Windows (cygwin_)\ [5]_ | - git | | | - ruby-devel | - curl | | | | | | | - rpm-build | | | - make | | | | | | | | - rsync | | | - gcc-g++ | | | | | | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ - | Windows (chocolatey_)\ [6]_ | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | + | Windows (chocolatey_)\ [5]_ | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | | | | | | | - maven | | | | - pip | | | | | | | | | | | - make | +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ @@ -130,11 +130,9 @@ Install the Dependencies .. [3] If you are on macOS, you additionally need to :ref:`dev-tr-mac-jdk`. -.. [4] On macOS, in order to add the ``sphinx-build`` binary to your ``PATH`` after installing the ``sphinx-doc`` package, run ``brew link --force sphinx-doc`` +.. [4] If you are on CentOS, you need to `download Go directly `_ instead of using a package manager in order to get the latest Go version. For most users, the desired architecture is AMD64/x86_64. -.. [5] If you are on CentOS, you need to `download Go directly `_ instead of using a package manager in order to get the latest Go version. For most users, the desired architecture is AMD64/x86_64. - -.. [6] If you are on Windows, you need to install both the Cygwin packages and the Chocolatey packages in order to build the Apache Traffic Control RPMs natively. +.. [5] If you are on Windows, you need to install both the Cygwin packages and the Chocolatey packages in order to build the Apache Traffic Control RPMs natively. .. |AdoptOpenJDK instructions| replace:: add the AdoptOpenJDK tap and install the ``adoptopenjdk8`` cask .. _AdoptOpenJDK instructions: https://github.com/AdoptOpenJDK/homebrew-openjdk#other-versions