diff --git a/.gitignore b/.gitignore index c415772126..9f72bff5b8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ cpanfile.snapshot rpm/build.number rpmbuild/ docs/build/* +/grove/grove +/grove/grovetccfg/grovetccfg local traffic_ops_extensions/log/* traffic_ops_extensions/lib/Extensions/InfluxDB/log/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8ec762d1..8969ae5a20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Router: Added support for topology-based delivery services - Updated /servers/details to use multiple interfaces in API v3 - Added [Edge Traffic Routing](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_router.html#edge-traffic-routing) feature which allows Traffic Router to localize more DNS record types than just the routing name for DNS delivery services +- Added the ability to speedily build development RPMs from any OS without needing Docker - Astats csv support - astats will now respond to `Accept: text/csv` and return a csv formatted stats list - Updated /deliveryservices/{{ID}}/servers to use multiple interfaces in API v3 - Updated /deliveryservices/{{ID}}/servers/eligible to use multiple interfaces in API v3 diff --git a/build/build.sh b/build/build.sh index 1f252111e6..ebb442a3f0 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,91 +11,98 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}"; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; # By default all sub-projects are built. Supply a list of projects to build if # only a subset is wanted. -# make sure we start out in traffic_control dir -topscript=$(readlink -f $0) -export TC_DIR=$(dirname $(dirname "$topscript")) -[[ -n $TC_DIR ]] && cd "$TC_DIR" || { echo "Could not cd $TC_DIR"; exit 1; } - . build/functions.sh -checkEnvironment - +# make sure we start out in traffic_control dir +topscript='' TC_DIR='' +topscript="$(realpath "$0")" +TC_DIR="$(dirname "$(dirname "$topscript")")" +export TC_DIR +if [ -z "$TC_DIR" ] || ! cd "$TC_DIR"; then + echo "Could not cd TC_DIR ${TC_DIR}"; + exit 1; +fi; -if [[ $# -gt 0 ]]; then - projects=( "$*" ) +if [ $# -gt 0 ]; then + projects="$*" else # get all subdirs containing build/build_rpm.sh - projects_to_build=( */build/build_rpm.sh ) + projects_to_build='*/build/build_rpm.sh' # Always build tarball when building everything.. - projects=(tarball) - for p in "${projects_to_build[@]}"; do - p=${p%%/*} - if [[ $p != "traffic_monitor_golang" ]]; then - projects+=($p) + projects=tarball + for p in ${projects_to_build}; do + p="${p%%/*}" + if [ "$p" != "traffic_monitor_golang" ]; then + projects="${projects} ${p}" fi done fi -declare -a badproj -declare -a goodproj -for p in "${projects[@]}"; do - if [[ $p == tarball ]]; then +badproj='' +goodproj='' +for p in ${projects}; do + if [ "$p" = tarball ]; then if isInGitTree; then echo "----- Building tarball ..." - tarball=$(createTarball "$TC_DIR") - ls -l $tarball + checkEnvironment -e rpmbuild + tarball="$(createTarball "$TC_DIR")" + ls -l "$tarball" else echo "---- Skipping tarball creation" fi continue fi - if [[ $p == docs ]]; then + if [ "$p" = docs ]; then if isInGitTree; then echo "----- Building docs ..." - pushd docs + checkEnvironment -i python3,make, -e rpmbuild + ( cd docs make html - popd + ) tarball=$(createDocsTarball "${TC_DIR}") - ls -l $tarball + ls -l "$tarball" else echo "---- Skipping docs creation" fi continue fi # strip trailing / - p=${p%/} - bldscript="$p/build/build_rpm.sh" - if [[ ! -x $bldscript ]]; then + p="${p%/}" + bldscript="${p}/build/build_rpm.sh" + if [ ! -x "$bldscript" ]; then echo "$bldscript not found" - badproj+=($p) + badproj="${badproj} ${p}" continue fi echo "----- Building $p ..." if $bldscript; then - goodproj+=($p) + goodproj="${goodproj} ${p}" else - echo "$p failed: $!" - badproj+=($p) + echo "${p} failed: ${bldscript}" + badproj="${badproj} ${p}" fi done -if [[ ${#goodproj[@]} -ne 0 ]]; then +if [ "$(echo "${goodproj}" | wc -w)" -ne 0 ]; then echo "The following subdirectories built successfully: " - for p in "${goodproj[@]}"; do + for p in ${goodproj}; do echo " $p" done echo "See $(pwd)/dist for newly built rpms." fi -if [[ ${#badproj[@]} -ne 0 ]]; then +if [ "$(echo "${badproj}" | wc -w)" -ne 0 ]; then echo "The following subdirectories had errors: " - for p in "${badproj[@]}"; do + for p in ${badproj}; do echo " $p" done exit 1 diff --git a/build/clean_build.sh b/build/clean_build.sh new file mode 100755 index 0000000000..8e36e04f2b --- /dev/null +++ b/build/clean_build.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env sh +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}"; cleanup; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; + + +# Fix ownership of output files +# $1 is file or dir with correct ownership +# remaining args are files/dirs to be fixed, recursively +setowner() { + own=$(stat -c%u:%g "$1" 2>/dev/null || stat -f%u:%g "$1") + shift + [ -n "$*" ] && chown -R "${own}" "$@" +} + +cleanup() { + setowner "$tc_volume" "${tc_volume}/dist" +} + +set -o xtrace; + +if ! script_path="$(readlink "$0")"; then + script_path="$0"; +fi; +cd "$(dirname "$script_path")"; +if ! tc_volume="$(git rev-parse --show-toplevel 2>/dev/null)"; then + tc_volume='/trafficcontrol'; # Default repo location for ATC builder Docker images +fi; + +# set owner of dist dir -- cleans up existing dist permissions... +export GOPATH=/tmp/go GOOS="${GOOS:-linux}"; +tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; +if which cygpath 2>/dev/null; then + GOPATH="$(cygpath -w "$GOPATH")" # cygwin compatibility +fi +(mkdir -p "$GOPATH" + cd "$GOPATH" + mkdir -p src pkg bin "$(dirname "$tc_dir")" +) +rsync -a --exclude=dist "${tc_volume}/" "$tc_dir"; +if ! [ -d ${tc_dir}/.git ]; then + rsync -a "${tc_volume}/.git" $tc_dir; # Docker for Windows compatibility +fi + +cd "$tc_dir" +# In case the mirrored repo already exists, remove gitignored files +git clean -fdX + +rm -rf "dist" +mkdir -p "${tc_volume}/dist" +ln -sf "${tc_volume}/dist" "dist" + +if [ $# -eq 0 ]; then + set -- tarball traffic_monitor traffic_ops traffic_ops_ort traffic_portal traffic_router traffic_stats grove grove/grovetccfg docs +fi + +for project in "$@"; do + ./build/build.sh "${project}" 2>&1 | tee "dist/build-${project//\//-}.log" +done diff --git a/build/functions.sh b/build/functions.sh index 90628b2a77..a60f66a70e 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,103 +11,176 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash + +if ! type -p realpath; then + # by default, macOS does not have realpath + realpath() { + ls "$( + cd "$(dirname "$0")" + pwd -P # -P resolves symlinks + )/$(basename "$0")" + } + export -f realpath +fi; + +if { ! stat -c%u . >/dev/null && stat -f%u .; } >/dev/null 2>&1; then + #BSD stat uses -f as its formatting flag instead of -c + stat() { + local format='' + while getopts c: option; do + case "$option" in + c) format="$OPTARG";; + *) return 1 + esac + done + shift $(( OPTIND - 1 )) + unset OPTIND + $(which stat) "-f${format}" "$@" + } + export -f stat +fi + +removeFirstArg() { + shift + echo "$@" +} # --------------------------------------- # versionOk checks version number against required version. # ``versionOk 1.2.3 2.0.4.7'' returns false value indicating # version you have is not at least version you need # if versionOk $haveversion $needversion; then -# echo "Need at least version $needversion"; exit 1 +# echo "Need at least version $needversion"; return 1 # fi -function versionOk() { - local h=$1 n=$2 +versionOk() { + local h="$1" n="$2" # string compare -- no need to do more if the same - [[ $h == $n ]] && return 0 + [ "$h" = "$n" ] && return 0 # split into fields - local have=(${h//\./ }) - local need=(${n//\./ }) + local have="${h//\./ }" + local need="${n//\./ }" # cmp first entry of each array. Bail when unequal. - while [[ -n $have && $have -eq $need ]]; do + while [ -n "$have" ] && [ "$have" = "$need" ]; do # pop 1st entry from each - have=("${have[@]:1}") - need=("${need[@]:1}") + have="$(removeFirstArg ${have})" + need="$(removeFirstArg ${need})" done - if [[ ${have:-0} -lt ${need:-0} ]]; then + have="${have%% *}"; + need="${need%% *}"; + if [ "${have:-0}" -lt "${need:-0}" ]; then return 1 fi return 0 } # --------------------------------------- -function getRevCount() { - local buildNum=$(getBuildNumber) - echo ${buildNum%.*} +getRevCount() { + local buildNum + buildNum=$(getBuildNumber) + echo "${buildNum%.*}" } # --------------------------------------- -function isInGitTree() { +isInGitTree() { # ignore output -- use exit status git rev-parse --is-inside-work-tree >/dev/null 2>&1 } # --------------------------------------- -function getBuildNumber() { - local in_git=$() +getBuildNumber() { + local in_git='' if isInGitTree; then - local commits=$(git rev-list HEAD 2>/dev/null | wc -l) - local sha=$(git rev-parse --short=8 HEAD) + local commits sha + commits="$(git rev-list HEAD 2>/dev/null | wc -l | awk '{print $1}')" # awk is for BSD compatibility + sha="$(git rev-parse --short=8 HEAD)" echo "$commits.$sha" else # Expect it's from the released tarball -- if BUILD_NUMBER file is not present, abort - if [[ ! -f $TC_DIR/BUILD_NUMBER ]]; then + if [ ! -f "${TC_DIR}/BUILD_NUMBER" ]; then echo "Not in git repository and no BUILD_NUMBER present -- aborting!" - exit 1 + return 1 fi - grep -v '^#' $TC_DIR/BUILD_NUMBER + grep -v '^#' "${TC_DIR}/BUILD_NUMBER" fi } # --------------------------------------- -function getVersion() { +getVersion() { local d="$1" local vf="$d/VERSION" - [ -r $vf ] || { echo "Could not read $vf: $!"; exit 1; } + [ -r "$vf" ] || { echo "Could not read $vf: $!"; return 1; } cat "$vf" } # --------------------------------------- -function getRhelVersion { - echo el$(rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)) +getRhelVersion() { + local releasever + local redhat_release=/etc/redhat-release + local default_version=7 + if [ -e $redhat_release ]; then + releasever="$(rpm -q --qf '%{version}' -f $redhat_release)" + else + echo "${redhat_release} not found, defaulting to major release ${default_version}" >/dev/stderr + releasever=${default_version} + fi; + + echo "el${releasever}" } # --------------------------------------- -function getCommit() { - local buildNum=$(getBuildNumber) - echo ${buildNum%.*} +getCommit() { + local buildNum + buildNum="$(getBuildNumber)" + echo "${buildNum%.*}" } # --------------------------------------- -function checkEnvironment { - export TC_VERSION=$(getVersion "$TC_DIR") - export BUILD_NUMBER=$(getBuildNumber) - export RHEL_VERSION=$(getRhelVersion) - export WORKSPACE=${WORKSPACE:-$TC_DIR} - export RPMBUILD="$WORKSPACE/rpmbuild" - export DIST="$WORKSPACE/dist" - - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } +checkEnvironment() { + include_programs='' exclude_programs='' + while getopts i:e: option; do + case "$option" in + i) include_programs="$OPTARG";; + e) exclude_programs="$OPTARG";; + *) return 1 + esac + done + unset OPTIND + include_file="$(mktemp)" + exclude_file="$(mktemp)" + printf '%s\n' git rpmbuild ${include_programs//,/ } | sort >"$include_file" + printf '%s\n' ${exclude_programs//,/ } | sort >"$exclude_file" + programs="$(comm -23 "$include_file" "$exclude_file")" + rm "$include_file" "$exclude_file" # verify required tools available in path -- extra tools required by subsystem are passed in - for pgm in git rpmbuild "$@"; do - type $pgm 2>/dev/null || { echo "$pgm not found in PATH"; } + for program in $programs; do + if ! type "$program"; then + echo "$program not found in PATH" + return 1 + fi done # verify git version requiredGitVersion=1.7.12 - if ! versionOk $(git --version | tr -dc 0-9. ) "$requiredGitVersion"; then + if ! versionOk "$(git --version | tr -dc 0-9. )" "$requiredGitVersion"; then echo "$(git --version) must be at least $requiredGitVersion" - exit 1 + return 1 fi + + TC_VERSION='' BUILD_NUMBER='' RHEL_VERSION='' RPMBUILD='' DIST='' + TC_VERSION="$(getVersion "$TC_DIR")" + BUILD_NUMBER="$(getBuildNumber)" + RHEL_VERSION="$(getRhelVersion)" + WORKSPACE="${WORKSPACE:-$TC_DIR}" + RPMBUILD="$WORKSPACE/rpmbuild" + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" + DIST="$WORKSPACE/dist" + export TC_VERSION BUILD_NUMBER RHEL_VERSION WORKSPACE RPMBUILD GOOS RPM_TARGET_OS DIST + + mkdir -p "$DIST" || { echo "Could not create ${DIST}: ${?}"; return 1; } + echo "Build environment has been verified." echo "==================================================" @@ -122,32 +192,41 @@ function checkEnvironment { } # --------------------------------------- -function createSourceDir() { - local target="$1-$TC_VERSION" +createSourceDir() { + local target="${1}-${TC_VERSION}" local srcpath="$RPMBUILD/SOURCES/$target" - mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; exit 1; } + mkdir -p "$srcpath" || { echo "Could not create $srcpath: $?"; return 1; } echo "$srcpath" } # --------------------------------------- -function buildRpm () { +buildRpm() { for package in "$@"; do local pre="${package}-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION}" - local rpm="${pre}.$(uname -m).rpm" + local rpm + rpm="${pre}.$(uname -m).rpm" local srpm="${pre}.src.rpm" echo "Building the rpm." - if [[ "$DEBUG_BUILD" == true ]]; then + { set +o nounset + set -- # Clear arguments for reuse + if [ "$DEBUG_BUILD" = true ]; then echo 'RPM will not strip binaries before packaging.'; - echo '%__os_install_post %{nil}' >> /etc/rpm/macros; # Do not strip binaries before packaging + set -- "$@" --define '%__os_install_post %{nil}' # Do not strip binaries before packaging fi; + set -- "$@" --define '%_source_payload w2.xzdio' # xz level 2 compression for text files + set -- "$@" --define '%_binary_payload w2.xzdio' # xz level 2 compression for binary files + set -o nounset; } - cd "$RPMBUILD" && \ + (cd "$RPMBUILD" && \ rpmbuild --define "_topdir $(pwd)" \ - --define "traffic_control_version $TC_VERSION" \ - --define "commit $(getCommit)" \ - --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ - -ba SPECS/$package.spec || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } + --define "traffic_control_version $TC_VERSION" \ + --define "commit $(getCommit)" \ + --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ + --define "_target_os $RPM_TARGET_OS" \ + -ba SPECS/$package.spec \ + "$@" # variable number of arguments + ) || \ + { echo "RPM BUILD FAILED: $?"; return 1; } echo echo "========================================================================================" @@ -155,80 +234,93 @@ function buildRpm () { echo "========================================================================================" echo - cp "$RPMBUILD/RPMS/$(uname -m)/$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } - cp "$RPMBUILD/SRPMS/$srpm" "$DIST/." || { echo "Could not copy $srpm to $DIST: $?"; exit 1; } + cp "$RPMBUILD/RPMS/$(uname -m)/$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; return 1; } + cp "$RPMBUILD/SRPMS/$srpm" "$DIST/." || { echo "Could not copy $srpm to $DIST: $?"; return 1; } done } # --------------------------------------- -function createTarball() { - local projDir=$(cd "$1"; pwd) +createTarball() { + local projDir + projDir="$(cd "$1"; pwd)" local projName=trafficcontrol - local version=$(getVersion "$TC_DIR") - local tarball="dist/apache-$projName-$version.tar.gz" - local tardir=$(basename $tarball .tar.gz) - - # Create a BULDNUMBER file and add to tarball - local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" - - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$projDir" --exclude-vcs --transform "s@^@$tardir/@S" $(git ls-files) - rm -r "$bndir" - echo "$tarball" + local version + version="$(getVersion "$TC_DIR")" + local tarball="dist/apache-${projName}-${version}.tar.gz" + local tarDir + tarDir="$(basename "$tarball" .tar.gz)" + local tarLink + ( trap 'rm -f "${projDir}/BUILD_NUMBER" "$projLink"' EXIT + projLink="$(dirname "$projDir")/${tarDir}" + ln -sf "$projDir" "$projLink" + + # Create a BUILDNUMBER file and add to tarball + getBuildNumber >"${projDir}/BUILD_NUMBER" + + # create the tarball from BUILD_NUMBER and files tracked in git + git ls-files | + sed "s|^|${tarDir}/|g" | + tar -czf "$tarball" -C "${projLink}/.." -T - "${tarDir}/BUILD_NUMBER" + # ^ read files from stdin to avoid tar argument limit + ) + echo "$tarball" } # --------------------------------------- -function createDocsTarball() { - local projDir=$(cd "$1"; pwd) +createDocsTarball() { + local projDir + projDir="$(cd "$1"; pwd)" local projName=trafficcontrol - local version=$(getVersion "$TC_DIR") + local version + version="$(getVersion "$TC_DIR")" local tarball="dist/apache-$projName-$version-docs.tar.gz" - local tardir="${projDir}/docs/build/" + local tarDir="${projDir}/docs/build/" - # Create a BULDNUMBER file and add to tarball - local bndir=$(mktemp -d) - getBuildNumber >"$bndir/BUILD_NUMBER" + # Create a BUILDNUMBER file and add to tarball + getBuildNumber >"${tarDir}/BUILD_NUMBER" - # create the tarball only from files in repo and BUILD_NUMBER - tar -czf "$tarball" -C "$bndir" BUILD_NUMBER -C "$tardir" . --exclude-vcs - rm -r "$bndir" - echo "$tarball" + # create the tarball only from files in repo and BUILD_NUMBER + tar -czf "$tarball" -C "$tarDir" . + rm "${tarDir}/BUILD_NUMBER" + echo "$tarball" } # ---------------------------------------- # verify if the go compiler is version 1.14 or higher, returns 0 if if not. returns 1 if it is. -# -function verify_and_set_go_version () { - GO_VERSION="none" - GO="none" - go_in_path=`type -p go` - for g in $go_in_path /usr/bin/go /usr/local/go/bin/go; do - if [[ -z $g ]] || [[ ! -x $g ]]; then - continue - fi - - go_version=`$g version | awk '{print $3}'` - - if [[ $go_version =~ go([1-9])\.([1-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 1 ]] && [[ ${BASH_REMATCH[2]} -ge 14 ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - GO=$g; export GO - PATH=`dirname $g`:$PATH; export PATH - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - echo "will use $g" - return 1 - else - if [[ $go_version =~ go([1-9])\.([1-9]+) ]]; then - GO_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"; export GO_VERSION - echo "go version for $g is ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - continue - fi - fi - done - - if [[ $GO == none ]]; then - echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" - return 0 - fi +# +verify_and_set_go_version() { + GO_VERSION="none" + GO="none" + go_in_path="$(type -p go)" + local group_1='' group_2='' + for g in "$go_in_path" /usr/bin/go /usr/local/go/bin/go; do + if [ -z "$g" ] || [ ! -x "$g" ]; then + continue + fi + + go_version="$($g version | awk '{print $3}')" + + version_pattern='.*go([1-9]+)\.([1-9]+).*' + if echo "$go_version" | grep -E "$version_pattern"; then + group_1="$(echo "$go_version" | sed -E "s/.*${version_pattern}/\1/")" + group_2="$(echo "$go_version" | sed -E "s/${version_pattern}/\2/")" + if [ ! "$group_1" -ge 1 ] || [ ! "$group_2" -ge 14 ]; then + GO_VERSION="${group_1}.${group_2}"; export GO_VERSION + echo "go version for $g is ${group_1}.${group_2}" + continue + fi + GO_VERSION="${group_1}.${group_2}"; export GO_VERSION + GO=$g; export GO + PATH="$(dirname "$g"):${PATH}"; export PATH + echo "go version for $g is ${group_1}.${group_2}" + echo "will use $g" + return 0 + fi + done + + if [ "$GO" = none ]; then + echo "ERROR: this build needs go 1.14 or greater and no usable go compiler was found, found GO_VERSION: $GO_VERSION" + return 1 + fi } diff --git a/docs/Makefile b/docs/Makefile index f48302f1da..958775ca91 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,7 +20,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD = python3 -m sphinx PAPER = letter BUILDDIR = build diff --git a/docs/source/admin/quick_howto/ciab.rst b/docs/source/admin/quick_howto/ciab.rst index 81693e8b68..9725101ac8 100644 --- a/docs/source/admin/quick_howto/ciab.rst +++ b/docs/source/admin/quick_howto/ciab.rst @@ -38,7 +38,9 @@ The CDN in a Box directory is found within the Traffic Control repository at :fi .. note:: These can also be specified via the ``RPM`` variable to a direct Docker build of the component - with the exception of Traffic Router, which instead accepts ``JDK8_RPM`` to specify a Java Development Kit RPM, ``TRAFFIC_ROUTER_RPM`` to specify a Traffic Router RPM, and ``TOMCAT_RPM`` to specify an Apache Tomcat RPM. -These can all be supplied manually via the steps in :ref:`dev-building` (for Traffic Control component RPMs) or via some external source. Alternatively, the :file:`infrastructure/cdn-in-a-box/Makefile` file contains recipes to build all of these - simply run :manpage:`make(1)`\ [2]_ from the :file:`infrastructure/cdn-in-a-box/` directory. Once all RPM dependencies have been satisfied, run ``docker-compose build`` from the :file:`infrastructure/cdn-in-a-box/` directory to construct the images needed to run CDN in a Box. +These can all be supplied manually via the steps in :ref:`dev-building` (for Traffic Control component RPMs) or via some external source. Alternatively, the :file:`infrastructure/cdn-in-a-box/Makefile` file contains recipes to build all of these - simply run :manpage:`make(1)` from the :file:`infrastructure/cdn-in-a-box/` directory. Once all RPM dependencies have been satisfied, run ``docker-compose build`` from the :file:`infrastructure/cdn-in-a-box/` directory to construct the images needed to run CDN in a Box. + +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make native`` instead of ``make`` to build the RPMs quickly. Another option is running ``make -j4`` to build 4 components at once, if your computer can handle it. .. tip:: When updating CDN-in-a-Box, there is no need to remove old images before building new ones. Docker detects which files are updated and only reuses cached layers that have not changed. @@ -66,7 +68,7 @@ In a typical scenario, if the steps in `Building`_ have been followed, all that' | Traffic Monitor | Web interface and API on port 80 | N/A | N/A | +---------------------------------+----------------------------------------------------------------+---------------------------------------+-------------------------------------------+ | Traffic Ops | Main API endpoints on port 6443, with a direct route to the | ``TO_ADMIN_USER`` in `variables.env`_ | ``TO_ADMIN_PASSWORD`` in `variables.env`_ | - | | Perl API on port 60443\ [3]_ | | | + | | Perl API on port 60443\ [2]_ | | | +---------------------------------+----------------------------------------------------------------+---------------------------------------+-------------------------------------------+ | Traffic Ops PostgresQL Database | PostgresQL connections accepted on port 5432 (database name: | ``DB_USER`` in `variables.env`_ | ``DB_USER_PASS`` in `variables.env`_ | | | ``DB_NAME`` in `variables.env`_) | | | @@ -131,8 +133,7 @@ variables.env .. note:: While these port settings can be changed without hampering the function of the CDN in a Box system, note that changing a port without also changing the matching port-mapping in :file:`infrastructure/cdn-in-a-box/docker-compose.yml` for the affected service *will* make it unreachable from the host. .. [1] It is perfectly possible to build and run all containers without Docker Compose, but it's not recommended and not covered in this guide. -.. [2] Consider ``make -j`` to build quickly, if your computer can handle multiple builds at once. -.. [3] Please do NOT use the Perl endpoints directly. The CDN will only work properly if everything hits the Go API, which will proxy to the Perl endpoints as needed. +.. [2] Please do NOT use the Perl endpoints directly. The CDN will only work properly if everything hits the Go API, which will proxy to the Perl endpoints as needed. X.509 SSL/TLS Certificates ========================== diff --git a/docs/source/development/building.rst b/docs/source/development/building.rst index f00601fbfa..58d11ad2ca 100644 --- a/docs/source/development/building.rst +++ b/docs/source/development/building.rst @@ -74,28 +74,102 @@ Output :file:`{component}-{version}.rpm` files, build logs and source tarballs w .. _build-with-dc: Build Using ``docker-compose`` -============================== +------------------------------ If the ``pkg`` script fails, ``docker-compose`` can still be used to build the projects directly. The compose file can be found at ``infrastructure/docker/build/docker-compose.yml`` under the repository's root directory. It can be passed directly to ``docker-compose``, either from the ``infrastructure/docker/build/`` directory or by explicitly passing a path to the ``infrastructure/docker/build/docker-compose.yml`` file via ``-f``. It is recommended that between builds ``docker-compose down -v`` is run to prevent caching of old build steps. The service names are the same as the project names described above in `Usage`_, and similar to the ``pkg`` script, the build results, logs and source tarballs may all be found in the ``dist`` directory after completion. .. note:: Calling ``docker-compose`` in the way described above will build _all_ projects, not just the default projects. .. seealso:: `The Docker Compose command line reference `_ -Building Individual Components -============================== -Each Traffic Control component can be individually built, and the instructions for doing so may be found in their respective component's development documentation. +.. _dev-building-natively: + +Build the RPMs Natively +======================= +A developer may end up building the RPMs several times to test or :ref:`debug ` code changes, so it can be desirable to build the RPMs quickly for this purpose. Natively building the RPMs has the lowest build time of any building method. + +Install the Dependencies +------------------------ + +.. table:: Build dependencies for Traffic Control + + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | | Common dependencies | :ref:`dev-traffic-monitor` | :ref:`dev-traffic-ops` | :ref:`dev-traffic-portal` | :ref:`dev-traffic-router` | :ref:`dev-traffic-stats` | Grove | Grove TC Config (grovetccfg) | :ref:`Docs ` | + +====================================+=====================+============================+========================+===========================+===========================+==========================+==========+==============================+==========================+ + | macOS (homebrew_)\ [3]_ | - rpm | - go | - go | - npm | - maven | - go | - go | - go | - python3 | + | | | | | - bower | | | | | | + | | | | | - grunt-cli | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | CentOS/Red Hat/Fedora (yum_)\ [4]_ | - git | | | - epel-release | - java-1.8.0-openjdk | | | | - python3-devel | + | | - rpm-build | | | - npm | - maven | | | | - gcc | + | | - rsync | | | - nodejs-grunt-cli | | | | | - make | + | | | | | - ruby-devel | | | | | | + | | | | | - gcc | | | | | | + | | | | | - make | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Arch Linux (pacman_) | - git | - go | - go | - npm | - jdk8-openjdk | - go | - go | - go | - python-pip | + | | - rpm-tools | | | - bower | - maven | | | | - python-sphinx | + | | - diff | | | - grunt-cli | | | | | - make | + | | - rsync | | | - ruby | | | | | | + | | | | | - gcc | | | | | | + | | | | | - make | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Windows (cygwin_)\ [5]_ | - git | | | - ruby-devel | - curl | | | | | + | | - rpm-build | | | - make | | | | | | + | | - rsync | | | - gcc-g++ | | | | | | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + | Windows (chocolatey_)\ [5]_ | | - golang | - golang | - nodejs | - openjdk8 | - golang | - golang | - golang | - python | + | | | | | | - maven | | | | - pip | + | | | | | | | | | | - make | + +------------------------------------+---------------------+----------------------------+------------------------+---------------------------+---------------------------+--------------------------+----------+------------------------------+--------------------------+ + +.. _homebrew: https://brew.sh/ +.. _yum: https://wiki.centos.org/PackageManagement/Yum +.. _pacman: https://www.archlinux.org/pacman/ +.. _cygwin: https://cygwin.com/ +.. _chocolatey: https://chocolatey.org/ + +.. [3] If you are on macOS, you additionally need to :ref:`dev-tr-mac-jdk`. + +.. [4] If you are on CentOS, you need to `download Go directly `_ instead of using a package manager in order to get the latest Go version. For most users, the desired architecture is AMD64/x86_64. + +.. [5] If you are on Windows, you need to install both the Cygwin packages and the Chocolatey packages in order to build the Apache Traffic Control RPMs natively. + +.. |AdoptOpenJDK instructions| replace:: add the AdoptOpenJDK tap and install the ``adoptopenjdk8`` cask +.. _AdoptOpenJDK instructions: https://github.com/AdoptOpenJDK/homebrew-openjdk#other-versions + +After installing the packages using your platform's package manager, + + - Install the :ref:`global NPM packages ` and :ref:`Compass ` to build Traffic Portal. + + - Install the Python 3 modules used to :ref:`build the documentation `. + +Run ``build/clean_build.sh`` directly +------------------------------------- + +In a terminal, navigate to the root directory of the repository. You can run ``build/clean_build.sh`` with no arguments to build all components. -.. _docs-build: +.. code-block:: shell + :caption: ``build/clean_build.sh`` with no arguments -Building This Documentation ---------------------------- -This documentation uses the `Sphinx documentation build system `_, and as such requires a Python3 version that is at least 3.4.1. It also has dependency on Sphinx, and Sphinx extensions and themes. All of these can be easily installed using `pip` by referencing the requirements file like so: + build/clean_build.sh + +This is the equivalent of running .. code-block:: shell - :caption: Run from the Repository's Root Directory + :caption: ``build/clean_build.sh`` with all components - python3 -m pip install --user -r docs/source/requirements.txt + build/clean_build.sh tarball traffic_monitor traffic_ops traffic_portal traffic_router traffic_stats grove grove/grovetccfg docs -Once all dependencies have been satisfied, build using the Makefile at ``docs/Makefile``. +If any component fails to build, no further component builds will be attempted. -Alternatively, it is also possible to :ref:`pkg` or to :ref:`build-with-dc`, both of which will output a documentation "tarball" to ``dist/``. +Regardless of which OS the RPMs were built on, they are meant to be installed on CentOS 7. The exception is if the RPMs were built on CentOS 8, in which case the target OS for the RPMs is CentOS 8. + +.. warning:: Although there are no known issues with natively-built RPMs, the official, supported method of building the RPMs is by using :ref:`pkg ` or :ref:`docker-compose `. Use natively-built RPMs at your own risk. + +Building Individual Components +============================== +Each Traffic Control component can be individually built, and the instructions for doing so may be found in their respective component's development documentation. + +Building This Documentation +--------------------------- +See instructions for :ref:`building the documentation `. diff --git a/docs/source/development/debugging.rst b/docs/source/development/debugging.rst index 953e1ea9b6..23c8628b71 100644 --- a/docs/source/development/debugging.rst +++ b/docs/source/development/debugging.rst @@ -40,6 +40,8 @@ Traffic Monitor make very-clean make debug +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make debug native`` instead of ``make debug`` to build the RPMs quickly. + * Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TM_DEBUG_ENABLE`` to ``true``. * Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficmonitor`` image without reusing any cached layers to make sure it uses our fresh ``traffic_monitor.rpm``. Then, start CDN-in-a-Box. @@ -87,6 +89,8 @@ Traffic Ops (Go) make very-clean make debug +.. tip:: If you have gone through the steps to :ref:`dev-building-natively`, you can run ``make debug native`` instead of ``make debug`` to build the RPMs quickly. + * Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TO_DEBUG_ENABLE`` to ``true``. * Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficops-go`` image without reusing any cached layers to make sure it uses our fresh ``traffic_ops.rpm``. Then, start CDN-in-a-Box. diff --git a/docs/source/development/documentation_guidelines.rst b/docs/source/development/documentation_guidelines.rst index e7319acc13..06c4558d4d 100644 --- a/docs/source/development/documentation_guidelines.rst +++ b/docs/source/development/documentation_guidelines.rst @@ -22,9 +22,20 @@ The Apache Traffic Control documentation is written in :abbr:`RST (reStructuredT .. seealso:: `The docutils RST reference `_. +.. _docs-build: + Building ======== -To build the documentation, see :ref:`docs-build`. +This documentation uses the `Sphinx documentation build system `_, and as such requires a Python3 version that is at least 3.4.1. It also has dependency on Sphinx, and Sphinx extensions and themes. All of these can be easily installed using `pip` by referencing the requirements file like so: + +.. code-block:: shell + :caption: Run from the Repository's Root Directory + + python3 -m pip install --user -r docs/source/requirements.txt + +Once all dependencies have been satisfied, build using the Makefile at ``docs/Makefile``. + +Alternatively, it is also possible to :ref:`pkg` or to :ref:`build-with-dc`, both of which will output a documentation "tarball" to ``dist/``. Writing ======= diff --git a/docs/source/development/traffic_monitor.rst b/docs/source/development/traffic_monitor.rst index f917c02bec..e4be3f9120 100644 --- a/docs/source/development/traffic_monitor.rst +++ b/docs/source/development/traffic_monitor.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-monitor: + *************** Traffic Monitor *************** diff --git a/docs/source/development/traffic_ops.rst b/docs/source/development/traffic_ops.rst index 95d6744511..9f6f97b1e4 100644 --- a/docs/source/development/traffic_ops.rst +++ b/docs/source/development/traffic_ops.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-ops: + *********** Traffic Ops *********** diff --git a/docs/source/development/traffic_portal.rst b/docs/source/development/traffic_portal.rst index 5f3e5a3d09..c6f8191d34 100644 --- a/docs/source/development/traffic_portal.rst +++ b/docs/source/development/traffic_portal.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-portal: + ************** Traffic Portal ************** @@ -21,6 +23,8 @@ Introduction ============ Traffic Portal is an `AngularJS `_ client served from a `Node.js `_ web server designed to consume the :ref:`to-api`. Traffic Portal is the official replacement for the legacy Traffic Ops UI. +.. _dev-tp-software-requirements: + Software Requirements ===================== To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly used) environment that has the following installed: @@ -32,54 +36,72 @@ To work on Traffic Portal you need a \*nix (MacOS and Linux are most commonly us * `Grunt CLI 1.2.0 or above `_ * Access to a working instance of Traffic Ops -.. note:: The Traffic Portal consumes the Traffic Ops API. Modify traffic_portal/conf/config.js to specify the location of Traffic Ops. +.. _dev-tp-global-npm: -Traffic Portal Project Tree Overview -===================================== -* **traffic_control/traffic_portal/app/src** - contains HTML, JavaScript and :abbr:`SCSS (Sassy CSS)` source files. +Install Global NPM Packages +--------------------------- -Installing The Traffic Portal Developer Environment -=================================================== -#. Clone the `Traffic Control Repository `_ -#. Navigate to the ``traffic_portal`` subdirectory of your cloned repository. -#. Run ``npm install`` to install application dependencies into ``traffic_portal/node_modules``. Only needs to be done the first time unless ``traffic_portal/package.json`` changes. -#. Run ``bower install`` to install client-side dependencies into ``traffic_portal/app/bower_components``. Only needs to be done the first time unless ``traffic_portal/bower.json`` changes. -#. Make sure that compass is installed and functioning correctly by running ``compass version``. If compass is not available, then it can be installed using ``gem``'s manually, or using ``bundle`` +Bower and the Grunt CLI can be installed using NPM. + +.. code-block:: shell + :caption: Install Bower and Grunt CLI + + npm -g install bower grunt-cli + +.. _dev-tp-compass: + +Install Compass +--------------- - .. tip:: Bundle will automitcally install the correct version of the gems +Compass can be installed using ``gem`` manually, or by using ``bundle`` - #. ``brew install ruby``/``apt-get install ruby``/``yum install ruby`` +.. tip:: Bundle will automatically install the correct version of the gems. - #. ``gem update --system`` +#. ``brew install ruby``/``apt-get install ruby``/``yum install ruby`` - #. At this point, you can either manually install the gems or use bundler +#. ``gem update --system`` - #. For manually: ``gem install sass compass`` +#. At this point, you can either manually install the gems or use bundler - #. For automatically: ``gem install bundle && bundle install`` + #. For manually: ``gem install sass compass`` - .. note:: Bundle requires ruby versions > 2.3.0, so if you're using a version of ruby < 2.3.0 then this will not work. + #. For automatically: ``gem install bundle && bundle install`` - #. Make sure that ``compass`` and ``sass`` are part of your ``PATH`` environment variable. + .. note:: Bundle requires ruby versions > 2.3.0, so if you're using a version of ruby < 2.3.0 then this will not work. - #. If not, you can see where gem installs ``compass`` and ``sass`` by running: - ``gem environment`` +#. Make sure that ``compass`` and ``sass`` are part of your ``PATH`` environment variable. - #. In there, you can see where ruby is installing all the gems. Add that path to your ``PATH`` environment variable. - For example, it is ``/usr/local/lib/ruby/gems/2.7.0/gems/compass-1.0.3/bin/`` for this test setup. +#. If not, you can see where gem installs ``compass`` and ``sass`` by running: + ``gem environment`` - #. Once you have installed ``compass`` successfully, make sure you can reach it by typing: - ``compass version`` - This should give a valid output. For example, for the test setup, the output is: +#. In there, you can see where ruby is installing all the gems. Add that path to your ``PATH`` environment variable. + For example, it is ``/usr/local/lib/ruby/gems/2.7.0/gems/compass-1.0.3/bin/`` for this test setup. - .. code-block:: text - :caption: Compass version output +#. Once you have installed ``compass`` successfully, make sure you can reach it by typing: + ``compass version`` + This should give a valid output. For example, for the test setup, the output is: - Compass 1.0.3 (Polaris) - Copyright (c) 2008-2020 Chris Eppstein - Released under the MIT License. - Compass is charityware. - Please make a tax deductable donation for a worthy cause: http://umdf.org/compass +.. code-block:: text + :caption: Compass version output + + Compass 1.0.3 (Polaris) + Copyright (c) 2008-2020 Chris Eppstein + Released under the MIT License. + Compass is charityware. + Please make a tax deductable donation for a worthy cause: http://umdf.org/compass + + +Traffic Portal Project Tree Overview +===================================== +* **traffic_control/traffic_portal/app/src** - contains HTML, JavaScript and :abbr:`SCSS (Sassy CSS)` source files. + +Installing The Traffic Portal Developer Environment +=================================================== +#. Clone the `Traffic Control Repository `_ +#. Navigate to the ``traffic_portal`` subdirectory of your cloned repository. +#. Run ``npm install`` to install application dependencies into ``traffic_portal/node_modules``. Only needs to be done the first time unless ``traffic_portal/package.json`` changes. +#. Run ``bower install`` to install client-side dependencies into ``traffic_portal/app/bower_components``. Only needs to be done the first time unless ``traffic_portal/bower.json`` changes. +#. Make sure that compass is installed and functioning correctly by running ``compass version``. If compass is not available, then it can be installed following the instructions under :ref:`dev-tp-compass`. #. Run ``grunt`` to package the application into ``traffic_portal/app/dist``, start a local HTTPS server (Express), and start a file watcher. #. Modify ``traffic_portal/conf/config.js``: @@ -90,3 +112,5 @@ Installing The Traffic Portal Developer Environment #. Modify ``log.stream`` to be ``./server/log/access.log``. #. Navigate to http(s)://localhost:[port|sslPort defined in ``traffic_portal/conf/config.js``] + +.. note:: The Traffic Portal consumes the Traffic Ops API. Modify traffic_portal/conf/config.js to specify the location of Traffic Ops. diff --git a/docs/source/development/traffic_router.rst b/docs/source/development/traffic_router.rst index 2a7970cc3b..321d874d97 100644 --- a/docs/source/development/traffic_router.rst +++ b/docs/source/development/traffic_router.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-router: + ************** Traffic Router ************** @@ -40,6 +42,30 @@ To work on Traffic Router you need a \*nix (MacOS and Linux are most commonly us * Tomcat Native >= 1.2.16 * Not Tomcat - You do not need a Tomcat installation for development. An embedded version is launched for development testing instead. +.. _dev-tr-mac-jdk: + +Get OpenJDK 8 on macOS +-------------------------- +If you are on macOS, OpenJDK 8 is not available from Homebrew by default, but it can still be installed using Homebrew with little effort. + +Using Homebrew, |AdoptOpenJDK instructions|_ + +.. code-block:: shell + :caption: Install OpenJDK 8 on macOS + + brew tap AdoptOpenJDK/openjdk + brew cask install adoptopenjdk8 + +Next, set the JAVA_HOME environment variable. Add this line to your ``~/.bash_profile``: + +.. code-block:: shell + :caption: Set JAVA_HOME environment variable + + export JAVA_HOME=$(/usr/libexec/java_home -v1.8) + +.. |AdoptOpenJDK instructions| replace:: add the AdoptOpenJDK tap and install the ``adoptopenjdk8`` cask +.. _AdoptOpenJDK instructions: https://github.com/AdoptOpenJDK/homebrew-openjdk#other-versions + Traffic Router Project Tree Overview ==================================== * ``traffic_control/traffic_traffic_router/`` - base directory for Traffic Router diff --git a/docs/source/development/traffic_stats.rst b/docs/source/development/traffic_stats.rst index d2663e1a1c..ea83360919 100644 --- a/docs/source/development/traffic_stats.rst +++ b/docs/source/development/traffic_stats.rst @@ -13,6 +13,8 @@ .. limitations under the License. .. +.. _dev-traffic-stats: + ************* Traffic Stats ************* diff --git a/grove/build/build_rpm.sh b/grove/build/build_rpm.sh index e02a533a90..7e49e0e915 100755 --- a/grove/build/build_rpm.sh +++ b/grove/build/build_rpm.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,39 +10,47 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } +importFunctions() { + [ -n "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; return 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } #---------------------------------------- -function checkGroveEnvironment() { +checkGroveEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - - export GROVE_DIR=$(dirname "$scriptdir") - export GROVE_VERSION=`cat ${GROVE_DIR}/VERSION` - export PACKAGE="grove" - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export RPMBUILD="${GROVE_DIR}/rpmbuild" - export DIST="${TC_DIR}/dist" - export RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + local script scriptdir + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + + GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' + GROVE_DIR=$(dirname "$scriptdir") + GROVE_VERSION="$(cat "${GROVE_DIR}/VERSION")" + PACKAGE="grove" + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + RPMBUILD="${GROVE_DIR}/rpmbuild" + DIST="${TC_DIR}/dist" + RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" + export GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM GOOS RPM_TARGET_OS # grove needs to be built with go 1.14 or greater - verify_and_set_go_version - if [[ $? -ne 1 ]]; then - exit 0 - fi + if ! verify_and_set_go_version; then + return $?; + fi; echo "==================================================" echo "GO_VERSION: $GO_VERSION" @@ -59,23 +66,27 @@ function checkGroveEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { cd "$GROVE_DIR" # prep build environment - [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; exit 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; exit 1; } + [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" + [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p BUILD RPMS SOURCES) || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- -function buildRpmGrove() { +buildRpmGrove() { # build - $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; exit 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build grove: $?" >&2; exit 1; } + ldflags='-s -w' + tags='osusergo netgo' + $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } + $GO build -v -ldflags "${ldflags} -X main.Version=$GROVE_VERSION" -tags "$tags" || { echo "Failed to build grove: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/grove-${GROVE_VERSION}.tgz grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; exit 1; } + tar -cvzf "${RPMBUILD}/SOURCES/grove-${GROVE_VERSION}.tgz" grove conf/grove.cfg build/grove.init build/grove.logrotate || { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -83,25 +94,33 @@ function buildRpmGrove() { spec=build/grove.spec spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) - if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/grove.spec + if ! id "$spec_owner" >/dev/null 2>&1; then + chown "$(id -u):$(id -g)" build/grove.spec - function give_spec_back { - chown ${spec_owner}:${spec_group} build/grove.spec - } - trap give_spec_back EXIT + give_spec_back() { + chown "${spec_owner}:${spec_group}" build/grove.spec + } + trap give_spec_back EXIT fi - # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/grove.spec || { echo "rpmbuild failed: $?" >&2; exit 1; } + # build RPM with xz level 2 compression + rpmbuild \ + --define "_topdir $RPMBUILD" \ + --define "version ${GROVE_VERSION}" \ + --define "build_number ${BUILD_NUMBER}" \ + --define "_target_os ${RPM_TARGET_OS}" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ + -ba build/grove.spec || + { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . - [ -e $DIST ] || mkdir -p $DIST - cp $RPMBUILD/RPMS/x86_64/${RPM} $DIST + [ -e "$DIST" ] || mkdir -p "$DIST" + cp "${RPMBUILD}/RPMS/x86_64/${RPM}" "$DIST" } importFunctions +checkEnvironment -i go checkGroveEnvironment initBuildArea buildRpmGrove - diff --git a/grove/build/grove.spec b/grove/build/grove.spec index 2f0f345aa4..31f24a46c3 100644 --- a/grove/build/grove.spec +++ b/grove/build/grove.spec @@ -10,18 +10,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -Summary: Grove HTTP Caching Proxy -Name: grove -Version: %{version} -Release: %{build_number} -License: Apache License, Version 2.0 -Group: Base System/System Tools -Prefix: /usr/sbin/%{name} -Source: %{_sourcedir}/%{name}-%{version}.tgz -URL: https://github.com/apache/trafficcontrol/%{name} +Summary: Grove HTTP Caching Proxy +Name: grove +Version: %{version} +Release: %{build_number} +License: Apache License, Version 2.0 +Group: Base System/System Tools +Prefix: /usr/sbin/%{name} +Source: %{_sourcedir}/%{name}-%{version}.tgz +URL: https://github.com/apache/trafficcontrol/%{name} Distribution: CentOS Linux -Vendor: Apache Software Foundation -BuildRoot: %{buildroot} +Vendor: Apache Software Foundation +BuildRoot: %{buildroot} # %define PACKAGEDIR %{prefix} diff --git a/grove/grovetccfg/build/build_rpm.sh b/grove/grovetccfg/build/build_rpm.sh index 16d0b6a540..4eb57b552b 100755 --- a/grove/grovetccfg/build/build_rpm.sh +++ b/grove/grovetccfg/build/build_rpm.sh @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,39 +10,47 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { - [ ! -z "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } +importFunctions() { + [ -n "$TC_DIR" ] || { echo "Cannot find repository root." >&2 ; exit 1; } export TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } #---------------------------------------- -function checkGroveEnvironment() { +checkGroveEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - - export GROVETC_DIR=$(dirname "$scriptdir") - export GROVE_DIR=$(dirname "$GROVETC_DIR") - export GROVE_VERSION=`cat ${GROVE_DIR}/VERSION` - export PACKAGE="grovetccfg" - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export RPMBUILD="${GROVE_DIR}/rpmbuild" - export DIST="${TC_DIR}/dist" - export RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + local script scriptdir + script=$(realpath "$0") + scriptdir=$(dirname "$script") + + GROVETC_DIR='' GROVE_DIR='' GROVE_VERSION='' PACKAGE='' RPMBUILD='' DIST='' RPM='' + GROVETC_DIR=$(dirname "$scriptdir") + GROVE_DIR=$(dirname "$GROVETC_DIR") + GROVE_VERSION="$(cat "${GROVE_DIR}/VERSION")" + PACKAGE="grovetccfg" + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + RPMBUILD="${GROVE_DIR}/rpmbuild" + DIST="${TC_DIR}/dist" + RPM="${PACKAGE}-${GROVE_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + GOOS="${GOOS:-linux}" + RPM_TARGET_OS="${RPM_TARGET_OS:-$GOOS}" + export GROVETC_DIR GROVE_DIR GROVE_VERSION PACKAGE BUILD_NUMBER RPMBUILD DIST RPM GOOS RPM_TARGET_OS # grovetccfg needs to be built with go 1.14 or greater - verify_and_set_go_version - if [[ $? -ne 1 ]]; then - exit 0 - fi + if ! verify_and_set_go_version; then + return $?; + fi; echo "==================================================" echo "GO_VERSION: $GO_VERSION" @@ -60,23 +67,27 @@ function checkGroveEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { cd "$GROVETC_DIR" # prep build environment - [ -e $RPMBUILD ] && rm -rf $RPMBUILD - [ ! -e $RPMBUILD ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; exit 1; } - mkdir -p $RPMBUILD/{BUILD,RPMS,SOURCES} || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; exit 1; } + [ -e "$RPMBUILD" ] && rm -rf "$RPMBUILD" + [ ! -e "$RPMBUILD" ] || { echo "Failed to clean up rpm build directory '$RPMBUILD': $?" >&2; return 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p BUILD RPMS SOURCES) || { echo "Failed to create build directory '$RPMBUILD': $?" >&2; return 1; } } # --------------------------------------- -function buildRpmGrove() { +buildRpmGrove() { # build - $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; exit 1; } - $GO build -v -ldflags "-X main.Version=$GROVE_VERSION" || { echo "Failed to build $PACKAGE: $?" >&2; exit 1; } + ldflags='-s -w' + tags='osusergo netgo' + $GO get -v -d . || { echo "Failed to go get dependencies: $?" >&2; return 1; } + $GO build -v -ldflags "${ldflags} -X main.Version=$GROVE_VERSION" -tags "$tags" || { echo "Failed to build $PACKAGE: $?" >&2; return 1; } # tar - tar -cvzf $RPMBUILD/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; exit 1; } + tar -cvzf "${RPMBUILD}/SOURCES/${PACKAGE}-${GROVE_VERSION}.tgz" ${PACKAGE}|| { echo "Failed to create archive for rpmbuild: $?" >&2; return 1; } # Work around bug in rpmbuild. Fixed in rpmbuild 4.13. # See: https://github.com/rpm-software-management/rpm/commit/916d528b0bfcb33747e81a57021e01586aa82139 @@ -84,24 +95,33 @@ function buildRpmGrove() { spec=build/${PACKAGE}.spec spec_owner=$(stat -c%u $spec) spec_group=$(stat -c%g $spec) - if ! id $spec_owner >/dev/null 2>&1; then - chown $(id -u):$(id -g) build/${PACKAGE}.spec + if ! id "$spec_owner" >/dev/null 2>&1; then + chown "$(id -u):$(id -g)" build/${PACKAGE}.spec - function give_spec_back { - chown ${spec_owner}:${spec_group} build/${PACKAGE}.spec - } - trap give_spec_back EXIT + give_spec_back() { + chown "${spec_owner}:${spec_group}" build/${PACKAGE}.spec + } + trap give_spec_back EXIT fi - # build RPM - rpmbuild --define "_topdir $RPMBUILD" --define "version ${GROVE_VERSION}" --define "build_number ${BUILD_NUMBER}" -ba build/${PACKAGE}.spec || { echo "rpmbuild failed: $?" >&2; exit 1; } + # build RPM with xz level 2 compression + rpmbuild \ + --define "_topdir $RPMBUILD" \ + --define "version ${GROVE_VERSION}" \ + --define "build_number ${BUILD_NUMBER}" \ + --define "_target_os ${RPM_TARGET_OS}" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ + -ba build/${PACKAGE}.spec || + { echo "rpmbuild failed: $?" >&2; return 1; } # copy build RPM to . - [ -e $DIST ] || mkdir -p $DIST - cp $RPMBUILD/RPMS/x86_64/${RPM} $DIST + [ -e "$DIST" ] || mkdir -p "$DIST" + cp "${RPMBUILD}/RPMS/x86_64/${RPM}" "$DIST" } importFunctions +checkEnvironment -i go checkGroveEnvironment initBuildArea buildRpmGrove diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index c3e2acc28a..176aa87e41 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -29,11 +29,13 @@ ifneq ($(PWD),$(makefile_dir)) $(error This makefile MUST be run from within its directory) endif +PKG_COMMAND := ../../pkg PKG_FLAGS := -v +BUILD_SUFFIX := _build BUILD_NUMBER := $(shell git rev-list HEAD 2>/dev/null | wc -l | tr -d '[[:space:]]').$(shell git rev-parse --short=8 HEAD) TC_VERSION := $(shell cat "../../VERSION") -TOMCAT_VERSION := $(shell grep 'export TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d '=' -f 2) -TOMCAT_RELEASE := $(shell grep 'export TOMCAT_RELEASE=' ../../traffic_router/build/build_rpm.sh | cut -d '=' -f 2) +TOMCAT_VERSION := $(shell grep '^\s*TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d= -f2) +TOMCAT_RELEASE := $(shell grep '^\s*TOMCAT_RELEASE=' ../../traffic_router/build/build_rpm.sh | cut -d= -f2) RHEL_VERSION := el7 @@ -49,17 +51,35 @@ TP_SOURCE := $(wildcard ../../traffic_portal/**/*) TR_SOURCE := $(wildcard ../../traffic_router/**/*) TS_SOURCE := $(wildcard ../../traffic_stats/**/*) -.PHONY: clean very-clean all nearly-all debug +.PHONY: clean very-clean all nearly-all debug native # Default target; builds all pre-requisite rpms from source trees all: cache/traffic_ops_ort.rpm traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm -debug: PKG_FLAGS += -d -ifneq ($(MAKECMDGOALS), debug) +ifneq ($(filter debug,$(MAKECMDGOALS)),) +PKG_FLAGS += -d +export DEBUG_BUILD = true +ifneq ($(MAKECMDGOALS),debug) +MAKECMDGOALS := $(filter-out debug,$(MAKECMDGOALS)) debug: $(MAKECMDGOALS) else debug: all endif +endif + +ifneq ($(filter native,$(MAKECMDGOALS)),) +PKG_COMMAND := ../../build/clean_build.sh +PKG_FLAGS := +BUILD_SUFFIX := +ifneq ($(MAKECMDGOALS),native) +MAKECMDGOALS := $(filter-out native,$(MAKECMDGOALS)) +native: $(MAKECMDGOALS) +else +native: all +endif +endif + +#.PHONY: native # Actual output rpm recipies traffic_monitor/traffic_monitor.rpm: ../../dist/traffic_monitor-$(SPECIAL_SAUCE) @@ -79,22 +99,22 @@ cache/traffic_ops_ort.rpm: ../../dist/traffic_ops_ort-$(SPECIAL_SAUCE) # Dist rpms ../../dist/traffic_monitor-$(SPECIAL_SAUCE): $(TM_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_monitor_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_monitor$(BUILD_SUFFIX) ../../dist/traffic_ops-$(SPECIAL_SAUCE): $(TO_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_ops_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_ops$(BUILD_SUFFIX) ../../dist/traffic_portal-$(SPECIAL_SAUCE): $(TP_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_portal_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_portal$(BUILD_SUFFIX) ../../dist/traffic_rou%er-$(SPECIAL_SAUCE) ../../dist/tomca%-$(SPECIAL_SEASONING): $(TR_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_router_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_router$(BUILD_SUFFIX) ../../dist/traffic_stats-$(SPECIAL_SAUCE): $(TS_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_stats_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_stats$(BUILD_SUFFIX) ../../dist/traffic_ops_ort-$(SPECIAL_SAUCE): $(ORT_SOURCE) - ../../pkg $(PKG_FLAGS) traffic_ops_ort_build + $(PKG_COMMAND) $(PKG_FLAGS) traffic_ops_ort$(BUILD_SUFFIX) clean: $(RM) traffic_monitor/traffic_monitor.rpm traffic_ops/traffic_ops.rpm traffic_portal/traffic_portal.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm cache/traffic_ops_ort.rpm traffic_stats/traffic_stats.rpm diff --git a/infrastructure/docker/build/Dockerfile-docs b/infrastructure/docker/build/Dockerfile-docs index 325e0bef06..9b9dae6bad 100644 --- a/infrastructure/docker/build/Dockerfile-docs +++ b/infrastructure/docker/build/Dockerfile-docs @@ -44,7 +44,7 @@ RUN python3 -m pip install --upgrade setuptools && \ python3 -m pip install -r /docs.requirements.txt ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh docs # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-grove b/infrastructure/docker/build/Dockerfile-grove index 878fc0a73c..4473934aae 100644 --- a/infrastructure/docker/build/Dockerfile-grove +++ b/infrastructure/docker/build/Dockerfile-grove @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / ENV GOPATH=/go RUN mkdir -p /go/src/github.com/apache && ln -s /tmp/trafficcontrol /go/src/github.com/apache/trafficcontrol diff --git a/infrastructure/docker/build/Dockerfile-grovetccfg b/infrastructure/docker/build/Dockerfile-grovetccfg index 6a3ffbc459..3e06710927 100644 --- a/infrastructure/docker/build/Dockerfile-grovetccfg +++ b/infrastructure/docker/build/Dockerfile-grovetccfg @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / ENV GOPATH=/go RUN mkdir -p /go/src/github.com/apache && ln -s /tmp/trafficcontrol /go/src/github.com/apache/trafficcontrol diff --git a/infrastructure/docker/build/Dockerfile-source b/infrastructure/docker/build/Dockerfile-source index 8d8e756d4f..e4e9d17f74 100644 --- a/infrastructure/docker/build/Dockerfile-source +++ b/infrastructure/docker/build/Dockerfile-source @@ -37,6 +37,7 @@ RUN yum -y install \ yum -y clean all WORKDIR /trafficcontrol -CMD ./build/build.sh tarball +ADD build/clean_build.sh / +CMD /clean_build.sh tarball # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_monitor b/infrastructure/docker/build/Dockerfile-traffic_monitor index b8e854fcf8..f2a67bc3cf 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_monitor +++ b/infrastructure/docker/build/Dockerfile-traffic_monitor @@ -45,7 +45,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_monitor # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_ops b/infrastructure/docker/build/Dockerfile-traffic_ops index f4acf8398e..99632f2407 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_ops +++ b/infrastructure/docker/build/Dockerfile-traffic_ops @@ -50,7 +50,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_ops # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_ops_ort b/infrastructure/docker/build/Dockerfile-traffic_ops_ort index 5909d6f775..77ebdf207f 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_ops_ort +++ b/infrastructure/docker/build/Dockerfile-traffic_ops_ort @@ -31,7 +31,7 @@ RUN curl -Lo go.tgz https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ln -s /usr/local/go/bin/go /usr/bin/go && \ rm go.tgz -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_ops_ort #vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_portal b/infrastructure/docker/build/Dockerfile-traffic_portal index 4049b4c9fa..86043e485d 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_portal +++ b/infrastructure/docker/build/Dockerfile-traffic_portal @@ -54,7 +54,7 @@ RUN echo '{ "allow_root": true }' > /root/.bowerrc ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_portal # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_router b/infrastructure/docker/build/Dockerfile-traffic_router index af7c7b83ec..b167371ff5 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_router +++ b/infrastructure/docker/build/Dockerfile-traffic_router @@ -47,7 +47,7 @@ RUN curl -L https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3 ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_router # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/Dockerfile-traffic_stats b/infrastructure/docker/build/Dockerfile-traffic_stats index 631960dc3e..aa1ddab67b 100644 --- a/infrastructure/docker/build/Dockerfile-traffic_stats +++ b/infrastructure/docker/build/Dockerfile-traffic_stats @@ -43,7 +43,7 @@ RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz && \ ### -ADD infrastructure/docker/build/clean_build.sh / +ADD build/clean_build.sh / CMD /clean_build.sh traffic_stats # vi:syntax=Dockerfile diff --git a/infrastructure/docker/build/clean_build.sh b/infrastructure/docker/build/clean_build.sh deleted file mode 100755 index 65729d17f5..0000000000 --- a/infrastructure/docker/build/clean_build.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Fix ownership of output files -# $1 is file or dir with correct ownership -# remaining args are files/dirs to be fixed, recursively -setowner() { - own=$(stat -c '%u:%g' "$1") - shift - [[ -n $* ]] && chown -R "${own}" "$@" -} - -cleanup() { - setowner /trafficcontrol /trafficcontrol/dist -} - -trap cleanup EXIT - -set -o xtrace; - -# set owner of dist dir -- cleans up existing dist permissions... -export GOPATH=/tmp/go; -tc_dir=${GOPATH}/src/github.com/apache/trafficcontrol; -mkdir -p ${GOPATH}/{src,pkg,bin} $tc_dir; -( set -o errexit; - rsync -a /trafficcontrol/ $tc_dir; - if ! [[ -d ${tc_dir}/.git ]]; then - rsync -a /trafficcontrol/.git $tc_dir; # Docker for Windows compatibility - fi; - rm -rf ${tc_dir}/dist; - mkdir -p /trafficcontrol/dist; - ln -s /trafficcontrol/dist ${tc_dir}/dist; ) && \ - cd $tc_dir && - ( ( ( (./build/build.sh "$1" 2>&1; echo $? >&3) | tee ./dist/build-"$1".log >&4) 3>&1) | (read -r x; exit "$x"); ) 4>&1 diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh index 35b438dd6e..fede55b6f9 100755 --- a/traffic_monitor/build/build_rpm.sh +++ b/traffic_monitor/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,29 +11,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; -function importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TM_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TM_DIR") +#---------------------------------------- +importFunctions() { + local script scriptdir + TM_DIR='' TC_DIR='' + script=$(realpath "$0") + scriptdir=$(dirname "$script") + TM_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TM_DIR")" + export TM_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local tm_dest=$(createSourceDir traffic_monitor) + local tm_dest + tm_dest="$(createSourceDir traffic_monitor)" cd "$TM_DIR" || \ - { echo "Could not cd to $TM_DIR: $?"; exit 1; } + { echo "Could not cd to $TM_DIR: $?"; return 1; } echo "PATH: $PATH" echo "GOPATH: $GOPATH" @@ -45,32 +52,38 @@ function initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/net/publicsuffix golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; exit 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_monitor - go_build=(go build -v) - if [[ "$DEBUG_BUILD" == true ]]; then + gcflags='' + ldflags="-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" + tags='osusergo netgo' + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; - go_build+=(-gcflags 'all=-N -l'); + gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; #strip binary fi; - "${go_build[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" || \ - { echo "Could not build traffic_monitor binary"; exit 1; } + set -o nounset; } + go build -v -gcflags "$gcflags" -ldflags "$ldflags" -tags "$tags" || \ + { echo "Could not build traffic_monitor binary"; return 1; } - rsync -av ./ "$tm_dest"/ || \ - { echo "Could not copy to $tm_dest: $?"; exit 1; } - cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + cp -av ./ "$tm_dest"/ || \ + { echo "Could not copy to $tm_dest: $?"; return 1; } + cp -av "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ + { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $tm_dest) || { echo "Could not create tar archive $tm_dest.tgz: $?"; exit 1; } - cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$tm_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$tm_dest")" || { echo "Could not create tar archive $tm_dest.tgz: $?"; return 1; } + cp "$TM_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } -function preBuildChecks() { - if [[ -e "$TM_DIR"/traffic_monitor ]]; then +preBuildChecks() { + if [ -e "$TM_DIR"/traffic_monitor ]; then echo "Found $TM_DIR/traffic_monitor, please remove before retrying to build" - exit 1 + return 1 fi } @@ -78,6 +91,6 @@ function preBuildChecks() { importFunctions preBuildChecks -checkEnvironment go +checkEnvironment -i go,rsync initBuildArea buildRpm traffic_monitor diff --git a/traffic_monitor/build/traffic_monitor.spec b/traffic_monitor/build/traffic_monitor.spec index abf8cfe0bf..4d89e54bb5 100644 --- a/traffic_monitor/build/traffic_monitor.spec +++ b/traffic_monitor/build/traffic_monitor.spec @@ -15,17 +15,17 @@ # # RPM spec file for the Go version of Traffic Monitor (tm). # -%define debug_package %{nil} -Name: traffic_monitor -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Monitor the caches -Packager: david_neuman2 at Cable dot Comcast dot com -Vendor: Apache Software Foundation -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol -Source: %{_sourcedir}/traffic_monitor-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_monitor +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Monitor the caches +Packager: david_neuman2 at Cable dot Comcast dot com +Vendor: Apache Software Foundation +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol +Source: %{_sourcedir}/traffic_monitor-%{traffic_control_version}.tgz %description Installs traffic_monitor @@ -38,8 +38,8 @@ Installs traffic_monitor # copy traffic_monitor binary godir=src/github.com/apache/trafficcontrol/traffic_monitor ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -r "$TC_DIR"/traffic_monitor/* . + cd "$godir" && \ + cp -r "$TC_DIR"/traffic_monitor/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } %install @@ -67,16 +67,13 @@ cp "$src"/build/traffic_monitor.logrotate "${RPM_BUILD_ROOT}"/etc/logrotate.d/t /usr/bin/getent group traffic_monitor >/dev/null if [ $? -ne 0 ]; then - /usr/sbin/groupadd -g 423 traffic_monitor fi /usr/bin/getent passwd traffic_monitor >/dev/null if [ $? -ne 0 ]; then - /usr/sbin/useradd -g traffic_monitor -u 423 -d /opt/traffic_monitor -M traffic_monitor - fi /usr/bin/passwd -l traffic_monitor >/dev/null @@ -89,10 +86,10 @@ fi #don't install over the top of java TM. This is a workaround since yum doesn't respect the Conflicts tag. if [[ $(rpm -q traffic_monitor --qf "%{VERSION}-%{RELEASE}") < 1.9.0 ]] then - echo -e "\n****************\n" - echo "A java version of traffic_monitor is installed. Please backup/remove that version before installing the golang version of traffic_monitor." - echo -e "\n****************\n" - exit 1 + echo -e "\n****************\n" + echo "A java version of traffic_monitor is installed. Please backup/remove that version before installing the golang version of traffic_monitor." + echo -e "\n****************\n" + exit 1 fi %post @@ -102,8 +99,8 @@ fi %files %defattr(644, traffic_monitor, traffic_monitor, 755) -%config(noreplace) /opt/traffic_monitor/conf/traffic_monitor.cfg -%config(noreplace) /opt/traffic_monitor/conf/traffic_ops.cfg +%config(noreplace) %attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/traffic_monitor.cfg +%config(noreplace) %attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/traffic_ops.cfg %config(noreplace) /etc/logrotate.d/traffic_monitor %dir /opt/traffic_monitor @@ -115,9 +112,10 @@ fi %dir /opt/traffic_monitor/var/log %dir /opt/traffic_monitor/var/run -%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/conf/* -%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/* -%attr(755, traffic_monitor, traffic_monitor) /opt/traffic_monitor/bin/* +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/index.html +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/script.js +%attr(600, traffic_monitor, traffic_monitor) /opt/traffic_monitor/static/style.css +%attr(755, traffic_monitor, traffic_monitor) /opt/traffic_monitor/bin/traffic_monitor %attr(755, traffic_monitor, traffic_monitor) /etc/init.d/traffic_monitor %preun diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index a2e4cbcce9..3fa1e42e7b 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,29 +11,38 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; +set -o xtrace #---------------------------------------- -function importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TO_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TO_DIR") +importFunctions() { + local script scriptdir + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + TO_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TO_DIR")" + export TO_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } - local dest=$(createSourceDir traffic_ops) + local dest + dest="$(createSourceDir traffic_ops)" cd "$TO_DIR" || \ - { echo "Could not cd to $TO_DIR: $?"; exit 1; } + { echo "Could not cd to $TO_DIR: $?"; return 1; } echo "PATH: $PATH" echo "GOPATH: $GOPATH" @@ -45,47 +51,54 @@ function initBuildArea() { # get x/* packages (everything else should be properly vendored) go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix || \ - { echo "Could not get go package dependencies"; exit 1; } + { echo "Could not get go package dependencies"; return 1; } # compile traffic_ops_golang - pushd traffic_ops_golang - go_build=(go build -v); - if [[ "$DEBUG_BUILD" == true ]]; then + cd traffic_ops_golang + gcflags='' + ldflags='' + tags='osusergo netgo' + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; - go_build+=(-gcflags 'all=-N -l'); + gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; # strip binary fi; - "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \ - { echo "Could not build traffic_ops_golang binary"; exit 1; } - popd + set -o nounset; } + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" -tags "$tags" || \ + { echo "Could not build traffic_ops_golang binary"; return 1; } + cd - # compile db/admin - pushd app/db - "${go_build[@]}" -o admin || \ - { echo "Could not build db/admin binary"; exit 1; } - popd + (cd app/db + go build -v -o admin -gcflags "$gcflags" -ldflags "$ldflags" -tags "$tags" || \ + { echo "Could not build db/admin binary"; return 1;}) # compile TO profile converter - pushd install/bin/convert_profile - "${go_build[@]}" || \ - { echo "Could not build convert_profile binary"; exit 1; } - popd + (cd install/bin/convert_profile + go build -v -gcflags "$gcflags" -ldflags "$ldflags" -tags="$tags" || \ + { echo "Could not build convert_profile binary"; return 1; }) rsync -av etc install "$dest"/ || \ - { echo "Could not copy to $dest: $?"; exit 1; } - rsync -av app/{bin,conf,cpanfile,db,lib,public,script,templates} "$dest"/app/ || \ - { echo "Could not copy to $dest/app: $?"; exit 1; } - tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES $(basename "$dest") || \ - { echo "Could not create tar archive $dest.tgz: $?"; exit 1; } + { echo "Could not copy to $dest: $?"; return 1; } + if ! (cd app; rsync -av bin conf cpanfile db lib public script templates "${dest}/app"); then + echo "Could not copy to $dest/app" + return 1 + fi + tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$dest")" || \ + { echo "Could not create tar archive $dest.tgz: $?"; return 1; } cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + { echo "Could not copy spec files: $?"; return 1; } - export PLUGINS=$(grep -l -P '(?/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail -o xtrace; #---------------------------------------- -function importFunctions() { - local script=$(readlink -f "$0"); - local scriptdir=$(dirname "$script"); - export ORT_DIR=$(dirname "$scriptdir"); - export TC_DIR=$(dirname "$ORT_DIR"); +importFunctions() { + local script scriptdir; + script="$(realpath "$0")"; + scriptdir="$(dirname "$script")"; + ORT_DIR="$(dirname "$scriptdir")"; + TC_DIR="$(dirname "$ORT_DIR")"; + export ORT_DIR TC_DIR; functions_sh="$TC_DIR/build/functions.sh"; - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" >&2; - exit 1; + return 1; fi . "$functions_sh"; } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area for Traffic Ops ORT"; - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } - local dest=$(createSourceDir traffic_ops_ort); + local dest; + dest=$(createSourceDir traffic_ops_ort); cd "$ORT_DIR"; echo "PATH: $PATH"; @@ -46,22 +49,26 @@ function initBuildArea() { go get -v golang.org/x/crypto/ed25519 golang.org/x/crypto/scrypt golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/sys/unix; - GC=(go build) - GFLAGS=(-v) - if [[ "$DEBUG_BUILD" == true ]]; then - echo "DEBUG_BUILD is enabled, building without optimization or inlining..."; - GFLAGS+=(--gcflags 'all=-N -l'); + gcflags='' + ldflags='' + tags='osusergo netgo' + { set +o nounset; + if [ "$DEBUG_BUILD" = true ]; then + echo 'DEBUG_BUILD is enabled, building without optimization or inlining...'; + gcflags="${gcflags} all=-N -l"; + else + ldflags="${ldflags} -s -w"; # strip binary fi; + set -o nounset; } - pushd atstccfg; - "${GC[@]}" "${GCFLAGS[@]}" -ldflags "-X main.GitRevision=`git rev-parse HEAD` -X main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%s'` -X main.Version=${TC_VERSION}"; - popd; + (cd atstccfg; + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" -tags "$tags") cp -p traffic_ops_ort.pl "$dest"; cp -p supermicro_udev_mapper.pl "$dest"; mkdir -p "${dest}/atstccfg"; cp -a atstccfg/* "${dest}/atstccfg"; - tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES $(basename "$dest"); + tar -czvf "$dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$dest")"; cp build/traffic_ops_ort.spec "$RPMBUILD"/SPECS/.; echo "The build area has been initialized."; diff --git a/traffic_ops_ort/build/traffic_ops_ort.spec b/traffic_ops_ort/build/traffic_ops_ort.spec index 1e14f6b900..cbbd99febe 100644 --- a/traffic_ops_ort/build/traffic_ops_ort.spec +++ b/traffic_ops_ort/build/traffic_ops_ort.spec @@ -16,16 +16,16 @@ # RPM spec file for Traffic Stats (tm). # %define debug_package %{nil} -Name: traffic_ops_ort -Summary: Installs ORT script for Traffic Control caches -Version: %{traffic_control_version} -Release: %{build_number} -License: Apache License, Version 2.0 -Group: Applications/Communications -Source0: traffic_ops_ort-%{version}.tgz -URL: https://github.com/apache/trafficcontrol/ -Vendor: Apache Software Foundation -Packager: dev at trafficcontrol dot Apache dot org +Name: traffic_ops_ort +Summary: Installs ORT script for Traffic Control caches +Version: %{traffic_control_version} +Release: %{build_number} +License: Apache License, Version 2.0 +Group: Applications/Communications +Source0: traffic_ops_ort-%{version}.tgz +URL: https://github.com/apache/trafficcontrol/ +Vendor: Apache Software Foundation +Packager: dev at trafficcontrol dot Apache dot org %{?el6:Requires: perl-JSON, perl-libwww-perl, perl-Crypt-SSLeay, perl-Digest-SHA} %{?el7:Requires: perl-JSON, perl-libwww-perl, perl-Crypt-SSLeay, perl-LWP-Protocol-https, perl-Digest-SHA} @@ -41,8 +41,8 @@ tar xvf %{SOURCE0} -C $RPM_SOURCE_DIR # copy atstccfg binary godir=src/github.com/apache/trafficcontrol/traffic_ops_ort/atstccfg ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp "$TC_DIR"/traffic_ops_ort/atstccfg/atstccfg . + cd "$godir" && \ + cp "$TC_DIR"/traffic_ops_ort/atstccfg/atstccfg . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } diff --git a/traffic_portal/build/build_rpm.sh b/traffic_portal/build/build_rpm.sh index a30f6f016e..7533e0c3aa 100755 --- a/traffic_portal/build/build_rpm.sh +++ b/traffic_portal/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash -# -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,37 +11,46 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TP_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TP_DIR") +importFunctions() { + local script scriptdir + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + TP_DIR='' TC_DIR='' + TP_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TP_DIR")" + export TP_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local ts_dest=$(createSourceDir traffic_portal) + local tp_dest + tp_dest="$(createSourceDir traffic_portal)" cd "$TP_DIR" || \ - { echo "Could not cd to $TP_DIR: $?"; exit 1; } - rsync -av ./ "$ts_dest"/ || \ - { echo "Could not copy to $to_dest: $?"; exit 1; } - cp -r "$TP_DIR"/ "$ts_dest" || { echo "Could not copy $TP_DIR to $ts_dest: $?"; exit 1; } + { echo "Could not cd to $TP_DIR: $?"; return 1; } + rsync -av ./ "$tp_dest"/ || \ + { echo "Could not copy to $to_dest: $?"; return 1; } + cp -r "$TP_DIR"/ "$tp_dest" || { echo "Could not copy $TP_DIR to $tp_dest: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; exit 1; } - cp "$TP_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$tp_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$tp_dest")" || { echo "Could not create tar archive ${tp_dest}.tgz: $?"; return 1; } + cp "$TP_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } @@ -52,6 +58,6 @@ function initBuildArea() { # --------------------------------------- importFunctions -checkEnvironment npm node +checkEnvironment -i npm,bower,grunt,compass,rsync initBuildArea buildRpm traffic_portal diff --git a/traffic_portal/build/traffic_portal.spec b/traffic_portal/build/traffic_portal.spec index f5e992252e..00635c0d78 100644 --- a/traffic_portal/build/traffic_portal.spec +++ b/traffic_portal/build/traffic_portal.spec @@ -15,15 +15,15 @@ # # RPM spec file for the Traffic Portal # -%define debug_package %{nil} -Name: traffic_portal -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Traffic Portal -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/ -Source: %{_sourcedir}/traffic_portal-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_portal +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Traffic Portal +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/ +Source: %{_sourcedir}/traffic_portal-%{traffic_control_version}.tgz AutoReqProv: no Requires: nodejs @@ -40,24 +40,25 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz %setup %build - /usr/bin/npm install - /usr/bin/bower install - /usr/bin/grunt dist + npm install + bower install + grunt dist %install - %__mkdir -p ${RPM_BUILD_ROOT}/etc/init.d - %__mkdir -p ${RPM_BUILD_ROOT}/etc/logrotate.d - %__mkdir -p ${RPM_BUILD_ROOT}/etc/traffic_portal - %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/public - %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/server - %__mkdir -p ${RPM_BUILD_ROOT}/var/log/traffic_portal + %__mkdir -p ${RPM_BUILD_ROOT}/etc/init.d + %__mkdir -p ${RPM_BUILD_ROOT}/etc/logrotate.d + %__mkdir -p ${RPM_BUILD_ROOT}/etc/traffic_portal + %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/public + %__mkdir -p ${RPM_BUILD_ROOT}%{traffic_portal_home}/server + %__mkdir -p ${RPM_BUILD_ROOT}/var/log/traffic_portal - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/server.js ${RPM_BUILD_ROOT}%{traffic_portal_home}/. - %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/conf ${RPM_BUILD_ROOT}/etc/traffic_portal/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/init.d/traffic_portal ${RPM_BUILD_ROOT}/etc/init.d/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal ${RPM_BUILD_ROOT}/etc/logrotate.d/. - %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal-access ${RPM_BUILD_ROOT}/etc/logrotate.d/. - %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/* ${RPM_BUILD_ROOT}%{traffic_portal_home}/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/server.js ${RPM_BUILD_ROOT}%{traffic_portal_home}/. + %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/conf ${RPM_BUILD_ROOT}/etc/traffic_portal/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/init.d/traffic_portal ${RPM_BUILD_ROOT}/etc/init.d/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal ${RPM_BUILD_ROOT}/etc/logrotate.d/. + %__cp ${RPM_BUILD_DIR}/traffic_portal-%{version}/build/etc/logrotate.d/traffic_portal-access ${RPM_BUILD_ROOT}/etc/logrotate.d/. + %__rm -f ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/package-lock.json + %__cp -r ${RPM_BUILD_DIR}/traffic_portal-%{version}/app/dist/* ${RPM_BUILD_ROOT}%{traffic_portal_home}/. # creates dynamic json file needed at runtime for traffic portal to display release info VERSION=%{version}-%{build_number} @@ -68,22 +69,26 @@ tar -xzvf $RPM_SOURCE_DIR/traffic_portal-%{version}.tgz echo -e $JSON_VERSION > ${RPM_BUILD_ROOT}%{traffic_portal_home}/public/traffic_portal_release.json %post - echo "Successfully installed the traffic_portal assets to " %{traffic_portal_home} - %__chmod +x %{traffic_portal_home}/node_modules/forever/bin/forever - %__chmod +x /etc/init.d/traffic_portal - echo "Successfully installed the 'traffic_portal' service" - /sbin/chkconfig traffic_portal on - echo "" - echo "Start with 'service traffic_portal start'" + echo "Successfully installed the traffic_portal assets to " %{traffic_portal_home} + %__chmod +x %{traffic_portal_home}/node_modules/forever/bin/forever + %__chmod +x /etc/init.d/traffic_portal + echo "Successfully installed the 'traffic_portal' service" + /sbin/chkconfig traffic_portal on + echo "" + echo "Start with 'service traffic_portal start'" %files %defattr(644,root,root,755) %attr(755,root,root) /etc/init.d/traffic_portal -%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/* +%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/forever +%attr(755,root,root) %{traffic_portal_home}/node_modules/forever/bin/monitor %config(noreplace)/etc/traffic_portal/conf/config.js %config(noreplace)%{traffic_portal_home}/public/traffic_portal_properties.json %dir /var/log/traffic_portal -%{traffic_portal_home}/* +%{traffic_portal_home}/node_modules +%{traffic_portal_home}/package.json +%{traffic_portal_home}/public +%{traffic_portal_home}/server +%{traffic_portal_home}/server.js /etc/logrotate.d/traffic_portal /etc/logrotate.d/traffic_portal-access -/etc/init.d/traffic_portal diff --git a/traffic_router/build/build_rpm.sh b/traffic_router/build/build_rpm.sh index 16be21eb3b..a6908e9be4 100755 --- a/traffic_router/build/build_rpm.sh +++ b/traffic_router/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,63 +11,73 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- -function importFunctions() { +importFunctions() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TR_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TR_DIR") + local script scriptdir + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + TR_DIR='' TC_DIR='' + TR_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TR_DIR")" + export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } #---------------------------------------- -function buildRpmTrafficRouter () { +buildRpmTrafficRouter () { echo "Building the rpm." export STARTUP_SCRIPT_DIR="/lib/systemd/system" export STARTUP_SCRIPT_LOC="../core/src/main/lib/systemd/system" - cd "$TR_DIR" || { echo "Could not cd to $TR_DIR: $?"; exit 1; } + cd "$TR_DIR" || { echo "Could not cd to $TR_DIR: $?"; return 1; } mvn -P rpm-build -Dmaven.test.skip=true -DminimumTPS=1 clean package || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } + { echo "RPM BUILD FAILED: $?"; return 1; } - local rpm=$(find -name \*.rpm) - if [[ -z $rpm ]]; then + local rpm + rpm="$(find . -name \*.rpm | head -n1)" + if [ -z "$rpm" ]; then echo "Could not find rpm file $RPM in $(pwd)" - exit 1; + return 1; fi echo "========================================================================================" echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." echo "========================================================================================" echo - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } + mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; return 1; } - cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } + cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; return 1; } } #---------------------------------------- -function adaptEnvironment() { +adaptEnvironment() { echo "Verifying the build configuration environment." # get traffic_control src path -- relative to build_rpm.sh script - export PACKAGE="traffic_router" - export TC_VERSION=$(getVersion "$TC_DIR") - export BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} - export BUILD_LOCK=$(getBuildNumber).${RHEL_VERSION} - export WORKSPACE=${WORKSPACE:-$TC_DIR} - export RPMBUILD="$WORKSPACE/rpmbuild" - export DIST="$WORKSPACE/dist" - export RPM="${PACKAGE}-${TC_VERSION}-${BUILD_NUMBER}.x86_64.rpm" - export TOMCAT_VERSION=8.5 - export TOMCAT_RELEASE=32 + PACKAGE='' TC_VERSION='' BUILD_LOCK='' RPMBUILD='' DIST='' RPM='' TOMCAT_VERSION='' TOMCAT_RELEASE='' + PACKAGE="traffic_router" + TC_VERSION=$(getVersion "$TC_DIR") + BUILD_NUMBER=${BUILD_NUMBER:-$(getBuildNumber)} + BUILD_LOCK=$(getBuildNumber).${RHEL_VERSION} + WORKSPACE=${WORKSPACE:-$TC_DIR} + RPMBUILD="$WORKSPACE/rpmbuild" + DIST="$WORKSPACE/dist" + RPM="${PACKAGE}-${TC_VERSION}-${BUILD_NUMBER}.x86_64.rpm" + RPM_TARGET_OS="${RPM_TARGET_OS:-linux}" + TOMCAT_VERSION=8.5 + TOMCAT_RELEASE=32 + export PACKAGE TC_VERSION BUILD_NUMBER BUILD_LOCK WORKSPACE RPMBUILD DIST RPM RPM_TARGET_OS TOMCAT_VERSION TOMCAT_RELEASE echo "==================================================" echo "WORKSPACE: $WORKSPACE" @@ -84,36 +91,38 @@ function adaptEnvironment() { } # --------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } tr_dest=$(createSourceDir traffic_router) export MVN_CMD="mvn versions:set -DnewVersion=$TC_VERSION" - echo $MVN_CMD - (cd $TR_DIR; $MVN_CMD) - cp -r "$TR_DIR"/{build,connector,core} "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; exit 1; } - cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; exit 1; } + echo "$MVN_CMD" + (cd "$TR_DIR"; $MVN_CMD) + cp -r "$TR_DIR"/build "$TR_DIR"/connector "$TR_DIR"/core "$tr_dest"/. || { echo "Could not copy to $tr_dest: $?"; return 1; } + cp "$TR_DIR"/pom.xml "$tr_dest" || { echo "Could not copy to $tr_dest: $?"; return 1; } # tar/gzip the source - tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" $(basename $tr_dest) || { echo "Could not create tar archive $tr_dest: $?"; exit 1; } + tar -czf "$tr_dest".tgz -C "$RPMBUILD/SOURCES" "$(basename "$tr_dest")" || { echo "Could not create tar archive $tr_dest: $?"; return 1; } echo "The build area has been initialized." } #---------------------------------------- -function buildRpmTomcat () { +buildRpmTomcat () { echo "Building the rpm for Tomcat." - cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; exit 1; } - ./build_rpm.sh + cd "$TR_DIR"/tomcat-rpm || { echo "Could not cd to $TR_DIR/tomcat-rpm: $?"; return 1; } + ./build_rpm.sh } # --------------------------------------- importFunctions -checkEnvironment +checkEnvironment -i mvn adaptEnvironment initBuildArea buildRpmTrafficRouter diff --git a/traffic_router/build/pom.xml b/traffic_router/build/pom.xml index 49caa35931..d7c5a080a2 100644 --- a/traffic_router/build/pom.xml +++ b/traffic_router/build/pom.xml @@ -92,6 +92,7 @@ TOMCAT_VERSION TOMCAT_RELEASE RHEL_VERSION + RPM_TARGET_OS STARTUP_SCRIPT_DIR STARTUP_SCRIPT_LOC @@ -133,6 +134,11 @@ ${project.parent.artifactId} ${env.BUILD_NUMBER}.${env.RHEL_VERSION} x86_64 + ${env.RPM_TARGET_OS} + + _source_payload w2.xzdio + _binary_payload w2.xzdio + ${deploy.dir} diff --git a/traffic_router/connector/pom.xml b/traffic_router/connector/pom.xml index 3a5f5a67b3..1f85a06883 100644 --- a/traffic_router/connector/pom.xml +++ b/traffic_router/connector/pom.xml @@ -43,7 +43,7 @@ true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/core/pom.xml b/traffic_router/core/pom.xml index 0cf0e096db..f39b743bc7 100644 --- a/traffic_router/core/pom.xml +++ b/traffic_router/core/pom.xml @@ -101,7 +101,7 @@ true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/core/src/main/scripts/postinstall.sh b/traffic_router/core/src/main/scripts/postinstall.sh index 1c017ec200..f50299778d 100644 --- a/traffic_router/core/src/main/scripts/postinstall.sh +++ b/traffic_router/core/src/main/scripts/postinstall.sh @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -keytool=$(dirname $(readlink -f $(which java)))/keytool +keytool=$(dirname $(realpath "$(which java)"))/keytool cd /opt/traffic_router/conf if [ -f /opt/traffic_router/conf/*.crt ]; then for file in *.crt; do alias=$(echo $file |sed -e 's/.crt//g' |tr [:upper:] [:lower:]) - cacerts=$(/bin/find $(dirname $(readlink -f $(which java)))/.. -name cacerts) + cacerts=$(/bin/find $(dirname $(realpath "$(which java)"))/.. -name cacerts) $keytool -list -alias $alias -keystore $cacerts -storepass changeit -noprompt > /dev/null if [ $? -ne 0 ]; then @@ -31,7 +31,7 @@ fi echo -e " cd /opt/traffic_router/conf -keytool=\$(dirname \$(readlink -f \$(which java)))/keytool +keytool=\$(dirname \$(realpath \$(which java)))/keytool if [ ! -f /opt/traffic_router/conf/keyStore.jks ]; then \n \$keytool -genkeypair -v -alias \$(hostname -f) -dname \"CN=\$(hostname -f), OU=APIDefault, O=Apache Traffic Control, L=Denver, ST=Colorado, C=US\" -keystore \$(pwd)/keyStore.jks -storepass changeit -keyalg RSA -ext KeyUsage=\"digitalSignature,keyEncipherment,keyCertSign\" -ext BasicConstraints:\"critical=ca:true\" -storetype JKS -validity 3650 diff --git a/traffic_router/shared/pom.xml b/traffic_router/shared/pom.xml index 6938dc2086..d7e86902ff 100644 --- a/traffic_router/shared/pom.xml +++ b/traffic_router/shared/pom.xml @@ -50,7 +50,7 @@ under the License. true 5 - file://${project.basedir}/build/pmd/ruleset.xml + ${project.basedir}${file.separator}build${file.separator}pmd${file.separator}ruleset.xml diff --git a/traffic_router/tomcat-rpm/build_rpm.sh b/traffic_router/tomcat-rpm/build_rpm.sh index 7a4a6ef17d..352fa99bd4 100755 --- a/traffic_router/tomcat-rpm/build_rpm.sh +++ b/traffic_router/tomcat-rpm/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash -# -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,16 +11,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; #---------------------------------------- -function checkEnvironment() { +checkEnvironment() { echo "Verifying the build configuration environment." - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TR_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TR_DIR") + local script scriptdir + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + TR_DIR='' TC_DIR='' + TR_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TR_DIR")" + export TR_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "Error: Can't find $functions_sh" exit 1 fi @@ -34,65 +37,73 @@ function checkEnvironment() { export PACKAGE="tomcat" export WORKSPACE=${WORKSPACE:-$TC_DIR} export RPMBUILD="$WORKSPACE/rpmbuild" + export RPM_TARGET_OS="${RPM_TARGET_OS:-linux}" export DIST="$WORKSPACE/dist" export RPM="${PACKAGE}-${TOMCAT_VERSION}.${TOMCAT_RELEASE}-${BUILD_NUMBER}.${RHEL_VERSION}.x86_64.rpm" echo "==================================================" echo "WORKSPACE: $WORKSPACE" - echo "TOMCAT_RELEASE: $TOMCAT_RELEASE" #defined in traffic_router - echo "TOMCAT_VERSION: $TOMCAT_VERSION" #defined in traffic_router - echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router - echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router + echo "TOMCAT_RELEASE: $TOMCAT_RELEASE" #defined in traffic_router + echo "TOMCAT_VERSION: $TOMCAT_VERSION" #defined in traffic_router + echo "BUILD_NUMBER: $BUILD_NUMBER" #defined in traffic_router + echo "BUILD_LOCK: $BUILD_LOCK" #defined in traffic_router echo "RPM: $RPM" echo "--------------------------------------------------" } # --------------------------------------- -function initBuildArea() { - echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } - export VERSION=$TOMCAT_VERSION - export RELEASE=$TOMCAT_RELEASE +initBuildArea() { + echo "Initializing the build area." + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } + export VERSION=$TOMCAT_VERSION + export RELEASE=$TOMCAT_RELEASE - echo "Downloading Tomcat $VERSION.$RELEASE..." - curl -fo "$RPMBUILD"/SOURCES/apache-tomcat-$VERSION.$RELEASE.tar.gz https://archive.apache.org/dist/tomcat/tomcat-8/v$VERSION.$RELEASE/bin/apache-tomcat-$VERSION.$RELEASE.tar.gz || \ - { echo "Could not download Tomcat $VERSION.$RELEASE: $?"; exit 1; } + echo "Downloading Tomcat $VERSION.$RELEASE..." + curl -fo "$RPMBUILD"/SOURCES/apache-tomcat-$VERSION.$RELEASE.tar.gz https://archive.apache.org/dist/tomcat/tomcat-8/v$VERSION.$RELEASE/bin/apache-tomcat-$VERSION.$RELEASE.tar.gz || \ + { echo "Could not download Tomcat $VERSION.$RELEASE: $?"; exit 1; } - cp "$TR_DIR/tomcat-rpm/tomcat.service" "$RPMBUILD/SOURCES/" || { echo "Could not copy source files: $?"; exit 1; } - cp "$TR_DIR/tomcat-rpm/tomcat.spec" "$RPMBUILD/SPECS/" || { echo "Could not copy spec files: $?"; exit 1; } + cp "$TR_DIR/tomcat-rpm/tomcat.service" "$RPMBUILD/SOURCES/" || { echo "Could not copy source files: $?"; exit 1; } + cp "$TR_DIR/tomcat-rpm/tomcat.spec" "$RPMBUILD/SPECS/" || { echo "Could not copy spec files: $?"; exit 1; } - echo "The build area has been initialized." + echo "The build area has been initialized." } #---------------------------------------- -function buildRpmTomcat () { +buildRpmTomcat () { export SPEC_FILE_NAME=tomcat.spec buildRpmForEl 7 } -function buildRpmForEl () { - echo "Building the rpm for "$RHEL_VERSION"." +buildRpmForEl () { + echo "Building the rpm for ${RHEL_VERSION}." - cd $RPMBUILD - rpmbuild --define "_topdir $(pwd)" \ - --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ - --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ - -ba SPECS/$SPEC_FILE_NAME || \ - { echo "RPM BUILD FAILED: $?"; exit 1; } - local rpm=$(find ./RPMS -name $RPM) - if [[ -z $rpm ]]; then - echo "Could not find rpm file $RPM in $(pwd)" - exit 1; - fi - echo "========================================================================================" - echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." - echo "========================================================================================" - echo - mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } + cd "$RPMBUILD" + # build RPM with xz level 2 compression + rpmbuild --define "_topdir $(pwd)" \ + --define "build_number $BUILD_NUMBER.$RHEL_VERSION" \ + --define "tomcat_version $TOMCAT_VERSION.$TOMCAT_RELEASE" \ + --define "_target_os ${RPM_TARGET_OS}" \ + --define '%_source_payload w2.xzdio' \ + --define '%_binary_payload w2.xzdio' \ + -ba SPECS/$SPEC_FILE_NAME || + { echo "RPM BUILD FAILED: $?"; exit 1; } + local rpm + rpm="$(find ./RPMS -name "$RPM")" + if [ -z "$rpm" ]; then + echo "Could not find rpm file $RPM in $(pwd)" + exit 1; + fi + echo "========================================================================================" + echo "RPM BUILD SUCCEEDED, See $DIST/$RPM for the newly built rpm." + echo "========================================================================================" + echo + mkdir -p "$DIST" || { echo "Could not create $DIST: $?"; exit 1; } - cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } + cp "$rpm" "$DIST/." || { echo "Could not copy $rpm to $DIST: $?"; exit 1; } } -checkEnvironment +checkEnvironment -i curl initBuildArea buildRpmTomcat diff --git a/traffic_router/tomcat-rpm/tomcat.spec b/traffic_router/tomcat-rpm/tomcat.spec index 119e89d6cf..a196fe230c 100644 --- a/traffic_router/tomcat-rpm/tomcat.spec +++ b/traffic_router/tomcat-rpm/tomcat.spec @@ -54,15 +54,15 @@ rm -rf ${RPM_BUILD_ROOT} # This here takes care of stopping and removing tomcat before installing new files %pretrans if [[ -e "/etc/init.d/tomcat" ]]; then - echo "Disabling and stopping SysV tomcat service..." - chkconfig tomcat off - service stop tomcat + echo "Disabling and stopping SysV tomcat service..." + chkconfig tomcat off + service stop tomcat fi if [ -d /opt/apache-tomcat-* ]; then - echo "Deleting unmanaged Tomcat install from < 2.3 version of Traffic Router" - rm -rf /opt/apache-tomcat-* - rm -rf /opt/tomcat + echo "Deleting unmanaged Tomcat install from < 2.3 version of Traffic Router" + rm -rf /opt/apache-tomcat-* + rm -rf /opt/tomcat fi %pre diff --git a/traffic_server/plugins/astats_over_http/astats-git-build.spec b/traffic_server/plugins/astats_over_http/astats-git-build.spec index 4e73069204..ae4f3466dc 100644 --- a/traffic_server/plugins/astats_over_http/astats-git-build.spec +++ b/traffic_server/plugins/astats_over_http/astats-git-build.spec @@ -21,18 +21,18 @@ %global no_commits %(git log astats_over_http.c | grep '^commit' | wc -l) %global _find_debuginfo_dwz_opts %{nil} -Name: astats_over_http -Version: 1.3.0 -Release: %{no_commits}.%{commit}%{?dist} -Epoch: 434 -Summary: Apache Traffic Server %{name} plugin -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: trafficserver >= 6011 -BuildRequires: trafficserver >= 6011 +Name: astats_over_http +Version: 1.3.0 +Release: %{no_commits}.%{commit}%{?dist} +Epoch: 434 +Summary: Apache Traffic Server %{name} plugin +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: trafficserver >= 6011 +BuildRequires: trafficserver >= 6011 %description Apache Traffic Server plugin diff --git a/traffic_server/plugins/astats_over_http/astats_over_http.spec b/traffic_server/plugins/astats_over_http/astats_over_http.spec index 0caaad09ae..8a26f97961 100644 --- a/traffic_server/plugins/astats_over_http/astats_over_http.spec +++ b/traffic_server/plugins/astats_over_http/astats_over_http.spec @@ -18,18 +18,18 @@ %global install_prefix "/opt" -Name: astats_over_http -Version: 1.3 -Release: 1%{?dist} -Summary: Apache Traffic Server %{name} plugin -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http -Source0: %{name}.tar.gz -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: trafficserver = 5.2.0 -BuildRequires: trafficserver = 5.2.0 +Name: astats_over_http +Version: 1.3 +Release: 1%{?dist} +Summary: Apache Traffic Server %{name} plugin +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol/tree/master/traffic_server/plugins/astats_over_http +Source0: %{name}.tar.gz +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: trafficserver = 5.2.0 +BuildRequires: trafficserver = 5.2.0 %description Apache Traffic Server plugin diff --git a/traffic_server/spec/trafficserver.spec b/traffic_server/spec/trafficserver.spec index 4c4afb2788..9090c86998 100644 --- a/traffic_server/spec/trafficserver.spec +++ b/traffic_server/spec/trafficserver.spec @@ -18,19 +18,19 @@ %global install_prefix "/opt" -Name: trafficserver -Version: 5.2.0 -Release: 1%{?dist} -Summary: Apache Traffic Server -Vendor: Comcast -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://trafficserver.apache.org/ +Name: trafficserver +Version: 5.2.0 +Release: 1%{?dist} +Summary: Apache Traffic Server +Vendor: Comcast +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://trafficserver.apache.org/ Source0: %{name}-%{version}.tar.bz2 -Patch: %{name}-%{version}-791ddc4.patch -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires: tcl, hwloc, pcre, openssl, libcap -BuildRequires: autoconf, automake, libtool, gcc-c++, glibc-devel, openssl-devel, expat-devel, pcre, libcap-devel, pcre-devel, perl-ExtUtils-MakeMaker, tcl-devel, hwloc-devel +Patch: %{name}-%{version}-791ddc4.patch +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: tcl, hwloc, pcre, openssl, libcap +BuildRequires: autoconf, automake, libtool, gcc-c++, glibc-devel, openssl-devel, expat-devel, pcre, libcap-devel, pcre-devel, perl-ExtUtils-MakeMaker, tcl-devel, hwloc-devel %description Apache Traffic Server with Comcast modifications and environment specific modifications diff --git a/traffic_stats/build/build_rpm.sh b/traffic_stats/build/build_rpm.sh index 416b080fdf..26bc56e5a6 100755 --- a/traffic_stats/build/build_rpm.sh +++ b/traffic_stats/build/build_rpm.sh @@ -1,7 +1,4 @@ -#!/bin/bash - -# -# +#!/usr/bin/env sh # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,73 +11,84 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# shellcheck shell=ash +trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "Error on line ${LINENO} of ${0}" >/dev/stderr; exit $exit_code' EXIT; +set -o errexit -o nounset -o pipefail; -function importFunctions() { - local script=$(readlink -f "$0") - local scriptdir=$(dirname "$script") - export TS_DIR=$(dirname "$scriptdir") - export TC_DIR=$(dirname "$TS_DIR") +#---------------------------------------- +importFunctions() { + local script scriptdir + TS_DIR='' TC_DIR='' + script="$(realpath "$0")" + scriptdir="$(dirname "$script")" + TS_DIR="$(dirname "$scriptdir")" + TC_DIR="$(dirname "$TS_DIR")" + export TS_DIR TC_DIR functions_sh="$TC_DIR/build/functions.sh" - if [[ ! -r $functions_sh ]]; then + if [ ! -r "$functions_sh" ]; then echo "error: can't find $functions_sh" - exit 1 + return 1 fi . "$functions_sh" } #---------------------------------------- -function initBuildArea() { +initBuildArea() { echo "Initializing the build area." - mkdir -p "$RPMBUILD"/{SPECS,SOURCES,RPMS,SRPMS,BUILD,BUILDROOT} || { echo "Could not create $RPMBUILD: $?"; exit 1; } + (mkdir -p "$RPMBUILD" + cd "$RPMBUILD" + mkdir -p SPECS SOURCES RPMS SRPMS BUILD BUILDROOT) || { echo "Could not create $RPMBUILD: $?"; return 1; } # tar/gzip the source - local ts_dest=$(createSourceDir traffic_stats) + local ts_dest + ts_dest="$(createSourceDir traffic_stats)" cd "$TS_DIR" || \ - { echo "Could not cd to $TS_DIR: $?"; exit 1; } + { echo "Could not cd to $TS_DIR: $?"; return 1; } - echo "PATH: $PATH" - echo "GOPATH: $GOPATH" - go version - go env + echo "PATH: $PATH" + echo "GOPATH: $GOPATH" + go version + go env + ldflags='-s -w' + tags='osusergo netgo' - # get x/* packages (everything else should be properly vendored) - go get -v golang.org/x/net/publicsuffix || \ - { echo "Could not get go package dependencies"; exit 1; } + # get x/* packages (everything else should be properly vendored) + go get -v golang.org/x/net/publicsuffix || \ + { echo "Could not get go package dependencies"; return 1; } - # compile traffic_stats - go build -v || \ - { echo "Could not build traffic_stats binary"; exit 1; } + # compile traffic_stats + go build -v -ldflags "$ldflags" -tags "$tags" || \ + { echo "Could not build traffic_stats binary"; return 1; } # compile influx_db_tools - pushd influxdb_tools - go build -v sync/sync_ts_databases.go || \ - { echo "Could not build sync_ts_databases binary"; exit 1; } - go build -v create/create_ts_databases.go || \ - { echo "Could not build create_ts_databases binary"; exit 1; } - popd + (cd influxdb_tools + go build -v -ldflags "$ldflags" -tags="$tags" sync/sync_ts_databases.go || \ + { echo "Could not build sync_ts_databases binary"; return 1; } + go build -v -ldflags "$ldflags" -tags="$tags" create/create_ts_databases.go || \ + { echo "Could not build create_ts_databases binary"; return 1; }) rsync -aLv ./ "$ts_dest"/ || \ - { echo "Could not copy to $ts_dest: $?"; exit 1; } + { echo "Could not copy to $ts_dest: $?"; return 1; } cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || \ - { echo "Could not copy spec files: $?"; exit 1; } + { echo "Could not copy spec files: $?"; return 1; } - tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES $(basename $ts_dest) || { echo "Could not create tar archive $ts_dest.tgz: $?"; exit 1; } - cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; exit 1; } + tar -czvf "$ts_dest".tgz -C "$RPMBUILD"/SOURCES "$(basename "$ts_dest")" || { echo "Could not create tar archive $ts_dest.tgz: $?"; return 1; } + cp "$TS_DIR"/build/*.spec "$RPMBUILD"/SPECS/. || { echo "Could not copy spec files: $?"; return 1; } echo "The build area has been initialized." } -function preBuildChecks() { - if [[ -e "$TS_DIR"/traffic_stats ]]; then - echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" - exit 1 - fi +preBuildChecks() { + if [ -e "${TS_DIR}/traffic_stats" ]; then + echo "Found $TS_DIR/traffic_stats, please remove before retrying to build" + return 1 + fi } # --------------------------------------- importFunctions preBuildChecks -checkEnvironment go +checkEnvironment -i go,rsync initBuildArea buildRpm traffic_stats diff --git a/traffic_stats/build/traffic_stats.spec b/traffic_stats/build/traffic_stats.spec index 52425eface..680224a4ab 100644 --- a/traffic_stats/build/traffic_stats.spec +++ b/traffic_stats/build/traffic_stats.spec @@ -15,17 +15,17 @@ # # RPM spec file for Traffic Stats (tm). # -%define debug_package %{nil} -Name: traffic_stats -Version: %{traffic_control_version} -Release: %{build_number} -Summary: Tool to pull data from traffic monitor and store in Influxdb -Packager: david_neuman2 at Cable dot Comcast dot com -Vendor: Apache Software Foundation -Group: Applications/Communications -License: Apache License, Version 2.0 -URL: https://github.com/apache/trafficcontrol -Source: %{_sourcedir}/traffic_stats-%{traffic_control_version}.tgz +%define debug_package %{nil} +Name: traffic_stats +Version: %{traffic_control_version} +Release: %{build_number} +Summary: Tool to pull data from traffic monitor and store in Influxdb +Packager: david_neuman2 at Cable dot Comcast dot com +Vendor: Apache Software Foundation +Group: Applications/Communications +License: Apache License, Version 2.0 +URL: https://github.com/apache/trafficcontrol +Source: %{_sourcedir}/traffic_stats-%{traffic_control_version}.tgz %description Installs traffic_stats which performs the follwing functions: @@ -40,15 +40,15 @@ Installs traffic_stats which performs the follwing functions: # copy traffic_stats client godir=src/github.com/apache/trafficcontrol/traffic_stats ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -L -r "$TC_DIR"/traffic_stats/* . + cd "$godir" && \ + cp -LR "$TC_DIR"/traffic_stats/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } # copy influxdb_tools godir=src/github.com/apache/trafficcontrol/traffic_stats/influxdb_tools ( mkdir -p "$godir" && \ - cd "$godir" && \ - cp -r "$TC_DIR"/traffic_stats/influxdb_tools/* . + cd "$godir" && \ + cp -R "$TC_DIR"/traffic_stats/influxdb_tools/* . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } %install @@ -70,8 +70,8 @@ cp "$src"/traffic_stats_seelog.xml "${RPM_BUILD_ROOT}"/opt/traffic_stats/conf/tr cp "$src"/traffic_stats.init "${RPM_BUILD_ROOT}"/etc/init.d/traffic_stats cp "$src"/traffic_stats.logrotate "${RPM_BUILD_ROOT}"/etc/logrotate.d/traffic_stats cp "$src"/grafana/*.js "${RPM_BUILD_ROOT}"/usr/share/grafana/public/dashboards/ -cp "$src"/influxdb_tools/sync_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ -cp "$src"/influxdb_tools/create_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ +cp "$src"/influxdb_tools/sync_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ +cp "$src"/influxdb_tools/create_ts_databases "${RPM_BUILD_ROOT}"/opt/traffic_stats/influxdb_tools/ %pre @@ -113,8 +113,8 @@ fi %files %defattr(644, traffic_stats, traffic_stats, 755) -%config(noreplace) /opt/traffic_stats/conf/traffic_stats.cfg -%config(noreplace) /opt/traffic_stats/conf/traffic_stats_seelog.xml +%config(noreplace) %attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/traffic_stats.cfg +%config(noreplace) %attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/traffic_stats_seelog.xml %config(noreplace) /etc/logrotate.d/traffic_stats %dir /opt/traffic_stats @@ -128,11 +128,14 @@ fi %dir /usr/share/grafana/public/dashboards %dir /opt/traffic_stats/influxdb_tools -%attr(600, traffic_stats, traffic_stats) /opt/traffic_stats/conf/* -%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/bin/* +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/bin/traffic_stats %attr(755, traffic_stats, traffic_stats) /etc/init.d/traffic_stats -%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/* -%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/* +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_cachegroup.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_deliveryservice.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_scripted.js +%attr(644, traffic_stats, traffic_stats) /usr/share/grafana/public/dashboards/traffic_ops_server.js +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/create_ts_databases +%attr(755, traffic_stats, traffic_stats) /opt/traffic_stats/influxdb_tools/sync_ts_databases %preun # args for hooks: https://www.ibm.com/developerworks/library/l-rpm2/