diff --git a/test/integration/smoke/test_deploy_vms_with_varied_deploymentplanners.py b/test/integration/smoke/test_deploy_vms_with_varied_deploymentplanners.py index 9fcb6431c1dc..041d23be056e 100644 --- a/test/integration/smoke/test_deploy_vms_with_varied_deploymentplanners.py +++ b/test/integration/smoke/test_deploy_vms_with_varied_deploymentplanners.py @@ -43,7 +43,7 @@ def setUpClass(cls): ) if cls.template == FAILED: - assert false, "get_template() failed to return template with description %s" % cls.services["ostype"] + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["template"] = cls.template.id diff --git a/tools/travis/after_failure.sh b/tools/travis/after_failure.sh index 1bc22aa720c7..df40f7336719 100755 --- a/tools/travis/after_failure.sh +++ b/tools/travis/after_failure.sh @@ -19,4 +19,4 @@ # As the filename suggests, this is executed on build failure # failure. # - +find /tmp//MarvinLogs -type f -exec echo -e "Printing logfile {} :\n" \; -exec cat {} \; diff --git a/tools/travis/before_install.sh b/tools/travis/before_install.sh index 31efc6536096..39790981e8bc 100755 --- a/tools/travis/before_install.sh +++ b/tools/travis/before_install.sh @@ -58,17 +58,15 @@ sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password passwor sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password your_password' sudo apt-get -q -y install mysql-server > /dev/null -sudo /etc/init.d/mysql start - -echo -e "\nInstalling Tomcat: " -wget -q -O tomcat.tar.gz http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz -sudo mkdir -p /opt/tomcat -sudo tar xfv tomcat.tar.gz -C /opt/tomcat --strip 1 > /dev/null +sudo start mysql echo -e "\nInstalling Development tools: " +RETRY_COUNT=3 sudo apt-get -q -y install uuid-runtime genisoimage python-setuptools python-pip netcat > /dev/null - +if [[ $? -ne 0 ]]; then + echo -e "\napt-get packages failed to install" +fi echo " @@ -82,5 +80,52 @@ echo " echo -e "\nInstalling some python packages: " -sudo pip install lxml > /dev/null -sudo pip install texttable > /dev/null +for ((i=0;i<$RETRY_COUNT;i++)) +do + sudo pip install --upgrade lxml texttable paramiko > /tmp/piplog + if [[ $? -eq 0 ]]; then + echo -e "\npython packages installed successfully" + break; + fi + echo -e "\npython packages failed to install" + cat /tmp/piplog +done + +#Download project dependencies in a way we can retry if there's a failure, without failing the whole build + +#Resolve plugins first +echo -e "\nDownloading Plugin dependencies" +for ((i=0;i<$RETRY_COUNT;i++)) +do + #The output file is used on the next phase by the downloadDeps.sh script + mvn org.apache.maven.plugins:maven-dependency-plugin:resolve-plugins | grep "Plugin Resolved:" | sort -u | awk '{print $4}' | tee /tmp/resolvedPlugins + if [[ $? -eq 0 ]]; then + echo -e "\nPlugin dependencies downloaded successfully" + break; + fi + echo -e "\nDependency download failed" + #Test DNS record + getent hosts repo1.maven.org + while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done +done + +#Resolve remaining deps +cd tools/travis +echo -e "\nDownloading Project dependencies" + +for ((i=0;i<$RETRY_COUNT;i++)) +do + ./downloadDeps.sh > /tmp/phase2 + if [[ $? -eq 0 ]]; then + echo -e "\n$(cat cleandeps.out |wc -l) project dependencies downloaded successfully" + break; + fi + echo -e "\nDependency download failed" + #Print out errors from failed run + cat /tmp/phase2 | grep -i -e "fail" -e "error" -e "exception" + #Test DNS record + getent hosts repo1.maven.org + while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done + echo -e "\nRetrying download" +done +cd ../.. diff --git a/tools/travis/before_script.sh b/tools/travis/before_script.sh index cba50e005fde..40e67001a276 100755 --- a/tools/travis/before_script.sh +++ b/tools/travis/before_script.sh @@ -35,12 +35,10 @@ if [ $MOD -ne 0 ]; then fi -export CATALINA_BASE=/opt/tomcat -export CATALINA_HOME=/opt/tomcat -export M2_HOME="/usr/local/maven-3.2.1/" -export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m" +export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom" +echo -e "\nStarting simulator" +mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log & -mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /dev/null & - -while ! nc -vz localhost 8096 2>&1 > /dev/null; do sleep 10; done +while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done +echo -e "\nStarting DataCenter deployment" python -m marvin.deployDataCenter -i setup/dev/advanced.cfg 2>&1 || true diff --git a/tools/travis/downloadDeps.sh b/tools/travis/downloadDeps.sh new file mode 100755 index 000000000000..3ad4517e8a25 --- /dev/null +++ b/tools/travis/downloadDeps.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This script should be used to install additional dependencies +# This includes: installing ubuntu packages, custom services +# or internet downloads. + +# Authored by Rafael da Fonseca + +#Get all dependency blocks from all pom.xml files in the project +for line in $(find ../../ -name pom.xml -exec sed -n '//{:a;n;/<\/dependencies>/b;p;ba}' {} \; | grep -e "artifactId" -e "groupId" -e "version" -e "dependency\>" -e "exclusion\>" -e "exclusions\>"| sed -e 's/\^M//'); do + + #Tokenize values + set -- $(echo $line | awk -v FS="(>|<)" '{print $2, $3}') + + #Start processing data + if [[ $1 == "dependency" ]]; then + #Create new artifact dep + unset ARTIFACT + unset VERSION + unset GROUP + elif [[ $1 == "/dependency" ]]; then + #Filter out project modules interdependencies + if [[ $GROUP != *org.apache.cloudstack* ]] && [[ $GROUP != *com.cloud* ]] && [[ $ARTIFACT != cloudstack-service-console-proxy-rdpclient ]]; then + if [[ -z $VERSION ]] ; then + VERSION=LATEST + #These dependencies don't support the LATEST keywork for some reason, and would cause mvn runs to file on dummy poms + if [[ $GROUP == jstl ]] || [[ $ARTIFACT == mysql-connector-java ]] || [[ $GROUP == org.apache.axis ]]; then + continue + fi + fi + #Output resolved dependency to a file, to be picked up later + echo "$GROUP $ARTIFACT $VERSION" >> deps.out + fi + elif [[ $1 == "version" ]]; then + #If version is a maven var, get the value from parent pom + if [[ $2 == \$\{* ]]; then + VERSION=$(grep \<$(echo $2 | awk -v FS="(}|{)" '{print $2 }') ../../pom.xml | awk -v FS="(>|<)" '{print $3}') + #If version tag is empty, add LATEST to avoid maven errors + elif [[ "$2" == "" ]]; then + VERSION="LATEST" + else + VERSION=$2 + fi + elif [[ $1 == "artifactId" ]]; then + #This avoids exclusions inside dependency block to overwrite original dependency + if [[ -z $ARTIFACT ]]; then + ARTIFACT=$2 + fi + elif [[ $1 == "groupId" ]]; then + #This avoids exclusions inside dependency block to overwrite original dependency + if [[ -z $GROUP ]]; then + GROUP=$2 + fi + fi +done + +#Add the resolved plugins to properly download their dependencies +while read line ; do + NAME=$(echo $line | sed -e 's/.jar$//') + VERSION=${NAME##*-} + ARTIFACT=${NAME%-*} + GROUP=$(find ~/.m2/repository -name ${NAME}.pom -exec sed -n "1,/${ARTIFACT}/p" {} \; | tac | grep -m 1 -e "" | sed -e 's/^[[:space:]]*//' -e 's/\^M//' | tr -d '\r' | awk -v FS="(>|<)" '{print $3}') + DATA="${GROUP} ${ARTIFACT} ${VERSION}" + echo $DATA >> deps.out +done < /tmp/resolvedPlugins + +#Remove duplicates and sort them, LANG export is needed to fix some sorting issue, sorting is needed for later function that relies on sorted input +cat deps.out | LANG=C sort -u > cleandeps.out + +#Define index of pomfiles, to avoid duplicate deps with different versions in pom.xml, several poms are created in case of more than one version of same artifact +LASTPOM=0 +echo '4.0.0org.apache.cloudstacktravis-build-depsDownload Deps for Travis CI1mido-maven-public-releasesmido-maven-public-releaseshttp://cs-maven.midokura.com/releasesjuniper-contrailhttp://juniper.github.io/contrail-maven/snapshots' > pom${LASTPOM}.xml +while read line ; do + set -- $line + #This relies on correct sorting, and distributes different versions of same dependency througout different pom files + if [[ $2 == $LASTARTIFACT ]]; then + POMID=$(($POMID+1)) + #If value is greater than current number of poms, create a new one + if [[ $POMID -gt $LASTPOM ]]; then + LASTPOM=$POMID + #This outputs the necessary structure to start a pom and also defines the extra repositories + echo '4.0.0org.apache.cloudstacktravis-build-depsDownload Deps for Travis CI1mido-maven-public-releasesmido-maven-public-releaseshttp://cs-maven.midokura.com/releasesjuniper-contrailhttp://juniper.github.io/contrail-maven/snapshots' > pom${LASTPOM}.xml + fi + else + POMID=0 + fi + LASTARTIFACT=$2 + echo "$1$2$3" >> pom${POMID}.xml +done < cleandeps.out + +RETURN_CODE=0 +for ((i=0;i<=$LASTPOM;i++)) +do + echo "" >> pom${i}.xml + mvn org.apache.maven.plugins:maven-dependency-plugin:resolve -f pom${i}.xml + if [[ $? -ne 0 ]]; then + RETURN_CODE=1 + fi +done + +#Run a few plugin goals to download some more deps + +#Hack to run maven-jaxb2-plugin generate, can be removed when we stop using such an old version... +mkdir -p src/main/resources +echo '' > src/main/resources/test.xsd + +#Declare plugin tests to run +declare -a arr=("maven-surefire-plugin test" "maven-pmd-plugin pmd" "maven-compiler-plugin compile" "maven-resources-plugin resources" "maven-checkstyle-plugin check" "maven-site-plugin attach-descriptor" "maven-surefire-plugin test" "maven-jar-plugin jar" "license-maven-plugin check" "maven-jgit-buildnumber-plugin extract-buildnumber" "maven-jaxb2-plugin generate" "maven-war-plugin war -DfailOnMissingWebXml=false" "gmaven-plugin compile") +for i in "${arr[@]}" +do + set -- $i + PLUGIN=$1 + MOJO=$2 + OPTION=$3 + #Get every listed version of the plugin and make a run for each version + while read line ; do + set -- $line + JOBS="${JOBS} ${1}:${2}:${3}:${MOJO} $OPTION" + done < <(grep $PLUGIN cleandeps.out) +done +echo "Running $JOBS" +#Call all the constructed plugin goals +mvn $JOBS -f pom0.xml +if [[ $? -ne 0 ]]; then + RETURN_CODE=1 +fi + +#Cleanup some files created in the run +rm -rf deps.out src target + +exit $RETURN_CODE diff --git a/tools/travis/install.sh b/tools/travis/install.sh index 3654e429b28c..28f4ed4fbe55 100755 --- a/tools/travis/install.sh +++ b/tools/travis/install.sh @@ -34,10 +34,7 @@ if [ $MOD -ne 0 ]; then fi fi -export CATALINA_BASE=/opt/tomcat -export CATALINA_HOME=/opt/tomcat -export M2_HOME="/usr/local/maven-3.2.1/" -export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m" +export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom" if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then mvn -q -Pimpatient -Dsimulator clean install diff --git a/tools/travis/script.sh b/tools/travis/script.sh index e1c75bb08331..7e7c49701687 100755 --- a/tools/travis/script.sh +++ b/tools/travis/script.sh @@ -39,7 +39,6 @@ mkdir -p integration-test-results/component for suite in $1; do - travis_wait 30 nosetests --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ; done diff --git a/tools/travis/xunit-reader.py b/tools/travis/xunit-reader.py index ffabbd45a6d2..c492c01e44b4 100755 --- a/tools/travis/xunit-reader.py +++ b/tools/travis/xunit-reader.py @@ -97,4 +97,4 @@ def parse_reports(file_path_list): if __name__ == "__main__": - main() \ No newline at end of file + main()