This is a quick fix to attempt to resolve most of the travis failures

test/integration/smoke/test_deploy_vms_with_varied_deploymentplanners.py

@@ -43,7 +43,7 @@ def setUpClass(cls):
         )
 
         if cls.template == FAILED:
-            assert false, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
 
         cls.services["virtual_machine"]["zoneid"] = cls.zone.id
         cls.services["template"] = cls.template.id

tools/travis/after_failure.sh

@@ -19,4 +19,4 @@
 # As the filename suggests, this is executed on build failure
 # failure.
 #
-
+find /tmp//MarvinLogs -type f -exec echo -e "Printing logfile {} :\n" \; -exec cat {} \;

tools/travis/before_install.sh

@@ -58,17 +58,15 @@ sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password passwor
 sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password your_password'
 sudo apt-get -q -y install mysql-server > /dev/null
 
-sudo /etc/init.d/mysql start
-
-echo -e "\nInstalling Tomcat: "
-wget -q -O tomcat.tar.gz http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz
-sudo mkdir -p /opt/tomcat
-sudo tar xfv tomcat.tar.gz -C /opt/tomcat --strip 1 > /dev/null
+sudo start mysql
 
 echo -e "\nInstalling Development tools: "
+RETRY_COUNT=3
 
 sudo apt-get -q -y install uuid-runtime genisoimage python-setuptools python-pip netcat > /dev/null
-
+if [[ $? -ne 0 ]]; then
+  echo -e "\napt-get packages failed to install"
+fi 
 echo "<settings>
   <mirrors>
     <mirror>
@@ -82,5 +80,52 @@ echo "<settings>
 
 echo -e "\nInstalling some python packages: "
 
-sudo pip install lxml > /dev/null
-sudo pip install texttable > /dev/null
+for ((i=0;i<$RETRY_COUNT;i++))
+do
+  sudo pip install --upgrade lxml texttable paramiko > /tmp/piplog
+  if [[ $? -eq 0 ]]; then
+    echo -e "\npython packages installed successfully"
+    break;
+  fi
+  echo -e "\npython packages failed to install"
+  cat /tmp/piplog
+done
+
+#Download project dependencies in a way we can retry if there's a failure, without failing the whole build
+
+#Resolve plugins first
+echo -e "\nDownloading Plugin dependencies"
+for ((i=0;i<$RETRY_COUNT;i++))
+do
+ #The output file is used on the next phase by the downloadDeps.sh script
+ mvn org.apache.maven.plugins:maven-dependency-plugin:resolve-plugins | grep "Plugin Resolved:" | sort -u | awk '{print $4}' | tee /tmp/resolvedPlugins
+ if [[ $? -eq 0 ]]; then
+   echo -e "\nPlugin dependencies downloaded successfully"
+   break;
+ fi
+ echo -e "\nDependency download failed"
+ #Test DNS record
+ getent hosts repo1.maven.org
+ while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done
+done
+
+#Resolve remaining deps
+cd tools/travis
+echo -e "\nDownloading Project dependencies"
+
+for ((i=0;i<$RETRY_COUNT;i++))
+do
+ ./downloadDeps.sh > /tmp/phase2
+ if [[ $? -eq 0 ]]; then
+   echo -e "\n$(cat cleandeps.out |wc -l) project dependencies downloaded successfully"
+   break;
+ fi
+ echo -e "\nDependency download failed"
+ #Print out errors from failed run
+ cat /tmp/phase2 | grep -i -e "fail" -e "error" -e "exception"
+ #Test DNS record
+ getent hosts repo1.maven.org
+ while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done
+ echo -e "\nRetrying download"
+done
+cd ../..

tools/travis/before_script.sh

@@ -35,12 +35,10 @@ if [ $MOD -ne 0 ]; then
 fi
 
 
-export CATALINA_BASE=/opt/tomcat
-export CATALINA_HOME=/opt/tomcat
-export M2_HOME="/usr/local/maven-3.2.1/"
-export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m"
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom"
+echo -e "\nStarting simulator"
+mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log &
 
-mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /dev/null &
-
-while ! nc -vz localhost 8096 2>&1 > /dev/null; do sleep 10; done
+while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done
+echo -e "\nStarting DataCenter deployment"
 python -m marvin.deployDataCenter -i setup/dev/advanced.cfg 2>&1 || true

tools/travis/downloadDeps.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This script should be used to install additional dependencies
+# This includes: installing ubuntu packages, custom services
+# or internet downloads.
+
+# Authored by Rafael da Fonseca <rsafonseca@gmail.com>
+
+#Get all dependency blocks from all pom.xml files in the project
+for line in $(find ../../ -name pom.xml -exec sed -n '/<dependencies>/{:a;n;/<\/dependencies>/b;p;ba}' {} \; | grep -e "artifactId" -e "groupId" -e "version" -e "dependency\>" -e "exclusion\>" -e "exclusions\>"| sed -e 's/\^M//'); do
+
+  #Tokenize values
+  set -- $(echo $line | awk -v FS="(>|<)" '{print $2, $3}')
+
+  #Start processing data
+  if [[ $1 == "dependency" ]]; then
+    #Create new artifact dep
+    unset ARTIFACT
+    unset VERSION
+    unset GROUP
+  elif [[ $1 == "/dependency" ]]; then
+    #Filter out project modules interdependencies
+    if [[ $GROUP != *org.apache.cloudstack* ]] && [[ $GROUP != *com.cloud* ]] && [[ $ARTIFACT != cloudstack-service-console-proxy-rdpclient ]]; then
+            if [[ -z $VERSION ]] ; then
+               VERSION=LATEST
+               #These dependencies don't support the LATEST keywork for some reason, and would cause mvn runs to file on dummy poms
+               if [[ $GROUP == jstl ]] || [[ $ARTIFACT == mysql-connector-java ]] || [[ $GROUP == org.apache.axis ]]; then
+                 continue
+               fi
+            fi
+            #Output resolved dependency to a file, to be picked up later
+            echo "$GROUP $ARTIFACT $VERSION" >> deps.out
+    fi
+  elif [[ $1 == "version" ]]; then
+    #If version is a maven var, get the value from parent pom
+    if [[ $2 == \$\{* ]]; then
+      VERSION=$(grep \<$(echo $2 | awk -v FS="(}|{)" '{print $2 }') ../../pom.xml | awk -v FS="(>|<)" '{print $3}')
+    #If version tag is empty, add LATEST to avoid maven errors
+    elif [[ "$2" == "" ]]; then
+      VERSION="LATEST"
+    else
+      VERSION=$2
+    fi
+  elif [[ $1 == "artifactId" ]]; then
+    #This avoids exclusions inside dependency block to overwrite original dependency
+    if [[ -z $ARTIFACT ]]; then
+      ARTIFACT=$2
+    fi
+  elif [[ $1 == "groupId" ]]; then
+    #This avoids exclusions inside dependency block to overwrite original dependency
+     if [[ -z $GROUP ]]; then
+       GROUP=$2
+     fi
+  fi
+done
+
+#Add the resolved plugins to properly download their dependencies
+while read line ; do
+    NAME=$(echo $line | sed -e 's/.jar$//')
+    VERSION=${NAME##*-}
+    ARTIFACT=${NAME%-*}
+    GROUP=$(find ~/.m2/repository -name ${NAME}.pom -exec sed -n "1,/${ARTIFACT}/p" {} \; | tac | grep -m 1 -e "<groupId>"  | sed -e 's/^[[:space:]]*//' -e 's/\^M//' | tr -d '\r' | awk -v FS="(>|<)" '{print $3}')
+    DATA="${GROUP} ${ARTIFACT} ${VERSION}"
+    echo $DATA >> deps.out
+done < /tmp/resolvedPlugins
+
+#Remove duplicates and sort them, LANG export is needed to fix some sorting issue, sorting is needed for later function that relies on sorted input
+cat deps.out | LANG=C sort -u > cleandeps.out
+
+#Define index of pomfiles, to avoid duplicate deps with different versions in pom.xml, several poms are created in case of more than one version of same artifact
+LASTPOM=0
+echo '<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>org.apache.cloudstack</groupId><artifactId>travis-build-deps</artifactId><name>Download Deps for Travis CI</name><version>1</version><repositories><repository><id>mido-maven-public-releases</id><name>mido-maven-public-releases</name><url>http://cs-maven.midokura.com/releases</url></repository><repository><id>juniper-contrail</id><url>http://juniper.github.io/contrail-maven/snapshots</url></repository></repositories><dependencies>' > pom${LASTPOM}.xml
+while read line ; do
+  set -- $line
+  #This relies on correct sorting, and distributes different versions of same dependency througout different pom files 
+  if [[ $2 == $LASTARTIFACT ]]; then
+    POMID=$(($POMID+1))
+    #If value is greater than current number of poms, create a new one
+    if [[ $POMID -gt $LASTPOM ]]; then
+       LASTPOM=$POMID
+       #This outputs the necessary structure to start a pom and also defines the extra repositories
+       echo '<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>org.apache.cloudstack</groupId><artifactId>travis-build-deps</artifactId><name>Download Deps for Travis CI</name><version>1</version><repositories><repository><id>mido-maven-public-releases</id><name>mido-maven-public-releases</name><url>http://cs-maven.midokura.com/releases</url></repository><repository><id>juniper-contrail</id><url>http://juniper.github.io/contrail-maven/snapshots</url></repository></repositories><dependencies>' > pom${LASTPOM}.xml
+    fi
+  else
+    POMID=0
+  fi
+  LASTARTIFACT=$2
+  echo "<dependency><groupId>$1</groupId><artifactId>$2</artifactId><version>$3</version></dependency>" >> pom${POMID}.xml
+done < cleandeps.out
+
+RETURN_CODE=0
+for ((i=0;i<=$LASTPOM;i++))
+do
+  echo "</dependencies></project>" >> pom${i}.xml
+  mvn org.apache.maven.plugins:maven-dependency-plugin:resolve -f pom${i}.xml
+  if [[ $? -ne 0 ]]; then
+    RETURN_CODE=1
+  fi
+done
+
+#Run a few plugin goals to download some more deps
+
+#Hack to run maven-jaxb2-plugin generate, can be removed when we stop using such an old version...
+mkdir -p src/main/resources
+echo '<?xml version="1.0" encoding="utf-8" ?><xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"></xs:schema>' > src/main/resources/test.xsd
+
+#Declare plugin tests to run
+declare -a arr=("maven-surefire-plugin test" "maven-pmd-plugin pmd" "maven-compiler-plugin compile" "maven-resources-plugin resources" "maven-checkstyle-plugin check" "maven-site-plugin attach-descriptor" "maven-surefire-plugin test" "maven-jar-plugin jar" "license-maven-plugin check" "maven-jgit-buildnumber-plugin extract-buildnumber" "maven-jaxb2-plugin generate" "maven-war-plugin war -DfailOnMissingWebXml=false" "gmaven-plugin compile")
+for i in "${arr[@]}"
+do
+    set -- $i
+    PLUGIN=$1
+    MOJO=$2
+    OPTION=$3
+    #Get every listed version of the plugin and make a run for each version
+    while read line ; do
+       set -- $line
+       JOBS="${JOBS} ${1}:${2}:${3}:${MOJO} $OPTION"
+    done < <(grep $PLUGIN cleandeps.out)
+done
+echo "Running $JOBS"
+#Call all the constructed plugin goals
+mvn $JOBS -f pom0.xml
+if [[ $? -ne 0 ]]; then
+  RETURN_CODE=1
+fi
+
+#Cleanup some files created in the run
+rm -rf deps.out src target
+
+exit $RETURN_CODE

tools/travis/install.sh

@@ -34,10 +34,7 @@ if [ $MOD -ne 0 ]; then
  fi
 fi
 
-export CATALINA_BASE=/opt/tomcat
-export CATALINA_HOME=/opt/tomcat
-export M2_HOME="/usr/local/maven-3.2.1/"
-export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m"
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom"
 
 if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then
    mvn -q -Pimpatient -Dsimulator clean install

tools/travis/script.sh

@@ -39,7 +39,6 @@ mkdir -p integration-test-results/component
 
 
 for suite in $1; do
-  travis_wait 30
   nosetests --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ;
 done
 

tools/travis/xunit-reader.py

@@ -97,4 +97,4 @@ def parse_reports(file_path_list):
 
 
 if __name__ == "__main__":
-    main()
+    main()