From 5001c15f9ed64674ba2fb1591ccdb86afc1897e8 Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Mon, 27 Aug 2018 16:45:12 -0500 Subject: [PATCH 1/5] Add portable builds to CI. --- netci.groovy | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/netci.groovy b/netci.groovy index 3e6736debb..c203db4b8c 100644 --- a/netci.groovy +++ b/netci.groovy @@ -55,16 +55,16 @@ def getDockerImageForOs(os) { return imageMap.get(os) } -def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) { +def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration, boolean portable) { job.with { steps { if (os == "Windows_NT") { batchFile("git submodule update --init --recursive"); - batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") + batchFile(".\\build.cmd /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") } else { shell("git submodule update --init --recursive"); - shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); + shell("./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); smokeTestExcludes = ""; if (os == "Fedora24" || os == "OSX10.12") { // Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560 @@ -78,29 +78,39 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) setMachineAffinity(job, os); } -def addPullRequestJob(String project, String branch, String os, String configuration, boolean runByDefault) +def addPullRequestJob(String project, String branch, String os, String configuration, boolean portable, boolean runByDefault) { - def newJobName = Utilities.getFullJobName(project, "${os}_${configuration}", true); + def config = configuration; + if (portable) { + config = "${configuration}_Portable" + } + def newJobName = Utilities.getFullJobName(project, "${os}_${config}", true); def contextString = "${os} ${configuration}"; + if (portable) { + contextString = "${os} ${configuration} Portable"; + } def triggerPhrase = "(?i).*test\\W+${contextString}.*"; def newJob = job(newJobName); - addBuildStepsAndSetMachineAffinity(newJob, os, configuration); + addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable); addArchival(newJob); Utilities.standardJobSetup(newJob, project, true, "*/${branch}"); Utilities.setJobTimeout(newJob, 180); Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase, !runByDefault); } -def addPushJob(String project, String branch, String os, String configuration) +def addPushJob(String project, String branch, String os, String configuration, boolean portable) { def shortJobName = "${os}_${configuration}"; + if (portable) { + shortJobName = "${os}_${configuration}_Portable"; + } def newJobName = Utilities.getFullJobName(project, shortJobName, false); def newJob = job(newJobName); - addBuildStepsAndSetMachineAffinity(newJob, os, configuration); + addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable); addArchival(newJob); Utilities.standardJobSetup(newJob, project, false, "*/${branch}"); Utilities.setJobTimeout(newJob, 180); @@ -108,14 +118,21 @@ def addPushJob(String project, String branch, String os, String configuration) } ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "CentOS7.1", "OSX10.12"].each { os -> - addPullRequestJob(project, branch, os, "Release", true); - addPullRequestJob(project, branch, os, "Debug", false); + addPullRequestJob(project, branch, os, "Release", false, true); + addPullRequestJob(project, branch, os, "Debug", false, false); }; +// do some but not all the portable builds +["Ubuntu16.04", "RHEL7.2", "OSX10.12"].each { os -> + addPullRequestJob(project, branch, os, "Release", true, true); +} + // Per push, run all the jobs ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os -> ["Release", "Debug"].each { configuration -> - addPushJob(project, branch, os, configuration); + [true, false].each { portability -> + addPushJob(project, branch, os, configuration, portability); + }; }; }; From f3f2b2978c5bf866d3376c3441a1477d87c59204 Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Tue, 28 Aug 2018 12:04:44 -0500 Subject: [PATCH 2/5] Address code review feedback. --- netci.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/netci.groovy b/netci.groovy index c203db4b8c..fb177020f1 100644 --- a/netci.groovy +++ b/netci.groovy @@ -82,12 +82,12 @@ def addPullRequestJob(String project, String branch, String os, String configura { def config = configuration; if (portable) { - config = "${configuration}_Portable" + config += "_Portable"; } def newJobName = Utilities.getFullJobName(project, "${os}_${config}", true); def contextString = "${os} ${configuration}"; if (portable) { - contextString = "${os} ${configuration} Portable"; + contextString += " Portable"; } def triggerPhrase = "(?i).*test\\W+${contextString}.*"; @@ -104,7 +104,7 @@ def addPushJob(String project, String branch, String os, String configuration, b { def shortJobName = "${os}_${configuration}"; if (portable) { - shortJobName = "${os}_${configuration}_Portable"; + shortJobName += "_Portable"; } def newJobName = Utilities.getFullJobName(project, shortJobName, false); @@ -130,8 +130,8 @@ def addPushJob(String project, String branch, String os, String configuration, b // Per push, run all the jobs ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os -> ["Release", "Debug"].each { configuration -> - [true, false].each { portability -> - addPushJob(project, branch, os, configuration, portability); + [true, false].each { portable -> + addPushJob(project, branch, os, configuration, portable); }; }; }; From 4bf161c778cd86c19c5a116e6f76a57501a3fbec Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Thu, 13 Sep 2018 11:47:05 -0500 Subject: [PATCH 3/5] Change to only do offline and tarball builds for portable since they require the online build anyway. --- netci.groovy | 133 +++++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 67 deletions(-) diff --git a/netci.groovy b/netci.groovy index fb177020f1..add4ff6fb6 100644 --- a/netci.groovy +++ b/netci.groovy @@ -122,11 +122,6 @@ def addPushJob(String project, String branch, String os, String configuration, b addPullRequestJob(project, branch, os, "Debug", false, false); }; -// do some but not all the portable builds -["Ubuntu16.04", "RHEL7.2", "OSX10.12"].each { os -> - addPullRequestJob(project, branch, os, "Release", true, true); -} - // Per push, run all the jobs ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os -> ["Release", "Debug"].each { configuration -> @@ -140,45 +135,47 @@ def addPushJob(String project, String branch, String os, String configuration, b [true, false].each { isPR -> ["RHEL7.2", "CentOS7.1"].each { os -> ["Release", "Debug"].each { configuration -> + [true, false].each { portable -> - def shortJobName = "${os}_Tarball_${configuration}"; - def contextString = "${os} Tarball ${configuration}"; - def triggerPhrase = "(?i).*test\\W+${contextString}.*"; + def shortJobName = "${os}_Tarball_${configuration}"; + def contextString = "${os} Tarball ${configuration}"; + def triggerPhrase = "(?i).*test\\W+${contextString}.*"; - def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){ - steps{ - shell("cd ./source-build;git submodule update --init --recursive"); - shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}"); - shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build"); + def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){ + steps{ + shell("cd ./source-build;git submodule update --init --recursive"); + shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} ${loggingOptions}"); + shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build"); - shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") - shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}") + shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") + shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}") + } } - } - setMachineAffinity(newJob, os); + setMachineAffinity(newJob, os); - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}"); + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}"); - // Increase timeout. The tarball builds can take longer than the 2 hour default. - Utilities.setJobTimeout(newJob, 240); + // Increase timeout. The tarball builds can take longer than the 2 hour default. + Utilities.setJobTimeout(newJob, 240); - // Clone into the source-build directory - Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build'); + // Clone into the source-build directory + Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build'); - addArchival(newJob); - if(isPR){ - if(configuration == "Release"){ - Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString); + addArchival(newJob); + if(isPR){ + if(configuration == "Release"){ + Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString); + } + else{ + Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase); + } } else{ - Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase); + Utilities.addGithubPushTrigger(newJob); } - } - else{ - Utilities.addGithubPushTrigger(newJob); - } + } } } } @@ -187,52 +184,54 @@ def addPushJob(String project, String branch, String os, String configuration, b [true, false].each { isPR -> ["RHEL7.2", "CentOS7.1"].each { os-> ["Release", "Debug"].each { configuration -> - - def shortJobName = "${os}_Unshared_${configuration}"; - def contextString = "${os} Unshared ${configuration}"; - def triggerPhrase = "(?i).*test\\W+${contextString}.*"; - def imageName = getDockerImageForOs(os); - - def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){ - steps{ - shell("cd ./source-build;git submodule update --init --recursive"); - // First build the product itself - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}"); - // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh - shell("mkdir tarball-output"); - // now build the tarball - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build"); - // now build from the tarball offline and without access to the regular non-tarball build - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); - // finally, run a smoke-test on the result - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}"); + [true, false].each { portable -> + + def shortJobName = "${os}_Unshared_${configuration}"; + def contextString = "${os} Unshared ${configuration}"; + def triggerPhrase = "(?i).*test\\W+${contextString}.*"; + def imageName = getDockerImageForOs(os); + + def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){ + steps{ + shell("cd ./source-build;git submodule update --init --recursive"); + // First build the product itself + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}"); + // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh + shell("mkdir tarball-output"); + // now build the tarball + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build"); + // now build from the tarball offline and without access to the regular non-tarball build + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); + // finally, run a smoke-test on the result + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}"); + } } - } - // Only Ubuntu Jenkins machines have Docker - setMachineAffinity(newJob, "Ubuntu16.04"); + // Only Ubuntu Jenkins machines have Docker + setMachineAffinity(newJob, "Ubuntu16.04"); - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}"); + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}"); - // Increase timeout. The offline build in Docker takes more than 2 hours. - Utilities.setJobTimeout(newJob, 240); + // Increase timeout. The offline build in Docker takes more than 2 hours. + Utilities.setJobTimeout(newJob, 240); - // Clone into the source-build directory - Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build'); + // Clone into the source-build directory + Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build'); - addArchival(newJob); - if(isPR){ - if(configuration == "Release"){ - Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString); + addArchival(newJob); + if(isPR){ + if(configuration == "Release"){ + Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString); + } + else{ + Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase); + } } else{ - Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase); + Utilities.addGithubPushTrigger(newJob); } - } - else{ - Utilities.addGithubPushTrigger(newJob); - } + } } } } From ee08499c768a62a64f900a0283cf8347a594be7f Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Thu, 13 Sep 2018 11:49:46 -0500 Subject: [PATCH 4/5] Fix tarball and offline build names. --- netci.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/netci.groovy b/netci.groovy index add4ff6fb6..e3ab4e5a3b 100644 --- a/netci.groovy +++ b/netci.groovy @@ -139,6 +139,12 @@ def addPushJob(String project, String branch, String os, String configuration, b def shortJobName = "${os}_Tarball_${configuration}"; def contextString = "${os} Tarball ${configuration}"; + + if (portable) { + shortJobName += "_Portable" + contextString += " Portable" + } + def triggerPhrase = "(?i).*test\\W+${contextString}.*"; def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){ @@ -188,6 +194,12 @@ def addPushJob(String project, String branch, String os, String configuration, b def shortJobName = "${os}_Unshared_${configuration}"; def contextString = "${os} Unshared ${configuration}"; + + if (portable) { + shortJobName += "_Portable" + contextString += " Portable" + } + def triggerPhrase = "(?i).*test\\W+${contextString}.*"; def imageName = getDockerImageForOs(os); From ad53bb58eed1071582a7a7b9036c50cc7292ae82 Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Thu, 13 Sep 2018 11:58:20 -0500 Subject: [PATCH 5/5] Add PR jobs that can be triggered for portable testing. --- netci.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/netci.groovy b/netci.groovy index e3ab4e5a3b..e4cd1af654 100644 --- a/netci.groovy +++ b/netci.groovy @@ -118,8 +118,14 @@ def addPushJob(String project, String branch, String os, String configuration, b } ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "CentOS7.1", "OSX10.12"].each { os -> + // Release non-portable run by default addPullRequestJob(project, branch, os, "Release", false, true); + // Debug non-portable can be triggered addPullRequestJob(project, branch, os, "Debug", false, false); + // Release portable can be triggered + addPullRequestJob(project, branch, os, "Release", true, false); + // Debug portable can be triggered + addPullRequestJob(project, branch, os, "Debug", true, false); }; // Per push, run all the jobs