Skip to content

[release/2.1] Add portable builds to CI. #718

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
crummel merged 5 commits into from
Sep 18, 2018
Merged

[release/2.1] Add portable builds to CI. #718

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5001c15
Add portable builds to CI.
crummel Aug 27, 2018
f3f2b29
Address code review feedback.
crummel Aug 28, 2018
4bf161c
Change to only do offline and tarball builds for portable since they …
crummel Sep 13, 2018
ee08499
Fix tarball and offline build names.
crummel Sep 13, 2018
ad53bb5
Add PR jobs that can be triggered for portable testing.
crummel Sep 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 106 additions & 72 deletions netci.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,16 @@ def getDockerImageForOs(os) {
return imageMap.get(os)
}

def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) {
def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration, boolean portable) {
job.with {
steps {
if (os == "Windows_NT") {
batchFile("git submodule update --init --recursive");
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
}
else {
shell("git submodule update --init --recursive");
shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
shell("./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
smokeTestExcludes = "";
if (os == "Fedora24" || os == "OSX10.12") {
// Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
Expand All @@ -78,90 +78,116 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
setMachineAffinity(job, os);
}

def addPullRequestJob(String project, String branch, String os, String configuration, boolean runByDefault)
def addPullRequestJob(String project, String branch, String os, String configuration, boolean portable, boolean runByDefault)
{
def newJobName = Utilities.getFullJobName(project, "${os}_${configuration}", true);
def config = configuration;
if (portable) {
config += "_Portable";
}
def newJobName = Utilities.getFullJobName(project, "${os}_${config}", true);
def contextString = "${os} ${configuration}";
if (portable) {
contextString += " Portable";
}
def triggerPhrase = "(?i).*test\\W+${contextString}.*";

def newJob = job(newJobName);

addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
addArchival(newJob);
Utilities.standardJobSetup(newJob, project, true, "*/${branch}");
Utilities.setJobTimeout(newJob, 180);
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase, !runByDefault);
}

def addPushJob(String project, String branch, String os, String configuration)
def addPushJob(String project, String branch, String os, String configuration, boolean portable)
{
def shortJobName = "${os}_${configuration}";
if (portable) {
shortJobName += "_Portable";
}

def newJobName = Utilities.getFullJobName(project, shortJobName, false);
def newJob = job(newJobName);

addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
addArchival(newJob);
Utilities.standardJobSetup(newJob, project, false, "*/${branch}");
Utilities.setJobTimeout(newJob, 180);
Utilities.addGithubPushTrigger(newJob);
}

["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "CentOS7.1", "OSX10.12"].each { os ->
addPullRequestJob(project, branch, os, "Release", true);
addPullRequestJob(project, branch, os, "Debug", false);
// Release non-portable run by default
addPullRequestJob(project, branch, os, "Release", false, true);
// Debug non-portable can be triggered
addPullRequestJob(project, branch, os, "Debug", false, false);
// Release portable can be triggered
addPullRequestJob(project, branch, os, "Release", true, false);
// Debug portable can be triggered
addPullRequestJob(project, branch, os, "Debug", true, false);
};

// Per push, run all the jobs
["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os ->
["Release", "Debug"].each { configuration ->
addPushJob(project, branch, os, configuration);
[true, false].each { portable ->
addPushJob(project, branch, os, configuration, portable);
};
};
};

// Tarball builds that are not enforced to be offline
[true, false].each { isPR ->
["RHEL7.2", "CentOS7.1"].each { os ->
["Release", "Debug"].each { configuration ->
[true, false].each { portable ->

def shortJobName = "${os}_Tarball_${configuration}";
def contextString = "${os} Tarball ${configuration}";
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
def shortJobName = "${os}_Tarball_${configuration}";
def contextString = "${os} Tarball ${configuration}";

def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
steps{
shell("cd ./source-build;git submodule update --init --recursive");
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
if (portable) {
shortJobName += "_Portable"
contextString += " Portable"
}

shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
def triggerPhrase = "(?i).*test\\W+${contextString}.*";

def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
steps{
shell("cd ./source-build;git submodule update --init --recursive");
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} ${loggingOptions}");
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");

shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
}
}
}

setMachineAffinity(newJob, os);
setMachineAffinity(newJob, os);

Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");

// Increase timeout. The tarball builds can take longer than the 2 hour default.
Utilities.setJobTimeout(newJob, 240);
// Increase timeout. The tarball builds can take longer than the 2 hour default.
Utilities.setJobTimeout(newJob, 240);

// Clone into the source-build directory
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
// Clone into the source-build directory
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');

addArchival(newJob);
if(isPR){
if(configuration == "Release"){
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
addArchival(newJob);
if(isPR){
if(configuration == "Release"){
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
}
else{
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
}
}
else{
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
Utilities.addGithubPushTrigger(newJob);
}
}
else{
Utilities.addGithubPushTrigger(newJob);
}

}
}
}
}
Expand All @@ -170,52 +196,60 @@ def addPushJob(String project, String branch, String os, String configuration)
[true, false].each { isPR ->
["RHEL7.2", "CentOS7.1"].each { os->
["Release", "Debug"].each { configuration ->
[true, false].each { portable ->

def shortJobName = "${os}_Unshared_${configuration}";
def contextString = "${os} Unshared ${configuration}";

def shortJobName = "${os}_Unshared_${configuration}";
def contextString = "${os} Unshared ${configuration}";
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
def imageName = getDockerImageForOs(os);

def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
steps{
shell("cd ./source-build;git submodule update --init --recursive");
// First build the product itself
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
shell("mkdir tarball-output");
// now build the tarball
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
// now build from the tarball offline and without access to the regular non-tarball build
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
// finally, run a smoke-test on the result
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
if (portable) {
shortJobName += "_Portable"
contextString += " Portable"
}
}

// Only Ubuntu Jenkins machines have Docker
setMachineAffinity(newJob, "Ubuntu16.04");
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
def imageName = getDockerImageForOs(os);

def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
steps{
shell("cd ./source-build;git submodule update --init --recursive");
// First build the product itself
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
shell("mkdir tarball-output");
// now build the tarball
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
// now build from the tarball offline and without access to the regular non-tarball build
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
// finally, run a smoke-test on the result
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
}
}

Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
// Only Ubuntu Jenkins machines have Docker
setMachineAffinity(newJob, "Ubuntu16.04");

// Increase timeout. The offline build in Docker takes more than 2 hours.
Utilities.setJobTimeout(newJob, 240);
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");

// Clone into the source-build directory
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
// Increase timeout. The offline build in Docker takes more than 2 hours.
Utilities.setJobTimeout(newJob, 240);

addArchival(newJob);
if(isPR){
if(configuration == "Release"){
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
// Clone into the source-build directory
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');

addArchival(newJob);
if(isPR){
if(configuration == "Release"){
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
}
else{
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
}
}
else{
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
Utilities.addGithubPushTrigger(newJob);
}
}
else{
Utilities.addGithubPushTrigger(newJob);
}

}
}
}
}
Expand Down