From c8ca5db1809d1ed206ed2875466e56bd671b6a46 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 10:47:02 +0100 Subject: [PATCH 001/127] Fetch build dependencies based on AD Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 33 +- .../pipelineBackend.config | 13 + .../utilities/fetchBuildDependencies.groovy | 359 ++++++++++++++++++ .../utilities/fetchBuildDependenciesUtils.sh | 90 +++++ 4 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy create mode 100644 Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 585d5054..5abfe9be 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -131,6 +131,7 @@ Help() { SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" +fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependenciesUtils.sh" # Customization - End # @@ -428,7 +429,37 @@ if [ $rc -eq 0 ]; then fi fi -# Ready to go TLD: Suggest in the section to echo as much as possible +# +# Validate to fetch external dependencies is based on the ApplicationDescriptor +retrieveBuildDependencies() { + + # extracting external dependencies is based on the application descriptor + applicationDescriptor="${AppDir}/applicationDescriptor.yml" + + # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) + externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" + mkdir -p "$(getLogDir)" + + # Set up to perform the clone of the Repo + if [ ! -f "${applicationDescriptor}" ]; then + rc=8 + ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc="$rc + echo $ERRMSG + else + fetchBuildDependencies + fi + + +} + +# Setup build environment and pull external dependencies if an ApplicationDescriptor is found +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then + retrieveBuildDependencies +fi + + +# +# Echo build configuration if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Started - DBB Build on HOST/USER: ${SYS}/${USER}" diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index a7b77fc0..cda95028 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -34,6 +34,19 @@ logsDir="logs" ## End of Shared Configuration parameters ######################################################## ##################################################################################################### + +##################################################################################################### +## Shared Configuration parameters ############################################################### +##################################################################################################### + +# Setting to enable fetching build dependencies configured within the Application Descriptor +# default: false +fetchBuildDependencies=true + +##################################################################################################### +## End of Shared Configuration parameters ######################################################## +##################################################################################################### + ##################################################################################################### ## DBB-BUILD.sh parameters ########################################################################## ##################################################################################################### diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy new file mode 100644 index 00000000..81992c30 --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -0,0 +1,359 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.metadata.* +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* +import com.ibm.dbb.build.report.* +import com.ibm.dbb.build.report.records.* +import groovy.yaml.YamlSlurper +import groovy.yaml.YamlBuilder +import groovy.lang.GroovyShell +import groovy.cli.commons.* +import groovy.util.* +import java.nio.file.* + + +// script properties +@Field Properties props = new Properties() +@Field def artifactRepositoryHelpers + +// Parse arguments from command-line +parseArgs(args) + +// Print parms +println("** Script configuration:") +props.each { k,v-> + println " $k -> $v" +} + +File artifactRepositoryHelpersScriptFile = new File(props.artifactRepositoryHelpersScript) +if (artifactRepositoryHelpersScriptFile.exists()) { + artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) +} else { + println("*! [ERROR] The Artifact Repository Helper script '${props.artifactRepositoryHelpersScript}' doesn't exist. Exiting.") + System.exit(1) +} + +File applicationDescriptorFile = new File(props.applicationDescriptor) +if (!applicationDescriptorFile.exists()) { + println("*! [ERROR] The Application Descriptor file '${props.applicationDescriptor}' doesn't exist. Exiting.") + System.exit(1) +} + +File applicationConfigurationFile = new File(props.applicationDependencyConfiguration) +if (!applicationConfigurationFile.exists()) { + println("*! [ERROR] The Application Configuration file '${props.applicationDependencyConfiguration}' doesn't exist. Exiting.") + System.exit(1) +} + +def yamlSlurper = new groovy.yaml.YamlSlurper() +// Parse the application descriptor and application configurations +applicationDescriptor = yamlSlurper.parse(applicationDescriptorFile) +applicationDependencyConfiguration = yamlSlurper.parse(applicationConfigurationFile) + + +ArrayList externalDependencies = new ArrayList<>() + +// If there are dependencies +if (applicationDescriptor.dependencies) { + + // Loop through all dependencies found in AD + applicationDescriptor.dependencies.each { dependency -> + + // Look for the dependency declaration in the Application Configuration file + applicationDependency = applicationDependencyConfiguration.find { it -> + it.application == dependency.name + } + + version = dependency.version + + // assess application dependency if found + if (applicationDependency) { + ExternalDependency externalDependency = new ExternalDependency() + // Map information to External Dependency Record + externalDependency.name = dependency.name // dependency name + externalDependency.type = dependency.type // dependency type + externalDependency.properties = new HashSet() + + if (dependency.type.equals("artifactrepository")) { + // download from artifactory + + if (version.equals("latest")) { + println("* Retrieving production version of application '${applicationDependency.application}'") + version = applicationDependency.productionVersion + } + + repositoryInfo = applicationDependency.repositories.find {it -> + it.name == dependency.type + } + repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) + + // this replicates the default format of package build outputs + + def String artifactUrl + def String artifactReference + def String artifactRelPath + if (dependency.version.startsWith("rel-")){ + artifactRelPath="${repositoryName}/main/release/${dependency.version}" + artifactReference="${artifactRelPath}/${dependency.name}.tar" + artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" + } else { + artifactRelPath="${repositoryName}" + artifactReference="${artifactRelPath}/${dependency.name}.tar" + artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" + } + println("*** Fetching package '${dependency.name}:${dependency.version}' ") + + // Generating information for packaging + Property p_uri = new Property() + p_uri.key = "uri" + p_uri.value = artifactReference + externalDependency.properties.add(p_uri) + Property p_version = new Property() + p_version.key = "version" + p_version.value = dependency.version + externalDependency.properties.add(p_version) + + importFolder = new File("$props.workspace/imports") + if (importFolder.exists()) importFolder.deleteDir() + importFolder.mkdirs() + + cacheFolder = new File("/tmp/.pkg") + if (!cacheFolder.exists()) cacheFolder.mkdirs() + + + String tarFile = "${cacheFolder}/${artifactReference}" + String includeFolder = "${importFolder}/${applicationDependency.application}" + + if (new File(tarFile).exists()) { + println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") + } else { + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + + if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() + + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") + def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) + println "download complete $rc" // TODO: Error handling in helper + } + + + File includeFolderFile = new File(includeFolder) + if (!includeFolderFile.exists()) { + includeFolderFile.mkdirs() + } + + + println("** Expanding tar file '$tarFile' to '$includeFolder' ") + + def processCmd = [ + "sh", + "-c", + "tar -C $includeFolder -xvf $tarFile" + ] + + def rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") + System.exit(1) + } + + } else { + println("* Dependency Types other than 'artifactrepository' are not yet implemented. Exiting.") + System.exit(1) + } + + // Store external dependency information + externalDependencies.add(externalDependency) + + } else { + println("*! [ERROR] Application dependency '${dependency.name}' not documented in Application Configurations File '${props.applicationConfigurations}'. Exiting.") + System.exit(1) + } + } +} + +// Do we actually need this or can this just be obtained from the dependencies? + +//Pulling in the baseline package +matchingBaseline = applicationDescriptor.baselines.find() { baseline -> + baseline.branch.equals(props.branch) +} +//if (matchingBaseline0) { +if ( 1 == 0 ){ + version = matchingBaseline.baseline + println("** Retrieving baseline package at version '${version}' for '${applicationDescriptor.application}'") + matchingDependency = applicationDependencyConfiguration.find { dependency -> + dependency.application == applicationDescriptor.application + } + if (matchingDependency) { + repositoryInfo = matchingDependency.repositories.find { repository -> + repository.name.equals("binary") + } + if (repositoryInfo) { + ExternalDependency externalDependency = new ExternalDependency() + // Map information to External Dependency Record + externalDependency.name = applicationDescriptor.application + externalDependency.type = "binary" + externalDependency.properties = new HashSet() + + // Adding key value pairs + Property p_uri = new Property() + p_uri.key = "uri" + p_uri.value = repositoryInfo.path + externalDependency.properties.add(p_uri) + Property p_version = new Property() + p_version.key = "version" + p_version.value = version + externalDependency.properties.add(p_version) + externalDependencies.add(externalDependency) + + String artifactUrl="${repositoryInfo.path}/${version}/${applicationDescriptor.application}-${version}.tar" + String tarFile = "${props.workspace}/imports/${applicationDescriptor.application}.tar" + String includeFolder = "${props.workspace}/imports/${applicationDescriptor.application}" + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) + + File includeFolderFile = new File(includeFolder) + if (!includeFolderFile.exists()) { + includeFolderFile.mkdirs() + } + + def processCmd = [ + "sh", + "-c", + "tar -C $includeFolder -xvf $tarFile" + ] + + def rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") + System.exit(1) + } + } else { + println("*! [ERROR] Couldn't find the application's 'binary' repository in the Application Configuration file. Exiting.") + System.exit(1) + } + } else { + println("*! [ERROR] Couldn't find the application '${applicationDescriptor.application}' in the Application Configuration file. Exiting.") + System.exit(1) + } +} else { + //println("*! [ERROR] Couldn't determine a baseline in the Application Descriptor file for application '${applicationDescriptor.application}'. Exiting.") + //System.exit(1) +} + +def yamlBuilder = new YamlBuilder() +// write file +if (props.externalDependenciesFilePath) { + + println("** Write ExternalDependency Record to ${props.externalDependenciesFilePath}") + yamlBuilder externalDependencies + + externalDependencyFile = new File(props.externalDependenciesFilePath) + externalDependencyFile.withWriter("IBM-1047") { extDepWriter -> + extDepWriter.write(yamlBuilder.toString()) + } +} + + +/** + * Parse CLI config + */ +def parseArgs(String[] args) { + + String usage = 'generateFetchStatements.groovy [options]' + + def cli = new CliBuilder(usage:usage) + // required sandbox options + cli.a(longOpt:'applicationDescriptor', args:1, 'Absolute path to the Application Descriptor file') + cli.w(longOpt:'workspace', args:1, 'Absolute path to the build workspace') + cli.d(longOpt:'externalDependenciesFilePath', args:1, 'Absolute path to the external dependencies file') + cli.p(longOpt:'pipelineBackendConfigFilePath', args:1, 'Absolute path to the pipelineBackend.config file') + cli.b(longOpt:'branch', args:1, 'Current branch of the application') + + + def opts = cli.parse(args) + if (!opts) { + System.exit(1) + } + + if (opts.a) { + props.applicationDescriptor = opts.a + } else { + println("*! [ERROR] Missing path to the Application Descriptor file. Exiting.") + System.exit(1) + } + + if (opts.w) { + props.workspace = opts.w + } else { + println("*! [ERROR] Missing path to the Workspace directory. Exiting.") + System.exit(1) + } + + if (opts.d) { + props.externalDependenciesFilePath = opts.d + } else { + println("*! [ERROR] Missing path to the External Dependencies file. Exiting.") + System.exit(1) + } + + if (opts.b) { + props.branch = opts.b + } else { + println("*! [ERROR] Missing current Branch name. Exiting.") + System.exit(1) + } + + if (opts.p) { + def pipelineBackendConfigFile = new File(opts.p) + if (pipelineBackendConfigFile.exists()) { + props.pipelineBackendConfigFile = opts.p + Properties temporaryProperties = new Properties() + pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } + props.put("applicationDependencyConfiguration", temporaryProperties.get("applicationDependencyConfiguration")) + props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) + props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) + props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) + props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) + props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) + } else { + println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") + System.exit(1) + } + } else { + println("*! [ERROR] Missing path to the pipelineBackend.config file. Exiting.") + System.exit(1) + } +} + +/********************************************************************************** + * run process + **********************************************************************************/ +def runProcess(ArrayList cmd){ + StringBuffer response = new StringBuffer() + StringBuffer error = new StringBuffer() + + def p = cmd.execute() + p.waitForProcessOutput(response, error) + + def rc = p.exitValue(); + if (rc != 0) { + println("*! [ERROR] Execution of command '$cmd' failed with rc=${rc} and message '${error.toString()}'") + } + return rc +} + +class ExternalDependency { + String name + String type + HashSet properties = new HashSet<>() +} + +class Property { + String key + String value +} \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh new file mode 100644 index 00000000..ceafb74e --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -0,0 +1,90 @@ +#!/bin/env bash +#=================================================================================== +# NAME: fetchDependencies.sh +# +# DESCRIPTION: The purpose of this script is to fetch all build dependencies +# +# SYNTAX: See Help() Section Below +# +# OPTIONS: See Help() Section Below +# +# RETURNS: +# +# rc - Return Code +# +# RETURN CODES: +# +# 0 - Successful +# 4 - Warning message(s) issued. See Console messages. +# 8 - Error encountered. See Console messages. +# +# NOTE(S): +# +# None +# +# Maintenance Log +# +# Date Who Vers Description +# ---------- --- ----- -------------------------------------------------------------- +# 2024/02/23 DB 1.0.0 Initial Release +#=================================================================================== + +fetchBuildDependencies() { + + # Read to go to fetch build dependencies configured in application descriptor + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] **************************************************************" + echo $PGM": [INFO] ** Start Fetch Build Dependencies on HOST/USER: ${SYS}/${USER}" + echo $PGM": [INFO] ** WorkDir:" $(getWorkDirectory) + echo $PGM": [INFO] ** Application:" ${application} + echo $PGM": [INFO] ** Branch:" ${branch} + echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} + if [ ! -z "${externalDependenciesLog}" ]; then + echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLog} + fi + echo $PGM": [INFO] **************************************************************" + echo "" + fi + # + + # Create import dir + if [ $rc -eq 0 ]; then + if [ ! -d "$(getWorkDirectory)/imports" ]; then + mkdir -p $(getWorkDirectory)/imports + fi + fi + + # Fetch Application Dependencies + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] ** Fetch Application Dependencies from Artifact Repository" + cmd="groovyz ${PIPELINE_SCRIPTS}/utilities/fetchBuildDependencies.groovy -w $(getWorkDirectory) -a ${applicationDescriptor} -p ${pipelineConfiguration} -b ${Branch}" + # + if [ ! -z "${artifactRepositoryUrl}" ]; then + CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" + fi + if [ ! -z "${artifactRepositoryUser}" ]; then + CMD="${CMD} --artifactRepositoryUser ${artifactRepositoryUser}" + fi + if [ ! -z "${artifactRepositoryPassword}" ]; then + CMD="${CMD} --artifactRepositoryPassword ${artifactRepositoryPassword}" + fi + if [ ! -z "${artifactRepositoryNamePattern}" ]; then + CMD="${CMD} --artifactRepositoryNamePattern ${artifactRepositoryNamePattern}" + fi + if [ ! -z "${artifactRepositoryDirectory}" ]; then + CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" + fi + if [ ! -z "${externalDependenciesLog}" ]; then + cmd="${cmd} -d ${externalDependenciesLog}" + fi + echo $PGM": [INFO] ** CMD : ${cmd}" + ${cmd} + rc=$? + fi + + if [ $rc -eq 0 ]; then + ERRMSG=$PGM": [INFO] Fetch Build Dependencies Completed. rc="$rc + echo $ERRMSG + fi + +} From 82a380932208cd49012159f26061cf3001040ac1 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 11:43:43 +0100 Subject: [PATCH 002/127] Missing import of Utilities Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 5abfe9be..df53787b 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -211,6 +211,15 @@ if [ $rc -eq 0 ]; then source $buildUtilities fi + # Read and import utilities + if [ ! -f "${fetchBuildDependenciesUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $fetchBuildDependenciesUtilities + fi + # # Get Options if [ $rc -eq 0 ]; then @@ -337,7 +346,7 @@ validateOptions() { # Check if application directory contains if [ -d "${AppDir}/${App}" ]; then - echo $PGM": [INFO] Detected the application respository (${App}) within the git repository layout structure." + echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." echo $PGM": [INFO] Assuming this as the new application location." AppDir="${AppDir}/${App}" nestedApplicationFolder="true" @@ -435,6 +444,8 @@ retrieveBuildDependencies() { # extracting external dependencies is based on the application descriptor applicationDescriptor="${AppDir}/applicationDescriptor.yml" + echo "######" $applicationDescriptor + # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" From e9eab81fbda3505b1532eeb04674500a14408bc5 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 13:11:11 +0100 Subject: [PATCH 003/127] Align fetchDependencies script with Application Descriptor Signed-off-by: Dennis Behm --- .../pipelineBackend.config | 39 ++- .../utilities/fetchBuildDependencies.groovy | 298 ++++++++---------- 2 files changed, 173 insertions(+), 164 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index cda95028..ffeabf40 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -53,9 +53,12 @@ fetchBuildDependencies=true # Path to main build script build.groovy. zAppBuild="$HOME/dbb-zappbuild" +# DBEHM: MyCustomization for Path to build.groovy. +zAppBuild="/var/dbb/dbb-zappbuild-DAT" # (Optional) Additional property files to customize the build framework zAppBuildPropFiles="" +zAppBuildPropFiles="/var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties,/var/dbb/dbb-zappbuild-config/buildFolder.properties" # High-Level Data Set Qualifier used for the DBB Build. # This prefix value will be used along with the @@ -121,7 +124,7 @@ featureBranchBuildBehaviour=merge-base # Path to PackageBuildOutputs.groovy script from the DBB community repository # e.g. PackagingScript="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -PackagingScript="/var/dbb/extensions/dbb-MD/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" +PackagingScript="/var/dbb/pipelineTemplates/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" ##################################################################################################### #### PackageBuildOutputs.groovy configuration #################################################### @@ -161,19 +164,22 @@ cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/v # Flag to indicate package upload to the provided Artifactory server. # Set to true to enable package upload # can be overridden by cli parm -p -publish=false +publish=true # URL to the Artifact repository server # e.q.: artifactRepositoryUrl=http://artifactoryserver:8081/artifactory -artifactRepositoryUrl="" +artifactRepositoryUrl=http://10.3.20.231:8081/artifactory # User to connect to the Artifact repository server # e.q.: artifactRepositoryUser=admin -artifactRepositoryUser="" +artifactRepositoryUser=admin # Password to connect to the Artifact repository server # e.q.: artifactRepositoryPassword=xxxxx -artifactRepositoryPassword="" +artifactRepositoryPassword=artifactoryadmin + +artifactRepositoryNamePattern=§application§-repo-local + ##################################################################################################### #### Methods enforcing naming conventions ######################################################## @@ -194,7 +200,14 @@ computeArtifactRepositoryDirectory() { if [ -z "${PipelineType}" ]; then echo "${Branch}" else + case $Branch in + main) echo "${Branch}/${PipelineType}" + ;; + *) + echo "${Branch}" + ;; + esac fi } @@ -256,6 +269,22 @@ ucdPassword="" ## End of UCD-Deploy.sh parameters ############################################################### ##################################################################################################### +##################################################################################################### +## fetchDependencies.sh parameters ########### +##################################################################################################### + +# Application configuration file +# this YAML format defines the references for the repositories for source or object-level +# dependencies +applicationDependencyConfiguration=/var/dbb/pipelineTemplates/dbb/Templates/Common-Backend-Scripts/config/applicationDependencyConfigurations.yaml + +# Location of the ArtifactRepositoryHelpers.groovy script +artifactRepositoryHelpersScript=/var/dbb/pipelineTemplates/dbb/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy + +##################################################################################################### +## End of fetchDependencies.sh parameters #### +##################################################################################################### + ##################################################################################################### ## wazideploy-generate.sh, wazideploy-deploy.sh and wazideploy-evidences.sh parameters ########### ##################################################################################################### diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 81992c30..06dffa22 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -40,16 +40,18 @@ if (!applicationDescriptorFile.exists()) { System.exit(1) } -File applicationConfigurationFile = new File(props.applicationDependencyConfiguration) -if (!applicationConfigurationFile.exists()) { - println("*! [ERROR] The Application Configuration file '${props.applicationDependencyConfiguration}' doesn't exist. Exiting.") - System.exit(1) -} +// setup import directory +File importFolder = new File("$props.workspace/imports") +if (importFolder.exists()) importFolder.deleteDir() +importFolder.mkdirs() + +// setup package cache +cacheFolder = new File("/tmp/.pkg") +if (!cacheFolder.exists()) cacheFolder.mkdirs() def yamlSlurper = new groovy.yaml.YamlSlurper() // Parse the application descriptor and application configurations applicationDescriptor = yamlSlurper.parse(applicationDescriptorFile) -applicationDependencyConfiguration = yamlSlurper.parse(applicationConfigurationFile) ArrayList externalDependencies = new ArrayList<>() @@ -60,51 +62,43 @@ if (applicationDescriptor.dependencies) { // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> - // Look for the dependency declaration in the Application Configuration file - applicationDependency = applicationDependencyConfiguration.find { it -> - it.application == dependency.name - } - version = dependency.version - // assess application dependency if found - if (applicationDependency) { - ExternalDependency externalDependency = new ExternalDependency() - // Map information to External Dependency Record - externalDependency.name = dependency.name // dependency name - externalDependency.type = dependency.type // dependency type - externalDependency.properties = new HashSet() - - if (dependency.type.equals("artifactrepository")) { - // download from artifactory - - if (version.equals("latest")) { - println("* Retrieving production version of application '${applicationDependency.application}'") - version = applicationDependency.productionVersion - } - - repositoryInfo = applicationDependency.repositories.find {it -> - it.name == dependency.type - } - repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) - - // this replicates the default format of package build outputs - - def String artifactUrl - def String artifactReference - def String artifactRelPath - if (dependency.version.startsWith("rel-")){ - artifactRelPath="${repositoryName}/main/release/${dependency.version}" - artifactReference="${artifactRelPath}/${dependency.name}.tar" - artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" - } else { - artifactRelPath="${repositoryName}" - artifactReference="${artifactRelPath}/${dependency.name}.tar" - artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" - } - println("*** Fetching package '${dependency.name}:${dependency.version}' ") - - // Generating information for packaging + if (dependency.type.equals("artifactrepository")) { + // download from artifactory + + // TODO: How do we deal with the latest available? + // if (version.equals("latest")) { + // println("* Retrieving production version of application '${applicationDependency.application}'") + // version = applicationDependency.productionVersion + // } + + // Construct the path within the Artifact repo + repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) + + def String artifactUrl + def String artifactReference + def String artifactRelPath + if (dependency.version.startsWith("rel-")){ + artifactRelPath="${repositoryName}/main/release/${dependency.version}" + artifactReference="${artifactRelPath}/${dependency.name}.tar" + artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" + } else { + artifactRelPath="${repositoryName}" + artifactReference="${artifactRelPath}/${dependency.name}.tar" + artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" + } + println("*** Fetching package '${dependency.name}:${dependency.version}' ") + + // Generating information for documentation in yaml file of retrieved dependencies + if (dependency.name != applicationDescriptor.application) { + + ExternalDependency externalDependency = new ExternalDependency() + // Map information to External Dependency Record + externalDependency.name = dependency.name // dependency name + externalDependency.type = dependency.type // dependency type + externalDependency.properties = new HashSet() + Property p_uri = new Property() p_uri.key = "uri" p_uri.value = artifactReference @@ -113,116 +107,38 @@ if (applicationDescriptor.dependencies) { p_version.key = "version" p_version.value = dependency.version externalDependency.properties.add(p_version) + + // Store external dependency information + externalDependencies.add(externalDependency) + } - importFolder = new File("$props.workspace/imports") - if (importFolder.exists()) importFolder.deleteDir() - importFolder.mkdirs() - - cacheFolder = new File("/tmp/.pkg") - if (!cacheFolder.exists()) cacheFolder.mkdirs() - - - String tarFile = "${cacheFolder}/${artifactReference}" - String includeFolder = "${importFolder}/${applicationDependency.application}" - - if (new File(tarFile).exists()) { - println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") - } else { - String user = props.artifactRepositoryUser - String password = props.artifactRepositoryPassword - - if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() - - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") - def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) - println "download complete $rc" // TODO: Error handling in helper - } - - - File includeFolderFile = new File(includeFolder) - if (!includeFolderFile.exists()) { - includeFolderFile.mkdirs() - } - - - println("** Expanding tar file '$tarFile' to '$includeFolder' ") - - def processCmd = [ - "sh", - "-c", - "tar -C $includeFolder -xvf $tarFile" - ] - - def rc = runProcess(processCmd) - if (rc != 0) { - println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") - System.exit(1) - } + String tarFile = "${cacheFolder}/${artifactReference}" + String includeFolder = "${importFolder}/${dependency.name}" + if (new File(tarFile).exists()) { + println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") } else { - println("* Dependency Types other than 'artifactrepository' are not yet implemented. Exiting.") - System.exit(1) - } - - // Store external dependency information - externalDependencies.add(externalDependency) + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword - } else { - println("*! [ERROR] Application dependency '${dependency.name}' not documented in Application Configurations File '${props.applicationConfigurations}'. Exiting.") - System.exit(1) - } - } -} + if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() -// Do we actually need this or can this just be obtained from the dependencies? + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") + def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) + println "download complete $rc" // TODO: Error handling in helper + } -//Pulling in the baseline package -matchingBaseline = applicationDescriptor.baselines.find() { baseline -> - baseline.branch.equals(props.branch) -} -//if (matchingBaseline0) { -if ( 1 == 0 ){ - version = matchingBaseline.baseline - println("** Retrieving baseline package at version '${version}' for '${applicationDescriptor.application}'") - matchingDependency = applicationDependencyConfiguration.find { dependency -> - dependency.application == applicationDescriptor.application - } - if (matchingDependency) { - repositoryInfo = matchingDependency.repositories.find { repository -> - repository.name.equals("binary") - } - if (repositoryInfo) { - ExternalDependency externalDependency = new ExternalDependency() - // Map information to External Dependency Record - externalDependency.name = applicationDescriptor.application - externalDependency.type = "binary" - externalDependency.properties = new HashSet() - - // Adding key value pairs - Property p_uri = new Property() - p_uri.key = "uri" - p_uri.value = repositoryInfo.path - externalDependency.properties.add(p_uri) - Property p_version = new Property() - p_version.key = "version" - p_version.value = version - externalDependency.properties.add(p_version) - externalDependencies.add(externalDependency) - - String artifactUrl="${repositoryInfo.path}/${version}/${applicationDescriptor.application}-${version}.tar" - String tarFile = "${props.workspace}/imports/${applicationDescriptor.application}.tar" - String includeFolder = "${props.workspace}/imports/${applicationDescriptor.application}" - String user = props.artifactRepositoryUser - String password = props.artifactRepositoryPassword - artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) File includeFolderFile = new File(includeFolder) if (!includeFolderFile.exists()) { includeFolderFile.mkdirs() } + + println("** Expanding tar file '$tarFile' to '$includeFolder' ") + def processCmd = [ - "sh", + "/bin/sh", "-c", "tar -C $includeFolder -xvf $tarFile" ] @@ -233,18 +149,82 @@ if ( 1 == 0 ){ System.exit(1) } } else { - println("*! [ERROR] Couldn't find the application's 'binary' repository in the Application Configuration file. Exiting.") + println("* Dependency Types other than 'artifactrepository' are not yet implemented. Exiting.") System.exit(1) } - } else { - println("*! [ERROR] Couldn't find the application '${applicationDescriptor.application}' in the Application Configuration file. Exiting.") - System.exit(1) } -} else { - //println("*! [ERROR] Couldn't determine a baseline in the Application Descriptor file for application '${applicationDescriptor.application}'. Exiting.") - //System.exit(1) } +// Do we actually need this or can this just be obtained from the dependencies? + +// Fetch baseline package for internal dependencies such as +// derived copybooks and object decks + +//baselineRecord = applicationDescriptor.baselines.find() { baseline -> +// baseline.branch.equals(props.branch) +//} +// +//if (baselineRecord){ +// version = matchingBaseline.baseline +// println("** Retrieving baseline package at version '${version}' for '${applicationDescriptor.application}'") +// +// baselineName=applicationDescriptor.application +// +// repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", baselineName) +// +// def String artifactUrl +// def String artifactReference +// def String artifactRelPath +// +// if (dependency.version.startsWith("rel-")){ +// artifactRelPath="${repositoryName}/main/release/${baseline.version}" +// artifactReference="${artifactRelPath}/${baselineName}.tar" +// artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" +// } else { +// artifactRelPath="${repositoryName}" +// artifactReference="${artifactRelPath}/${baselineName}.tar" +// artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" +// } +// println("*** Fetching baseline package '${baselineName}:${baseline.version}' ") +// +// String tarFile = "${cacheFolder}/${artifactReference}" +// String includeFolder = "${importFolder}/${baselineName}" +// +// if (new File(tarFile).exists()) { +// println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") +// } else { +// String user = props.artifactRepositoryUser +// String password = props.artifactRepositoryPassword +// +// if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() +// +// println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") +// def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) +// println "download complete $rc" // TODO: Error handling in helper +// } +// +// // subdir for each dependency +// File includeFolderFile = new File(includeFolder) +// if (!includeFolderFile.exists()) { +// includeFolderFile.mkdirs() +// } +// +// // Expand tar file +// def processCmd = [ +// "/bin/sh", +// "-c", +// "tar -C $includeFolder -xvf $tarFile" +// ] +// +// def rc = runProcess(processCmd) +// if (rc != 0) { +// println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") +// System.exit(1) +// } +//} else { +// println("** [INFO] No baseline record found for application '${applicationDescriptor.application}'.") +//} + def yamlBuilder = new YamlBuilder() // write file if (props.externalDependenciesFilePath) { @@ -314,12 +294,12 @@ def parseArgs(String[] args) { props.pipelineBackendConfigFile = opts.p Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } - props.put("applicationDependencyConfiguration", temporaryProperties.get("applicationDependencyConfiguration")) - props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) - props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) - props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) - props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) - props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) + if(temporaryProperties.get("applicationDependencyConfiguration")) props.put("applicationDependencyConfiguration", temporaryProperties.get("applicationDependencyConfiguration")) + if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) + if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) + if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) + if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) + if(temporaryProperties.get("artifactRepositoryNamePattern")) props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) } else { println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") System.exit(1) From 8de77944308a98ee0eba2ce22114101e0930d9cb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 15:50:53 +0100 Subject: [PATCH 004/127] initial documentation Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index e24ea8c4..9f6a973e 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -209,7 +209,7 @@ gitClone.sh: [INFO] Clone Repository Complete. rc=0 ## Build stage -### Conventions +### Conventions and capabilities #### Git branch naming conventions @@ -236,6 +236,17 @@ Note that the location of the baselineReferences.config file can be customized i [baselineReference.config](samples/baselineReference.config) is a sample, that indicates the baseline for the `main` and `release maintenance` branches. +#### Fetching build dependencies + +Both implementations for the build stage are enabled to pull external dependencies into the build workspace based on the dependency definitions specified in the *Application Descriptor*. + +The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository that are necessary inputs to the build. Please read more about the Application Descriptor at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which generates it and provides insights about cross application dependencies. + +Each application package can export shared components such as shared copybooks and even build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/) script. + + + +##### sad ### dbbBuild.sh for zAppBuild framework From f584cfc99adc2c5f94184c600b6f0753690796a5 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 15:51:16 +0100 Subject: [PATCH 005/127] initial documentation Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 9f6a973e..298c23e5 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -238,15 +238,15 @@ Note that the location of the baselineReferences.config file can be customized i #### Fetching build dependencies -Both implementations for the build stage are enabled to pull external dependencies into the build workspace based on the dependency definitions specified in the *Application Descriptor*. +Both implementations for the build stage are enabled to optionally pull external dependencies into the build workspace based on the dependency definitions specified in the *Application Descriptor*. The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository that are necessary inputs to the build. Please read more about the Application Descriptor at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which generates it and provides insights about cross application dependencies. -Each application package can export shared components such as shared copybooks and even build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/) script. +Each application package can export shared components such as shared copybooks and even build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the Artifact repository through the Common Backend scripts. Fetching the dependencies is powered on the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) +Packages will be stored at a cache location to improve performance. - -##### sad +Fetched packages will be expanded to the pipeline working directory. ### dbbBuild.sh for zAppBuild framework From 9e98a02b70cf9ce4f0dcbf3bbbd3b562fcf44b0d Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 17:02:21 +0100 Subject: [PATCH 006/127] Consolidate fetch script for both dbbBuild.sh and zBuilder.sh Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 33 ++----------------- .../utilities/fetchBuildDependenciesUtils.sh | 26 ++++++++++++++- Templates/Common-Backend-Scripts/zBuilder.sh | 23 +++++++++++-- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index df53787b..57af35e1 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -125,14 +125,13 @@ Help() { } # -# Build Type Customization # Configuration file leveraged by the backend scripts # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +# Utility scripts buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependenciesUtils.sh" -# Customization - End # # Internal Variables @@ -140,7 +139,7 @@ fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependencies #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.10" +PGMVERS="1.20" USER=$USER SYS=$(uname -Ia) @@ -155,7 +154,6 @@ PipelineType="" HELP=$1 # Local Variables -# TLD: Always a good idea to initialize any local varables AppDir="" # Derived Application Directory HLQ="" # Derived High Level Qualifier HLQPrefix="" # Prefix of HLQ, either specified via the cli option -q or via configuration file @@ -438,37 +436,12 @@ if [ $rc -eq 0 ]; then fi fi -# -# Validate to fetch external dependencies is based on the ApplicationDescriptor -retrieveBuildDependencies() { - - # extracting external dependencies is based on the application descriptor - applicationDescriptor="${AppDir}/applicationDescriptor.yml" - echo "######" $applicationDescriptor - - - # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) - externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" - mkdir -p "$(getLogDir)" - - # Set up to perform the clone of the Repo - if [ ! -f "${applicationDescriptor}" ]; then - rc=8 - ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc="$rc - echo $ERRMSG - else - fetchBuildDependencies - fi - - -} # Setup build environment and pull external dependencies if an ApplicationDescriptor is found if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then - retrieveBuildDependencies + fetchBuildDependencies fi - # # Echo build configuration if [ $rc -eq 0 ]; then diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index ceafb74e..8067dd36 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -29,7 +29,8 @@ # 2024/02/23 DB 1.0.0 Initial Release #=================================================================================== -fetchBuildDependencies() { +# Internal script to fetch +runFetchLogic() { # Read to go to fetch build dependencies configured in application descriptor if [ $rc -eq 0 ]; then @@ -88,3 +89,26 @@ fetchBuildDependencies() { fi } + +# +# Validate to fetch external dependencies is based on the ApplicationDescriptor +fetchBuildDependencies() { + + # extracting external dependencies is based on the application descriptor + applicationDescriptor="${AppDir}/applicationDescriptor.yml" + echo "######" $applicationDescriptor + + + # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) + externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" + mkdir -p "$(getLogDir)" + + # Set up to perform the clone of the Repo + if [ ! -f "${applicationDescriptor}" ]; then + rc=8 + ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc="$rc + echo $ERRMSG + else + runFetchLogic + fi +} \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh index 861638d1..c588b675 100644 --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -128,8 +128,9 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +# Utility scripts buildUtilities="${SCRIPT_HOME}/utilities/dbbzBuilderUtils.sh" - +fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependenciesUtils.sh" # # Internal Variables @@ -137,7 +138,7 @@ buildUtilities="${SCRIPT_HOME}/utilities/dbbzBuilderUtils.sh" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$USER SYS=$(uname -Ia) @@ -205,6 +206,15 @@ if [ $rc -eq 0 ]; then source $buildUtilities fi + # Read and import utilities + if [ ! -f "${fetchBuildDependenciesUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $fetchBuildDependenciesUtilities + fi + # # Get Options if [ $rc -eq 0 ]; then @@ -438,7 +448,14 @@ if [ $rc -eq 0 ]; then fi fi -# Ready to go: Suggest in the section to echo as much as possible + +# Setup build environment and pull external dependencies if an ApplicationDescriptor is found +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then + fetchBuildDependencies +fi + +# +# Echo build configuration if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Started - DBB Build on HOST/USER: ${SYS}/${USER}" From 7b49b4ec9b005fcdb991c6e371147921b001ca27 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 30 Jan 2025 17:22:47 +0100 Subject: [PATCH 007/127] Set execute bits for zBuilder.sh Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/zBuilder.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 Templates/Common-Backend-Scripts/zBuilder.sh diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh old mode 100644 new mode 100755 From ca23c007c91069cfd2f1096cc980558a039eba71 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Feb 2025 09:34:09 +0100 Subject: [PATCH 008/127] Update release info and add page break Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependenciesUtils.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index 8067dd36..9546fa55 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -26,7 +26,7 @@ # # Date Who Vers Description # ---------- --- ----- -------------------------------------------------------------- -# 2024/02/23 DB 1.0.0 Initial Release +# 2025/01/30 DB 1.0.0 Initial Release #=================================================================================== # Internal script to fetch @@ -86,6 +86,7 @@ runFetchLogic() { if [ $rc -eq 0 ]; then ERRMSG=$PGM": [INFO] Fetch Build Dependencies Completed. rc="$rc echo $ERRMSG + echo "" fi } @@ -96,9 +97,7 @@ fetchBuildDependencies() { # extracting external dependencies is based on the application descriptor applicationDescriptor="${AppDir}/applicationDescriptor.yml" - echo "######" $applicationDescriptor - - + # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" mkdir -p "$(getLogDir)" From 98e9de0205122dc077f3ea6ac0df59976c9698cc Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 5 Feb 2025 16:47:46 +0100 Subject: [PATCH 009/127] Doc updates from code review Co-authored-by: Mathieu Dalbin --- Templates/Common-Backend-Scripts/README.md | 4 ++-- Templates/Common-Backend-Scripts/dbbBuild.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 298c23e5..971fe24c 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -240,9 +240,9 @@ Note that the location of the baselineReferences.config file can be customized i Both implementations for the build stage are enabled to optionally pull external dependencies into the build workspace based on the dependency definitions specified in the *Application Descriptor*. -The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository that are necessary inputs to the build. Please read more about the Application Descriptor at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which generates it and provides insights about cross application dependencies. +The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository, which are necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. -Each application package can export shared components such as shared copybooks and even build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the Artifact repository through the Common Backend scripts. Fetching the dependencies is powered on the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) +Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. Fetching the dependencies is powered by the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script. Packages will be stored at a cache location to improve performance. diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 57af35e1..fe577e51 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -437,7 +437,7 @@ if [ $rc -eq 0 ]; then fi -# Setup build environment and pull external dependencies if an ApplicationDescriptor is found +# Setup build environment and pull external dependencies if an Application Descriptor is found if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then fetchBuildDependencies fi From b87255e8997f543ab802786b5b478b3fd9197efc Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 5 Feb 2025 17:20:17 +0100 Subject: [PATCH 010/127] Configure Cache directory Signed-off-by: Dennis Behm --- .../pipelineBackend.config | 14 ++++++++-- .../utilities/fetchBuildDependencies.groovy | 27 +++++++++++-------- .../utilities/fetchBuildDependenciesUtils.sh | 21 +++------------ 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index ffeabf40..0592a74a 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -36,15 +36,25 @@ logsDir="logs" ##################################################################################################### -## Shared Configuration parameters ############################################################### +## Fetch Build Dependencies based on Application Descriptor ######################################## ##################################################################################################### # Setting to enable fetching build dependencies configured within the Application Descriptor # default: false fetchBuildDependencies=true +# Settings for caching packages on build machine + +# Setting to control if packages should be stored on the build machine +# default: false +enablePackageCache=true + +# Location for cachine packages on the build machine +packageCacheLocation=$HOME/cbs/.packagecache + + ##################################################################################################### -## End of Shared Configuration parameters ######################################################## +## End of Fetch Build Dependencies based on Application Descriptor ################################# ##################################################################################################### ##################################################################################################### diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 06dffa22..e53f9162 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -46,8 +46,8 @@ if (importFolder.exists()) importFolder.deleteDir() importFolder.mkdirs() // setup package cache -cacheFolder = new File("/tmp/.pkg") -if (!cacheFolder.exists()) cacheFolder.mkdirs() +tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean() && props.packageCacheLocation) ? new File(props.packageCacheLocation) : new File("$props.workspace/imports_download/") +if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() def yamlSlurper = new groovy.yaml.YamlSlurper() // Parse the application descriptor and application configurations @@ -112,18 +112,18 @@ if (applicationDescriptor.dependencies) { externalDependencies.add(externalDependency) } - String tarFile = "${cacheFolder}/${artifactReference}" + String tarFile = "${tmpPackageDir}/${artifactReference}" String includeFolder = "${importFolder}/${dependency.name}" if (new File(tarFile).exists()) { - println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") + println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") } else { String user = props.artifactRepositoryUser String password = props.artifactRepositoryPassword - if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() + if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) println "download complete $rc" // TODO: Error handling in helper } @@ -148,6 +148,10 @@ if (applicationDescriptor.dependencies) { println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") System.exit(1) } + + // Delete temporary download location if cache is not used + if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) {tmpPackageDir.deleteDir()} + } else { println("* Dependency Types other than 'artifactrepository' are not yet implemented. Exiting.") System.exit(1) @@ -187,18 +191,18 @@ if (applicationDescriptor.dependencies) { // } // println("*** Fetching baseline package '${baselineName}:${baseline.version}' ") // -// String tarFile = "${cacheFolder}/${artifactReference}" +// String tarFile = "${tmpPackageDir}/${artifactReference}" // String includeFolder = "${importFolder}/${baselineName}" // // if (new File(tarFile).exists()) { -// println("** Package was already found in package cache at '${cacheFolder}/${artifactRelPath}'") +// println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") // } else { // String user = props.artifactRepositoryUser // String password = props.artifactRepositoryPassword // -// if (!(new File("${cacheFolder}/${artifactRelPath}").exists())) (new File("${cacheFolder}/${artifactRelPath}")).mkdirs() +// if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() // -// println("** Downloading application package '$artifactUrl' from Artifact Repository into ${cacheFolder}/${artifactRelPath}.") +// println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") // def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) // println "download complete $rc" // TODO: Error handling in helper // } @@ -254,7 +258,6 @@ def parseArgs(String[] args) { cli.p(longOpt:'pipelineBackendConfigFilePath', args:1, 'Absolute path to the pipelineBackend.config file') cli.b(longOpt:'branch', args:1, 'Current branch of the application') - def opts = cli.parse(args) if (!opts) { System.exit(1) @@ -300,6 +303,8 @@ def parseArgs(String[] args) { if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) if(temporaryProperties.get("artifactRepositoryNamePattern")) props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) + if(temporaryProperties.get("packageCacheLocation")) props.put("packageCacheLocation", temporaryProperties.get("packageCacheLocation")) + if(temporaryProperties.get("enablePackageCache")) props.put("enablePackageCache", temporaryProperties.get("enablePackageCache")) } else { println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") System.exit(1) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index 9546fa55..eb1e54e1 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -40,9 +40,9 @@ runFetchLogic() { echo $PGM": [INFO] ** Application:" ${application} echo $PGM": [INFO] ** Branch:" ${branch} echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} - if [ ! -z "${externalDependenciesLog}" ]; then - echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLog} - fi + echo $PGM": [INFO] ** Use Package Cache :" ${enablePackageCache} + echo $PGM": [INFO] ** Package Cache Location :" ${packageCacheLocation} + echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLog} echo $PGM": [INFO] **************************************************************" echo "" fi @@ -60,21 +60,6 @@ runFetchLogic() { echo $PGM": [INFO] ** Fetch Application Dependencies from Artifact Repository" cmd="groovyz ${PIPELINE_SCRIPTS}/utilities/fetchBuildDependencies.groovy -w $(getWorkDirectory) -a ${applicationDescriptor} -p ${pipelineConfiguration} -b ${Branch}" # - if [ ! -z "${artifactRepositoryUrl}" ]; then - CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" - fi - if [ ! -z "${artifactRepositoryUser}" ]; then - CMD="${CMD} --artifactRepositoryUser ${artifactRepositoryUser}" - fi - if [ ! -z "${artifactRepositoryPassword}" ]; then - CMD="${CMD} --artifactRepositoryPassword ${artifactRepositoryPassword}" - fi - if [ ! -z "${artifactRepositoryNamePattern}" ]; then - CMD="${CMD} --artifactRepositoryNamePattern ${artifactRepositoryNamePattern}" - fi - if [ ! -z "${artifactRepositoryDirectory}" ]; then - CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" - fi if [ ! -z "${externalDependenciesLog}" ]; then cmd="${cmd} -d ${externalDependenciesLog}" fi From 9a1f583b107355c876edc5f51b9513527aea5518 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 5 Feb 2025 17:31:52 +0100 Subject: [PATCH 011/127] Configure Cache directory Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 6 ++++++ .../utilities/fetchBuildDependenciesUtils.sh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index e53f9162..d96707a4 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -257,6 +257,8 @@ def parseArgs(String[] args) { cli.d(longOpt:'externalDependenciesFilePath', args:1, 'Absolute path to the external dependencies file') cli.p(longOpt:'pipelineBackendConfigFilePath', args:1, 'Absolute path to the pipelineBackend.config file') cli.b(longOpt:'branch', args:1, 'Current branch of the application') + cli.c(longOpt:'packageCacheLocation', args:1, 'Location of the Package cache') + def opts = cli.parse(args) if (!opts) { @@ -291,6 +293,10 @@ def parseArgs(String[] args) { System.exit(1) } + if(opts.c){ + props.packageCacheLocation=opts.c + } + if (opts.p) { def pipelineBackendConfigFile = new File(opts.p) if (pipelineBackendConfigFile.exists()) { diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index eb1e54e1..84a7c791 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -63,6 +63,10 @@ runFetchLogic() { if [ ! -z "${externalDependenciesLog}" ]; then cmd="${cmd} -d ${externalDependenciesLog}" fi + + if [ ! -z "${packageCacheLocation}" ]; then + cmd="${cmd} -c ${packageCacheLocation}" + fi echo $PGM": [INFO] ** CMD : ${cmd}" ${cmd} rc=$? From e923fd2eebb4a513415b6a3e1732c1df70b4710e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 5 Feb 2025 17:36:04 +0100 Subject: [PATCH 012/127] Configure Cache directory Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index d96707a4..fb377354 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -294,9 +294,9 @@ def parseArgs(String[] args) { } if(opts.c){ - props.packageCacheLocation=opts.c + props.packageCacheLocation = opts.c } - + if (opts.p) { def pipelineBackendConfigFile = new File(opts.p) if (pipelineBackendConfigFile.exists()) { @@ -309,7 +309,6 @@ def parseArgs(String[] args) { if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) if(temporaryProperties.get("artifactRepositoryNamePattern")) props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) - if(temporaryProperties.get("packageCacheLocation")) props.put("packageCacheLocation", temporaryProperties.get("packageCacheLocation")) if(temporaryProperties.get("enablePackageCache")) props.put("enablePackageCache", temporaryProperties.get("enablePackageCache")) } else { println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") From 84998167cc1155981ee95ce0b11f9bb12ceddda1 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 5 Feb 2025 17:39:40 +0100 Subject: [PATCH 013/127] Configure Cache directory Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependenciesUtils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index 84a7c791..9a453768 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -37,8 +37,8 @@ runFetchLogic() { echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Start Fetch Build Dependencies on HOST/USER: ${SYS}/${USER}" echo $PGM": [INFO] ** WorkDir:" $(getWorkDirectory) - echo $PGM": [INFO] ** Application:" ${application} - echo $PGM": [INFO] ** Branch:" ${branch} + echo $PGM": [INFO] ** Application:" ${App} + echo $PGM": [INFO] ** Branch:" ${Branch} echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} echo $PGM": [INFO] ** Use Package Cache :" ${enablePackageCache} echo $PGM": [INFO] ** Package Cache Location :" ${packageCacheLocation} From 4a32af5c6d3389307963c122efae4baa05fdc476 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Feb 2025 09:43:45 +0100 Subject: [PATCH 014/127] Add dbbCommunityRepoRoot Signed-off-by: Dennis Behm --- .../pipelineBackend.config | 30 +++++++++---------- .../utilities/fetchBuildDependencies.groovy | 4 +-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 0592a74a..5ac515e3 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -28,7 +28,12 @@ buildRootDir="$PIPELINE_WORKSPACE" # Relative directory name for outputs # default: logsDir="logs" -logsDir="logs" +logsDir="logs" + +# root directory of the location of the cloned DBB community repository +# https://github.com/IBM/dbb on the build machine +# Note: Cannot reference environment variables +dbbCommunityRepoRootDir=/var/dbb/pipelineTemplates/dbb ##################################################################################################### ## End of Shared Configuration parameters ######################################################## @@ -43,6 +48,10 @@ logsDir="logs" # default: false fetchBuildDependencies=true +# Relative path of the ArtifactRepositoryHelpers.groovy script which is part of the dbb community repository +# Note: Cannot reference environment variables +artifactRepositoryHelpersScript=Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy + # Settings for caching packages on build machine # Setting to control if packages should be stored on the build machine @@ -52,6 +61,10 @@ enablePackageCache=true # Location for cachine packages on the build machine packageCacheLocation=$HOME/cbs/.packagecache +# Relative path of the applicationDescriptorHelperUtils which is part of the dbb community repository +# Note: Cannot reference environment variables +# default +applicationDescriptorHelperUtils=Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy ##################################################################################################### ## End of Fetch Build Dependencies based on Application Descriptor ################################# @@ -279,21 +292,6 @@ ucdPassword="" ## End of UCD-Deploy.sh parameters ############################################################### ##################################################################################################### -##################################################################################################### -## fetchDependencies.sh parameters ########### -##################################################################################################### - -# Application configuration file -# this YAML format defines the references for the repositories for source or object-level -# dependencies -applicationDependencyConfiguration=/var/dbb/pipelineTemplates/dbb/Templates/Common-Backend-Scripts/config/applicationDependencyConfigurations.yaml - -# Location of the ArtifactRepositoryHelpers.groovy script -artifactRepositoryHelpersScript=/var/dbb/pipelineTemplates/dbb/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy - -##################################################################################################### -## End of fetchDependencies.sh parameters #### -##################################################################################################### ##################################################################################################### ## wazideploy-generate.sh, wazideploy-deploy.sh and wazideploy-evidences.sh parameters ########### diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index fb377354..d1dac761 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -26,7 +26,7 @@ props.each { k,v-> println " $k -> $v" } -File artifactRepositoryHelpersScriptFile = new File(props.artifactRepositoryHelpersScript) +File artifactRepositoryHelpersScriptFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.artifactRepositoryHelpersScript}") if (artifactRepositoryHelpersScriptFile.exists()) { artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) } else { @@ -303,7 +303,7 @@ def parseArgs(String[] args) { props.pipelineBackendConfigFile = opts.p Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } - if(temporaryProperties.get("applicationDependencyConfiguration")) props.put("applicationDependencyConfiguration", temporaryProperties.get("applicationDependencyConfiguration")) + if(temporaryProperties.get("dbbCommunityRepoRootDir")) props.put("dbbCommunityRepoRootDir", temporaryProperties.get("dbbCommunityRepoRootDir")) if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) From 27587cf676eafc460a1638b5f4af832e34e9113b Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Feb 2025 09:55:31 +0100 Subject: [PATCH 015/127] Read Application Descriptor via utils Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index d1dac761..485e5c40 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -15,7 +15,8 @@ import java.nio.file.* // script properties @Field Properties props = new Properties() -@Field def artifactRepositoryHelpers +@Field def artifactRepositoryHelpers // Helpers to download +@Field def applicationDescriptorUtils // Helper to parse Application Descriptor // Parse arguments from command-line parseArgs(args) @@ -26,6 +27,7 @@ props.each { k,v-> println " $k -> $v" } +// Load and verify helpers File artifactRepositoryHelpersScriptFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.artifactRepositoryHelpersScript}") if (artifactRepositoryHelpersScriptFile.exists()) { artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) @@ -34,6 +36,15 @@ if (artifactRepositoryHelpersScriptFile.exists()) { System.exit(1) } +File applicationDescriptorUtilsFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.applicationDescriptorHelperUtils}") + +if (applicationDescriptorUtilsFile.exists()) { + applicationDescriptorUtils = loadScript(applicationDescriptorUtilsFile) +} else { + println("*! [ERROR] The Application Descriptor Helper script '${props.applicationDescriptorHelperUtils}' was not found. Exiting.") + System.exit(1) +} + File applicationDescriptorFile = new File(props.applicationDescriptor) if (!applicationDescriptorFile.exists()) { println("*! [ERROR] The Application Descriptor file '${props.applicationDescriptor}' doesn't exist. Exiting.") @@ -50,8 +61,9 @@ tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean( if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() def yamlSlurper = new groovy.yaml.YamlSlurper() -// Parse the application descriptor and application configurations -applicationDescriptor = yamlSlurper.parse(applicationDescriptorFile) + +// Parse the application descriptor and application configurations based on the defined schema +applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) ArrayList externalDependencies = new ArrayList<>() From 7e1ac6d99cc2fd1bfd64c981268afcd259b0e7d6 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Feb 2025 09:59:16 +0100 Subject: [PATCH 016/127] Read Application Descriptor via utils Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 485e5c40..0eb06502 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -316,7 +316,10 @@ def parseArgs(String[] args) { Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } if(temporaryProperties.get("dbbCommunityRepoRootDir")) props.put("dbbCommunityRepoRootDir", temporaryProperties.get("dbbCommunityRepoRootDir")) + if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) + if(temporaryProperties.get("applicationDescriptorHelperUtils")) props.put("applicationDescriptorHelperUtils", temporaryProperties.get("applicationDescriptorHelperUtils")) + if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) From 88f2df32c8de6795d4ea7f6901a275aae476b8fd Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Feb 2025 10:00:57 +0100 Subject: [PATCH 017/127] Read Application Descriptor via utils Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 0eb06502..cbc5dfc9 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -23,7 +23,7 @@ parseArgs(args) // Print parms println("** Script configuration:") -props.each { k,v-> +props.sort().each { k,v-> println " $k -> $v" } @@ -316,10 +316,8 @@ def parseArgs(String[] args) { Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } if(temporaryProperties.get("dbbCommunityRepoRootDir")) props.put("dbbCommunityRepoRootDir", temporaryProperties.get("dbbCommunityRepoRootDir")) - if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) if(temporaryProperties.get("applicationDescriptorHelperUtils")) props.put("applicationDescriptorHelperUtils", temporaryProperties.get("applicationDescriptorHelperUtils")) - if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) From 2881037385494e672ea6eef94c2c749d3db50bcb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Feb 2025 14:46:41 +0100 Subject: [PATCH 018/127] Update Packaging process Signed-off-by: Dennis Behm --- .../PackageBuildOutputs.groovy | 2 +- .../packageBuildOutputs.sh | 43 ++++++++----------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 1e87657b..9130d60e 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -965,7 +965,7 @@ def parseInput(String[] cliArgs){ // Artifact repository options :: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') - cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') + cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server. (Optional)') // Artifact repository info cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)') diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 30bca6d8..cf91d973 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -57,11 +57,6 @@ Help() { echo " " echo " Optional parameters " echo " " - echo " -t - Name of the package tar file " - echo " (Optional) " - echo " " - echo " Ex: package.tar " - echo " " echo " -u - flag to enable uploading " echo " outputs to configured " echo " artifact repository " @@ -111,6 +106,8 @@ Help() { # Central configuration file leveraged by the backend scripts SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" + # Path and File Name to the advanced debug options. #log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" @@ -180,6 +177,17 @@ if [ $rc -eq 0 ]; then fi fi +# Source helper +if [ $rc -eq 0 ]; then + if [ ! -f "${packageUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Packaging Utils (${packageUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $packageUtilities + fi +fi + # # Get Options if [ $rc -eq 0 ]; then @@ -210,17 +218,6 @@ if [ $rc -eq 0 ]; then fi App="$argument" ;; - t) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc - echo $ERRMSG - break - fi - tarFileName="$argument" - ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -411,16 +408,10 @@ if [ $rc -eq 0 ]; then validateOptions fi -# compute parameters +# compute packaging parameters if [ $rc -eq 0 ]; then - # Compute artifactRepositoryName based on function in packageBuildOutputs.config - artifactRepositoryName=$(computeArtifactRepositoryName) - artifactRepositoryDirectory=$(computeArtifactRepositoryDirectory) - - # set default PipelineType for pipelines on main - if [ "${Branch}" == "main" ] && [ -z "${PipelineType}" ]; then - PipelineType="build" - fi + # invoke function in packageUtils + computePackageInformation fi # Call validate publishing options @@ -453,7 +444,7 @@ if [ $rc -eq 0 ]; then fi if [ ! -z "${artifactVersionName}" ]; then - echo $PGM": [INFO] ** Artifact name:" ${artifactVersionName} + echo $PGM": [INFO] ** Artifact Version:" ${artifactVersionName} fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} From d4bd7dd96489c2be69695998278e51f91c6ba7ec Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Feb 2025 14:48:51 +0100 Subject: [PATCH 019/127] Update Packaging notes Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index cf91d973..e0df7dd2 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -177,7 +177,7 @@ if [ $rc -eq 0 ]; then fi fi -# Source helper +# Source packaging helper if [ $rc -eq 0 ]; then if [ ! -f "${packageUtilities}" ]; then rc=8 From 4aba150fea2000fa1f0acb54e1f02450e4228aaa Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Feb 2025 15:06:28 +0100 Subject: [PATCH 020/127] packageUtils Signed-off-by: Dennis Behm --- .../utilities/packageUtils.sh | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 Templates/Common-Backend-Scripts/utilities/packageUtils.sh diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh new file mode 100755 index 00000000..b5e625c6 --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# internal variables +mainBranchSegment="" +secondBranchSegment="" +rc=0 + +# outputs +artifactRepositoryAbsoluteUrl="" +artifactRepositoryName="" +artifactRepositoryDirectory="" + +computePackageInformation() { + + branchConvention=(${Branch//// }) + + if [ $rc -eq 0 ]; then + if [ ${#branchConvention[@]} -gt 3 ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Script is only managing branch name with up to 3 segments (${Branch}) . See recommended naming conventions. rc="$rc + echo $ERRMSG + fi + fi + + if [ $rc -eq 0 ]; then + + # split the segments + mainBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $1 }') + secondBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $2 }') + thirdBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $3 }') + + # remove chars (. -) from the name + mainBranchSegmentTrimmed=$(echo ${mainBranchSegment} | tr -d '.-' | tr '[:lower:]' '[:upper:]') + + artifactRepositoryName=$(echo "${artifactRepositoryRepoPattern}") + + # evaluate main segment + case $mainBranchSegmentTrimmed in + "MASTER" | "MAIN" | REL*) + if [ "${PipelineType}" == "release" ]; then + artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternReleaseBuilds}) + + else + artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternSnapshotBuilds}) + fi + ;; + *) + artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternSnapshotBuilds}) + ;; + esac + + # Fixed convention + tarFileName="${App}-${artifactVersionName}.tar" + + artifactRepositoryAbsoluteUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" + + # unset internal variables + mainBranchSegment="" + secondBranchSegment="" + thirdBranchSegment="" + + fi +} + + + +if [ $rc -eq 0 ]; then + while getopts ":a:b:p:v:" opt; do + case $opt in + a) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + App="$argument" + ;; + b) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc + echo $ERRMSG + break + fi + Branch="$argument" + ;; + p) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc + echo $INFO + break + fi + PipelineType="$argument" + ;; + v) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the artifact version in Artifact repository is required. rc="$rc + echo $ERRMSG + break + fi + artifactVersionName="$argument" + ;; + :) + rc=4 + ERRMSG=$PGM": [WARNING] Option -$OPTARG requires an argument. rc="$rc + echo $ERRMSG + break + ;; + esac + done + + # Invoke package configuration + if [ ! -z "${artifactRepositoryUrl}" ] && [ ! -z "${App}" ] && [ ! -z "${artifactVersionName}" ] && [ ! -z "${PipelineType}" ] && [ ! -z "${Branch}" ]; then + computePackageInformation + echo $artifactRepositoryAbsoluteUrl + #echo $artifactRepositoryName + #echo $artifactRepositoryDirectory + fi + +fi \ No newline at end of file From 43797c7fcdc28522d34b6effbb186b8a08a817a4 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Feb 2025 18:02:23 +0100 Subject: [PATCH 021/127] Add eyecatcher to build frameworks Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 2 +- Templates/Common-Backend-Scripts/zBuilder.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index fe577e51..7813df43 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -508,7 +508,7 @@ if [ $rc -eq 0 ]; then CMD="${CMD} ${Type}" # Append zAppBuild Build Type echo $PGM": [INFO] ${CMD}" - ${CMD} #TLD: I commented this out for testing purposed + ${CMD} | awk '{print "dbb (groovyz) [zAppBuild] " $0}' #DEBUG: I commented this out for testing purposed rc=$? #exit 0 diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh index c588b675..6ab6c48f 100755 --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -511,7 +511,7 @@ if [ $rc -eq 0 ]; then fi echo $PGM": [INFO] ${CMD}" - ${CMD} # run build + ${CMD} | awk '{print "dbb [zBuilder] " $0}' # run build rc=$? if [ $rc -eq 0 ]; then From 4b25d17693020bccf097bc1b2f3ab576246ac139 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Feb 2025 15:09:57 +0100 Subject: [PATCH 022/127] why on earth again Signed-off-by: Dennis Behm --- Templates/GitlabCIPipeline/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/GitlabCIPipeline/.gitlab-ci.yml b/Templates/GitlabCIPipeline/.gitlab-ci.yml index 1f095b16..2dbbe23b 100644 --- a/Templates/GitlabCIPipeline/.gitlab-ci.yml +++ b/Templates/GitlabCIPipeline/.gitlab-ci.yml @@ -569,7 +569,7 @@ Packaging: wdEvidencesDir: ${wdEvidencesRoot}/${application}/production artifacts: - name: “productionDeploymentReport-${CI_PIPELINE_ID}" + name: productionDeploymentReport-${CI_PIPELINE_ID}" when: always paths: - "${productionReportDir}/deployment-report.html" From 2d9da76da02a4e3beabfc0d0586a9419cce32752 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Feb 2025 15:10:35 +0100 Subject: [PATCH 023/127] bla Signed-off-by: Dennis Behm --- Templates/GitlabCIPipeline/.gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Templates/GitlabCIPipeline/.gitlab-ci.yml b/Templates/GitlabCIPipeline/.gitlab-ci.yml index 2dbbe23b..d0ac3beb 100644 --- a/Templates/GitlabCIPipeline/.gitlab-ci.yml +++ b/Templates/GitlabCIPipeline/.gitlab-ci.yml @@ -569,7 +569,11 @@ Packaging: wdEvidencesDir: ${wdEvidencesRoot}/${application}/production artifacts: +<<<<<<< HEAD name: productionDeploymentReport-${CI_PIPELINE_ID}" +======= + name: "productionDeploymentReport-${CI_PIPELINE_ID}" +>>>>>>> bb3af91 ( remove odd char) when: always paths: - "${productionReportDir}/deployment-report.html" From 333c0f79f3f1d451738c309f950dfe04499faa98 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Feb 2025 13:28:01 +0100 Subject: [PATCH 024/127] add buildIdentifier and releaseIdentifier Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index e0df7dd2..55e333aa 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -117,7 +117,7 @@ packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) @@ -139,6 +139,8 @@ generateSBOM="" sbomAuthor="" publish="" +buildIdentifier="" +releaseIdentifier="" artifactVersionName="" # required for publishing to artifact repo artifactRepositoryUrl="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryUser="" # required if artifactRepositoryPropertyFile not specified @@ -188,10 +190,11 @@ if [ $rc -eq 0 ]; then fi fi + # # Get Options if [ $rc -eq 0 ]; then - while getopts ":h:w:a:t:b:v:p:us:" opt; do + while getopts ":h:w:a:b:i:r:v:p:us:" opt; do case $opt in h) Help @@ -218,6 +221,17 @@ if [ $rc -eq 0 ]; then fi App="$argument" ;; + i) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -255,16 +269,22 @@ if [ $rc -eq 0 ]; then fi PipelineType="$argument" ;; - v) + r) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the artifact version in Artifact repository is required. rc="$rc + ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc echo $ERRMSG break fi - artifactVersionName="$argument" + releaseIdentifier="$argument" + ;; + v) + argument="$OPTARG" + rc=4 + ERRMSG=$PGM": [WARNING] The argument (-v) for naming the version is no longer supported. Please switch to supply the build identifier argument (-i) and for release pipelines the release identifier argument (-r). rc="$rc + echo $ERRMSG ;; \?) Help @@ -281,7 +301,7 @@ if [ $rc -eq 0 ]; then done fi # -# Validate options + validateOptions() { if [ -z "${Workspace}" ]; then rc=8 @@ -299,6 +319,12 @@ validateOptions() { fi fi + if [ -z "${buildIdentifier}" ]; then + ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" + echo $ERRMSG + buildIdentifier=$(date +%Y%m%d_%H%M%S) + fi + # Validate Packaging script if [ ! -f "${PackagingScript}" ]; then rc=8 @@ -340,12 +366,6 @@ validatePublishingOptions() { fi else - if [ -z "${artifactVersionName}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Name of the artifact version (artifactVersionName) is required. rc="$rc - echo $ERRMSG - fi - if [ -z "${artifactRepositoryUrl}" ]; then rc=8 ERRMSG=$PGM": [ERROR] URL to artifact repository (artifactRepositoryUrl) is required. rc="$rc @@ -444,7 +464,7 @@ if [ $rc -eq 0 ]; then fi if [ ! -z "${artifactVersionName}" ]; then - echo $PGM": [INFO] ** Artifact Version:" ${artifactVersionName} + echo $PGM": [INFO] ** Artifact Version:" ${artifactVersionName} fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} @@ -554,7 +574,7 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ${CMD}" - /bin/env bash -c "${CMD}" + ${CMD} rc=$? if [ $rc -eq 0 ]; then From 2eea9b1ecc2bc4c1543db67747144bb4333b244e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Feb 2025 13:47:00 +0100 Subject: [PATCH 025/127] Compute publishing parameters for packaging Signed-off-by: Dennis Behm --- .../utilities/packageUtils.sh | 129 +++++++----------- 1 file changed, 47 insertions(+), 82 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh index b5e625c6..6db190fa 100755 --- a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh @@ -1,17 +1,24 @@ -#!/usr/bin/env bash - # internal variables mainBranchSegment="" secondBranchSegment="" rc=0 -# outputs -artifactRepositoryAbsoluteUrl="" -artifactRepositoryName="" -artifactRepositoryDirectory="" -computePackageInformation() { +computePackageInformation() { + ############################################# + # output environment variables + ############################################# + artifactRepositoryName="" # identifier of the artifact repo + artifactRepositoryDirectory="" # root directory folder in repo + artifactVersionName="" # subfolder in repo path identifying version / origin branch + tarFileName="" # computed tarFileName how it is stored in the artifact repository + artifactRepositoryUrl="" # absolute URL + ############################################# + + # configuration variable defining the Artifactory repository name pattern + artifactRepositoryRepoPattern="${App}-repo-local" + branchConvention=(${Branch//// }) if [ $rc -eq 0 ]; then @@ -38,21 +45,46 @@ computePackageInformation() { case $mainBranchSegmentTrimmed in "MASTER" | "MAIN" | REL*) if [ "${PipelineType}" == "release" ]; then - artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternReleaseBuilds}) - + ############################################# + # Conventions for release builds: + # ///--.tar + # MortgageApplication-repo-local/release/1.2.3/MortgageApplication-1.2.3-1234567890.tar + ############################################# + + # Release builds are captured in the release directory of the artifact repo + artifactRepositoryDirectory="release" + + # artifactVersionName is second identifier in the folder structure and represents the + artifactVersionName=${releaseIdentifier} + + # building up the tarFileName + tarFileName="${App}-${releaseIdentifier}-${buildIdentifier}.tar" else - artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternSnapshotBuilds}) + ############################################# + # Conventions for snapshot builds: + # ///-.tar + # Mortgage-repo-local/build/feature/123-enhance-something/Mortgage-123456.tar + ############################################# + + artifactRepositoryDirectory="build" + artifactVersionName=${Branch} + tarFileName="${App}-${buildIdentifier}.tar" fi ;; *) - artifactRepositoryDirectory=$(echo ${artifactRepositoryRepoDirectoryPatternSnapshotBuilds}) + ############################################# + ### similar to snapshot builds + ############################################# + artifactRepositoryDirectory="build" + artifactVersionName=${Branch} + tarFileName="${App}-${buildIdentifier}.tar" ;; esac - # Fixed convention - tarFileName="${App}-${artifactVersionName}.tar" - - artifactRepositoryAbsoluteUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" + ############################################# + ### Construct the absolute repository URL + ############################################# + artifactRepositoryUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" # unset internal variables mainBranchSegment="" @@ -62,70 +94,3 @@ computePackageInformation() { fi } - - -if [ $rc -eq 0 ]; then - while getopts ":a:b:p:v:" opt; do - case $opt in - a) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc - echo $ERRMSG - break - fi - App="$argument" - ;; - b) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc - echo $ERRMSG - break - fi - Branch="$argument" - ;; - p) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc - echo $INFO - break - fi - PipelineType="$argument" - ;; - v) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] The name of the artifact version in Artifact repository is required. rc="$rc - echo $ERRMSG - break - fi - artifactVersionName="$argument" - ;; - :) - rc=4 - ERRMSG=$PGM": [WARNING] Option -$OPTARG requires an argument. rc="$rc - echo $ERRMSG - break - ;; - esac - done - - # Invoke package configuration - if [ ! -z "${artifactRepositoryUrl}" ] && [ ! -z "${App}" ] && [ ! -z "${artifactVersionName}" ] && [ ! -z "${PipelineType}" ] && [ ! -z "${Branch}" ]; then - computePackageInformation - echo $artifactRepositoryAbsoluteUrl - #echo $artifactRepositoryName - #echo $artifactRepositoryDirectory - fi - -fi \ No newline at end of file From 068cc8b21ce84c3bb3a28c15f811a3f39f41da46 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Feb 2025 13:55:37 +0100 Subject: [PATCH 026/127] fix duplicate variable usage Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/utilities/packageUtils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh index 6db190fa..aa4e0fcc 100755 --- a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh @@ -13,7 +13,7 @@ computePackageInformation() { artifactRepositoryDirectory="" # root directory folder in repo artifactVersionName="" # subfolder in repo path identifying version / origin branch tarFileName="" # computed tarFileName how it is stored in the artifact repository - artifactRepositoryUrl="" # absolute URL + artifactRepositoryAbsoluteUrl="" # absolute URL ############################################# # configuration variable defining the Artifactory repository name pattern @@ -84,7 +84,7 @@ computePackageInformation() { ############################################# ### Construct the absolute repository URL ############################################# - artifactRepositoryUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" + artifactRepositoryAbsoluteUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" # unset internal variables mainBranchSegment="" From fd398d21e21494bf518a93f930b8c526286da5a9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Feb 2025 15:03:08 +0100 Subject: [PATCH 027/127] introduce packageBuildIdentifier Signed-off-by: Dennis Behm --- .../PackageBuildOutputs/PackageBuildOutputs.groovy | 8 ++++++-- .../utilities/WaziDeployManifestGenerator.groovy | 2 +- .../Common-Backend-Scripts/packageBuildOutputs.sh | 10 ++++++++-- .../Common-Backend-Scripts/utilities/packageUtils.sh | 4 ++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 9130d60e..0c970316 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -636,7 +636,7 @@ if (rc == 0) { if (props.publish && props.publish.toBoolean()) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") - packageInfo.put("name", props.versionName) + packageInfo.put("name", props.packageBuildIdentifier) packageUrl = computeAbsoluteRepositoryUrl() if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) @@ -950,8 +950,10 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') + cli.bi(longOpt:'packageBuildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier metadata stored in Wazi Deploys Application Manifest file.') cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') + // Concert Build Manifest generation cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.') @@ -1022,7 +1024,9 @@ def parseInput(String[] cliArgs){ if (opts.il) props.includeLogs = opts.il if (opts.a) props.application = opts.a if (opts.b) props.branch = opts.b - + if (opts.bi) props.packageBuildIdentifier = opts.bi + + // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : (props.generateWaziDeployAppManifest ? props.generateWaziDeployAppManifest : 'false') props.generateConcertBuildManifest = (opts.ic) ? 'true' : (props.generateConcertBuildManifest ? props.generateConcertBuildManifest : 'false') diff --git a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy index a4a98efd..47e50e18 100644 --- a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy @@ -29,7 +29,7 @@ def initWaziDeployManifestGenerator(Properties props) { } // Metadata information - wdManifest.metadata.version = (props.versionName) ? props.versionName : props.startTime + wdManifest.metadata.version = (props.packageBuildIdentifier) ? props.packageBuildIdentifier : props.startTime if (props.application) wdManifest.metadata.name = props.application // Annotations diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 55e333aa..b483982e 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -463,8 +463,8 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} fi - if [ ! -z "${artifactVersionName}" ]; then - echo $PGM": [INFO] ** Artifact Version:" ${artifactVersionName} + if [ ! -z "${packageBuildIdentifier}" ]; then + echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} @@ -538,6 +538,11 @@ if [ $rc -eq 0 ]; then if [ ! -z "${artifactVersionName}" ]; then CMD="${CMD} --versionName ${artifactVersionName}" fi + + # Wazi Deploy build identifier + if [ ! -z "${packageBuildIdentifier}" ]; then + CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" + fi # publishing options if [ "$publish" == "true" ]; then @@ -562,6 +567,7 @@ if [ $rc -eq 0 ]; then if [ ! -z "${artifactRepositoryDirectory}" ]; then CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" fi + fi # SBOM options diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh index aa4e0fcc..36eda29d 100755 --- a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh @@ -14,6 +14,7 @@ computePackageInformation() { artifactVersionName="" # subfolder in repo path identifying version / origin branch tarFileName="" # computed tarFileName how it is stored in the artifact repository artifactRepositoryAbsoluteUrl="" # absolute URL + packageBuildIdentifier="" # Identifier for Wazi Deploy Application Manifest file ############################################# # configuration variable defining the Artifactory repository name pattern @@ -59,6 +60,7 @@ computePackageInformation() { # building up the tarFileName tarFileName="${App}-${releaseIdentifier}-${buildIdentifier}.tar" + packageBuildIdentifier="${releaseIdentifier}-${buildIdentifier}" else ############################################# # Conventions for snapshot builds: @@ -69,6 +71,7 @@ computePackageInformation() { artifactRepositoryDirectory="build" artifactVersionName=${Branch} tarFileName="${App}-${buildIdentifier}.tar" + packageBuildIdentifier="${buildIdentifier}" fi ;; *) @@ -78,6 +81,7 @@ computePackageInformation() { artifactRepositoryDirectory="build" artifactVersionName=${Branch} tarFileName="${App}-${buildIdentifier}.tar" + packageBuildIdentifier="${buildIdentifier}" ;; esac From d09cdacb9b084e406240746ece727dce3e442bc3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 10:31:55 +0100 Subject: [PATCH 028/127] Refactored PackageBuildOutputs to return url Signed-off-by: Dennis Behm --- .../PackageBuildOutputs.groovy | 1316 +++++++++-------- 1 file changed, 668 insertions(+), 648 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 0c970316..22daef86 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -66,9 +66,11 @@ import com.ibm.jzos.ZFile; ************************************************************************************/ // start create & publish package -@Field Properties props = null +@Field Properties props = new Properties() def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) +@Field def artifactRepositoryHelpers = loadScript(new File("${scriptDir}/ArtifactRepositoryHelpers.groovy")) + @Field def applicationDescriptorUtils @Field def applicationDescriptor @Field def sbomUtilities @@ -81,740 +83,736 @@ def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).p @Field String libSubfolder = "lib" @Field String binSubfolder = "bin" @Field def tempLoadDir +@Field def String tarFileLabel = "Default" +@Field def String tarFileName = "" -def startTime = new Date() -props = parseInput(args) +parseInput(args) -props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") -println("** PackageBuildOutputs start at $props.startTime") -println("** Properties at startup:") -props.sort().each { k,v-> - if ( k == "artifactRepository.password" ) - println " $k -> xxxxxx " - else - println " $k -> $v" -} -if (rc != 0) { - println("*! [ERROR] One or several properties were missing in the configuration. Review the console output.") - System.exit(rc) -} +def packageOutputs(){ + + startTime = new Date() + props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") + println("** PackageBuildOutputs start at $props.startTime") + println("** Properties at startup:") + props.sort().each { k,v-> + if ( k == "artifactRepository.password" ) + println " $k -> xxxxxx " + else + println " $k -> $v" + } + if (rc != 0) { + println("*! [ERROR] One or several properties were missing in the configuration. Review the console output.") + System.exit(rc) + } -// Enable file tagging -BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging + // Enable file tagging + BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging -// Map of last level dataset qualifier to DBB CopyToHFS CopyMode. -def copyModeMap = parseCopyModeMap(props.copyModeMap) + // Map of last level dataset qualifier to DBB CopyToHFS CopyMode. + def copyModeMap = parseCopyModeMap(props.copyModeMap) -// Hashmap of BuildOutput to Record -Map buildOutputsMap = new HashMap() + // Hashmap of BuildOutput to Record + Map buildOutputsMap = new HashMap() -// Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. -@Field def String tarFileLabel = "Default" -@Field def String tarFileName = "" -// Field to store build number, set to default when its not decipherable from build report -def String buildNumber = "UNKNOWN" - -// Object to store scm information for Wazi Deploy Application Manifest file -HashMap scmInfo = new HashMap() - -// Package the public Include Files and service submodules -// if Path to Application Descriptor file is specified -if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { - File applicationDescriptorFile = new File("${props.applicationFolderPath}/applicationDescriptor.yml") - if (applicationDescriptorFile.exists()) { - applicationDescriptorUtils = loadScript(new File("utilities/applicationDescriptorUtils.groovy")) - applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) - } else { - println("*! [ERROR] No Application Descriptor file '${props.applicationFolderPath}/applicationDescriptor.yml' found. Exiting.") - System.exit(1) - } -} + // Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. -// iterate over all build reports to obtain build output -props.buildReportOrder.each { buildReportFile -> - Map temporaryBuildOutputsMap = new HashMap() - println("** Read build report data from '${buildReportFile}'.") - def jsonOutputFile = new File(buildReportFile) + // Field to store build number, set to default when its not decipherable from build report + def String buildNumber = "UNKNOWN" - if (!jsonOutputFile.exists()){ - println("*! [ERROR] Build Report '$buildReportFile' not found.") - rc = 1 - } else { - def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) - - // Read buildInfo to obtain build information - def buildInfo = buildReport.getRecords().findAll{ - try { - it.getType()==DefaultRecordFactory.TYPE_BUILD_RESULT - } catch (Exception e){} - } - if (buildInfo.size() != 0) { - tarFileLabel = buildInfo[0].label - buildNumber = buildInfo[0].label - } - - // retrieve the buildResultPropertiesRecord - def buildResultPropertiesRecord = buildReport.getRecords().find { - try { - it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getId()=="DBB.BuildResultProperties" - } catch (Exception e){} - } - - // finds all the build outputs with a deployType - def buildRecords = buildReport.getRecords().findAll{ - try { - (it.getType()==DefaultRecordFactory.TYPE_EXECUTE || it.getType()==DefaultRecordFactory.TYPE_COPY_TO_PDS) && - !it.getOutputs().isEmpty() - } catch (Exception e){} - } - - // finds all the build outputs with a deployType - // Today the USS_RECORD type is built using an AnyTypeRecord record - // An Idea is currently opened to have an official USS_RECORD: https://ideas.ibm.com/ideas/DBB-I-43 - def ussBuildRecords = buildReport.getRecords().findAll{ - try { - it.getType()=="USS_RECORD" && !it.getAttribute("outputs").isEmpty() - } catch (Exception e){} - } + // Object to store scm information for Wazi Deploy Application Manifest file + HashMap scmInfo = new HashMap() - // find all deletions using the DELETE_RECORD of zAppBuild - def deletionRecords = buildReport.getRecords().findAll { - try { - // Obtain delete records, which got added by zAppBuild - it.getType() == "DELETE_RECORD" - } catch (Exception e) { - println e - } + // Package the public Include Files and service submodules + // if Path to Application Descriptor file is specified + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + File applicationDescriptorFile = new File("${props.applicationFolderPath}/applicationDescriptor.yml") + if (applicationDescriptorFile.exists()) { + applicationDescriptorUtils = loadScript(new File("utilities/applicationDescriptorUtils.groovy")) + applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) + } else { + println("*! [ERROR] No Application Descriptor file '${props.applicationFolderPath}/applicationDescriptor.yml' found. Exiting.") + System.exit(1) } - - if (props.deployTypeFilter) { - println("** Filter Output Records on following deployTypes: ${props.deployTypeFilter}...") - buildRecords.each { - // filtered executes - def filteredOutputs = it.getOutputs().findAll { o -> - o.deployType != null && (props.deployTypeFilter).split(',').contains(o.deployType) - } - // Manipulating the scope of build outputs - it.getOutputs().clear() - it.getOutputs().addAll(filteredOutputs) + } + + // iterate over all build reports to obtain build output + props.buildReportOrder.each { buildReportFile -> + Map temporaryBuildOutputsMap = new HashMap() + println("** Read build report data from '${buildReportFile}'.") + def jsonOutputFile = new File(buildReportFile) + + if (!jsonOutputFile.exists()){ + println("*! [ERROR] Build Report '$buildReportFile' not found.") + rc = 1 + } else { + def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) + + // Read buildInfo to obtain build information + def buildInfo = buildReport.getRecords().findAll{ + try { + it.getType()==DefaultRecordFactory.TYPE_BUILD_RESULT + } catch (Exception e){} } - ussBuildRecords.each { - ArrayList outputs = [] - it.getAttribute("outputs").split(';').collectEntries { entry -> - outputs += entry.replaceAll('\\[|\\]', '').split(',') - } - - ArrayList filteredOutputs = [] - outputs.each { output -> - rootDir = output[0].trim() - file = output[1].trim() - deployType = output[2].trim() - if (!(props.deployTypeFilter).split(',').contains(deployType)) { - filteredOutputs += output.toString() - } - } - - def filteredOutputsStrings = String.join(";", filteredOutputs) - it.setAttribute("outputs", filteredOutputsStrings) + if (buildInfo.size() != 0) { + tarFileLabel = buildInfo[0].label + buildNumber = buildInfo[0].label } - } else { - // Remove outputs without deployType + ZUNIT-TESTCASEs - println("** Remove output records without deployType or with deployType=ZUNIT-TESTCASE") - buildRecords.each { - def unwantedOutputs = it.getOutputs().findAll{ o -> - o.deployType == null || o.deployType == 'ZUNIT-TESTCASE' + // retrieve the buildResultPropertiesRecord + def buildResultPropertiesRecord = buildReport.getRecords().find { + try { + it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getId()=="DBB.BuildResultProperties" + } catch (Exception e){} + } + + // finds all the build outputs with a deployType + def buildRecords = buildReport.getRecords().findAll{ + try { + (it.getType()==DefaultRecordFactory.TYPE_EXECUTE || it.getType()==DefaultRecordFactory.TYPE_COPY_TO_PDS) && + !it.getOutputs().isEmpty() + } catch (Exception e){} + } + + // finds all the build outputs with a deployType + // Today the USS_RECORD type is built using an AnyTypeRecord record + // An Idea is currently opened to have an official USS_RECORD: https://ideas.ibm.com/ideas/DBB-I-43 + def ussBuildRecords = buildReport.getRecords().findAll{ + try { + it.getType()=="USS_RECORD" && !it.getAttribute("outputs").isEmpty() + } catch (Exception e){} + } + + // find all deletions using the DELETE_RECORD of zAppBuild + def deletionRecords = buildReport.getRecords().findAll { + try { + // Obtain delete records, which got added by zAppBuild + it.getType() == "DELETE_RECORD" + } catch (Exception e) { + println e } - it.getOutputs().removeAll(unwantedOutputs) } - } - - buildRecords += ussBuildRecords // append USS records - - def datasetMembersCount = 0 - def zFSFilesCount = 0 - def deletionCount = 0 - - // adding files and executes with outputs to Hashmap to remove redundant data - buildRecords.each{ buildRecord -> - if (buildRecord.getType()=="USS_RECORD") { - if (!buildRecord.getAttribute("outputs").isEmpty()) { + + if (props.deployTypeFilter) { + println("** Filter Output Records on following deployTypes: ${props.deployTypeFilter}...") + buildRecords.each { + // filtered executes + def filteredOutputs = it.getOutputs().findAll { o -> + o.deployType != null && (props.deployTypeFilter).split(',').contains(o.deployType) + } + // Manipulating the scope of build outputs + it.getOutputs().clear() + it.getOutputs().addAll(filteredOutputs) + } + ussBuildRecords.each { ArrayList outputs = [] - buildRecord.getAttribute("outputs").split(';').collectEntries { entry -> + it.getAttribute("outputs").split(';').collectEntries { entry -> outputs += entry.replaceAll('\\[|\\]', '').split(',') } - zFSFilesCount += outputs.size() - outputs.each{ output -> + + ArrayList filteredOutputs = [] + outputs.each { output -> rootDir = output[0].trim() file = output[1].trim() deployType = output[2].trim() - def dependencySetRecord = buildReport.getRecords().find { - it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) + if (!(props.deployTypeFilter).split(',').contains(deployType)) { + filteredOutputs += output.toString() } - - temporaryBuildOutputsMap.put(new DeployableArtifact(file, deployType, "zFSFile"), [ - container: rootDir, - owningApplication: props.application, - record: buildRecord, - propertiesRecord: buildResultPropertiesRecord, - dependencySetRecord: dependencySetRecord - ]) } + + def filteredOutputsStrings = String.join(";", filteredOutputs) + it.setAttribute("outputs", filteredOutputsStrings) } } else { - if (buildRecord.getOutputs().size() != 0) { - buildRecord.getOutputs().each { output -> - def (dataset, member) = getDatasetName(output.dataset) - def fileUsage - if (applicationDescriptor && output.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) + // Remove outputs without deployType + ZUNIT-TESTCASEs + println("** Remove output records without deployType or with deployType=ZUNIT-TESTCASE") + buildRecords.each { + def unwantedOutputs = it.getOutputs().findAll{ o -> + o.deployType == null || o.deployType == 'ZUNIT-TESTCASE' + } + it.getOutputs().removeAll(unwantedOutputs) + } + } + + buildRecords += ussBuildRecords // append USS records + + def datasetMembersCount = 0 + def zFSFilesCount = 0 + def deletionCount = 0 + + // adding files and executes with outputs to Hashmap to remove redundant data + buildRecords.each{ buildRecord -> + if (buildRecord.getType()=="USS_RECORD") { + if (!buildRecord.getAttribute("outputs").isEmpty()) { + ArrayList outputs = [] + buildRecord.getAttribute("outputs").split(';').collectEntries { entry -> + outputs += entry.replaceAll('\\[|\\]', '').split(',') } - // If the artifact is not an Object Deck or has no usage or its usage is not main - if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { - datasetMembersCount++ - String file = buildRecord.getFile() + zFSFilesCount += outputs.size() + outputs.each{ output -> + rootDir = output[0].trim() + file = output[1].trim() + deployType = output[2].trim() def dependencySetRecord = buildReport.getRecords().find { it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) } - temporaryBuildOutputsMap.put(new DeployableArtifact(member, output.deployType, "DatasetMember"), [ - container: dataset, + + temporaryBuildOutputsMap.put(new DeployableArtifact(file, deployType, "zFSFile"), [ + container: rootDir, owningApplication: props.application, record: buildRecord, propertiesRecord: buildResultPropertiesRecord, dependencySetRecord: dependencySetRecord ]) - } else { - if (props.verbose) println("*! Build output ${output.dataset} with deployType '${output.deployType}' has been excluded from packaging.") + } + } + } else { + if (buildRecord.getOutputs().size() != 0) { + buildRecord.getOutputs().each { output -> + def (dataset, member) = getDatasetName(output.dataset) + def fileUsage + if (applicationDescriptor && output.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) + } + // If the artifact is not an Object Deck or has no usage or its usage is not main + if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { + datasetMembersCount++ + String file = buildRecord.getFile() + def dependencySetRecord = buildReport.getRecords().find { + it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) + } + temporaryBuildOutputsMap.put(new DeployableArtifact(member, output.deployType, "DatasetMember"), [ + container: dataset, + owningApplication: props.application, + record: buildRecord, + propertiesRecord: buildResultPropertiesRecord, + dependencySetRecord: dependencySetRecord + ]) + } else { + if (props.verbose) println("*! Build output ${output.dataset} with deployType '${output.deployType}' has been excluded from packaging.") + } } } } } - } - deletionRecords.each { deleteRecord -> - deletionCount += deleteRecord.getAttributeAsList("deletedBuildOutputs").size() - deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> - - String cleansedDeletedFile = ((String) deletedFile).replace('"', ''); - def (dataset, member) = getDatasetName(cleansedDeletedFile) - - // search for an existing deployableArtifacts record - ArrayList filteredDeployableArtifacts = new ArrayList() - - temporaryBuildOutputsMap.each { DeployableArtifact deployableArtifact, Map info -> - if (deployableArtifact.file == deleteRecord.getAttribute("file")) { - filteredDeployableArtifacts.add(deployableArtifact, info) + deletionRecords.each { deleteRecord -> + deletionCount += deleteRecord.getAttributeAsList("deletedBuildOutputs").size() + deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> + + String cleansedDeletedFile = ((String) deletedFile).replace('"', ''); + def (dataset, member) = getDatasetName(cleansedDeletedFile) + + // search for an existing deployableArtifacts record + ArrayList filteredDeployableArtifacts = new ArrayList() + + temporaryBuildOutputsMap.each { DeployableArtifact deployableArtifact, Map info -> + if (deployableArtifact.file == deleteRecord.getAttribute("file")) { + filteredDeployableArtifacts.add(deployableArtifact, info) + } } - } - - if (filteredDeployableArtifacts){ - filteredDeployableArtifacts.each {deployableArtifact, info -> - String container = info.get("container") - if (container == dataset && member == deployableArtifact.file) { - deployType = deployableArtifact.deployType - // remove any existing change - temporaryBuildOutputsMap.remove(deployableArtifact) - // add deletion - temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ - container: dataset, - owningApplication: props.application, - record: buildRecord, - propertiesRecord: buildResultPropertiesRecord, - dependencySetRecord: dependencySetRecord + + if (filteredDeployableArtifacts){ + filteredDeployableArtifacts.each {deployableArtifact, info -> + String container = info.get("container") + if (container == dataset && member == deployableArtifact.file) { + deployType = deployableArtifact.deployType + // remove any existing change + temporaryBuildOutputsMap.remove(deployableArtifact) + // add deletion + temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ + container: dataset, + owningApplication: props.application, + record: buildRecord, + propertiesRecord: buildResultPropertiesRecord, + dependencySetRecord: dependencySetRecord ]) + } } - } - } else { - deployType = dataset.replaceAll(/.*\.([^.]*)/, "\$1") // DELETE_RECORD does not contain deployType attribute. Use LLQ - temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ - container: dataset, - owningApplication: props.application, - record: deleteRecord, - propertiesRecord: buildResultPropertiesRecord + } else { + deployType = dataset.replaceAll(/.*\.([^.]*)/, "\$1") // DELETE_RECORD does not contain deployType attribute. Use LLQ + temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ + container: dataset, + owningApplication: props.application, + record: deleteRecord, + propertiesRecord: buildResultPropertiesRecord ]) + } } - } - } - - - // Print summary of BuildReport - if ( datasetMembersCount + zFSFilesCount == 0 ) { - println("** No items to package in '$buildReportFile'.") - } else { - println("** ${temporaryBuildOutputsMap.size()} Build outputs detected in '$buildReportFile':") - temporaryBuildOutputsMap.each { deployableArtifact, info -> - String container = info.get("container") - String owningApplication = info.get("owningApplication") - Record record = info.get("record") - PropertiesRecord propertiesRecord = info.get("propertiesRecord") - DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") - println("\t'${deployableArtifact.file}' from '${container}' with Deploy Type '${deployableArtifact.deployType}'") } - } - - // Log detected deleted files - if (deletionCount != 0) { - println("** Deleted files detected in '$buildReportFile':") - deletionRecords.each { it.getAttributeAsList("deletedBuildOutputs").each { println(" ${it}")}} - } - - buildOutputsMap.putAll(temporaryBuildOutputsMap) - - // generate scmInfo for Wazi Deploy Application Manifest file - if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || - (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { - - if (props.buildReportOrder.size() == 1) { - scmInfo.put("type", "git") - gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") - if (gitUrl) scmInfo.put("uri", gitUrl) - gitHash = retrieveBuildResultProperty (buildResultPropertiesRecord, "githash") - if (gitHash) scmInfo.put("shortCommit", gitHash) - scmInfo.put("branch", props.branch) + + + // Print summary of BuildReport + if ( datasetMembersCount + zFSFilesCount == 0 ) { + println("** No items to package in '$buildReportFile'.") } else { - scmInfo.put("shortCommit", "multipleBuildReports") - scmInfo.put("uri", "multipleBuildReports") + println("** ${temporaryBuildOutputsMap.size()} Build outputs detected in '$buildReportFile':") + temporaryBuildOutputsMap.each { deployableArtifact, info -> + String container = info.get("container") + String owningApplication = info.get("owningApplication") + Record record = info.get("record") + PropertiesRecord propertiesRecord = info.get("propertiesRecord") + DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") + println("\t'${deployableArtifact.file}' from '${container}' with Deploy Type '${deployableArtifact.deployType}'") + } } - } - } -} -if (rc == 0) { + // Log detected deleted files + if (deletionCount != 0) { + println("** Deleted files detected in '$buildReportFile':") + deletionRecords.each { it.getAttributeAsList("deletedBuildOutputs").each { println(" ${it}")}} + } - if (buildOutputsMap.size() == 0) { - println("** There are no build outputs found in all provided build reports. Exiting.") - rc = 0 - } else { - // Initialize Wazi Deploy Manifest Generator - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest - wdManifestGeneratorUtilities.setScmInfo(scmInfo) - if (props.externalDependenciesEvidences) { - File externalDependenciesEvidenceFile = new File("${props.externalDependenciesEvidences}") - if (externalDependenciesEvidenceFile.exists()){ - wdManifestGeneratorUtilities.setExternalDependencies(externalDependenciesEvidenceFile) + buildOutputsMap.putAll(temporaryBuildOutputsMap) + + // generate scmInfo for Wazi Deploy Application Manifest file + if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || + (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { + + if (props.buildReportOrder.size() == 1) { + scmInfo.put("type", "git") + gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") + if (gitUrl) scmInfo.put("uri", gitUrl) + gitHash = retrieveBuildResultProperty (buildResultPropertiesRecord, "githash") + if (gitHash) scmInfo.put("shortCommit", gitHash) + scmInfo.put("branch", props.branch) } else { - println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") - rc=4 - } + scmInfo.put("shortCommit", "multipleBuildReports") + scmInfo.put("uri", "multipleBuildReports") + } } } + } - // Initialize SBOM - if (props.generateSBOM && props.generateSBOM.toBoolean()) { - sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) - sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() - sbomFileName = "${buildNumber}_sbom.json" - sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) + if (rc == 0) { - } + if (buildOutputsMap.size() == 0) { + println("** There are no build outputs found in all provided build reports. Exiting.") + rc = 0 + } else { + // Initialize Wazi Deploy Manifest Generator + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest + wdManifestGeneratorUtilities.setScmInfo(scmInfo) + if (props.externalDependenciesEvidences) { + File externalDependenciesEvidenceFile = new File("${props.externalDependenciesEvidences}") + if (externalDependenciesEvidenceFile.exists()){ + wdManifestGeneratorUtilities.setExternalDependencies(externalDependenciesEvidenceFile) + } else { + println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") + rc=4 + } + } + } - // Initialize Concert Build Manifest Generator - if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { - // Concert Build Manifest - - concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) - concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() - concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) - concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) - } - - // Local variables - tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" - def tarFile = "$props.workDir/${tarFileName}" - - //Create a temporary directory on zFS to copy the load modules from data sets to - tempLoadDir = new File("$props.workDir/tempPackageDir") - !tempLoadDir.exists() ?: tempLoadDir.deleteDir() - tempLoadDir.mkdirs() - - // A baseline Package has been specified, we then extract it in the $tempLoadDir folder - if (props.baselinePackageFilePath) { - File baselinePackageFile = new File(props.baselinePackageFilePath) - if (baselinePackageFile.exists()) { - println("** Extract the baseline package from '${props.baselinePackageFilePath}'") - def processCmd = [ - "sh", - "-c", - "tar -xUXf ${props.baselinePackageFilePath}" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Baseline Package '${props.baselinePackageFilePath}' successfully extracted.") - - // Read the existing Wazi Deploy Manifest if any - File wdManifestFile = new File("$tempLoadDir/wazideploy_manifest.yml") - if (wdManifestFile.exists()) { - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Read the manifest file if it exists - wdManifestGeneratorUtilities.readWaziDeployManifestFile(wdManifestFile, props) - wdManifestGeneratorUtilities.setScmInfo(scmInfo) + // Initialize SBOM + if (props.generateSBOM && props.generateSBOM.toBoolean()) { + sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) + sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() + sbomFileName = "${buildNumber}_sbom.json" + sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) + } + + // Initialize Concert Build Manifest Generator + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + // Concert Build Manifest + + concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) + concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() + concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) + concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) + } + + // Local variables + tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" + def tarFile = "$props.workDir/${tarFileName}" + + //Create a temporary directory on zFS to copy the load modules from data sets to + tempLoadDir = new File("$props.workDir/tempPackageDir") + !tempLoadDir.exists() ?: tempLoadDir.deleteDir() + tempLoadDir.mkdirs() + + // A baseline Package has been specified, we then extract it in the $tempLoadDir folder + if (props.baselinePackageFilePath) { + File baselinePackageFile = new File(props.baselinePackageFilePath) + if (baselinePackageFile.exists()) { + println("** Extract the baseline package from '${props.baselinePackageFilePath}'") + def processCmd = [ + "sh", + "-c", + "tar -xUXf ${props.baselinePackageFilePath}" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Baseline Package '${props.baselinePackageFilePath}' successfully extracted.") + + // Read the existing Wazi Deploy Manifest if any + File wdManifestFile = new File("$tempLoadDir/wazideploy_manifest.yml") + if (wdManifestFile.exists()) { + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + // Read the manifest file if it exists + wdManifestGeneratorUtilities.readWaziDeployManifestFile(wdManifestFile, props) + wdManifestGeneratorUtilities.setScmInfo(scmInfo) + } else { + wdManifestFile.delete() + } } else { - wdManifestFile.delete() - } - } else { - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Otherwise initialize an empty manifest - wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props) - wdManifestGeneratorUtilities.setScmInfo(scmInfo) + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + // Otherwise initialize an empty manifest + wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props) + wdManifestGeneratorUtilities.setScmInfo(scmInfo) + } } - } - - // Search in all subfolders of the archive except the folders - // that contains includes "$includeSubfolder" and binaries "$binSubfolder" - // Copy the artifacts found to comply with the right structure - // All the artifact that don't comply will end up in the binSubfolder - tempLoadDir.eachDir() { subfolder -> - if (!subfolder.getName().equals(includeSubfolder) && !subfolder.getName().equals(libSubfolder)) { - subfolder.eachFileRecurse(FileType.FILES) { file -> - String fileName = file.getName() - def fileNameParts = fileName.split("\\.") - if (fileNameParts.size() > 1) { - fileName = fileNameParts.first() - String fileDeployType = fileNameParts.last() - if (props.fullPackage && props.fullPackage.toBoolean()) { - String expectedFilePath = "$tempLoadDir/$binSubfolder/$fileDeployType/$fileName" - try { - Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") - Path destinationDirPath = destinationPath.getParent() - destinationDirPath.toFile().mkdirs() - Path sourcePath = file.toPath() - copyFiles(sourcePath.toString(), destinationPath.toString()) - println("\tCopy file '${sourcePath}' to '${destinationPath}'") + + // Search in all subfolders of the archive except the folders + // that contains includes "$includeSubfolder" and binaries "$binSubfolder" + // Copy the artifacts found to comply with the right structure + // All the artifact that don't comply will end up in the binSubfolder + tempLoadDir.eachDir() { subfolder -> + if (!subfolder.getName().equals(includeSubfolder) && !subfolder.getName().equals(libSubfolder)) { + subfolder.eachFileRecurse(FileType.FILES) { file -> + String fileName = file.getName() + def fileNameParts = fileName.split("\\.") + if (fileNameParts.size() > 1) { + fileName = fileNameParts.first() + String fileDeployType = fileNameParts.last() + if (props.fullPackage && props.fullPackage.toBoolean()) { + String expectedFilePath = "$tempLoadDir/$binSubfolder/$fileDeployType/$fileName" + try { + Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") + Path destinationDirPath = destinationPath.getParent() + destinationDirPath.toFile().mkdirs() + Path sourcePath = file.toPath() + copyFiles(sourcePath.toString(), destinationPath.toString()) + println("\tCopy file '${sourcePath}' to '${destinationPath}'") + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + // Update Path for the moved file in Wazi Deploy Manifest + rc = rc + wdManifestGeneratorUtilities.updateArtifactPathToManifest(fileName, fileDeployType, "$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") + } + } catch (IOException e) { + println("!* [ERROR] Error when moving file '${sourcePath}' to '${destinationPath}' during baseline package extraction.") + rc = 1 + } + } else { + file.delete() if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Update Path for the moved file in Wazi Deploy Manifest - rc = rc + wdManifestGeneratorUtilities.updateArtifactPathToManifest(fileName, fileDeployType, "$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") + wdManifestGeneratorUtilities.removeArtifactFromManifest(fileName, fileDeployType) } - } catch (IOException e) { - println("!* [ERROR] Error when moving file '${sourcePath}' to '${destinationPath}' during baseline package extraction.") - rc = 1 } - } else { - file.delete() - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.removeArtifactFromManifest(fileName, fileDeployType) - } } } + subfolder.deleteDir() + } + if (subfolder.getName().equals("tmp")) { + subfolder.deleteDir() } - subfolder.deleteDir() } - if (subfolder.getName().equals("tmp")) { - subfolder.deleteDir() - } + } else { + println("*! [ERROR] Error when extracting baseline package '${created}' with rc=$rc.") + rc = 1 } } else { - println("*! [ERROR] Error when extracting baseline package '${created}' with rc=$rc.") + println("*! [ERROR] The Baseline Package '${props.baselinePackageFilePath}' was not found.") rc = 1 - } - } else { - println("*! [ERROR] The Baseline Package '${props.baselinePackageFilePath}' was not found.") - rc = 1 - } - } - - if (rc == 0) { - - println("** Total number of build outputs to package: ${buildOutputsMap.size()}") + } + } - def publicInterfacesDeployTypes - def privateInterfacesDeployTypes - def processedArtifacts = 0 // used as a checksum that all files got categorized + if (rc == 0) { - if (props.publicInterfacesDeployTypes) { - publicInterfacesDeployTypes = props.publicInterfacesDeployTypes.split(",") // split comma separated list into list - } else { - println("*! [WARNING] Property 'publicInterfacesDeployTypes' not defined, using default types 'OBJ'.") - publicInterfacesDeployTypes = ["OBJ"] - } - if (props.privateInterfacesDeployTypes) { - privateInterfacesDeployTypes = props.privateInterfacesDeployTypes.split(",") // split comma separated list into list - } else { - println("*! [WARNING] Property 'privateInterfacesDeployTypes' not defined, using default types 'OBJ,BMSCOPY'.") - privateInterfacesDeployTypes = "OBJ,BMSCOPY".split(",") - } + println("** Total number of build outputs to package: ${buildOutputsMap.size()}") - def deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - !((publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (privateInterfacesDeployTypes && privateInterfacesDeployTypes.contains(deployableArtifact.deployType))) - } - if (deployableOutputs && !deployableOutputs.isEmpty()) { - println("** Copy ${deployableOutputs.size()} deployable artifacts to temporary package directory '$tempLoadDir/$binSubfolder'") - copyArtifactsToUSS(deployableOutputs, binSubfolder, copyModeMap) - processedArtifacts += deployableOutputs.size() - } + def publicInterfacesDeployTypes + def privateInterfacesDeployTypes + def processedArtifacts = 0 // used as a checksum that all files got categorized - if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + if (props.publicInterfacesDeployTypes) { + publicInterfacesDeployTypes = props.publicInterfacesDeployTypes.split(",") // split comma separated list into list + } else { + println("*! [WARNING] Property 'publicInterfacesDeployTypes' not defined, using default types 'OBJ'.") + publicInterfacesDeployTypes = ["OBJ"] + } + if (props.privateInterfacesDeployTypes) { + privateInterfacesDeployTypes = props.privateInterfacesDeployTypes.split(",") // split comma separated list into list + } else { + println("*! [WARNING] Property 'privateInterfacesDeployTypes' not defined, using default types 'OBJ,BMSCOPY'.") + privateInterfacesDeployTypes = "OBJ,BMSCOPY".split(",") + } - def publicInterfaces - if (publicInterfacesDeployTypes) { - // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor - publicInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> - if (deployableArtifact.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - publicInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") - } else { - publicInterfacesDeployTypes.contains(deployableArtifact.deployType) + def deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + !((publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (privateInterfacesDeployTypes && privateInterfacesDeployTypes.contains(deployableArtifact.deployType))) + } + if (deployableOutputs && !deployableOutputs.isEmpty()) { + println("** Copy ${deployableOutputs.size()} deployable artifacts to temporary package directory '$tempLoadDir/$binSubfolder'") + copyArtifactsToUSS(deployableOutputs, binSubfolder, copyModeMap) + processedArtifacts += deployableOutputs.size() + } + + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + + def publicInterfaces + if (publicInterfacesDeployTypes) { + // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor + publicInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> + if (deployableArtifact.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + publicInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") + } else { + publicInterfacesDeployTypes.contains(deployableArtifact.deployType) + } + } + if (publicInterfaces && !publicInterfaces.isEmpty()) { + println("** Copy ${publicInterfaces.size()} public interfaces to temporary package directory '$tempLoadDir/$includeSubfolder'") + copyArtifactsToUSS(publicInterfaces, includeSubfolder, copyModeMap) + processedArtifacts += publicInterfaces.size() } } - if (publicInterfaces && !publicInterfaces.isEmpty()) { - println("** Copy ${publicInterfaces.size()} public interfaces to temporary package directory '$tempLoadDir/$includeSubfolder'") - copyArtifactsToUSS(publicInterfaces, includeSubfolder, copyModeMap) - processedArtifacts += publicInterfaces.size() + def privateInterfaces + if (privateInterfacesDeployTypes) { + // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor + privateInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> + if (deployableArtifact.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + privateInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") + } else { + privateInterfacesDeployTypes.contains(deployableArtifact.deployType) + } + } + println("** Copy ${privateInterfaces.size()} private interfaces to temporary package directory '$tempLoadDir/$libSubfolder'") + copyArtifactsToUSS(privateInterfaces, libSubfolder, copyModeMap) + processedArtifacts += privateInterfaces.size() } - } - def privateInterfaces - if (privateInterfacesDeployTypes) { - // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor - privateInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> - if (deployableArtifact.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - privateInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") - } else { - privateInterfacesDeployTypes.contains(deployableArtifact.deployType) + if ((publicInterfaces && !publicInterfaces.isEmpty()) || + (privateInterfaces && !privateInterfaces.isEmpty())) { + // Checks if all binary interfaces (submodules) are in the archive or not + println("** Validate if all interfaces known in Application Descriptor are packaged.") + checkBinaryInterfaces(tempLoadDir, applicationDescriptor) + } + + ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") + ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") + ArrayList allIncludeFiles = new ArrayList() + + if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { + allIncludeFiles.addAll(publicIncludeFiles) + } + if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { + allIncludeFiles.addAll(sharedIncludeFiles) + } + + if (!allIncludeFiles.isEmpty()) { + println("** Copy ${allIncludeFiles.size()} public/shared Include Files from Application Folder to temporary package directory '$tempLoadDir'") + + allIncludeFiles.forEach() { includeFile -> + Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") + Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") + try { + //Create target parent folder if it doesn't exist + def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() + if (!targetIncludeFilesFolder.exists()) { + targetIncludeFilesFolder.mkdirs() + } + println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") + copyFiles(includeFilePath.toString(), targetIncludeFilePath.toString()) + } catch (IOException exception) { + println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" + rc = Math.max(rc, 1) + } } } - println("** Copy ${privateInterfaces.size()} private interfaces to temporary package directory '$tempLoadDir/$libSubfolder'") - copyArtifactsToUSS(privateInterfaces, libSubfolder, copyModeMap) - processedArtifacts += privateInterfaces.size() } - if ((publicInterfaces && !publicInterfaces.isEmpty()) || - (privateInterfaces && !privateInterfaces.isEmpty())) { - // Checks if all binary interfaces (submodules) are in the archive or not - println("** Validate if all interfaces known in Application Descriptor are packaged.") - checkBinaryInterfaces(tempLoadDir, applicationDescriptor) + + if (processedArtifacts != buildOutputsMap.size()) { + println("*! [WARNING] The number of copied artifacts ($processedArtifacts) doesn't match the number of identified build outputs (${buildOutputsMap.size()}). Some files might have an incorrect 'usage' in the Application Descriptor.") } + } - ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") - ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") - ArrayList allIncludeFiles = new ArrayList() + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) + } - if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { - allIncludeFiles.addAll(publicIncludeFiles) + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { + if (props.publish && props.publish.toBoolean()) { + HashMap packageInfo = new HashMap() + packageInfo.put("type", "artifactRepository") + packageInfo.put("name", props.packageBuildIdentifier) + packageUrl = computeAbsoluteRepositoryUrl() + if (packageUrl) packageInfo.put("uri", packageUrl) + wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } - if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { - allIncludeFiles.addAll(sharedIncludeFiles) + // print application manifest + // wazideploy_manifest.yml is the default name of the manifest file + wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) + } + + if (rc == 0) { + + // log buildReportOrder file and add build reports to tar file + File buildReportOrder = new File("$tempLoadDir/buildReportOrder.txt") + ArrayList buildReportOrderLines = new ArrayList() + + if (buildReportOrder.exists()) { + String line + buildReportOrder.withReader(props.fileEncoding) { reader -> + while ((line = reader.readLine()) != null && !line.equals("")) { + buildReportOrderLines.add(line) + } + } } - if (!allIncludeFiles.isEmpty()) { - println("** Copy ${allIncludeFiles.size()} public/shared Include Files from Application Folder to temporary package directory '$tempLoadDir'") - - allIncludeFiles.forEach() { includeFile -> - Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") - Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") - try { - //Create target parent folder if it doesn't exist - def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() - if (!targetIncludeFilesFolder.exists()) { - targetIncludeFilesFolder.mkdirs() - } - println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") - copyFiles(includeFilePath.toString(), targetIncludeFilePath.toString()) - - } catch (IOException exception) { - println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" - rc = Math.max(rc, 1) + println("** Generate package build report order file to '$buildReportOrder'") + + props.buildReportOrder.each { buildReportFile -> + Path buildReportFilePath = Paths.get(buildReportFile) + + // Always prefix the buildreport with sequence number + int nextIndex = buildReportOrderLines.size() + 1 + Path copiedBuildReportFilePath = Paths.get(tempLoadDir.getPath() + "/" + "$nextIndex".padLeft(3, "0") + "_" + buildReportFilePath.getFileName().toString()) + + copyFiles(buildReportFilePath.toString(), copiedBuildReportFilePath.toString()) + buildReportOrderLines.add("${copiedBuildReportFilePath.getFileName().toString()}\n") + } + buildReportOrder.withWriter(props.fileEncoding) { writer -> + buildReportOrderLines.each { line -> + if (!line.isEmpty()) { + File buildReportFilePath = new File(line) + String buildReportFileName = buildReportFilePath.getName() + writer.write("$buildReportFileName\n") } } } - } - if (processedArtifacts != buildOutputsMap.size()) { - println("*! [WARNING] The number of copied artifacts ($processedArtifacts) doesn't match the number of identified build outputs (${buildOutputsMap.size()}). Some files might have an incorrect 'usage' in the Application Descriptor.") - } - } - - if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) - } + Path packagingPropertiesFilePath = Paths.get(props.packagingPropertiesFile) + Path copiedPackagingPropertiesFilePath = Paths.get(tempLoadDir.getPath() + "/" + packagingPropertiesFilePath.getFileName().toString()) + if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") + copyFiles(packagingPropertiesFilePath.toString(), copiedPackagingPropertiesFilePath.toString()) + + if (props.owner) { + def processCmd = [ + "sh", + "-c", + "chown -R ${props.owner} $tempLoadDir/*" + ] + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Ownership of files in '$tempLoadDir' successfully changed to '${props.owner}'.") + } else { + println("*! [ERROR] Error when changing ownership to '${props.owner}' with rc=$rc.") + } + } - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { - if (props.publish && props.publish.toBoolean()) { - HashMap packageInfo = new HashMap() - packageInfo.put("type", "artifactRepository") - packageInfo.put("name", props.packageBuildIdentifier) - packageUrl = computeAbsoluteRepositoryUrl() - if (packageUrl) packageInfo.put("uri", packageUrl) - wdManifestGeneratorUtilities.setPackageInfo(packageInfo) - } - // print application manifest - // wazideploy_manifest.yml is the default name of the manifest file - wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) - } - - if (rc == 0) { - - // log buildReportOrder file and add build reports to tar file - File buildReportOrder = new File("$tempLoadDir/buildReportOrder.txt") - ArrayList buildReportOrderLines = new ArrayList() - - if (buildReportOrder.exists()) { - String line - buildReportOrder.withReader(props.fileEncoding) { reader -> - while ((line = reader.readLine()) != null && !line.equals("")) { - buildReportOrderLines.add(line) + if (rc == 0) { + println("** Create tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar cUXf $tarFile *" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully created.") + } else { + println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") } } } - - println("** Generate package build report order file to '$buildReportOrder'") - - props.buildReportOrder.each { buildReportFile -> - Path buildReportFilePath = Paths.get(buildReportFile) - // Always prefix the buildreport with sequence number - int nextIndex = buildReportOrderLines.size() + 1 - Path copiedBuildReportFilePath = Paths.get(tempLoadDir.getPath() + "/" + "$nextIndex".padLeft(3, "0") + "_" + buildReportFilePath.getFileName().toString()) - - copyFiles(buildReportFilePath.toString(), copiedBuildReportFilePath.toString()) - buildReportOrderLines.add("${copiedBuildReportFilePath.getFileName().toString()}\n") - } - buildReportOrder.withWriter(props.fileEncoding) { writer -> - buildReportOrderLines.each { line -> - if (!line.isEmpty()) { - File buildReportFilePath = new File(line) - String buildReportFileName = buildReportFilePath.getName() - writer.write("$buildReportFileName\n") + //Package additional outputs to tar file. + if (props.includeLogs && rc == 0) { + (props.includeLogs).split(",").each { logPattern -> + println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") + processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile $logPattern" + ] + + processRC = runProcess(processCmd, new File(props.workDir)) + rc = Math.max(rc, processRC) + if (rc != 0) { + println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") } } } - - Path packagingPropertiesFilePath = Paths.get(props.packagingPropertiesFile) - Path copiedPackagingPropertiesFilePath = Paths.get(tempLoadDir.getPath() + "/" + packagingPropertiesFilePath.getFileName().toString()) - if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") - copyFiles(packagingPropertiesFilePath.toString(), copiedPackagingPropertiesFilePath.toString()) - - if (props.owner) { - def processCmd = [ + + if (props.verbose && props.verbose.toBoolean() && rc == 0) { + println ("** List package contents.") + + processCmd = [ "sh", "-c", - "chown -R ${props.owner} $tempLoadDir/*" + "tar tvf $tarFile" ] - def processRC = runProcess(processCmd, tempLoadDir) + + processRC = runProcess(processCmd, new File(props.workDir)) rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Ownership of files in '$tempLoadDir' successfully changed to '${props.owner}'.") - } else { - println("*! [ERROR] Error when changing ownership to '${props.owner}' with rc=$rc.") + if (rc != 0) { + println("*! [ERROR] Error when listing contents of Package '${tarFile}' with rc=$rc.") } } - - if (rc == 0) { - println("** Create tar file at ${tarFile}") - // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file - // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX - def processCmd = [ - "sh", - "-c", - "tar cUXf $tarFile *" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Package '${tarFile}' successfully created.") - } else { - println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") + + //Set up the artifact repository information to publish the tar file + if (props.publish && props.publish.toBoolean() && rc == 0){ + // Configuring artifact repositoryHelper parms + def url = computeAbsoluteRepositoryUrl() + + def apiKey = props.'artifactRepository.user' + def user = props.'artifactRepository.user' + def password = props.'artifactRepository.password' + def httpClientVersion = props.'artifactRepository.httpClientVersion' + def repo = props.get('artifactRepository.repo') as String + + println ("** Upload package to Artifact Repository '$url'.") + artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + + // generate PackageInfo for Concert Manifest file + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) } } - } - - //Package additional outputs to tar file. - if (props.includeLogs && rc == 0) { - (props.includeLogs).split(",").each { logPattern -> - println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") - processCmd = [ - "sh", - "-c", - "tar rUXf $tarFile $logPattern" - ] - - processRC = runProcess(processCmd, new File(props.workDir)) - rc = Math.max(rc, processRC) - if (rc != 0) { - println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") + + if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { + // concert_build_manifest.yaml is the default name of the manifest file + + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) } - } - } - - if (props.verbose && props.verbose.toBoolean() && rc == 0) { - println ("** List package contents.") - - processCmd = [ - "sh", - "-c", - "tar tvf $tarFile" - ] - - processRC = runProcess(processCmd, new File(props.workDir)) - rc = Math.max(rc, processRC) - if (rc != 0) { - println("*! [ERROR] Error when listing contents of Package '${tarFile}' with rc=$rc.") - } - } - - //Set up the artifact repository information to publish the tar file - if (props.publish && props.publish.toBoolean() && rc == 0){ - // Configuring artifact repositoryHelper parms - def url = computeAbsoluteRepositoryUrl() - - def apiKey = props.'artifactRepository.user' - def user = props.'artifactRepository.user' - def password = props.'artifactRepository.password' - def httpClientVersion = props.'artifactRepository.httpClientVersion' - def repo = props.get('artifactRepository.repo') as String - - //Call the artifactRepositoryHelpers to publish the tar file - File artifactRepoHelpersFile = new File("$scriptDir/ArtifactRepositoryHelpers.groovy") - Class artifactRepositoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactRepoHelpersFile) - GroovyObject artifactRepositoryHelpers = (GroovyObject) artifactRepositoryHelpersClass.newInstance() - - println ("** Upload package to Artifact Repository '$url'.") - artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) + println("** Add concert build config yaml to tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile concert_build_manifest.yaml" + ] - // generate PackageInfo for Concert Manifest file - if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { - concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) - } - } - - if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { - // concert_build_manifest.yaml is the default name of the manifest file - - if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) - } - concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) - println("** Add concert build config yaml to tar file at ${tarFile}") - // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file - // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX - def processCmd = [ - "sh", - "-c", - "tar rUXf $tarFile concert_build_manifest.yaml" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Package '${tarFile}' successfully appended with concert manifest yaml.") - } else { - println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully appended with concert manifest yaml.") + } else { + println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") + } } } } -} -if (rc > 0) { - println ("*! [ERROR] Packaging failed with rc=$rc. Review the console output.") -} else { - println ("** Packaging completed successfully.") + if (rc > 0) { + println ("*! [ERROR] Packaging failed with rc=$rc. Review the console output.") + } else { + println ("** Packaging completed successfully.") + } + System.exit(rc) } -System.exit(rc) def copyArtifactsToUSS(Map buildOutputsMap, String tarSubfolder, HashMap copyModeMap) { buildOutputsMap.each { deployableArtifact, info -> @@ -823,7 +821,7 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS Record record = info.get("record") PropertiesRecord propertiesRecord = info.get("propertiesRecord") DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") - + def relativeFilePath = "" if (deployableArtifact.artifactType.equals("zFSFile")) { relativeFilePath = "$binSubfolder/uss" @@ -886,7 +884,6 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { wdManifestGeneratorUtilities.appendArtifactToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, dependencySetRecord, propertiesRecord) } - } else { println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' doesn't exist." rc = Math.max(rc, 1) @@ -896,10 +893,10 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS rc = Math.max(rc, 1) } } else if (deployableArtifact.artifactType.equals("DatasetMemberDelete")) { - // generate delete instruction for Wazi Deploy - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) - } + // generate delete instruction for Wazi Deploy + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) + } } if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { @@ -953,7 +950,7 @@ def parseInput(String[] cliArgs){ cli.bi(longOpt:'packageBuildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier metadata stored in Wazi Deploys Application Manifest file.') cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') - + // Concert Build Manifest generation cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.') @@ -967,7 +964,7 @@ def parseInput(String[] cliArgs){ // Artifact repository options :: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') - cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server. (Optional)') + cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server. (Optional)') // Artifact repository info cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)') @@ -978,6 +975,9 @@ def parseInput(String[] cliArgs){ cli.ah(longOpt:'artifactRepositoryHttpClientProtocolVersion', args:1, argName: 'httpClientProtocolVersion', 'HttpClient.Version setting to override the HTTP protocol version. (Optional)') cli.aprop(longOpt:'artifactRepositoryPropertyFile', args:1, argName:'propertyFile', 'Path of a property file containing application specific artifact repository details. (Optional) ** (Deprecated)') + // Compute Flag to recompute url + cli.cPkgUrl(longOpt:'computePackageUrl', 'Flag to recompute the artifact Url based on existing parameters') + // Tracing cli.verb(longOpt:'verbose', 'Flag to provide more log output. (Optional)') @@ -994,13 +994,12 @@ def parseInput(String[] cliArgs){ cli.h(longOpt:'help', 'Prints this message') def opts = cli.parse(cliArgs) - if (opts.h) { // if help option used, print usage and exit + if (opts.h) { + // if help option used, print usage and exit cli.usage() - System.exit(0) + System.exit(0) } - def props = new Properties() - // read properties file if (opts.properties) { def propertiesFile = new File(opts.properties) @@ -1008,7 +1007,8 @@ def parseInput(String[] cliArgs){ props.packagingPropertiesFile = opts.properties propertiesFile.withInputStream { props.load(it) } } - } else { // read default sample properties file shipped with the script + } else { + // read default sample properties file shipped with the script def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent def defaultPackagePropFile = new File("$scriptDir/packageBuildOutputs.properties") if (defaultPackagePropFile.exists()) { @@ -1025,8 +1025,8 @@ def parseInput(String[] cliArgs){ if (opts.a) props.application = opts.a if (opts.b) props.branch = opts.b if (opts.bi) props.packageBuildIdentifier = opts.bi - - + if (opts.cPkgUrl) props.computePackageUrl = true + // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : (props.generateWaziDeployAppManifest ? props.generateWaziDeployAppManifest : 'false') props.generateConcertBuildManifest = (opts.ic) ? 'true' : (props.generateConcertBuildManifest ? props.generateConcertBuildManifest : 'false') @@ -1038,7 +1038,7 @@ def parseInput(String[] cliArgs){ if (opts.o) props.owner = opts.o props.verbose = (opts.verb) ? 'true' : 'false' - + if (opts.af) { props.applicationFolderPath = opts.af if (opts.bp) props.baselinePackageFilePath = opts.bp @@ -1083,7 +1083,6 @@ def parseInput(String[] cliArgs){ println("*! [ERROR] Missing required property 'tarFilename'. 'tarFilename' is only optional when no build report order is specified.") rc = 2 } - } if (opts.bO) { opts.bO.split(',').each{ @@ -1102,7 +1101,7 @@ def parseInput(String[] cliArgs){ if (opts.sbomAuthor) { props.sbomAuthor = opts.sbomAuthor - } + } if (!props.workDir) { println("*! [ERROR] Missing required Working Directory parameter ('--workDir'). ") @@ -1119,7 +1118,7 @@ def parseInput(String[] cliArgs){ println("*! [ERROR] Missing Application ('-a') property required when generating SBOM.") rc = 2 } - + // validate publishing options if (props.publish && props.publish.toBoolean()){ if (!props.'artifactRepository.url') { @@ -1152,7 +1151,7 @@ def parseInput(String[] cliArgs){ } } - // assess required options to generate Concert Build manifest + // assess required options to generate Concert Build manifest if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { if (!props.branch) { println("*! [ERROR] Missing Branch parameter ('--branch'). It is required for generating the Concert Build Manifest file.") @@ -1167,7 +1166,7 @@ def parseInput(String[] cliArgs){ rc = 2 } } - + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { if (!props.applicationFolderPath) { println("*! [ERROR] Missing Application Folder Path parameter ('--applicationFolderPath'). It is required for publishing intefaces in the archive.") @@ -1175,7 +1174,27 @@ def parseInput(String[] cliArgs){ } } - return props + // invoke processing + if (props.computePackageUrl && props.computePackageUrl.toBoolean()) { + assert props.tarFileName : "Missing tarFileName" + assert props.get('artifactRepository.url') : "Missing artifactRepository.url" + assert props.get('artifactRepository.repo'): "artifactRepository.repo" + +// dependencies: +// - name: "retirementCalculator" +// reference: "release" +// version: "1.2.3" +// buildid: "875487" + + packageUrl = computeAbsoluteRepositoryUrl() + println packageUrl + + } else { + // continue with standard packaging + packageOutputs() + } + + } /* @@ -1205,28 +1224,29 @@ def parseCopyModeMap(String copyModeMapString) { } /* - * build package url + * build package url */ def computeAbsoluteRepositoryUrl() { - def String remotePath = (props.versionName) ? (props.versionName + "/" + tarFileName) : (tarFileLabel + "/" + tarFileName) + def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL return url } + /* * checksBinaryInterfaces - Checks if all interfaces * (public/shared Include Files and submodules) * are present in the archive. * If not, issue a warning message */ - + def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { ArrayList binaryPublicInterfaces = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Program", "service submodule") ArrayList binaryPrivateInterfaces = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Program", "internal submodule") binaryPublicInterfaces.each { binaryInterface -> String sourceFileName = Paths.get(binaryInterface).getFileName().toString() String member = sourceFileName.split("\\.")[0].toUpperCase() - + String expectedInterfaceFileName = "$includeSubfolder/bin/" + member + ".OBJ" File expectedInterfaceFile = new File("${tempLoadDir.getAbsolutePath()}/$expectedInterfaceFileName") if (!expectedInterfaceFile.exists()) { @@ -1236,7 +1256,7 @@ def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { binaryPrivateInterfaces.each { binaryInterface -> String sourceFileName = Paths.get(binaryInterface).getFileName().toString() String member = sourceFileName.split("\\.")[0].toUpperCase() - + String expectedInterfaceFileName = "$libSubfolder/bin/" + member + ".OBJ" File expectedInterfaceFile = new File("${tempLoadDir.getAbsolutePath()}/$expectedInterfaceFileName") if (!expectedInterfaceFile.exists()) { From 86a5b61e39512e9bc9a7a508cd727237a0f2f7d9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 10:32:26 +0100 Subject: [PATCH 029/127] reference PackagingScript Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 5ac515e3..6b1b7523 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -34,6 +34,8 @@ logsDir="logs" # https://github.com/IBM/dbb on the build machine # Note: Cannot reference environment variables dbbCommunityRepoRootDir=/var/dbb/pipelineTemplates/dbb +dbbCommunityRepoRootDir=/u/dbehm/git/dbb + ##################################################################################################### ## End of Shared Configuration parameters ######################################################## @@ -147,7 +149,7 @@ featureBranchBuildBehaviour=merge-base # Path to PackageBuildOutputs.groovy script from the DBB community repository # e.g. PackagingScript="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -PackagingScript="/var/dbb/pipelineTemplates/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" +PackagingScript="${dbbCommunityRepoRootDir}/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" ##################################################################################################### #### PackageBuildOutputs.groovy configuration #################################################### From 31a24393f0d1fc72ee8da88aa38ba6194b184433 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 14:14:36 +0100 Subject: [PATCH 030/127] document external dependencies Signed-off-by: Dennis Behm --- .../applicationDescriptorUtils.groovy | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 53aab543..f3ddc76b 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -46,10 +46,23 @@ class Baseline { String baseline } +/* +- name: "retirementCalculator" + reference: "release" + version: "1.2.3" + buildid: "875487" +- name: "GenApp" + reference: "build" + version: "feature/789-enhance-something" + buildid: "123456" +*/ + + class DependencyDescriptor { String name + String reference String version - String type + String buildid } /** @@ -193,18 +206,20 @@ def removeFileDefinition(ApplicationDescriptor applicationDescriptor, String sou * Method to add an application dependency */ -def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String version, String type) { +def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String reference, String version, String buildid) { if (!applicationDescriptor.dependencies) { applicationDescriptor.dependencies = new ArrayList() } + // skip readding same/similar entries def existingDependencies = applicationDescriptor.dependencies.findAll() { - it.name.equals(applicationDependency) & it.type.equals(type) + it.name.equals(applicationDependency) } if (!existingDependencies) { def dependency = new DependencyDescriptor() dependency.name = applicationDependency + dependency.reference = reference dependency.version = version - dependency.type = type + dependency.buildid = buildid applicationDescriptor.dependencies.add(dependency) applicationDescriptor.dependencies.sort { it.name From 14caa4f04019b1ae9250ffef3309e322013fc541 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 14:16:28 +0100 Subject: [PATCH 031/127] polish cli options Signed-off-by: Dennis Behm --- .../ArtifactRepositoryHelpers.groovy | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 96606766..36ef2209 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -174,13 +174,18 @@ def evaluateHttpResponse (HttpResponse response, String action, boolean verbose) def run(String[] cliArgs) { def cli = new CliBuilder(usage: "ArtifactRepositoryHelpers.groovy [options]", header: '', stopAtNonOption: false) cli.h(longOpt:'help', 'Prints this message') - cli.u(longOpt:'url', args:1, required:true, 'Artifactory file uri location') + cli.u(longOpt:'url', args:1,'Absolute artifact repository url location to store package') cli.fU(longOpt:'fileToUpload', args:1, 'The full path of the file to upload') cli.fD(longOpt:'fileToDownload', args:1, 'The full path of the file to download') - cli.U(longOpt:'user', args:1, required:true, 'Artifactory user id') - cli.P(longOpt:'password', args:1, required:true, 'Artifactory password') + cli.U(longOpt:'user', args:1,'Artifact repository user id or token') + cli.P(longOpt:'password', args:1, 'Artifact repository password') cli.ht(longOpt:'httpClientVersion', args:1, 'HTTP Client protocol version') cli.v(longOpt:'verbose', 'Flag to turn on script trace') + + // recompute options + cli.c(longOpt:'computeArtifactUrl', 'Action Flag to identify to recompute the uri of a given package') + cli.aRU(longOpt:'artifactRepository.url', args:1, 'Artifact repository Url') + cli.aRN(longOpt:'artifactRepositoryName', args:1, '') def opts = cli.parse(cliArgs) // if opt parsing fails, exit @@ -194,8 +199,19 @@ def run(String[] cliArgs) { } if ( opts.fU) { + // assert required CLI options for upload + assert opts.u : "Missing option: Absolute artifact repository url location to store package" + assert opts.U : "Missing option: Artifact repository user id or token" + assert opts.P : "Missing option: Artifactory password" upload(opts.u, opts.fU, opts.U, opts.P, opts.v) - } else { + } else if (opts.fD) { + // assert required CLI options for download + assert opts.u : "Missing option: Absolute artifact repository url location to store package" + assert opts.U : "Missing option: Artifact repository user id or token" + assert opts.P : "Missing option: Artifactory password" download(opts.u, opts.fD, opts.U, opts.P, opts.v) - } -} \ No newline at end of file + } else { + println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload') "); + } +} + From cb713bb0a53e6afb23c3b021167b0c7bb505f945 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 14:17:02 +0100 Subject: [PATCH 032/127] generalize repo url computation Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 22daef86..6909fd5d 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -639,7 +639,7 @@ def packageOutputs(){ HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") packageInfo.put("name", props.packageBuildIdentifier) - packageUrl = computeAbsoluteRepositoryUrl() + packageUrl = computeAbsoluteRepositoryUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } @@ -762,7 +762,7 @@ def packageOutputs(){ //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def url = computeAbsoluteRepositoryUrl() + def url = computeAbsoluteRepositoryUrl(props) def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' @@ -1186,7 +1186,7 @@ def parseInput(String[] cliArgs){ // version: "1.2.3" // buildid: "875487" - packageUrl = computeAbsoluteRepositoryUrl() + packageUrl = computeAbsoluteRepositoryUrl(props) println packageUrl } else { @@ -1226,7 +1226,7 @@ def parseCopyModeMap(String copyModeMapString) { /* * build package url */ -def computeAbsoluteRepositoryUrl() { +def computeAbsoluteRepositoryUrl(Properties props) { def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL return url From 725c4f21c4bca371ff5700ee67b7d8278af4fcac Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 14:19:17 +0100 Subject: [PATCH 033/127] switch to updated AD dependency declaration Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 222 ++++++++++-------- 1 file changed, 126 insertions(+), 96 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index cbc5dfc9..a41fbb94 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -17,6 +17,7 @@ import java.nio.file.* @Field Properties props = new Properties() @Field def artifactRepositoryHelpers // Helpers to download @Field def applicationDescriptorUtils // Helper to parse Application Descriptor +@Field def packageBuildOutputs // Helpers to download // Parse arguments from command-line parseArgs(args) @@ -36,6 +37,15 @@ if (artifactRepositoryHelpersScriptFile.exists()) { System.exit(1) } +// Load and verify helpers +File packageBuildOutputsFile = new File("${props.PackagingScript}") +if (packageBuildOutputsFile.exists()) { + packageBuildOutputs = loadScript(packageBuildOutputsFile) +} else { + println("*! [ERROR] The Package Build Outputs script '${props.PackagingScript}' doesn't exist. Exiting.") + System.exit(1) +} + File applicationDescriptorUtilsFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.applicationDescriptorHelperUtils}") if (applicationDescriptorUtilsFile.exists()) { @@ -68,106 +78,124 @@ applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(app ArrayList externalDependencies = new ArrayList<>() -// If there are dependencies if (applicationDescriptor.dependencies) { // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> - version = dependency.version - - if (dependency.type.equals("artifactrepository")) { - // download from artifactory - - // TODO: How do we deal with the latest available? - // if (version.equals("latest")) { - // println("* Retrieving production version of application '${applicationDependency.application}'") - // version = applicationDependency.productionVersion - // } - - // Construct the path within the Artifact repo - repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) - - def String artifactUrl - def String artifactReference - def String artifactRelPath - if (dependency.version.startsWith("rel-")){ - artifactRelPath="${repositoryName}/main/release/${dependency.version}" - artifactReference="${artifactRelPath}/${dependency.name}.tar" - artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" - } else { - artifactRelPath="${repositoryName}" - artifactReference="${artifactRelPath}/${dependency.name}.tar" - artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" - } - println("*** Fetching package '${dependency.name}:${dependency.version}' ") - - // Generating information for documentation in yaml file of retrieved dependencies - if (dependency.name != applicationDescriptor.application) { - - ExternalDependency externalDependency = new ExternalDependency() - // Map information to External Dependency Record - externalDependency.name = dependency.name // dependency name - externalDependency.type = dependency.type // dependency type - externalDependency.properties = new HashSet() - - Property p_uri = new Property() - p_uri.key = "uri" - p_uri.value = artifactReference - externalDependency.properties.add(p_uri) - Property p_version = new Property() - p_version.key = "version" - p_version.value = dependency.version - externalDependency.properties.add(p_version) - - // Store external dependency information - externalDependencies.add(externalDependency) - } - - String tarFile = "${tmpPackageDir}/${artifactReference}" - String includeFolder = "${importFolder}/${dependency.name}" - - if (new File(tarFile).exists()) { - println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") - } else { - String user = props.artifactRepositoryUser - String password = props.artifactRepositoryPassword - - if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() - - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") - def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) - println "download complete $rc" // TODO: Error handling in helper - } - - - File includeFolderFile = new File(includeFolder) - if (!includeFolderFile.exists()) { - includeFolderFile.mkdirs() - } - - - println("** Expanding tar file '$tarFile' to '$includeFolder' ") - - def processCmd = [ - "/bin/sh", - "-c", - "tar -C $includeFolder -xvf $tarFile" - ] - - def rc = runProcess(processCmd) - if (rc != 0) { - println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") - System.exit(1) - } - - // Delete temporary download location if cache is not used - if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) {tmpPackageDir.deleteDir()} +/* - name: "retirementCalculator" + reference: "release" + version: "1.2.3" + buildid: "875487" + - name: "GenApp" + reference: "build" + version: "feature/789-enhance-something" + buildid: "123456"*/ + + // compute tar file name based on build type + if (dependency.reference.equalsIgnoreCase("release")) { + assert dependency.version : "Missing dependency version in dependency record" + assert dependency.buildid : "Missing buildid in dependency record" + props.put("tarFileName","${dependency.name}-${dependency.version}-${dependency.buildid}.tar") + } else { + props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") + } + props.put("versionName","${dependency.version}") // compute the version name being part of the path + props.put("artifactRepository.directory", "${dependency.reference}") // compute the main directory to classify builds + props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) + + // The absolute url the package in artifact repo + artifactUrl = packageBuildOutputs.computeAbsoluteRepositoryUrl(props) + + println artifactUrl + // TODO: How do we deal with the latest available? + + // Construct the path within the Artifact repo + repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) + // retrieve path without artifact url + artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") + + println("*** Fetching package '${dependency.name}:${artifactUrl}' ") + + // Generating information for documentation in yaml file of retrieved dependencies + if (dependency.name != applicationDescriptor.application) { + + ExternalDependency externalDependency = new ExternalDependency() + // Map information to External Dependency Record + externalDependency.name = dependency.name // dependency name + externalDependency.properties = new HashSet() + + // Add url + Property p_uri = new Property() + p_uri.key = "uri" + p_uri.value = artifactUrl + externalDependency.properties.add(p_uri) + + // type + Property p_version = new Property() + p_version.key = "version" + p_version.value = dependency.version + externalDependency.properties.add(p_version) + + // reference + Property p_reference = new Property() + p_reference.key = "reference" + p_reference.value = dependency.reference + externalDependency.properties.add(p_reference) + + // buildid + Property p_buildid = new Property() + p_buildid.key = "buildid" + p_buildid.value = dependency.buildid + externalDependency.properties.add(p_buildid) + + // Store external dependency information + externalDependencies.add(externalDependency) + } + + // download from artifact repo + + // foldername in workspace directory + String includeFolder = "${importFolder}/${dependency.name}" + if (new File(props.tarFileName).exists()) { + println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") } else { - println("* Dependency Types other than 'artifactrepository' are not yet implemented. Exiting.") + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + + if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() + + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") + def rc = artifactRepositoryHelpers.download(artifactUrl, props.tarFileName, user, password, true) + println "download complete $rc" // TODO: Error handling in helper + } + + + File includeFolderFile = new File(includeFolder) + if (!includeFolderFile.exists()) { + includeFolderFile.mkdirs() + } + + + println("** Expanding tar file '${props.tarFileName}' to '$includeFolder' ") + + def processCmd = [ + "/bin/sh", + "-c", + "tar -C $includeFolder -xvf ${props.tarFileName}" + ] + + def rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") System.exit(1) } + + // Delete temporary download location if cache is not used + if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) { + tmpPackageDir.deleteDir() + } } } @@ -307,7 +335,7 @@ def parseArgs(String[] args) { if(opts.c){ props.packageCacheLocation = opts.c - } + } if (opts.p) { def pipelineBackendConfigFile = new File(opts.p) @@ -316,11 +344,14 @@ def parseArgs(String[] args) { Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } if(temporaryProperties.get("dbbCommunityRepoRootDir")) props.put("dbbCommunityRepoRootDir", temporaryProperties.get("dbbCommunityRepoRootDir")) + // helper scripts if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) if(temporaryProperties.get("applicationDescriptorHelperUtils")) props.put("applicationDescriptorHelperUtils", temporaryProperties.get("applicationDescriptorHelperUtils")) - if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepositoryUrl", temporaryProperties.get("artifactRepositoryUrl")) - if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepositoryUser", temporaryProperties.get("artifactRepositoryUser")) - if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepositoryPassword", temporaryProperties.get("artifactRepositoryPassword")) + if(temporaryProperties.get("PackagingScript")) props.put("PackagingScript", temporaryProperties.get("PackagingScript")) + // artifact repo configuration properties / Map CBS pipelineBackend.config to script properties + if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepository.url", temporaryProperties.get("artifactRepositoryUrl")) + if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepository.user", temporaryProperties.get("artifactRepositoryUser")) + if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepository.password", temporaryProperties.get("artifactRepositoryPassword")) if(temporaryProperties.get("artifactRepositoryNamePattern")) props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) if(temporaryProperties.get("enablePackageCache")) props.put("enablePackageCache", temporaryProperties.get("enablePackageCache")) } else { @@ -352,7 +383,6 @@ def runProcess(ArrayList cmd){ class ExternalDependency { String name - String type HashSet properties = new HashSet<>() } From cbf88b123741f3a422907d50e125b280eece7580 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 3 Mar 2025 14:22:15 +0100 Subject: [PATCH 034/127] updated reference for testing Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 6b1b7523..bd5f8719 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -149,7 +149,7 @@ featureBranchBuildBehaviour=merge-base # Path to PackageBuildOutputs.groovy script from the DBB community repository # e.g. PackagingScript="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -PackagingScript="${dbbCommunityRepoRootDir}/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" +PackagingScript=/u/dbehm/git/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy ##################################################################################################### #### PackageBuildOutputs.groovy configuration #################################################### From 5a8296ee70eaee943b28134e9318498cbfa5c8b8 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Mar 2025 08:37:18 +0100 Subject: [PATCH 035/127] Consider packageUrl config file Signed-off-by: Dennis Behm --- .../wazideploy-generate.sh | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index a84824ee..26994977 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -132,6 +132,8 @@ Workspace="" Debug="" HELP=$1 +tarFileName="" # variable to store the package tar + if [ "$HELP" = "?" ]; then Help fi @@ -332,6 +334,15 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi + + if [ -z "${packageUrl}" ]; then + echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + PackageInputFile="${packageUrl}" + ## Take the last segment of the URL ... + tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') + PackageOutputFile="$(getLogDir)/${tarFileName}" + fi + # validate package input file if [ -z "${PackageInputFile}" ]; then rc=8 @@ -344,12 +355,7 @@ validateOptions() { PackageInputFile="$(getLogDir)/${PackageInputFile}" fi fi - if [ ! -f "${PackageInputFile}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Package Input File (${PackageInputFile}) was not found. rc="$rc - echo $ERRMSG - fi - + # validate config file if [ -z "${ConfigFile}" ]; then ConfigFile="${wdDeployArtifactoryConfig}" @@ -358,14 +364,13 @@ validateOptions() { if [ ! -z "${ConfigFile}" ]; then if [ ! -f "${ConfigFile}" ]; then rc=8 - ERRMSG=$PGM": [ERROR] Specified Wazi Deploy Artifactory Configuration file (${ConfigFile}) was not found. rc="$rc + ERRMSG=$PGM": [ERROR] Specified Wazi Deploy Artifact repository configuration file (${ConfigFile}) was not found. rc="$rc echo $ERRMSG fi fi # compute the output file if [ -z "${PackageOutputFile}" ] && [ ! -z "${ConfigFile}" ]; then - # c checkWorkspace PackageOutputFile="$(wdDeployPackageDir)" fi @@ -375,6 +380,14 @@ validateOptions() { fi } +# When publishing is enabled, try reading the tempVersionFile +# that needs to be computed before this step. +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + if [ -f "$$(getLogDir)/${tempVersionFile}" ]; then + source $(getLogDir)/${tempVersionFile} + fi +fi + # Call validate Options if [ $rc -eq 0 ]; then validateOptions @@ -435,7 +448,7 @@ if [ $rc -eq 0 ]; then CommandLine+=${Debug} fi echo ${CommandLine} 2>&1 - ${CommandLine} 2>&1 + #${CommandLine} 2>&1 rc=$? if [ $rc -ne 0 ]; then From 172f7a2136eccc6c76340170d441bd40ae017c4a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Mar 2025 08:37:57 +0100 Subject: [PATCH 036/127] change eyecatcher Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 6909fd5d..ecf0815b 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -1187,7 +1187,7 @@ def parseInput(String[] cliArgs){ // buildid: "875487" packageUrl = computeAbsoluteRepositoryUrl(props) - println packageUrl + println "packageUrl=$packageUrl" } else { // continue with standard packaging From 50e9934a40c1c701ca751ff727ab234a6485f3f5 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Mar 2025 08:39:25 +0100 Subject: [PATCH 037/127] new standalone cbs to recompute the package url Signed-off-by: Dennis Behm --- .../computePackageUrl.sh | 473 ++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100755 Templates/Common-Backend-Scripts/computePackageUrl.sh diff --git a/Templates/Common-Backend-Scripts/computePackageUrl.sh b/Templates/Common-Backend-Scripts/computePackageUrl.sh new file mode 100755 index 00000000..066d1f99 --- /dev/null +++ b/Templates/Common-Backend-Scripts/computePackageUrl.sh @@ -0,0 +1,473 @@ +#!/bin/env bash +#=================================================================================== +# NAME: computePackageUrl.sh +# +# DESCRIPTION: The purpose of this script is to recompute the URL, +# where a package got published/stored. +# +# SYNTAX: See Help() section below for usage +# +# RETURNS: +# +# rc - Return Code +# +# RETURN CODES: +# +# 0 - Successful +# 4 - Warning message(s) issued. See Console messages +# 8 - Error encountered. See Console messages +# +# NOTE(S): +# +# 1. Review the common naming conventions where tar files get stored. +# +#=================================================================================== +Help() { + echo $PGM" - Invoke Compute Package URL ("$PGMVERS") " + echo " " + echo "DESCRIPTION: The purpose of this script is to compute " + echo "the URL where a package got published/stored " + echo " " + echo "Syntax: " + echo " " + echo " "$PGM" [Options] " + echo " " + echo "Options: " + echo " " + echo " Mandatory parameters " + echo " " + echo " " + echo " -w - Directory Path to a unique " + echo " working directory " + echo " Either an absolute path " + echo " or relative path. " + echo " If a relative path is provided, " + echo " buildRootDir and the workspace " + echo " path are combined " + echo " Default=None, Required. " + echo " " + echo " Ex: MortgageApplication/main/build-1 " + echo " " + echo " -a - Application name " + echo " Used to compute " + echo " Artifact repository name. " + echo " " + echo " Ex: MortgageApplication " + echo " " + echo " -p - Type of the pipeline to " + echo " control in which directory builds " + echo " are stored in the artifact repo " + echo " Accepted values: " + echo " build - " + echo " development builds " + echo " release - " + echo " builds with options for " + echo " performance optimized " + echo " executables for production env " + echo " " + echo " -b - Name of the git branch. " + echo " " + echo " Ex: main " + echo " " + echo " " + echo " -i - A unique build identifier " + echo " typically the buildID of the " + echo " pipeline " + echo " " + echo " -r - The release identifier for " + echo " release pipeline builds " + echo " (optional) " + echo " " + echo " " + echo " -h - Display this Help. " + + exit 0 +} + +# Customization +# Central configuration file leveraged by the backend scripts +SCRIPT_HOME="$(dirname "$0")" +pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" + +# Path and File Name to the advanced debug options. +#log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" + +# +# Internal Variables +#set -x # Uncomment to enable shell script debug +#export BASH_XTRACEFD=1 # Write set -x trace to file descriptor + +PGM=$(basename "$0") +PGMVERS="1.00" +USER=$(whoami) +SYS=$(uname -Ia) + +# +# Set initialization +# +rc=0 +ERRMSG="" +App="" +tarFileName="" +PipelineType="" +Branch="" + +buildIdentifier="" +releaseIdentifier="" + +artifactVersionName="" # required for publishing to artifact repo +artifactRepositoryUrl="" # required if artifactRepositoryPropertyFile not specified +artifactRepositoryName="" # required if artifactRepositoryPropertyFile not specified +artifactRepositoryDirectory="" # required if artifactRepositoryPropertyFile not specified +artifactRepositoryPropertyFile="" # alternative to above cli parms + +outputFile="" + +HELP=$1 + +if [ "$HELP" = "?" ]; then + Help +fi + +# Validate Shell environment +currentShell=$(ps -p $$ | grep bash) +if [ -z "${currentShell}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] The scripts are designed to run in bash. You are running a different shell. rc=${rc}. \n. $(ps -p $$)." + echo $ERRMSG +fi +# + +# Script label +if [ $rc -eq 0 ]; then + echo $PGM": [INFO] Compute Package Url. Version="$PGMVERS +fi + +# Read and import pipeline configuration +if [ $rc -eq 0 ]; then + if [ ! -f "${pipelineConfiguration}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Pipeline Configuration File (${pipelineConfiguration}) was not found. rc="$rc + echo $ERRMSG + else + source $pipelineConfiguration + fi +fi + +# Source packaging helper +if [ $rc -eq 0 ]; then + if [ ! -f "${packageUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Packaging Utils (${packageUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $packageUtilities + fi +fi + +# +# Get Options +if [ $rc -eq 0 ]; then + while getopts ":h:w:a:b:i:r:p:" opt; do + case $opt in + h) + Help + ;; + w) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Build Workspace Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + Workspace="$argument" + ;; + a) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + App="$argument" + ;; + i) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; + b) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc + echo $ERRMSG + break + fi + Branch="$argument" + ;; + p) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc + echo $INFO + break + fi + PipelineType="$argument" + ;; + r) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + echo $ERRMSG + break + fi + releaseIdentifier="$argument" + ;; + \?) + Help + rc=1 + break + ;; + :) + rc=4 + ERRMSG=$PGM": [WARNING] Option -$OPTARG requires an argument. rc="$rc + echo $ERRMSG + break + ;; + esac + done +fi +# + +validateOptions() { + + if [ -z "${Workspace}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Unique Workspace parameter (-w) is required. rc="$rc + echo $ERRMSG + else + + # Compute the logDir parameter + logDir=$(getLogDir) + + if [ ! -d "$logDir" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Build Log Directory ($logDir) was not found. rc="$rc + echo $ERRMSG + fi + + outputFile=$(getLogDir)/${tempVersionFile} + fi + + if [ -z "${buildIdentifier}" ]; then + ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" + echo $ERRMSG + buildIdentifier=$(date +%Y%m%d_%H%M%S) + fi + + # Validate Packaging script + if [ ! -f "${PackagingScript}" ]; then + rc=8 + ERRMSG=$PGM": [ERR] Unable to locate ${PackagingScript}. rc="$rc + echo $ERRMSG + fi + +} + +# function to validate publishing input options +validatePackagingOptions() { + + if [ -z "${App}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc + echo $ERRMSG + fi + + if [ -z "${Branch}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Branch Name parameter (-b) is required. rc="$rc + echo $ERRMSG + fi + + if [ -z "${artifactRepositoryUrl}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] URL to artifact repository (artifactRepositoryUrl) is required. rc="$rc + echo $ERRMSG + fi + + if [ -z "${artifactRepositoryName}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] artifact repository name to store the build (artifactRepositoryName) is required. rc="$rc + echo $ERRMSG + fi + + if [ -z "${artifactRepositoryDirectory}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Directory path in the repository to store the build (artifactRepositoryDirectory) is required. rc="$rc + echo $ERRMSG + fi + + # If pipeline type is specified, evaluate the value + if [ ! -z "${PipelineType}" ]; then + tmp1=$(echo $PipelineType | tr '[:upper:]' '[:lower:]') + + case $tmp1 in + "build") + PipelineType=$tmp1 + ;; + "release") + PipelineType=$tmp1 + ;; + "preview") + rc=4 + ERRMSG=$PGM": [WARN] Default Pipeline Type : ${PipelineType} not supported for packaging." + echo $ERRMSG + ;; + *) + rc=4 + ERRMSG=$PGM": [WARN] Inavlid Pipeline Type : ${PipelineType} specified." + echo $ERRMSG + ;; + esac + + fi +} + +# Call validate input options +if [ $rc -eq 0 ]; then + validateOptions +fi + +# compute packaging parameters +if [ $rc -eq 0 ]; then + # invoke function in packageUtils + computePackageInformation +fi + +# Call validate packaging options +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + validatePackagingOptions +fi + +# +# Ready to go +if [ $rc -eq 0 ]; then + echo $PGM": [INFO] **************************************************************" + echo $PGM": [INFO] ** Started - Package Build Outputs on HOST/USER: ${SYS}/${USER}" + if [ ! -z "${App}" ]; then + echo $PGM": [INFO] ** Application:" ${App} + fi + if [ ! -z "${Branch}" ]; then + echo $PGM": [INFO] ** Branch:" ${Branch} + fi + if [ ! -z "${PipelineType}" ]; then + echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} + fi + if [ ! -z "${tarFileName}" ]; then + echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} + fi + echo $PGM": [INFO] ** PackagingScript Path:" ${PackagingScript} + + if [ ! -z "${packageBuildIdentifier}" ]; then + echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} + fi + + if [ ! -z "${artifactRepositoryPropertyFile}" ]; then + echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} + fi + + if [ ! -z "${artifactRepositoryUrl}" ]; then + echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} + fi + + if [ ! -z "${artifactRepositoryName}" ]; then + echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} + fi + if [ ! -z "${artifactRepositoryDirectory}" ]; then + echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} + fi + + echo $PGM": [INFO] ** Output file:" ${outputFile} + + echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} + echo $PGM": [INFO] **************************************************************" + echo "" +fi + +# +# Invoke the Package Build Outputs script +if [ $rc -eq 0 ]; then + echo $PGM": [INFO] Invoking the Package Build Outputs script to compute Package Url." + + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${PackagingScript} --computePackageUrl --workDir /tmp" + + # add tarfile name + if [ ! -z "${tarFileName}" ]; then + CMD="${CMD} --tarFileName ${tarFileName}" + fi + + # application name + if [ ! -z "${App}" ]; then + CMD="${CMD} --application ${App}" + fi + + # branch name + if [ ! -z "${Branch}" ]; then + CMD="${CMD} --branch ${Branch}" + fi + + # artifactVersionName + if [ ! -z "${artifactVersionName}" ]; then + CMD="${CMD} --versionName ${artifactVersionName}" + fi + + # Wazi Deploy build identifier + if [ ! -z "${packageBuildIdentifier}" ]; then + CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" + fi + + # Artifact repo options + if [ ! -z "${artifactRepositoryUrl}" ]; then + CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" + fi + + if [ ! -z "${artifactRepositoryName}" ]; then + CMD="${CMD} --artifactRepositoryName ${artifactRepositoryName}" + fi + if [ ! -z "${artifactRepositoryDirectory}" ]; then + CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" + fi + + echo $PGM": [INFO] ${CMD}" + ${CMD} | grep "packageUrl=" > $outputFile + rc=$? + + if [ $rc -eq 0 ]; then + ERRMSG=$PGM": [INFO] Compute Package Url Complete. Results stored in $outputFile. Printing file contents. rc="$rc + echo $ERRMSG + cat $outputFile + else + ERRMSG=$PGM": [ERR] Compute Package Url Failed. Check Console for details. rc="$rc + echo $ERRMSG + rc=12 + fi +fi + +exit $rc From aabd600d3c498d2cbb211ec92ba66c1d6624eb48 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Mar 2025 08:40:06 +0100 Subject: [PATCH 038/127] set default name for packageVersionConfig Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index bd5f8719..cac56c8f 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -176,6 +176,9 @@ sbomAuthor= # Example: /var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar +# File name that is stored in the logs directory containing +# the computed absolute package url +tempVersionFile=packageVersionConfig.txt ##################################################################################################### #### Artifact repository upload options ########################################################## From 601dc33ca9840ea53d85efc28906804b6a3ea2da Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 7 Mar 2025 09:00:09 +0100 Subject: [PATCH 039/127] minor fixes Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/wazideploy-generate.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 26994977..f762d600 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -334,9 +334,11 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi - - if [ -z "${packageUrl}" ]; then + echo "${packageUrl}" + + if [ ! -z "${packageUrl}" ]; then echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + PackageInputFile="${packageUrl}" ## Take the last segment of the URL ... tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') @@ -383,8 +385,11 @@ validateOptions() { # When publishing is enabled, try reading the tempVersionFile # that needs to be computed before this step. if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - if [ -f "$$(getLogDir)/${tempVersionFile}" ]; then - source $(getLogDir)/${tempVersionFile} + if [ -f "$(getLogDir)/${tempVersionFile}" ]; then + echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" + source "$(getLogDir)/${tempVersionFile}" + else + echo $PGM": [INFO] ** $(getLogDir)/${tempVersionFile} not found" fi fi From f8e4b6b3f505a7846a17f07413d424cac6ba2066 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 10 Mar 2025 17:51:33 +0100 Subject: [PATCH 040/127] Consolidate package url computation into wazideploy-generate.sh Signed-off-by: Dennis Behm --- .../{ => utilities}/computePackageUrl.sh | 26 +-- .../wazideploy-generate.sh | 162 +++++++++++++++--- 2 files changed, 151 insertions(+), 37 deletions(-) rename Templates/Common-Backend-Scripts/{ => utilities}/computePackageUrl.sh (99%) diff --git a/Templates/Common-Backend-Scripts/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh similarity index 99% rename from Templates/Common-Backend-Scripts/computePackageUrl.sh rename to Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh index 066d1f99..925d7c42 100755 --- a/Templates/Common-Backend-Scripts/computePackageUrl.sh +++ b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh @@ -87,8 +87,8 @@ Help() { # Customization # Central configuration file leveraged by the backend scripts SCRIPT_HOME="$(dirname "$0")" -pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" -packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" +pipelineConfiguration="${SCRIPT_HOME}/../pipelineBackend.config" +packageUtilities="${SCRIPT_HOME}/packageUtils.sh" # Path and File Name to the advanced debug options. #log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" @@ -196,17 +196,6 @@ if [ $rc -eq 0 ]; then fi App="$argument" ;; - i) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc - echo $ERRMSG - break - fi - buildIdentifier="$argument" - ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -229,6 +218,17 @@ if [ $rc -eq 0 ]; then fi PipelineType="$argument" ;; + i) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; r) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index f762d600..49fbe11b 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -61,8 +61,8 @@ Help() { echo " If a relative path is provided," echo " the log directory is suffixed " echo " where PackageBuildOutputs " - echo " stores outputs " - echo " Default=None, Required. " + echo " stores outputs " + echo " Default=None, Required. " echo " " echo " Ex: MortgageApplication.tar " echo " " @@ -97,6 +97,33 @@ Help() { echo " See wdDeployArtifactoryConfig " echo " in pipelineBackend.config " echo " " + echo " -P - Type of the pipeline to " + echo " control in which directory builds " + echo " are stored in the artifact repo " + echo " Accepted values: " + echo " build - " + echo " development builds " + echo " release - " + echo " builds with options for " + echo " performance optimized " + echo " executables for production env " + echo " (optional) " + echo " " + echo " -b - Name of the git branch. " + echo " (optional) " + echo " " + echo " Ex: main " + echo " " + echo " " + echo " -I - A unique build identifier " + echo " typically the buildID of the " + echo " pipeline " + echo " (optional) " + echo " " + echo " -R - The release identifier for " + echo " release pipeline builds " + echo " (optional) " + echo " " echo " -d - Debug tracing flag " echo " " exit 0 @@ -108,6 +135,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +computePackageUrlUtil="${SCRIPT_HOME}/utilities/computePackageUrl.sh" # Customization - End # @@ -122,13 +150,23 @@ SYS=$(uname -Ia) rc=0 ERRMSG="" +# Wazi Deploy configuration variables DeploymentMethod="" DeploymentPlan="" DeploymentPlanReport="" PackageInputFile="" PackageOutputFile="" ConfigFile="" + +# CBS configuration variables Workspace="" +PipelineType="" # takes cli option P +Branch="" # takes cli option b + +# Package identifier variables +buildIdentifier="" # takes cli option I +releaseIdentifier="" # takes cli option R + Debug="" HELP=$1 @@ -162,10 +200,19 @@ if [ $rc -eq 0 ]; then fi fi +# Read and import pipeline configuration +if [ $rc -eq 0 ]; then + if [ ! -f "${computePackageUrlUtil}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] The Package Url Util (${computePackageUrlUtil}) was not found. rc="$rc + echo $ERRMSG + fi +fi + # # Get Options if [ $rc -eq 0 ]; then - while getopts "hdw:m:p:r:i:o:c:" opt; do + while getopts "hdw:m:p:r:i:o:c:I:R:P:b:" opt; do case $opt in h) Help @@ -258,6 +305,50 @@ if [ $rc -eq 0 ]; then Debug=" -d" ;; + b) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc + echo $ERRMSG + break + fi + Branch="$argument" + ;; + P) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc + echo $INFO + break + fi + PipelineType="$argument" + ;; + I) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; + R) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + echo $ERRMSG + break + fi + releaseIdentifier="$argument" + ;; \?) Help rc=1 @@ -318,11 +409,10 @@ validateOptions() { fi fi # if relative path - if [[ ! ${DeploymentPlan:0:1} == "/" ]] ; then + if [[ ! ${DeploymentPlan:0:1} == "/" ]]; then DeploymentPlan="$(wdDeployPackageDir)/${DeploymentPlan}" fi - # compute deployment plan report if not specified if [ -z "${DeploymentPlanReport}" ]; then # compute default based on configuration @@ -330,19 +420,19 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${wdDeploymentPlanReportName}" fi # if relative path - if [[ ! ${DeploymentPlanReport:0:1} == "/" ]] ; then + if [[ ! ${DeploymentPlanReport:0:1} == "/" ]]; then DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi echo "${packageUrl}" - + if [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - - PackageInputFile="${packageUrl}" - ## Take the last segment of the URL ... - tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(getLogDir)/${tarFileName}" + echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + + PackageInputFile="${packageUrl}" + ## Take the last segment of the URL ... + tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') + PackageOutputFile="$(getLogDir)/${tarFileName}" fi # validate package input file @@ -352,12 +442,12 @@ validateOptions() { echo $ERRMSG else # check for relative path - if [[ ! ${PackageInputFile:0:1} == "/" ]] ; then - checkWorkspace - PackageInputFile="$(getLogDir)/${PackageInputFile}" + if [[ ! ${PackageInputFile:0:1} == "/" ]]; then + checkWorkspace + PackageInputFile="$(getLogDir)/${PackageInputFile}" fi fi - + # validate config file if [ -z "${ConfigFile}" ]; then ConfigFile="${wdDeployArtifactoryConfig}" @@ -377,20 +467,44 @@ validateOptions() { PackageOutputFile="$(wdDeployPackageDir)" fi # if relative path - if [[ ! ${PackageOutputFile:0:1} == "/" ]] && [[ ! -z "${PackageOutputFile}" ]] ; then + if [[ ! ${PackageOutputFile:0:1} == "/" ]] && [[ ! -z "${PackageOutputFile}" ]]; then PackageOutputFile="$(wdDeployPackageDir)/${PackageOutputFile}" fi } # When publishing is enabled, try reading the tempVersionFile # that needs to be computed before this step. -if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then +if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ] && ; then + checkWorkspace + CMD="${computePackageUrlUtil} -w $Workspace -a $App -b $Branch -i $buildIdentifier" + + + if [ ! -z "${PipelineType}" ]; then + CMD+=" -p ${PipelineType}" + else + rc=8 + ERRMSG=$PGM": [ERROR] To compute the Package Url to automatically download the tar file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc + echo $ERRMSG + fi + + if [ ! -z "${releaseIdentifier}" ]; then + CMD+=" -r ${releaseIdentifier}" + fi + + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] ** Compute Package Url based on existing conventions using command" + echo $PGM": ${CMD}" + ${CMD} + if [ -f "$(getLogDir)/${tempVersionFile}" ]; then - echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" - source "$(getLogDir)/${tempVersionFile}" - else - echo $PGM": [INFO] ** $(getLogDir)/${tempVersionFile} not found" - fi + echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" + source "$(getLogDir)/${tempVersionFile}" + else + rc=4 + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${tempVersionFile} was not found. Check previous console output. rc="$rc + echo $ERRMSG + fi + fi fi # Call validate Options From 1da2b20c20cfebbdddc6751dab8363e009837027 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 10 Mar 2025 18:05:33 +0100 Subject: [PATCH 041/127] merged wazideploy-gen Signed-off-by: Dennis Behm --- .../wazideploy-generate.sh | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 49fbe11b..9d21b8af 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -424,15 +424,15 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi - echo "${packageUrl}" + # When a the packageUrl environment variable found in the file if [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - - PackageInputFile="${packageUrl}" - ## Take the last segment of the URL ... - tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(getLogDir)/${tarFileName}" + echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + + PackageInputFile="${packageUrl}" + ## Take the last segment of the URL to define the tarFileName + tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') + PackageOutputFile="$(getLogDir)/${tarFileName}" fi # validate package input file @@ -442,9 +442,9 @@ validateOptions() { echo $ERRMSG else # check for relative path - if [[ ! ${PackageInputFile:0:1} == "/" ]]; then - checkWorkspace - PackageInputFile="$(getLogDir)/${PackageInputFile}" + if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${packageUrl}" ] ; then + checkWorkspace + PackageInputFile="$(getLogDir)/${PackageInputFile}" fi fi @@ -567,7 +567,7 @@ if [ $rc -eq 0 ]; then CommandLine+=${Debug} fi echo ${CommandLine} 2>&1 - #${CommandLine} 2>&1 + ${CommandLine} 2>&1 rc=$? if [ $rc -ne 0 ]; then From e78bf4757882d17620a6bcaeec5ccecccd4947b0 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 10 Mar 2025 18:01:04 +0100 Subject: [PATCH 042/127] update config for testing Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index cac56c8f..b59b6be4 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -79,7 +79,7 @@ applicationDescriptorHelperUtils=Pipeline/PackageBuildOutputs/utilities/applicat # Path to main build script build.groovy. zAppBuild="$HOME/dbb-zappbuild" # DBEHM: MyCustomization for Path to build.groovy. -zAppBuild="/var/dbb/dbb-zappbuild-DAT" +zAppBuild="/var/dbb/dbb-zappbuild_300" # (Optional) Additional property files to customize the build framework zAppBuildPropFiles="" @@ -310,7 +310,7 @@ wdWorkDir=waziDeployDir # # wazideploy deployment method file # wdDeploymentMethod= -wdDeploymentMethod="" +wdDeploymentMethod="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/external-repos/deployment-method/deployment-method.yml" # # Default name of the wazideploy **Deployment Plan** file @@ -326,7 +326,7 @@ wdDeploymentPlanReportName="deploymentPlanReport.html" # # (Optional) Only necessary when wazideploy-generate is used to download the package # wdDeployArtifactoryConfig= -wdDeployArtifactoryConfig= +wdDeployArtifactoryConfig="/var/WaziDeploy/config/WaziDeploy-ConfigFile.yml" # # Path to the environment configurations for Wazi Deploy From 95475230b8b43b447b66ca10a22ca824d005f088 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 10 Mar 2025 18:01:43 +0100 Subject: [PATCH 043/127] make WD manifest default Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/packageBuildOutputs.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties index 9ba3781e..eb65ca11 100644 --- a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -71,7 +71,7 @@ addExtension=true # Boolean setting to define if the Wazi Deploy Application Manifest file should be generated # Please note that the cli option `generateWaziDeployAppManifest` can override this setting and activate it. # Default: false -generateWaziDeployAppManifest=false +generateWaziDeployAppManifest=true # Boolean setting to define if the IBM Concert Build Manifest file should be generated # Please note that the cli option `generateConcertBuildManifest` can override this setting and activate it. From 0a17ed0d77d56a9297ee7c42c7d7bb66760e926e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 09:39:40 +0100 Subject: [PATCH 044/127] move package url computation into ArtifactRepositoryHelpers Signed-off-by: Dennis Behm --- .../ArtifactRepositoryHelpers.groovy | 69 +- .../PackageBuildOutputs.groovy | 1175 ++++++++--------- .../utilities/computePackageUrl.sh | 32 +- 3 files changed, 626 insertions(+), 650 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 36ef2209..ff865de7 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -10,6 +10,7 @@ import java.net.http.HttpResponse.BodyHandler import java.util.concurrent.CompletableFuture import java.nio.file.Paths +import java.nio.file.Path /** Very basic script to upload/download from an artifact repository server @@ -46,6 +47,8 @@ run(args) def upload(String url, String fileName, String user, String password, boolean verbose, String httpClientVersion) throws IOException { System.setProperty("jdk.httpclient.allowRestrictedHeaders", "Connection") + Path testing = Paths.get(fileName) + println "asdadasdasdasd ----- ${testing.toString()}" println( "** ArtifactRepositoryHelper started for upload of $fileName to $url" ); // create http client @@ -92,18 +95,18 @@ def upload(String url, String fileName, String user, String password, boolean ve // submit request CompletableFuture> response = httpClient.sendAsync(request, handler).thenComposeAsync(r -> tryResend(httpClient, request, handler, 1, r)); HttpResponse finalResponse = response.get() - - if (verbose) - println("** Response: " + finalResponse); - - def rc = evaluateHttpResponse(finalResponse, "upload", verbose) - - if (rc == 0 ) { - println("** Upload completed."); - } - else { - println("*! Upload failed."); - } +// +// if (verbose) +// println("** Response: " + finalResponse); +// +// def rc = evaluateHttpResponse(finalResponse, "upload", verbose) +// +// if (rc == 0 ) { +// println("** Upload completed."); +// } +// else { +// println("*! Upload failed."); +// } } def download(String url, String fileName, String user, String password, boolean verbose) throws IOException { @@ -173,6 +176,7 @@ def evaluateHttpResponse (HttpResponse response, String action, boolean verbose) //Parsing the command line def run(String[] cliArgs) { def cli = new CliBuilder(usage: "ArtifactRepositoryHelpers.groovy [options]", header: '', stopAtNonOption: false) + def Properties props = new Properties() cli.h(longOpt:'help', 'Prints this message') cli.u(longOpt:'url', args:1,'Absolute artifact repository url location to store package') cli.fU(longOpt:'fileToUpload', args:1, 'The full path of the file to upload') @@ -183,15 +187,30 @@ def run(String[] cliArgs) { cli.v(longOpt:'verbose', 'Flag to turn on script trace') // recompute options - cli.c(longOpt:'computeArtifactUrl', 'Action Flag to identify to recompute the uri of a given package') - cli.aRU(longOpt:'artifactRepository.url', args:1, 'Artifact repository Url') - cli.aRN(longOpt:'artifactRepositoryName', args:1, '') + // Compute Flag to recompute url + + cli.c(longOpt:'computePackageUrl', 'Action Flag to identify to recompute the uri of a given package') + cli.t(longOpt:'tarFileName', args:1, argName:'filename', 'Name of the package tar file. (Optional unless using --buildReportOrder or --buildReportOrderFile)') + cli.aRU(longOpt:'artifactRepositoryUrl', args:1, 'Artifact repository url') + cli.aRN(longOpt:'artifactRepositoryName', args:1, 'Artifact repository name') + cli.aRD(longOpt:'artifactRepositoryDirectory', args:1, 'Artifact repository directory') + cli.aVN(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server.') + def opts = cli.parse(cliArgs) // if opt parsing fails, exit if (opts == null || !opts) { System.exit(1) } + + if (opts.c) props.computePackageUrl = true + if (opts.t) props.tarFileName = opts.t + if (opts.aRU) props.put('artifactRepository.url', opts.aRU) + if (opts.aRN) props.put('artifactRepository.repo', opts.aRN) + if (opts.aRD) props.put('artifactRepository.directory', opts.aRD) + if (opts.aVN) props.versionName = opts.aVN + + if (opts.h) { cli.usage() @@ -210,8 +229,24 @@ def run(String[] cliArgs) { assert opts.U : "Missing option: Artifact repository user id or token" assert opts.P : "Missing option: Artifactory password" download(opts.u, opts.fD, opts.U, opts.P, opts.v) - } else { - println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload') "); + } else if (props.computePackageUrl){ + + // invoke processing + if (props.computePackageUrl && props.computePackageUrl.toBoolean()) { + // check requires cli arguments for this operation + assert props.tarFileName : "Missing option tarFileName (--tarFileName)" + assert props.versionName : "Missing option versionName (--versionName)" + assert props.get('artifactRepository.url') : "Missing option artifactRepository.url (--artifactRepositoryUrl)" + assert props.get('artifactRepository.repo'): "Missing option artifactRepository.repo (--artifactRepositoryName)" + assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" + + // load script + def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + def artifactRepositoryPathUtilities = loadScript(new File("${scriptDir}/utilities/ArtifactRepositoryPathUtilities.groovy")) + packageUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) + println "packageUrl=$packageUrl" + } else + println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computePackageUrl') "); } } diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index ecf0815b..17bdfaec 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -69,6 +69,7 @@ import com.ibm.jzos.ZFile; @Field Properties props = new Properties() def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) +@Field def artifactRepositoryPathUtilities = loadScript(new File("${scriptDir}/utilities/ArtifactRepositoryPathUtilities.groovy")) @Field def artifactRepositoryHelpers = loadScript(new File("${scriptDir}/ArtifactRepositoryHelpers.groovy")) @Field def applicationDescriptorUtils @@ -89,731 +90,729 @@ def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).p parseInput(args) -def packageOutputs(){ - - startTime = new Date() - props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") - println("** PackageBuildOutputs start at $props.startTime") - println("** Properties at startup:") - props.sort().each { k,v-> - if ( k == "artifactRepository.password" ) - println " $k -> xxxxxx " - else - println " $k -> $v" - } - if (rc != 0) { - println("*! [ERROR] One or several properties were missing in the configuration. Review the console output.") - System.exit(rc) - } +startTime = new Date() +props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") +println("** PackageBuildOutputs start at $props.startTime") +println("** Properties at startup:") +props.sort().each { k,v-> + if ( k == "artifactRepository.password" ) + println " $k -> xxxxxx " + else + println " $k -> $v" +} +if (rc != 0) { + println("*! [ERROR] One or several properties were missing in the configuration. Review the console output.") + System.exit(rc) +} - // Enable file tagging - BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging - // Map of last level dataset qualifier to DBB CopyToHFS CopyMode. - def copyModeMap = parseCopyModeMap(props.copyModeMap) +// Enable file tagging +BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging +// Map of last level dataset qualifier to DBB CopyToHFS CopyMode. +def copyModeMap = parseCopyModeMap(props.copyModeMap) - // Hashmap of BuildOutput to Record - Map buildOutputsMap = new HashMap() - // Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. +// Hashmap of BuildOutput to Record +Map buildOutputsMap = new HashMap() - // Field to store build number, set to default when its not decipherable from build report - def String buildNumber = "UNKNOWN" +// Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. - // Object to store scm information for Wazi Deploy Application Manifest file - HashMap scmInfo = new HashMap() +// Field to store build number, set to default when its not decipherable from build report +def String buildNumber = "UNKNOWN" - // Package the public Include Files and service submodules - // if Path to Application Descriptor file is specified - if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { - File applicationDescriptorFile = new File("${props.applicationFolderPath}/applicationDescriptor.yml") - if (applicationDescriptorFile.exists()) { - applicationDescriptorUtils = loadScript(new File("utilities/applicationDescriptorUtils.groovy")) - applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) - } else { - println("*! [ERROR] No Application Descriptor file '${props.applicationFolderPath}/applicationDescriptor.yml' found. Exiting.") - System.exit(1) - } +// Object to store scm information for Wazi Deploy Application Manifest file +HashMap scmInfo = new HashMap() + +// Package the public Include Files and service submodules +// if Path to Application Descriptor file is specified +if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + File applicationDescriptorFile = new File("${props.applicationFolderPath}/applicationDescriptor.yml") + if (applicationDescriptorFile.exists()) { + applicationDescriptorUtils = loadScript(new File("utilities/applicationDescriptorUtils.groovy")) + applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) + } else { + println("*! [ERROR] No Application Descriptor file '${props.applicationFolderPath}/applicationDescriptor.yml' found. Exiting.") + System.exit(1) } +} - // iterate over all build reports to obtain build output - props.buildReportOrder.each { buildReportFile -> - Map temporaryBuildOutputsMap = new HashMap() - println("** Read build report data from '${buildReportFile}'.") - def jsonOutputFile = new File(buildReportFile) +// iterate over all build reports to obtain build output +props.buildReportOrder.each { buildReportFile -> + Map temporaryBuildOutputsMap = new HashMap() + println("** Read build report data from '${buildReportFile}'.") + def jsonOutputFile = new File(buildReportFile) - if (!jsonOutputFile.exists()){ - println("*! [ERROR] Build Report '$buildReportFile' not found.") - rc = 1 - } else { - def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) + if (!jsonOutputFile.exists()){ + println("*! [ERROR] Build Report '$buildReportFile' not found.") + rc = 1 + } else { + def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) - // Read buildInfo to obtain build information - def buildInfo = buildReport.getRecords().findAll{ - try { - it.getType()==DefaultRecordFactory.TYPE_BUILD_RESULT - } catch (Exception e){} - } - if (buildInfo.size() != 0) { - tarFileLabel = buildInfo[0].label - buildNumber = buildInfo[0].label - } + // Read buildInfo to obtain build information + def buildInfo = buildReport.getRecords().findAll{ + try { + it.getType()==DefaultRecordFactory.TYPE_BUILD_RESULT + } catch (Exception e){} + } + if (buildInfo.size() != 0) { + tarFileLabel = buildInfo[0].label + buildNumber = buildInfo[0].label + } - // retrieve the buildResultPropertiesRecord - def buildResultPropertiesRecord = buildReport.getRecords().find { - try { - it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getId()=="DBB.BuildResultProperties" - } catch (Exception e){} - } + // retrieve the buildResultPropertiesRecord + def buildResultPropertiesRecord = buildReport.getRecords().find { + try { + it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getId()=="DBB.BuildResultProperties" + } catch (Exception e){} + } - // finds all the build outputs with a deployType - def buildRecords = buildReport.getRecords().findAll{ - try { - (it.getType()==DefaultRecordFactory.TYPE_EXECUTE || it.getType()==DefaultRecordFactory.TYPE_COPY_TO_PDS) && - !it.getOutputs().isEmpty() - } catch (Exception e){} - } + // finds all the build outputs with a deployType + def buildRecords = buildReport.getRecords().findAll{ + try { + (it.getType()==DefaultRecordFactory.TYPE_EXECUTE || it.getType()==DefaultRecordFactory.TYPE_COPY_TO_PDS) && + !it.getOutputs().isEmpty() + } catch (Exception e){} + } - // finds all the build outputs with a deployType - // Today the USS_RECORD type is built using an AnyTypeRecord record - // An Idea is currently opened to have an official USS_RECORD: https://ideas.ibm.com/ideas/DBB-I-43 - def ussBuildRecords = buildReport.getRecords().findAll{ - try { - it.getType()=="USS_RECORD" && !it.getAttribute("outputs").isEmpty() - } catch (Exception e){} + // finds all the build outputs with a deployType + // Today the USS_RECORD type is built using an AnyTypeRecord record + // An Idea is currently opened to have an official USS_RECORD: https://ideas.ibm.com/ideas/DBB-I-43 + def ussBuildRecords = buildReport.getRecords().findAll{ + try { + it.getType()=="USS_RECORD" && !it.getAttribute("outputs").isEmpty() + } catch (Exception e){} + } + + // find all deletions using the DELETE_RECORD of zAppBuild + def deletionRecords = buildReport.getRecords().findAll { + try { + // Obtain delete records, which got added by zAppBuild + it.getType() == "DELETE_RECORD" + } catch (Exception e) { + println e } + } - // find all deletions using the DELETE_RECORD of zAppBuild - def deletionRecords = buildReport.getRecords().findAll { - try { - // Obtain delete records, which got added by zAppBuild - it.getType() == "DELETE_RECORD" - } catch (Exception e) { - println e + if (props.deployTypeFilter) { + println("** Filter Output Records on following deployTypes: ${props.deployTypeFilter}...") + buildRecords.each { + // filtered executes + def filteredOutputs = it.getOutputs().findAll { o -> + o.deployType != null && (props.deployTypeFilter).split(',').contains(o.deployType) } + // Manipulating the scope of build outputs + it.getOutputs().clear() + it.getOutputs().addAll(filteredOutputs) } + ussBuildRecords.each { + ArrayList outputs = [] + it.getAttribute("outputs").split(';').collectEntries { entry -> + outputs += entry.replaceAll('\\[|\\]', '').split(',') + } - if (props.deployTypeFilter) { - println("** Filter Output Records on following deployTypes: ${props.deployTypeFilter}...") - buildRecords.each { - // filtered executes - def filteredOutputs = it.getOutputs().findAll { o -> - o.deployType != null && (props.deployTypeFilter).split(',').contains(o.deployType) + ArrayList filteredOutputs = [] + outputs.each { output -> + rootDir = output[0].trim() + file = output[1].trim() + deployType = output[2].trim() + if (!(props.deployTypeFilter).split(',').contains(deployType)) { + filteredOutputs += output.toString() } - // Manipulating the scope of build outputs - it.getOutputs().clear() - it.getOutputs().addAll(filteredOutputs) } - ussBuildRecords.each { + + def filteredOutputsStrings = String.join(";", filteredOutputs) + it.setAttribute("outputs", filteredOutputsStrings) + } + } else { + // Remove outputs without deployType + ZUNIT-TESTCASEs + println("** Remove output records without deployType or with deployType=ZUNIT-TESTCASE") + buildRecords.each { + def unwantedOutputs = it.getOutputs().findAll{ o -> + o.deployType == null || o.deployType == 'ZUNIT-TESTCASE' + } + it.getOutputs().removeAll(unwantedOutputs) + } + } + + buildRecords += ussBuildRecords // append USS records + + def datasetMembersCount = 0 + def zFSFilesCount = 0 + def deletionCount = 0 + + // adding files and executes with outputs to Hashmap to remove redundant data + buildRecords.each{ buildRecord -> + if (buildRecord.getType()=="USS_RECORD") { + if (!buildRecord.getAttribute("outputs").isEmpty()) { ArrayList outputs = [] - it.getAttribute("outputs").split(';').collectEntries { entry -> + buildRecord.getAttribute("outputs").split(';').collectEntries { entry -> outputs += entry.replaceAll('\\[|\\]', '').split(',') } - - ArrayList filteredOutputs = [] - outputs.each { output -> + zFSFilesCount += outputs.size() + outputs.each{ output -> rootDir = output[0].trim() file = output[1].trim() deployType = output[2].trim() - if (!(props.deployTypeFilter).split(',').contains(deployType)) { - filteredOutputs += output.toString() + def dependencySetRecord = buildReport.getRecords().find { + it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) } - } - def filteredOutputsStrings = String.join(";", filteredOutputs) - it.setAttribute("outputs", filteredOutputsStrings) - } - } else { - // Remove outputs without deployType + ZUNIT-TESTCASEs - println("** Remove output records without deployType or with deployType=ZUNIT-TESTCASE") - buildRecords.each { - def unwantedOutputs = it.getOutputs().findAll{ o -> - o.deployType == null || o.deployType == 'ZUNIT-TESTCASE' + temporaryBuildOutputsMap.put(new DeployableArtifact(file, deployType, "zFSFile"), [ + container: rootDir, + owningApplication: props.application, + record: buildRecord, + propertiesRecord: buildResultPropertiesRecord, + dependencySetRecord: dependencySetRecord + ]) } - it.getOutputs().removeAll(unwantedOutputs) } - } - - buildRecords += ussBuildRecords // append USS records - - def datasetMembersCount = 0 - def zFSFilesCount = 0 - def deletionCount = 0 - - // adding files and executes with outputs to Hashmap to remove redundant data - buildRecords.each{ buildRecord -> - if (buildRecord.getType()=="USS_RECORD") { - if (!buildRecord.getAttribute("outputs").isEmpty()) { - ArrayList outputs = [] - buildRecord.getAttribute("outputs").split(';').collectEntries { entry -> - outputs += entry.replaceAll('\\[|\\]', '').split(',') + } else { + if (buildRecord.getOutputs().size() != 0) { + buildRecord.getOutputs().each { output -> + def (dataset, member) = getDatasetName(output.dataset) + def fileUsage + if (applicationDescriptor && output.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) } - zFSFilesCount += outputs.size() - outputs.each{ output -> - rootDir = output[0].trim() - file = output[1].trim() - deployType = output[2].trim() + // If the artifact is not an Object Deck or has no usage or its usage is not main + if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { + datasetMembersCount++ + String file = buildRecord.getFile() def dependencySetRecord = buildReport.getRecords().find { it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) } - - temporaryBuildOutputsMap.put(new DeployableArtifact(file, deployType, "zFSFile"), [ - container: rootDir, + temporaryBuildOutputsMap.put(new DeployableArtifact(member, output.deployType, "DatasetMember"), [ + container: dataset, owningApplication: props.application, record: buildRecord, propertiesRecord: buildResultPropertiesRecord, dependencySetRecord: dependencySetRecord ]) - } - } - } else { - if (buildRecord.getOutputs().size() != 0) { - buildRecord.getOutputs().each { output -> - def (dataset, member) = getDatasetName(output.dataset) - def fileUsage - if (applicationDescriptor && output.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) - } - // If the artifact is not an Object Deck or has no usage or its usage is not main - if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { - datasetMembersCount++ - String file = buildRecord.getFile() - def dependencySetRecord = buildReport.getRecords().find { - it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) - } - temporaryBuildOutputsMap.put(new DeployableArtifact(member, output.deployType, "DatasetMember"), [ - container: dataset, - owningApplication: props.application, - record: buildRecord, - propertiesRecord: buildResultPropertiesRecord, - dependencySetRecord: dependencySetRecord - ]) - } else { - if (props.verbose) println("*! Build output ${output.dataset} with deployType '${output.deployType}' has been excluded from packaging.") - } + } else { + if (props.verbose) println("*! Build output ${output.dataset} with deployType '${output.deployType}' has been excluded from packaging.") } } } } + } - deletionRecords.each { deleteRecord -> - deletionCount += deleteRecord.getAttributeAsList("deletedBuildOutputs").size() - deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> + deletionRecords.each { deleteRecord -> + deletionCount += deleteRecord.getAttributeAsList("deletedBuildOutputs").size() + deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> - String cleansedDeletedFile = ((String) deletedFile).replace('"', ''); - def (dataset, member) = getDatasetName(cleansedDeletedFile) + String cleansedDeletedFile = ((String) deletedFile).replace('"', ''); + def (dataset, member) = getDatasetName(cleansedDeletedFile) - // search for an existing deployableArtifacts record - ArrayList filteredDeployableArtifacts = new ArrayList() + // search for an existing deployableArtifacts record + ArrayList filteredDeployableArtifacts = new ArrayList() - temporaryBuildOutputsMap.each { DeployableArtifact deployableArtifact, Map info -> - if (deployableArtifact.file == deleteRecord.getAttribute("file")) { - filteredDeployableArtifacts.add(deployableArtifact, info) - } + temporaryBuildOutputsMap.each { DeployableArtifact deployableArtifact, Map info -> + if (deployableArtifact.file == deleteRecord.getAttribute("file")) { + filteredDeployableArtifacts.add(deployableArtifact, info) } + } - if (filteredDeployableArtifacts){ - filteredDeployableArtifacts.each {deployableArtifact, info -> - String container = info.get("container") - if (container == dataset && member == deployableArtifact.file) { - deployType = deployableArtifact.deployType - // remove any existing change - temporaryBuildOutputsMap.remove(deployableArtifact) - // add deletion - temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ - container: dataset, - owningApplication: props.application, - record: buildRecord, - propertiesRecord: buildResultPropertiesRecord, - dependencySetRecord: dependencySetRecord - ]) - } + if (filteredDeployableArtifacts){ + filteredDeployableArtifacts.each {deployableArtifact, info -> + String container = info.get("container") + if (container == dataset && member == deployableArtifact.file) { + deployType = deployableArtifact.deployType + // remove any existing change + temporaryBuildOutputsMap.remove(deployableArtifact) + // add deletion + temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ + container: dataset, + owningApplication: props.application, + record: buildRecord, + propertiesRecord: buildResultPropertiesRecord, + dependencySetRecord: dependencySetRecord + ]) } - } else { - deployType = dataset.replaceAll(/.*\.([^.]*)/, "\$1") // DELETE_RECORD does not contain deployType attribute. Use LLQ - temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ - container: dataset, - owningApplication: props.application, - record: deleteRecord, - propertiesRecord: buildResultPropertiesRecord - ]) } + } else { + deployType = dataset.replaceAll(/.*\.([^.]*)/, "\$1") // DELETE_RECORD does not contain deployType attribute. Use LLQ + temporaryBuildOutputsMap.put(new DeployableArtifact(member, deployType, "DatasetMemberDelete"), [ + container: dataset, + owningApplication: props.application, + record: deleteRecord, + propertiesRecord: buildResultPropertiesRecord + ]) } } + } - // Print summary of BuildReport - if ( datasetMembersCount + zFSFilesCount == 0 ) { - println("** No items to package in '$buildReportFile'.") - } else { - println("** ${temporaryBuildOutputsMap.size()} Build outputs detected in '$buildReportFile':") - temporaryBuildOutputsMap.each { deployableArtifact, info -> - String container = info.get("container") - String owningApplication = info.get("owningApplication") - Record record = info.get("record") - PropertiesRecord propertiesRecord = info.get("propertiesRecord") - DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") - println("\t'${deployableArtifact.file}' from '${container}' with Deploy Type '${deployableArtifact.deployType}'") - } + // Print summary of BuildReport + if ( datasetMembersCount + zFSFilesCount == 0 ) { + println("** No items to package in '$buildReportFile'.") + } else { + println("** ${temporaryBuildOutputsMap.size()} Build outputs detected in '$buildReportFile':") + temporaryBuildOutputsMap.each { deployableArtifact, info -> + String container = info.get("container") + String owningApplication = info.get("owningApplication") + Record record = info.get("record") + PropertiesRecord propertiesRecord = info.get("propertiesRecord") + DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") + println("\t'${deployableArtifact.file}' from '${container}' with Deploy Type '${deployableArtifact.deployType}'") } + } - // Log detected deleted files - if (deletionCount != 0) { - println("** Deleted files detected in '$buildReportFile':") - deletionRecords.each { it.getAttributeAsList("deletedBuildOutputs").each { println(" ${it}")}} - } + // Log detected deleted files + if (deletionCount != 0) { + println("** Deleted files detected in '$buildReportFile':") + deletionRecords.each { it.getAttributeAsList("deletedBuildOutputs").each { println(" ${it}")}} + } - buildOutputsMap.putAll(temporaryBuildOutputsMap) + buildOutputsMap.putAll(temporaryBuildOutputsMap) - // generate scmInfo for Wazi Deploy Application Manifest file - if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || - (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { + // generate scmInfo for Wazi Deploy Application Manifest file + if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || + (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { - if (props.buildReportOrder.size() == 1) { - scmInfo.put("type", "git") - gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") - if (gitUrl) scmInfo.put("uri", gitUrl) - gitHash = retrieveBuildResultProperty (buildResultPropertiesRecord, "githash") - if (gitHash) scmInfo.put("shortCommit", gitHash) - scmInfo.put("branch", props.branch) - } else { - scmInfo.put("shortCommit", "multipleBuildReports") - scmInfo.put("uri", "multipleBuildReports") - } + if (props.buildReportOrder.size() == 1) { + scmInfo.put("type", "git") + gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") + if (gitUrl) scmInfo.put("uri", gitUrl) + gitHash = retrieveBuildResultProperty (buildResultPropertiesRecord, "githash") + if (gitHash) scmInfo.put("shortCommit", gitHash) + scmInfo.put("branch", props.branch) + } else { + scmInfo.put("shortCommit", "multipleBuildReports") + scmInfo.put("uri", "multipleBuildReports") } } } +} - if (rc == 0) { +if (rc == 0) { - if (buildOutputsMap.size() == 0) { - println("** There are no build outputs found in all provided build reports. Exiting.") - rc = 0 - } else { - // Initialize Wazi Deploy Manifest Generator - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest - wdManifestGeneratorUtilities.setScmInfo(scmInfo) - if (props.externalDependenciesEvidences) { - File externalDependenciesEvidenceFile = new File("${props.externalDependenciesEvidences}") - if (externalDependenciesEvidenceFile.exists()){ - wdManifestGeneratorUtilities.setExternalDependencies(externalDependenciesEvidenceFile) - } else { - println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") - rc=4 - } + if (buildOutputsMap.size() == 0) { + println("** There are no build outputs found in all provided build reports. Exiting.") + rc = 0 + } else { + // Initialize Wazi Deploy Manifest Generator + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest + wdManifestGeneratorUtilities.setScmInfo(scmInfo) + if (props.externalDependenciesEvidences) { + File externalDependenciesEvidenceFile = new File("${props.externalDependenciesEvidences}") + if (externalDependenciesEvidenceFile.exists()){ + wdManifestGeneratorUtilities.setExternalDependencies(externalDependenciesEvidenceFile) + } else { + println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") + rc=4 } } + } - // Initialize SBOM - if (props.generateSBOM && props.generateSBOM.toBoolean()) { - sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) - sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() - sbomFileName = "${buildNumber}_sbom.json" - sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) - } + // Initialize SBOM + if (props.generateSBOM && props.generateSBOM.toBoolean()) { + sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) + sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() + sbomFileName = "${buildNumber}_sbom.json" + sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) + } - // Initialize Concert Build Manifest Generator - if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { - // Concert Build Manifest + // Initialize Concert Build Manifest Generator + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + // Concert Build Manifest - concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) - concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() - concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) - concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) - } + concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) + concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() + concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) + concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) + } - // Local variables - tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" - def tarFile = "$props.workDir/${tarFileName}" - - //Create a temporary directory on zFS to copy the load modules from data sets to - tempLoadDir = new File("$props.workDir/tempPackageDir") - !tempLoadDir.exists() ?: tempLoadDir.deleteDir() - tempLoadDir.mkdirs() - - // A baseline Package has been specified, we then extract it in the $tempLoadDir folder - if (props.baselinePackageFilePath) { - File baselinePackageFile = new File(props.baselinePackageFilePath) - if (baselinePackageFile.exists()) { - println("** Extract the baseline package from '${props.baselinePackageFilePath}'") - def processCmd = [ - "sh", - "-c", - "tar -xUXf ${props.baselinePackageFilePath}" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Baseline Package '${props.baselinePackageFilePath}' successfully extracted.") - - // Read the existing Wazi Deploy Manifest if any - File wdManifestFile = new File("$tempLoadDir/wazideploy_manifest.yml") - if (wdManifestFile.exists()) { - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Read the manifest file if it exists - wdManifestGeneratorUtilities.readWaziDeployManifestFile(wdManifestFile, props) - wdManifestGeneratorUtilities.setScmInfo(scmInfo) - } else { - wdManifestFile.delete() - } + // Local variables + tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" + def tarFile = "$props.workDir/${tarFileName}" + + //Create a temporary directory on zFS to copy the load modules from data sets to + tempLoadDir = new File("$props.workDir/tempPackageDir") + !tempLoadDir.exists() ?: tempLoadDir.deleteDir() + tempLoadDir.mkdirs() + + // A baseline Package has been specified, we then extract it in the $tempLoadDir folder + if (props.baselinePackageFilePath) { + File baselinePackageFile = new File(props.baselinePackageFilePath) + if (baselinePackageFile.exists()) { + println("** Extract the baseline package from '${props.baselinePackageFilePath}'") + def processCmd = [ + "sh", + "-c", + "tar -xUXf ${props.baselinePackageFilePath}" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Baseline Package '${props.baselinePackageFilePath}' successfully extracted.") + + // Read the existing Wazi Deploy Manifest if any + File wdManifestFile = new File("$tempLoadDir/wazideploy_manifest.yml") + if (wdManifestFile.exists()) { + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + // Read the manifest file if it exists + wdManifestGeneratorUtilities.readWaziDeployManifestFile(wdManifestFile, props) + wdManifestGeneratorUtilities.setScmInfo(scmInfo) } else { - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Otherwise initialize an empty manifest - wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props) - wdManifestGeneratorUtilities.setScmInfo(scmInfo) - } + wdManifestFile.delete() } + } else { + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + // Otherwise initialize an empty manifest + wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props) + wdManifestGeneratorUtilities.setScmInfo(scmInfo) + } + } - // Search in all subfolders of the archive except the folders - // that contains includes "$includeSubfolder" and binaries "$binSubfolder" - // Copy the artifacts found to comply with the right structure - // All the artifact that don't comply will end up in the binSubfolder - tempLoadDir.eachDir() { subfolder -> - if (!subfolder.getName().equals(includeSubfolder) && !subfolder.getName().equals(libSubfolder)) { - subfolder.eachFileRecurse(FileType.FILES) { file -> - String fileName = file.getName() - def fileNameParts = fileName.split("\\.") - if (fileNameParts.size() > 1) { - fileName = fileNameParts.first() - String fileDeployType = fileNameParts.last() - if (props.fullPackage && props.fullPackage.toBoolean()) { - String expectedFilePath = "$tempLoadDir/$binSubfolder/$fileDeployType/$fileName" - try { - Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") - Path destinationDirPath = destinationPath.getParent() - destinationDirPath.toFile().mkdirs() - Path sourcePath = file.toPath() - copyFiles(sourcePath.toString(), destinationPath.toString()) - println("\tCopy file '${sourcePath}' to '${destinationPath}'") - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - // Update Path for the moved file in Wazi Deploy Manifest - rc = rc + wdManifestGeneratorUtilities.updateArtifactPathToManifest(fileName, fileDeployType, "$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") - } - } catch (IOException e) { - println("!* [ERROR] Error when moving file '${sourcePath}' to '${destinationPath}' during baseline package extraction.") - rc = 1 - } - } else { - file.delete() + // Search in all subfolders of the archive except the folders + // that contains includes "$includeSubfolder" and binaries "$binSubfolder" + // Copy the artifacts found to comply with the right structure + // All the artifact that don't comply will end up in the binSubfolder + tempLoadDir.eachDir() { subfolder -> + if (!subfolder.getName().equals(includeSubfolder) && !subfolder.getName().equals(libSubfolder)) { + subfolder.eachFileRecurse(FileType.FILES) { file -> + String fileName = file.getName() + def fileNameParts = fileName.split("\\.") + if (fileNameParts.size() > 1) { + fileName = fileNameParts.first() + String fileDeployType = fileNameParts.last() + if (props.fullPackage && props.fullPackage.toBoolean()) { + String expectedFilePath = "$tempLoadDir/$binSubfolder/$fileDeployType/$fileName" + try { + Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") + Path destinationDirPath = destinationPath.getParent() + destinationDirPath.toFile().mkdirs() + Path sourcePath = file.toPath() + copyFiles(sourcePath.toString(), destinationPath.toString()) + println("\tCopy file '${sourcePath}' to '${destinationPath}'") if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.removeArtifactFromManifest(fileName, fileDeployType) + // Update Path for the moved file in Wazi Deploy Manifest + rc = rc + wdManifestGeneratorUtilities.updateArtifactPathToManifest(fileName, fileDeployType, "$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") } + } catch (IOException e) { + println("!* [ERROR] Error when moving file '${sourcePath}' to '${destinationPath}' during baseline package extraction.") + rc = 1 + } + } else { + file.delete() + if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.removeArtifactFromManifest(fileName, fileDeployType) } } } - subfolder.deleteDir() - } - if (subfolder.getName().equals("tmp")) { - subfolder.deleteDir() } + subfolder.deleteDir() + } + if (subfolder.getName().equals("tmp")) { + subfolder.deleteDir() } - } else { - println("*! [ERROR] Error when extracting baseline package '${created}' with rc=$rc.") - rc = 1 } } else { - println("*! [ERROR] The Baseline Package '${props.baselinePackageFilePath}' was not found.") + println("*! [ERROR] Error when extracting baseline package '${created}' with rc=$rc.") rc = 1 } + } else { + println("*! [ERROR] The Baseline Package '${props.baselinePackageFilePath}' was not found.") + rc = 1 } + } - if (rc == 0) { + if (rc == 0) { - println("** Total number of build outputs to package: ${buildOutputsMap.size()}") + println("** Total number of build outputs to package: ${buildOutputsMap.size()}") - def publicInterfacesDeployTypes - def privateInterfacesDeployTypes - def processedArtifacts = 0 // used as a checksum that all files got categorized + def publicInterfacesDeployTypes + def privateInterfacesDeployTypes + def processedArtifacts = 0 // used as a checksum that all files got categorized - if (props.publicInterfacesDeployTypes) { - publicInterfacesDeployTypes = props.publicInterfacesDeployTypes.split(",") // split comma separated list into list - } else { - println("*! [WARNING] Property 'publicInterfacesDeployTypes' not defined, using default types 'OBJ'.") - publicInterfacesDeployTypes = ["OBJ"] - } - if (props.privateInterfacesDeployTypes) { - privateInterfacesDeployTypes = props.privateInterfacesDeployTypes.split(",") // split comma separated list into list - } else { - println("*! [WARNING] Property 'privateInterfacesDeployTypes' not defined, using default types 'OBJ,BMSCOPY'.") - privateInterfacesDeployTypes = "OBJ,BMSCOPY".split(",") - } + if (props.publicInterfacesDeployTypes) { + publicInterfacesDeployTypes = props.publicInterfacesDeployTypes.split(",") // split comma separated list into list + } else { + println("*! [WARNING] Property 'publicInterfacesDeployTypes' not defined, using default types 'OBJ'.") + publicInterfacesDeployTypes = ["OBJ"] + } + if (props.privateInterfacesDeployTypes) { + privateInterfacesDeployTypes = props.privateInterfacesDeployTypes.split(",") // split comma separated list into list + } else { + println("*! [WARNING] Property 'privateInterfacesDeployTypes' not defined, using default types 'OBJ,BMSCOPY'.") + privateInterfacesDeployTypes = "OBJ,BMSCOPY".split(",") + } - def deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - !((publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (privateInterfacesDeployTypes && privateInterfacesDeployTypes.contains(deployableArtifact.deployType))) - } - if (deployableOutputs && !deployableOutputs.isEmpty()) { - println("** Copy ${deployableOutputs.size()} deployable artifacts to temporary package directory '$tempLoadDir/$binSubfolder'") - copyArtifactsToUSS(deployableOutputs, binSubfolder, copyModeMap) - processedArtifacts += deployableOutputs.size() - } + def deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + !((publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (privateInterfacesDeployTypes && privateInterfacesDeployTypes.contains(deployableArtifact.deployType))) + } + if (deployableOutputs && !deployableOutputs.isEmpty()) { + println("** Copy ${deployableOutputs.size()} deployable artifacts to temporary package directory '$tempLoadDir/$binSubfolder'") + copyArtifactsToUSS(deployableOutputs, binSubfolder, copyModeMap) + processedArtifacts += deployableOutputs.size() + } - if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { - - def publicInterfaces - if (publicInterfacesDeployTypes) { - // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor - publicInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> - if (deployableArtifact.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - publicInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") - } else { - publicInterfacesDeployTypes.contains(deployableArtifact.deployType) - } - } - if (publicInterfaces && !publicInterfaces.isEmpty()) { - println("** Copy ${publicInterfaces.size()} public interfaces to temporary package directory '$tempLoadDir/$includeSubfolder'") - copyArtifactsToUSS(publicInterfaces, includeSubfolder, copyModeMap) - processedArtifacts += publicInterfaces.size() + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + + def publicInterfaces + if (publicInterfacesDeployTypes) { + // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor + publicInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> + if (deployableArtifact.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + publicInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") + } else { + publicInterfacesDeployTypes.contains(deployableArtifact.deployType) } } - def privateInterfaces - if (privateInterfacesDeployTypes) { - // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor - privateInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> - if (deployableArtifact.deployType.equals("OBJ")) { - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - privateInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") - } else { - privateInterfacesDeployTypes.contains(deployableArtifact.deployType) - } - } - println("** Copy ${privateInterfaces.size()} private interfaces to temporary package directory '$tempLoadDir/$libSubfolder'") - copyArtifactsToUSS(privateInterfaces, libSubfolder, copyModeMap) - processedArtifacts += privateInterfaces.size() + if (publicInterfaces && !publicInterfaces.isEmpty()) { + println("** Copy ${publicInterfaces.size()} public interfaces to temporary package directory '$tempLoadDir/$includeSubfolder'") + copyArtifactsToUSS(publicInterfaces, includeSubfolder, copyModeMap) + processedArtifacts += publicInterfaces.size() } - if ((publicInterfaces && !publicInterfaces.isEmpty()) || - (privateInterfaces && !privateInterfaces.isEmpty())) { - // Checks if all binary interfaces (submodules) are in the archive or not - println("** Validate if all interfaces known in Application Descriptor are packaged.") - checkBinaryInterfaces(tempLoadDir, applicationDescriptor) + } + def privateInterfaces + if (privateInterfacesDeployTypes) { + // build outputs that are mapped to a public deployType and are flagged as 'service submodule' in the application descriptor + privateInterfaces = buildOutputsMap.findAll { deployableArtifact, info -> + if (deployableArtifact.deployType.equals("OBJ")) { + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + privateInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") + } else { + privateInterfacesDeployTypes.contains(deployableArtifact.deployType) + } } + println("** Copy ${privateInterfaces.size()} private interfaces to temporary package directory '$tempLoadDir/$libSubfolder'") + copyArtifactsToUSS(privateInterfaces, libSubfolder, copyModeMap) + processedArtifacts += privateInterfaces.size() + } + if ((publicInterfaces && !publicInterfaces.isEmpty()) || + (privateInterfaces && !privateInterfaces.isEmpty())) { + // Checks if all binary interfaces (submodules) are in the archive or not + println("** Validate if all interfaces known in Application Descriptor are packaged.") + checkBinaryInterfaces(tempLoadDir, applicationDescriptor) + } - ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") - ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") - ArrayList allIncludeFiles = new ArrayList() + ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") + ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") + ArrayList allIncludeFiles = new ArrayList() - if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { - allIncludeFiles.addAll(publicIncludeFiles) - } - if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { - allIncludeFiles.addAll(sharedIncludeFiles) - } + if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { + allIncludeFiles.addAll(publicIncludeFiles) + } + if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { + allIncludeFiles.addAll(sharedIncludeFiles) + } - if (!allIncludeFiles.isEmpty()) { - println("** Copy ${allIncludeFiles.size()} public/shared Include Files from Application Folder to temporary package directory '$tempLoadDir'") - - allIncludeFiles.forEach() { includeFile -> - Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") - Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") - try { - //Create target parent folder if it doesn't exist - def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() - if (!targetIncludeFilesFolder.exists()) { - targetIncludeFilesFolder.mkdirs() - } - println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") - copyFiles(includeFilePath.toString(), targetIncludeFilePath.toString()) - } catch (IOException exception) { - println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" - rc = Math.max(rc, 1) + if (!allIncludeFiles.isEmpty()) { + println("** Copy ${allIncludeFiles.size()} public/shared Include Files from Application Folder to temporary package directory '$tempLoadDir'") + + allIncludeFiles.forEach() { includeFile -> + Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") + Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") + try { + //Create target parent folder if it doesn't exist + def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() + if (!targetIncludeFilesFolder.exists()) { + targetIncludeFilesFolder.mkdirs() } + println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") + copyFiles(includeFilePath.toString(), targetIncludeFilePath.toString()) + } catch (IOException exception) { + println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" + rc = Math.max(rc, 1) } } } - - if (processedArtifacts != buildOutputsMap.size()) { - println("*! [WARNING] The number of copied artifacts ($processedArtifacts) doesn't match the number of identified build outputs (${buildOutputsMap.size()}). Some files might have an incorrect 'usage' in the Application Descriptor.") - } } - if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) + if (processedArtifacts != buildOutputsMap.size()) { + println("*! [WARNING] The number of copied artifacts ($processedArtifacts) doesn't match the number of identified build outputs (${buildOutputsMap.size()}). Some files might have an incorrect 'usage' in the Application Descriptor.") } + } - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { - if (props.publish && props.publish.toBoolean()) { - HashMap packageInfo = new HashMap() - packageInfo.put("type", "artifactRepository") - packageInfo.put("name", props.packageBuildIdentifier) - packageUrl = computeAbsoluteRepositoryUrl(props) - if (packageUrl) packageInfo.put("uri", packageUrl) - wdManifestGeneratorUtilities.setPackageInfo(packageInfo) - } - // print application manifest - // wazideploy_manifest.yml is the default name of the manifest file - wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) + } + + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { + if (props.publish && props.publish.toBoolean()) { + HashMap packageInfo = new HashMap() + packageInfo.put("type", "artifactRepository") + packageInfo.put("name", props.packageBuildIdentifier) + packageUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) + if (packageUrl) packageInfo.put("uri", packageUrl) + wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } + // print application manifest + // wazideploy_manifest.yml is the default name of the manifest file + wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) + } - if (rc == 0) { + if (rc == 0) { - // log buildReportOrder file and add build reports to tar file - File buildReportOrder = new File("$tempLoadDir/buildReportOrder.txt") - ArrayList buildReportOrderLines = new ArrayList() + // log buildReportOrder file and add build reports to tar file + File buildReportOrder = new File("$tempLoadDir/buildReportOrder.txt") + ArrayList buildReportOrderLines = new ArrayList() - if (buildReportOrder.exists()) { - String line - buildReportOrder.withReader(props.fileEncoding) { reader -> - while ((line = reader.readLine()) != null && !line.equals("")) { - buildReportOrderLines.add(line) - } + if (buildReportOrder.exists()) { + String line + buildReportOrder.withReader(props.fileEncoding) { reader -> + while ((line = reader.readLine()) != null && !line.equals("")) { + buildReportOrderLines.add(line) } } + } - println("** Generate package build report order file to '$buildReportOrder'") + println("** Generate package build report order file to '$buildReportOrder'") - props.buildReportOrder.each { buildReportFile -> - Path buildReportFilePath = Paths.get(buildReportFile) + props.buildReportOrder.each { buildReportFile -> + Path buildReportFilePath = Paths.get(buildReportFile) - // Always prefix the buildreport with sequence number - int nextIndex = buildReportOrderLines.size() + 1 - Path copiedBuildReportFilePath = Paths.get(tempLoadDir.getPath() + "/" + "$nextIndex".padLeft(3, "0") + "_" + buildReportFilePath.getFileName().toString()) + // Always prefix the buildreport with sequence number + int nextIndex = buildReportOrderLines.size() + 1 + Path copiedBuildReportFilePath = Paths.get(tempLoadDir.getPath() + "/" + "$nextIndex".padLeft(3, "0") + "_" + buildReportFilePath.getFileName().toString()) - copyFiles(buildReportFilePath.toString(), copiedBuildReportFilePath.toString()) - buildReportOrderLines.add("${copiedBuildReportFilePath.getFileName().toString()}\n") - } - buildReportOrder.withWriter(props.fileEncoding) { writer -> - buildReportOrderLines.each { line -> - if (!line.isEmpty()) { - File buildReportFilePath = new File(line) - String buildReportFileName = buildReportFilePath.getName() - writer.write("$buildReportFileName\n") - } + copyFiles(buildReportFilePath.toString(), copiedBuildReportFilePath.toString()) + buildReportOrderLines.add("${copiedBuildReportFilePath.getFileName().toString()}\n") + } + buildReportOrder.withWriter(props.fileEncoding) { writer -> + buildReportOrderLines.each { line -> + if (!line.isEmpty()) { + File buildReportFilePath = new File(line) + String buildReportFileName = buildReportFilePath.getName() + writer.write("$buildReportFileName\n") } } + } - Path packagingPropertiesFilePath = Paths.get(props.packagingPropertiesFile) - Path copiedPackagingPropertiesFilePath = Paths.get(tempLoadDir.getPath() + "/" + packagingPropertiesFilePath.getFileName().toString()) - if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") - copyFiles(packagingPropertiesFilePath.toString(), copiedPackagingPropertiesFilePath.toString()) - - if (props.owner) { - def processCmd = [ - "sh", - "-c", - "chown -R ${props.owner} $tempLoadDir/*" - ] - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Ownership of files in '$tempLoadDir' successfully changed to '${props.owner}'.") - } else { - println("*! [ERROR] Error when changing ownership to '${props.owner}' with rc=$rc.") - } - } + Path packagingPropertiesFilePath = Paths.get(props.packagingPropertiesFile) + Path copiedPackagingPropertiesFilePath = Paths.get(tempLoadDir.getPath() + "/" + packagingPropertiesFilePath.getFileName().toString()) + if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") + copyFiles(packagingPropertiesFilePath.toString(), copiedPackagingPropertiesFilePath.toString()) + if (props.owner) { + def processCmd = [ + "sh", + "-c", + "chown -R ${props.owner} $tempLoadDir/*" + ] + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) if (rc == 0) { - println("** Create tar file at ${tarFile}") - // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file - // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX - def processCmd = [ - "sh", - "-c", - "tar cUXf $tarFile *" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Package '${tarFile}' successfully created.") - } else { - println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") - } + println("** Ownership of files in '$tempLoadDir' successfully changed to '${props.owner}'.") + } else { + println("*! [ERROR] Error when changing ownership to '${props.owner}' with rc=$rc.") } } - //Package additional outputs to tar file. - if (props.includeLogs && rc == 0) { - (props.includeLogs).split(",").each { logPattern -> - println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") - processCmd = [ - "sh", - "-c", - "tar rUXf $tarFile $logPattern" - ] - - processRC = runProcess(processCmd, new File(props.workDir)) - rc = Math.max(rc, processRC) - if (rc != 0) { - println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") - } + if (rc == 0) { + println("** Create tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar cUXf $tarFile *" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully created.") + } else { + println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") } } + } - if (props.verbose && props.verbose.toBoolean() && rc == 0) { - println ("** List package contents.") - + //Package additional outputs to tar file. + if (props.includeLogs && rc == 0) { + (props.includeLogs).split(",").each { logPattern -> + println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") processCmd = [ "sh", "-c", - "tar tvf $tarFile" + "tar rUXf $tarFile $logPattern" ] processRC = runProcess(processCmd, new File(props.workDir)) rc = Math.max(rc, processRC) if (rc != 0) { - println("*! [ERROR] Error when listing contents of Package '${tarFile}' with rc=$rc.") + println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") } } + } - //Set up the artifact repository information to publish the tar file - if (props.publish && props.publish.toBoolean() && rc == 0){ - // Configuring artifact repositoryHelper parms - def url = computeAbsoluteRepositoryUrl(props) - - def apiKey = props.'artifactRepository.user' - def user = props.'artifactRepository.user' - def password = props.'artifactRepository.password' - def httpClientVersion = props.'artifactRepository.httpClientVersion' - def repo = props.get('artifactRepository.repo') as String + if (props.verbose && props.verbose.toBoolean() && rc == 0) { + println ("** List package contents.") - println ("** Upload package to Artifact Repository '$url'.") - artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + processCmd = [ + "sh", + "-c", + "tar tvf $tarFile" + ] - // generate PackageInfo for Concert Manifest file - if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { - concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) - } + processRC = runProcess(processCmd, new File(props.workDir)) + rc = Math.max(rc, processRC) + if (rc != 0) { + println("*! [ERROR] Error when listing contents of Package '${tarFile}' with rc=$rc.") } + } - if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { - // concert_build_manifest.yaml is the default name of the manifest file + //Set up the artifact repository information to publish the tar file + if (props.publish && props.publish.toBoolean() && rc == 0){ + // Configuring artifact repositoryHelper parms + def url = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) - if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) - } - concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) - println("** Add concert build config yaml to tar file at ${tarFile}") - // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file - // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX - def processCmd = [ - "sh", - "-c", - "tar rUXf $tarFile concert_build_manifest.yaml" - ] + def apiKey = props.'artifactRepository.user' + def user = props.'artifactRepository.user' + def password = props.'artifactRepository.password' + def httpClientVersion = props.'artifactRepository.httpClientVersion' + def repo = props.get('artifactRepository.repo') as String - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) - if (rc == 0) { - println("** Package '${tarFile}' successfully appended with concert manifest yaml.") - } else { - println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") - } + println ("** Upload package to Artifact Repository '$url'.") + artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + + // generate PackageInfo for Concert Manifest file + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) } } - } - if (rc > 0) { - println ("*! [ERROR] Packaging failed with rc=$rc. Review the console output.") - } else { - println ("** Packaging completed successfully.") + if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { + // concert_build_manifest.yaml is the default name of the manifest file + + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) + } + concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) + println("** Add concert build config yaml to tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile concert_build_manifest.yaml" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully appended with concert manifest yaml.") + } else { + println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") + } + } } - System.exit(rc) } +if (rc > 0) { + println ("*! [ERROR] Packaging failed with rc=$rc. Review the console output.") +} else { + println ("** Packaging completed successfully.") +} +System.exit(rc) + def copyArtifactsToUSS(Map buildOutputsMap, String tarSubfolder, HashMap copyModeMap) { buildOutputsMap.each { deployableArtifact, info -> String container = info.get("container") @@ -975,9 +974,6 @@ def parseInput(String[] cliArgs){ cli.ah(longOpt:'artifactRepositoryHttpClientProtocolVersion', args:1, argName: 'httpClientProtocolVersion', 'HttpClient.Version setting to override the HTTP protocol version. (Optional)') cli.aprop(longOpt:'artifactRepositoryPropertyFile', args:1, argName:'propertyFile', 'Path of a property file containing application specific artifact repository details. (Optional) ** (Deprecated)') - // Compute Flag to recompute url - cli.cPkgUrl(longOpt:'computePackageUrl', 'Flag to recompute the artifact Url based on existing parameters') - // Tracing cli.verb(longOpt:'verbose', 'Flag to provide more log output. (Optional)') @@ -1025,7 +1021,6 @@ def parseInput(String[] cliArgs){ if (opts.a) props.application = opts.a if (opts.b) props.branch = opts.b if (opts.bi) props.packageBuildIdentifier = opts.bi - if (opts.cPkgUrl) props.computePackageUrl = true // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : (props.generateWaziDeployAppManifest ? props.generateWaziDeployAppManifest : 'false') @@ -1174,27 +1169,6 @@ def parseInput(String[] cliArgs){ } } - // invoke processing - if (props.computePackageUrl && props.computePackageUrl.toBoolean()) { - assert props.tarFileName : "Missing tarFileName" - assert props.get('artifactRepository.url') : "Missing artifactRepository.url" - assert props.get('artifactRepository.repo'): "artifactRepository.repo" - -// dependencies: -// - name: "retirementCalculator" -// reference: "release" -// version: "1.2.3" -// buildid: "875487" - - packageUrl = computeAbsoluteRepositoryUrl(props) - println "packageUrl=$packageUrl" - - } else { - // continue with standard packaging - packageOutputs() - } - - } /* @@ -1223,15 +1197,6 @@ def parseCopyModeMap(String copyModeMapString) { return copyModeMap } -/* - * build package url - */ -def computeAbsoluteRepositoryUrl(Properties props) { - def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) - def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL - return url -} - /* * checksBinaryInterfaces - Checks if all interfaces diff --git a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh index 925d7c42..26c0b3dd 100755 --- a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh +++ b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh @@ -383,15 +383,7 @@ if [ $rc -eq 0 ]; then if [ ! -z "${tarFileName}" ]; then echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} fi - echo $PGM": [INFO] ** PackagingScript Path:" ${PackagingScript} - - if [ ! -z "${packageBuildIdentifier}" ]; then - echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} - fi - - if [ ! -z "${artifactRepositoryPropertyFile}" ]; then - echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} - fi + echo $PGM": [INFO] ** Artifact Repository Helpers:" ${artifactRepositoryHelpersScript} if [ ! -z "${artifactRepositoryUrl}" ]; then echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} @@ -416,33 +408,18 @@ fi if [ $rc -eq 0 ]; then echo $PGM": [INFO] Invoking the Package Build Outputs script to compute Package Url." - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${PackagingScript} --computePackageUrl --workDir /tmp" + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${dbbCommunityRepoRootDir}/${artifactRepositoryHelpersScript} --computePackageUrl" # add tarfile name if [ ! -z "${tarFileName}" ]; then CMD="${CMD} --tarFileName ${tarFileName}" fi - # application name - if [ ! -z "${App}" ]; then - CMD="${CMD} --application ${App}" - fi - - # branch name - if [ ! -z "${Branch}" ]; then - CMD="${CMD} --branch ${Branch}" - fi - # artifactVersionName if [ ! -z "${artifactVersionName}" ]; then CMD="${CMD} --versionName ${artifactVersionName}" fi - # Wazi Deploy build identifier - if [ ! -z "${packageBuildIdentifier}" ]; then - CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" - fi - # Artifact repo options if [ ! -z "${artifactRepositoryUrl}" ]; then CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" @@ -460,11 +437,10 @@ if [ $rc -eq 0 ]; then rc=$? if [ $rc -eq 0 ]; then - ERRMSG=$PGM": [INFO] Compute Package Url Complete. Results stored in $outputFile. Printing file contents. rc="$rc + ERRMSG=$PGM": [INFO] Compute Package Url completed. Results stored in $outputFile. rc="$rc echo $ERRMSG - cat $outputFile else - ERRMSG=$PGM": [ERR] Compute Package Url Failed. Check Console for details. rc="$rc + ERRMSG=$PGM": [ERR] Compute Package Url failed. Check Console for details. rc="$rc echo $ERRMSG rc=12 fi From 1bf2035aa6609672fd4b946e69b21eadb90a8fdb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 09:40:22 +0100 Subject: [PATCH 045/127] Use computation in ArtifactRepoHelpers Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index a41fbb94..7227e9d9 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -17,6 +17,7 @@ import java.nio.file.* @Field Properties props = new Properties() @Field def artifactRepositoryHelpers // Helpers to download @Field def applicationDescriptorUtils // Helper to parse Application Descriptor +@Field def artifactRepositoryPathUtilities // Util to compute the URL path @Field def packageBuildOutputs // Helpers to download // Parse arguments from command-line @@ -38,11 +39,11 @@ if (artifactRepositoryHelpersScriptFile.exists()) { } // Load and verify helpers -File packageBuildOutputsFile = new File("${props.PackagingScript}") -if (packageBuildOutputsFile.exists()) { - packageBuildOutputs = loadScript(packageBuildOutputsFile) +File artifactRepositoryPathUtilitiesFile = new File("${props.artifactRepositoryPathUtilities}") +if (artifactRepositoryPathUtilitiesFile.exists()) { + artifactRepositoryPathUtilities = loadScript(artifactRepositoryPathUtilitiesFile) } else { - println("*! [ERROR] The Package Build Outputs script '${props.PackagingScript}' doesn't exist. Exiting.") + println("*! [ERROR] The Artifact Repo Path Utility script '${props.artifactRepositoryPathUtilities}' doesn't exist. Exiting.") System.exit(1) } @@ -100,21 +101,24 @@ if (applicationDescriptor.dependencies) { } else { props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") } + props.put("versionName","${dependency.version}") // compute the version name being part of the path props.put("artifactRepository.directory", "${dependency.reference}") // compute the main directory to classify builds props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) - + // The absolute url the package in artifact repo - artifactUrl = packageBuildOutputs.computeAbsoluteRepositoryUrl(props) - + artifactUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) println artifactUrl - // TODO: How do we deal with the latest available? - + // Construct the path within the Artifact repo repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") + // File in cache / workspace + tarFile="${tmpPackageDir}/${artifactRelPath}" + tarFileDir=tarFile.replaceAll(props.tarFileName, "") + println("*** Fetching package '${dependency.name}:${artifactUrl}' ") // Generating information for documentation in yaml file of retrieved dependencies @@ -158,16 +162,16 @@ if (applicationDescriptor.dependencies) { // foldername in workspace directory String includeFolder = "${importFolder}/${dependency.name}" - if (new File(props.tarFileName).exists()) { - println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") + if (new File(tarFile).exists()) { + println("** Package was already found in package cache at '${tarFile}'") } else { String user = props.artifactRepositoryUser String password = props.artifactRepositoryPassword - if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() + if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") - def rc = artifactRepositoryHelpers.download(artifactUrl, props.tarFileName, user, password, true) + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tarFileDir}.") + def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) println "download complete $rc" // TODO: Error handling in helper } @@ -178,12 +182,12 @@ if (applicationDescriptor.dependencies) { } - println("** Expanding tar file '${props.tarFileName}' to '$includeFolder' ") + println("** Expanding tar file '${tarFile}' to '$includeFolder' ") def processCmd = [ "/bin/sh", "-c", - "tar -C $includeFolder -xvf ${props.tarFileName}" + "tar -C $includeFolder -xvf ${tarFile}" ] def rc = runProcess(processCmd) @@ -347,6 +351,8 @@ def parseArgs(String[] args) { // helper scripts if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) if(temporaryProperties.get("applicationDescriptorHelperUtils")) props.put("applicationDescriptorHelperUtils", temporaryProperties.get("applicationDescriptorHelperUtils")) + if(temporaryProperties.get("artifactRepositoryPathUtilities")) props.put("artifactRepositoryPathUtilities", temporaryProperties.get("artifactRepositoryPathUtilities")) + if(temporaryProperties.get("PackagingScript")) props.put("PackagingScript", temporaryProperties.get("PackagingScript")) // artifact repo configuration properties / Map CBS pipelineBackend.config to script properties if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepository.url", temporaryProperties.get("artifactRepositoryUrl")) From da200ea151997d10bb4b7001739fda7e117f2bbf Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 09:41:02 +0100 Subject: [PATCH 046/127] include package url computation in wazideploy-generate Signed-off-by: Dennis Behm --- .../wazideploy-generate.sh | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 9d21b8af..ae6fcee6 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -60,8 +60,6 @@ Help() { echo " used as input " echo " If a relative path is provided," echo " the log directory is suffixed " - echo " where PackageBuildOutputs " - echo " stores outputs " echo " Default=None, Required. " echo " " echo " Ex: MortgageApplication.tar " @@ -160,6 +158,7 @@ ConfigFile="" # CBS configuration variables Workspace="" +App="" # Application name - takes cli option a PipelineType="" # takes cli option P Branch="" # takes cli option b @@ -212,7 +211,7 @@ fi # # Get Options if [ $rc -eq 0 ]; then - while getopts "hdw:m:p:r:i:o:c:I:R:P:b:" opt; do + while getopts "hdw:m:p:r:i:o:c:a:I:R:P:b:" opt; do case $opt in h) Help @@ -304,7 +303,19 @@ if [ $rc -eq 0 ]; then # Add command to produce debug output with Wazi Deploy Debug=" -d" ;; - + + a) + # Application argument + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + App="$argument" + ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -427,8 +438,7 @@ validateOptions() { # When a the packageUrl environment variable found in the file if [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - + echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." PackageInputFile="${packageUrl}" ## Take the last segment of the URL to define the tarFileName tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') @@ -474,7 +484,7 @@ validateOptions() { # When publishing is enabled, try reading the tempVersionFile # that needs to be computed before this step. -if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ] && ; then +if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; then checkWorkspace CMD="${computePackageUrlUtil} -w $Workspace -a $App -b $Branch -i $buildIdentifier" @@ -493,16 +503,18 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ] && if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Compute Package Url based on existing conventions using command" - echo $PGM": ${CMD}" + echo $PGM": [INFO] ** Invoking subtask ${CMD}" ${CMD} - - if [ -f "$(getLogDir)/${tempVersionFile}" ]; then - echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" - source "$(getLogDir)/${tempVersionFile}" - else - rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${tempVersionFile} was not found. Check previous console output. rc="$rc - echo $ERRMSG + rc=$? + if [ $rc -eq 0 ]; then + if [ -f "$(getLogDir)/${tempVersionFile}" ]; then + echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" + source "$(getLogDir)/${tempVersionFile}" + else + rc=4 + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${tempVersionFile} was not found. Check previous console output. rc="$rc + echo $ERRMSG + fi fi fi fi From 2740ac79f17450dcb83679f1ad9f77bb3b84dc44 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:21:52 +0100 Subject: [PATCH 047/127] remove script configuration from ucd integrations Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/ucdDeploy.sh | 28 +++++++++++++------ .../Common-Backend-Scripts/ucdPackaging.sh | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Templates/Common-Backend-Scripts/ucdDeploy.sh b/Templates/Common-Backend-Scripts/ucdDeploy.sh index 1ffc0493..2fa2820a 100755 --- a/Templates/Common-Backend-Scripts/ucdDeploy.sh +++ b/Templates/Common-Backend-Scripts/ucdDeploy.sh @@ -76,6 +76,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +deployScript="${SCRIPT_HOME}/../../../Pipeline/DeployUCDComponentVersion/ucd-deploy.groovy" # Customization - End # @@ -269,6 +270,15 @@ if [ $rc -eq 0 ]; then fi fi +if [ $rc -eq 0 ]; then + # Validate Packaging script + if [ ! -f "${deployScript}" ]; then + rc=8 + ERRMSG=$PGM": [ERR] Unable to locate ${deployScript}. rc="$rc + echo $ERRMSG + fi +fi + # # Set up Environment if [ $rc -eq 0 ]; then @@ -280,13 +290,13 @@ fi if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Start UCD Component Deploy on HOST/USER: ${SYS}/${USER}" - echo $PGM": [INFO] ** Location of ucd-deploy.groovy:" ${deployScriptLoc} - echo $PGM": [INFO] ** UCD Application Name:" ${Application} - echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} - echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} - echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} - echo $PGM": [INFO] ** UCD Component Version:" ${Version} - echo $PGM": [INFO] ** UCD Application Process Name:" ${Process} + echo $PGM": [INFO] ** Create UCD component version script:" ${deployScript} + echo $PGM": [INFO] ** UCD Application Name:" ${Application} + echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} + echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} + echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} + echo $PGM": [INFO] ** UCD Component Version:" ${Version} + echo $PGM": [INFO] ** UCD Application Process Name:" ${Process} if [ ! -z "${Timeout}" ]; then echo $PGM": [INFO] ** UCD Timeout (seconds):" ${Timeout} @@ -315,8 +325,8 @@ fi # # Set up to execute the UCD Deploy groovy script if [ $rc -eq 0 ]; then - echo $DBB_HOME/bin/groovyz ${deployScriptLoc}/ucd-deploy.groovy -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u ${ucdServerURL} -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 - $DBB_HOME/bin/groovyz ${deployScriptLoc}/ucd-deploy.groovy -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u '"'${ucdServerURL}'"' -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 + echo $DBB_HOME/bin/groovyz ${deployScript} -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u ${ucdServerURL} -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 + $DBB_HOME/bin/groovyz ${deployScript} -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u '"'${ucdServerURL}'"' -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 rc=$? if [ $rc -ne 0 ]; then diff --git a/Templates/Common-Backend-Scripts/ucdPackaging.sh b/Templates/Common-Backend-Scripts/ucdPackaging.sh index b32dd5f7..d6f201e2 100755 --- a/Templates/Common-Backend-Scripts/ucdPackaging.sh +++ b/Templates/Common-Backend-Scripts/ucdPackaging.sh @@ -26,7 +26,6 @@ # # 2. Review the Customization Section in the pipelineBackend.config file : # -# ucdPackagingScript - Location of the dbb-ucd-packaging.groovy # BuzTool - Location of the UCD buztool.sh # #=================================================================================== @@ -96,6 +95,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +ucdPackagingScript="${SCRIPT_HOME}/../../../Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" # Customization - End # Path and File Name to the advanced debug options. From d439114e089bedfc8542e20d4c42148e2e7828fc Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:22:43 +0100 Subject: [PATCH 048/127] remove script configuration from packaging integration Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/packageBuildOutputs.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index b483982e..8cef9df6 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -23,9 +23,6 @@ # 1. Review and update the Customization Section to reference the # central configuration file pipelineBackend.config # -# 2. Review the Customization Section in the pipelineBackend.config file : -# -# PackagingScript - Location of the PackageBuildOutputs.groovy # #=================================================================================== Help() { @@ -106,6 +103,7 @@ Help() { # Central configuration file leveraged by the backend scripts SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +packagingScript="${SCRIPT_HOME}/../../Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" # Path and File Name to the advanced debug options. @@ -326,9 +324,9 @@ validateOptions() { fi # Validate Packaging script - if [ ! -f "${PackagingScript}" ]; then + if [ ! -f "${packagingScript}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${PackagingScript}. rc="$rc + ERRMSG=$PGM": [ERR] Unable to locate ${packagingScript}. rc="$rc echo $ERRMSG fi @@ -458,7 +456,7 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} fi echo $PGM": [INFO] ** BuildReport Location:" ${logDir} - echo $PGM": [INFO] ** PackagingScript Path:" ${PackagingScript} + echo $PGM": [INFO] ** PackagingScript Path:" ${packagingScript} if [ ! -z "${PkgPropFile}" ]; then echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} fi @@ -507,7 +505,7 @@ if [ $rc -eq 0 ]; then cycloneDXlibraries="-cp ${cycloneDXlibraries}" fi - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${cycloneDXlibraries} ${PackagingScript} --workDir ${logDir}" + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${cycloneDXlibraries} ${packagingScript} --workDir ${logDir}" # add tarfile name if [ ! -z "${tarFileName}" ]; then From 1c3725571fb0783b10b03e16e139d1d4c1870fd1 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:23:35 +0100 Subject: [PATCH 049/127] remove script configuration from pipelinebackend config Signed-off-by: Dennis Behm --- .../utilities/computePackageUrl.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh index 26c0b3dd..fe6979d9 100755 --- a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh +++ b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh @@ -89,6 +89,7 @@ Help() { SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/../pipelineBackend.config" packageUtilities="${SCRIPT_HOME}/packageUtils.sh" +artifactRepositoryHelpersScript="${SCRIPT_HOME}/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy" # Path and File Name to the advanced debug options. #log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" @@ -282,12 +283,13 @@ validateOptions() { buildIdentifier=$(date +%Y%m%d_%H%M%S) fi - # Validate Packaging script - if [ ! -f "${PackagingScript}" ]; then + if [ ! -f "${artifactRepositoryHelpersScript}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${PackagingScript}. rc="$rc + ERRMSG=$PGM": [ERR] Unable to locate ${artifactRepositoryHelpersScript}. rc="$rc echo $ERRMSG fi + + } @@ -408,7 +410,7 @@ fi if [ $rc -eq 0 ]; then echo $PGM": [INFO] Invoking the Package Build Outputs script to compute Package Url." - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${dbbCommunityRepoRootDir}/${artifactRepositoryHelpersScript} --computePackageUrl" + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computePackageUrl" # add tarfile name if [ ! -z "${tarFileName}" ]; then From 52e2ea984da70403587fbf2ae16d07fc02eea707 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:23:47 +0100 Subject: [PATCH 050/127] remove script configuration from pipelinebackend config Signed-off-by: Dennis Behm --- .../pipelineBackend.config | 71 +++++-------------- 1 file changed, 18 insertions(+), 53 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index b59b6be4..14446609 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -30,48 +30,11 @@ buildRootDir="$PIPELINE_WORKSPACE" # default: logsDir="logs" logsDir="logs" -# root directory of the location of the cloned DBB community repository -# https://github.com/IBM/dbb on the build machine -# Note: Cannot reference environment variables -dbbCommunityRepoRootDir=/var/dbb/pipelineTemplates/dbb -dbbCommunityRepoRootDir=/u/dbehm/git/dbb - - ##################################################################################################### ## End of Shared Configuration parameters ######################################################## ##################################################################################################### -##################################################################################################### -## Fetch Build Dependencies based on Application Descriptor ######################################## -##################################################################################################### - -# Setting to enable fetching build dependencies configured within the Application Descriptor -# default: false -fetchBuildDependencies=true - -# Relative path of the ArtifactRepositoryHelpers.groovy script which is part of the dbb community repository -# Note: Cannot reference environment variables -artifactRepositoryHelpersScript=Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy - -# Settings for caching packages on build machine - -# Setting to control if packages should be stored on the build machine -# default: false -enablePackageCache=true - -# Location for cachine packages on the build machine -packageCacheLocation=$HOME/cbs/.packagecache - -# Relative path of the applicationDescriptorHelperUtils which is part of the dbb community repository -# Note: Cannot reference environment variables -# default -applicationDescriptorHelperUtils=Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy - -##################################################################################################### -## End of Fetch Build Dependencies based on Application Descriptor ################################# -##################################################################################################### - ##################################################################################################### ## DBB-BUILD.sh parameters ########################################################################## ##################################################################################################### @@ -144,15 +107,28 @@ featureBranchBuildBehaviour=merge-base ##################################################################################################### ##################################################################################################### -## packageBuildOutputs.sh parameters ############################################################# +## Fetch Build Dependencies based on Application Descriptor ######################################## ##################################################################################################### -# Path to PackageBuildOutputs.groovy script from the DBB community repository -# e.g. PackagingScript="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -PackagingScript=/u/dbehm/git/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +# Setting to enable fetching build dependencies configured within the Application Descriptor +# default: false +fetchBuildDependencies=true + +# Settings for caching packages on build machine + +# Setting to control if packages should be stored on the build machine +# default: false +enablePackageCache=true + +# Location for cachine packages on the build machine +packageCacheLocation=$HOME/cbs/.packagecache + +##################################################################################################### +## End of Fetch Build Dependencies based on Application Descriptor ################################# +##################################################################################################### ##################################################################################################### -#### PackageBuildOutputs.groovy configuration #################################################### +## packageBuildOutputs.sh parameters and configuration ############################################ ##################################################################################################### # (Optional) Absolute path of a property file containing default application packaging details. @@ -208,7 +184,6 @@ artifactRepositoryPassword=artifactoryadmin artifactRepositoryNamePattern=§application§-repo-local - ##################################################################################################### #### Methods enforcing naming conventions ######################################################## ##################################################################################################### @@ -255,10 +230,6 @@ artifactRepositoryPropertyFile= ## ucdPackaging.sh parameters #################################################################### ##################################################################################################### -# Path to dbb-ucd-packaging.groovy script from the DBB community repository -# e.g. ucdPackagingScript="/var/dbb/extensions/dbb20/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" -ucdPackagingScript="" - # Path to UCD buztool.sh script # e.g. BuzTool="/var/ucd-agent/bin/buztool.sh" BuzTool="/var/ucd-agent/bin/buztool.sh" @@ -271,11 +242,6 @@ BuzTool="/var/ucd-agent/bin/buztool.sh" ## UCD-Deploy.sh parameters ###################################################################### ##################################################################################################### -# -# Location of UCD Deploy Script -# e.g. deployScriptLoc=/var/dbb/extensions/dbb20/Pipeline/DeployUCDComponentVersion -deployScriptLoc="" - # # UCD Server URL # e.g. ucdServerURL=https://ucd.server:8443/ @@ -292,7 +258,6 @@ ucdUserName="" # Protect the file by specifying the correct file permissions ucdPassword="" - ##################################################################################################### ## End of UCD-Deploy.sh parameters ############################################################### ##################################################################################################### From d73c75529b2252fb7895e5744e4b53e05b5abf0b Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:24:27 +0100 Subject: [PATCH 051/127] remove script configuration from pipeline configuration Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 7227e9d9..c5972a66 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -30,7 +30,10 @@ props.sort().each { k,v-> } // Load and verify helpers -File artifactRepositoryHelpersScriptFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.artifactRepositoryHelpersScript}") +def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + +// Artifact RepositoryHelpers are used to download the package +File artifactRepositoryHelpersScriptFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy") if (artifactRepositoryHelpersScriptFile.exists()) { artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) } else { @@ -38,28 +41,27 @@ if (artifactRepositoryHelpersScriptFile.exists()) { System.exit(1) } -// Load and verify helpers -File artifactRepositoryPathUtilitiesFile = new File("${props.artifactRepositoryPathUtilities}") +// ArtifactRepositoryPathUtilities are used to recompute the URL +File artifactRepositoryPathUtilitiesFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/utilities/ArtifactRepositoryPathUtilities.groovy") if (artifactRepositoryPathUtilitiesFile.exists()) { artifactRepositoryPathUtilities = loadScript(artifactRepositoryPathUtilitiesFile) } else { - println("*! [ERROR] The Artifact Repo Path Utility script '${props.artifactRepositoryPathUtilities}' doesn't exist. Exiting.") - System.exit(1) + println("*! [ERROR] The Artifact Repo Path Utility script '$artifactRepositoryPathUtilitiesFile.getName()' doesn't exist. Exiting.") + System.exit(8) } -File applicationDescriptorUtilsFile = new File("${props.dbbCommunityRepoRootDir}" + "/" + "${props.applicationDescriptorHelperUtils}") - +File applicationDescriptorUtilsFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy") if (applicationDescriptorUtilsFile.exists()) { applicationDescriptorUtils = loadScript(applicationDescriptorUtilsFile) } else { - println("*! [ERROR] The Application Descriptor Helper script '${props.applicationDescriptorHelperUtils}' was not found. Exiting.") - System.exit(1) + println("*! [ERROR] The Application Descriptor Helper script '$applicationDescriptorUtilsFile.getName()' was not found. Exiting.") + System.exit(8) } File applicationDescriptorFile = new File(props.applicationDescriptor) if (!applicationDescriptorFile.exists()) { - println("*! [ERROR] The Application Descriptor file '${props.applicationDescriptor}' doesn't exist. Exiting.") - System.exit(1) + println("*! [ERROR] The Application Descriptor file '${applicationDescriptorFile.getName()}' doesn't exist. Exiting.") + System.exit(8) } // setup import directory @@ -347,13 +349,6 @@ def parseArgs(String[] args) { props.pipelineBackendConfigFile = opts.p Properties temporaryProperties = new Properties() pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } - if(temporaryProperties.get("dbbCommunityRepoRootDir")) props.put("dbbCommunityRepoRootDir", temporaryProperties.get("dbbCommunityRepoRootDir")) - // helper scripts - if(temporaryProperties.get("artifactRepositoryHelpersScript")) props.put("artifactRepositoryHelpersScript", temporaryProperties.get("artifactRepositoryHelpersScript")) - if(temporaryProperties.get("applicationDescriptorHelperUtils")) props.put("applicationDescriptorHelperUtils", temporaryProperties.get("applicationDescriptorHelperUtils")) - if(temporaryProperties.get("artifactRepositoryPathUtilities")) props.put("artifactRepositoryPathUtilities", temporaryProperties.get("artifactRepositoryPathUtilities")) - - if(temporaryProperties.get("PackagingScript")) props.put("PackagingScript", temporaryProperties.get("PackagingScript")) // artifact repo configuration properties / Map CBS pipelineBackend.config to script properties if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepository.url", temporaryProperties.get("artifactRepositoryUrl")) if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepository.user", temporaryProperties.get("artifactRepositoryUser")) From 9c602a2979a2f73dd5fa98723ca5db01ca1b40c2 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:38:14 +0100 Subject: [PATCH 052/127] consolidate how the url is computed Signed-off-by: Dennis Behm --- .../ArtifactRepositoryHelpers.groovy | 37 +++++++++++-------- .../PackageBuildOutputs.groovy | 5 +-- .../utilities/fetchBuildDependencies.groovy | 12 +----- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index ff865de7..da939924 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -45,6 +45,7 @@ def CompletableFuture> run(args) +// public methods def upload(String url, String fileName, String user, String password, boolean verbose, String httpClientVersion) throws IOException { System.setProperty("jdk.httpclient.allowRestrictedHeaders", "Connection") Path testing = Paths.get(fileName) @@ -95,18 +96,18 @@ def upload(String url, String fileName, String user, String password, boolean ve // submit request CompletableFuture> response = httpClient.sendAsync(request, handler).thenComposeAsync(r -> tryResend(httpClient, request, handler, 1, r)); HttpResponse finalResponse = response.get() -// -// if (verbose) -// println("** Response: " + finalResponse); -// -// def rc = evaluateHttpResponse(finalResponse, "upload", verbose) -// -// if (rc == 0 ) { -// println("** Upload completed."); -// } -// else { -// println("*! Upload failed."); -// } + + if (verbose) + println("** Response: " + finalResponse); + + def rc = evaluateHttpResponse(finalResponse, "upload", verbose) + + if (rc == 0 ) { + println("** Upload completed."); + } + else { + println("*! Upload failed."); + } } def download(String url, String fileName, String user, String password, boolean verbose) throws IOException { @@ -159,6 +160,14 @@ def download(String url, String fileName, String user, String password, boolean } } +// Method directly accessed by PackageBuildOutputs and Common Backend script functionality +def computeAbsoluteRepositoryUrl(Properties props) { + def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) + def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL + return url +} + +// private methods def evaluateHttpResponse (HttpResponse response, String action, boolean verbose) { int rc = 0 def statusCode = response.statusCode() @@ -242,9 +251,7 @@ def run(String[] cliArgs) { // load script def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent - def artifactRepositoryPathUtilities = loadScript(new File("${scriptDir}/utilities/ArtifactRepositoryPathUtilities.groovy")) - packageUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) - println "packageUrl=$packageUrl" + packageUrl = computeAbsoluteRepositoryUrl(props) } else println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computePackageUrl') "); } diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 17bdfaec..fe1b9ffa 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -69,7 +69,6 @@ import com.ibm.jzos.ZFile; @Field Properties props = new Properties() def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) -@Field def artifactRepositoryPathUtilities = loadScript(new File("${scriptDir}/utilities/ArtifactRepositoryPathUtilities.groovy")) @Field def artifactRepositoryHelpers = loadScript(new File("${scriptDir}/ArtifactRepositoryHelpers.groovy")) @Field def applicationDescriptorUtils @@ -639,7 +638,7 @@ if (rc == 0) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") packageInfo.put("name", props.packageBuildIdentifier) - packageUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) + packageUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } @@ -762,7 +761,7 @@ if (rc == 0) { //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def url = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) + def url = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index c5972a66..7197f62c 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -17,7 +17,6 @@ import java.nio.file.* @Field Properties props = new Properties() @Field def artifactRepositoryHelpers // Helpers to download @Field def applicationDescriptorUtils // Helper to parse Application Descriptor -@Field def artifactRepositoryPathUtilities // Util to compute the URL path @Field def packageBuildOutputs // Helpers to download // Parse arguments from command-line @@ -38,15 +37,6 @@ if (artifactRepositoryHelpersScriptFile.exists()) { artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) } else { println("*! [ERROR] The Artifact Repository Helper script '${props.artifactRepositoryHelpersScript}' doesn't exist. Exiting.") - System.exit(1) -} - -// ArtifactRepositoryPathUtilities are used to recompute the URL -File artifactRepositoryPathUtilitiesFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/utilities/ArtifactRepositoryPathUtilities.groovy") -if (artifactRepositoryPathUtilitiesFile.exists()) { - artifactRepositoryPathUtilities = loadScript(artifactRepositoryPathUtilitiesFile) -} else { - println("*! [ERROR] The Artifact Repo Path Utility script '$artifactRepositoryPathUtilitiesFile.getName()' doesn't exist. Exiting.") System.exit(8) } @@ -109,7 +99,7 @@ if (applicationDescriptor.dependencies) { props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) // The absolute url the package in artifact repo - artifactUrl = artifactRepositoryPathUtilities.computeAbsoluteRepositoryUrl(props) + artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) println artifactUrl // Construct the path within the Artifact repo From 15cfcbde735908e823fe5dbb9407527528c00ef1 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 10:42:37 +0100 Subject: [PATCH 053/127] print result to be picked up by cbs Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index da939924..bf6ac26b 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -252,6 +252,8 @@ def run(String[] cliArgs) { // load script def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent packageUrl = computeAbsoluteRepositoryUrl(props) + // the println is used in a script by the CBS to grep the packageUrl + println "packageUrl=$packageUrl" } else println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computePackageUrl') "); } From 79f6ca05818522107ce2001af2071c21ca0a1b74 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:24:46 +0100 Subject: [PATCH 054/127] return rc on upload Signed-off-by: Dennis Behm --- .../PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index bf6ac26b..f95113f4 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -49,7 +49,6 @@ run(args) def upload(String url, String fileName, String user, String password, boolean verbose, String httpClientVersion) throws IOException { System.setProperty("jdk.httpclient.allowRestrictedHeaders", "Connection") Path testing = Paths.get(fileName) - println "asdadasdasdasd ----- ${testing.toString()}" println( "** ArtifactRepositoryHelper started for upload of $fileName to $url" ); // create http client @@ -70,7 +69,7 @@ def upload(String url, String fileName, String user, String password, boolean ve // } HttpClient httpClient = httpClientBuilder.build(); - + // build http request HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() .uri(URI.create("$url")) @@ -78,7 +77,7 @@ def upload(String url, String fileName, String user, String password, boolean ve .header("Connection","Keep-Alive") .PUT(BodyPublishers.ofFile(Paths.get(fileName))); - // set http client version if set + // set http client version if set if (httpClientVersion) { def httpVer = HttpClient.Version.valueOf(httpClientVersion) if (httpVer) { @@ -94,9 +93,10 @@ def upload(String url, String fileName, String user, String password, boolean ve HttpResponse.BodyHandler handler = HttpResponse.BodyHandlers.ofString(); // submit request + CompletableFuture> response = httpClient.sendAsync(request, handler).thenComposeAsync(r -> tryResend(httpClient, request, handler, 1, r)); HttpResponse finalResponse = response.get() - + if (verbose) println("** Response: " + finalResponse); @@ -108,6 +108,7 @@ def upload(String url, String fileName, String user, String password, boolean ve else { println("*! Upload failed."); } + return rc } def download(String url, String fileName, String user, String password, boolean verbose) throws IOException { From b8f5edfd7defe6b1ee89939c85b01e98fa4e0005 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:25:30 +0100 Subject: [PATCH 055/127] return rc on download Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index f95113f4..8a45a0cd 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -159,6 +159,8 @@ def download(String url, String fileName, String user, String password, boolean } else { println("*! Download failed."); } + + return rc } // Method directly accessed by PackageBuildOutputs and Common Backend script functionality From 4431915414d248f44c1ac299f41377a274ae5911 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:26:14 +0100 Subject: [PATCH 056/127] capture the rc from upload Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index fe1b9ffa..26a0ca86 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -768,9 +768,8 @@ if (rc == 0) { def password = props.'artifactRepository.password' def httpClientVersion = props.'artifactRepository.httpClientVersion' def repo = props.get('artifactRepository.repo') as String - println ("** Upload package to Artifact Repository '$url'.") - artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + rc = artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) // generate PackageInfo for Concert Manifest file if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { From 53af6505c868e825e08a7d7503f2f1fcd9cf6f07 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:31:42 +0100 Subject: [PATCH 057/127] evaluate return code Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 7197f62c..5e00eedd 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -164,7 +164,13 @@ if (applicationDescriptor.dependencies) { println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tarFileDir}.") def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) - println "download complete $rc" // TODO: Error handling in helper + + if (rc != 0) { + println "** Download of application package '$artifactUrl' failed. Process exists. Return code:"$rc + exitFetchDependencyProcess() + } else { + //println "* Download successful." + } } @@ -278,6 +284,10 @@ if (props.externalDependenciesFilePath) { } } +exitFetchDependencyProcess(){ + println("** [ERROR] fetchBuildDependencies encountered a problem. Please review log. Exiting") + System.exit(1) +} /** * Parse CLI config From ac1c4bc14d8f531592a727f1cf494abfc23234db Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:36:15 +0100 Subject: [PATCH 058/127] fix syntax error Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 5e00eedd..f1e9fd7f 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -284,7 +284,7 @@ if (props.externalDependenciesFilePath) { } } -exitFetchDependencyProcess(){ +def exitFetchDependencyProcess(){ println("** [ERROR] fetchBuildDependencies encountered a problem. Please review log. Exiting") System.exit(1) } From 62f4091cefd4d3d827b914d2d5beaf69983b409e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:44:45 +0100 Subject: [PATCH 059/127] switch to suffix pattern Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/pipelineBackend.config | 10 ++++++++-- .../utilities/fetchBuildDependencies.groovy | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 14446609..48e47f41 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -111,6 +111,8 @@ featureBranchBuildBehaviour=merge-base ##################################################################################################### # Setting to enable fetching build dependencies configured within the Application Descriptor +# this leverages the configuration for the publishing script PackageBuildOutputs +# to retrieve build dependencies from the Artifact repo. # default: false fetchBuildDependencies=true @@ -123,6 +125,12 @@ enablePackageCache=true # Location for cachine packages on the build machine packageCacheLocation=$HOME/cbs/.packagecache +# Name conventions for the Artifact repository within the Artifact repository server +# script will prefix it with the application name +# default: artifactRepositoryNameSuffix=repo-local +# leading to application-repo-local +artifactRepositoryNameSuffix=repo-local + ##################################################################################################### ## End of Fetch Build Dependencies based on Application Descriptor ################################# ##################################################################################################### @@ -182,8 +190,6 @@ artifactRepositoryUser=admin # e.q.: artifactRepositoryPassword=xxxxx artifactRepositoryPassword=artifactoryadmin -artifactRepositoryNamePattern=§application§-repo-local - ##################################################################################################### #### Methods enforcing naming conventions ######################################################## ##################################################################################################### diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index f1e9fd7f..09c684c2 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -103,7 +103,7 @@ if (applicationDescriptor.dependencies) { println artifactUrl // Construct the path within the Artifact repo - repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", dependency.name) + repositoryName="${dependency.name}-${props.artifactRepositoryNameSuffix}" // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") @@ -216,7 +216,7 @@ if (applicationDescriptor.dependencies) { // // baselineName=applicationDescriptor.application // -// repositoryName="${props.artifactRepositoryNamePattern}".replaceAll("§application§", baselineName) +// repositoryName="${props.artifactRepositoryNameSuffix}".replaceAll("§application§", baselineName) // // def String artifactUrl // def String artifactReference @@ -353,7 +353,7 @@ def parseArgs(String[] args) { if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepository.url", temporaryProperties.get("artifactRepositoryUrl")) if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepository.user", temporaryProperties.get("artifactRepositoryUser")) if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepository.password", temporaryProperties.get("artifactRepositoryPassword")) - if(temporaryProperties.get("artifactRepositoryNamePattern")) props.put("artifactRepositoryNamePattern", temporaryProperties.get("artifactRepositoryNamePattern")) + if(temporaryProperties.get("artifactRepositoryNameSuffix")) props.put("artifactRepositoryNameSuffix", temporaryProperties.get("artifactRepositoryNameSuffix")) if(temporaryProperties.get("enablePackageCache")) props.put("enablePackageCache", temporaryProperties.get("enablePackageCache")) } else { println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") From 9f6aab6f7bc9069d5eed8a3d381a58e4d151cc73 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:51:59 +0100 Subject: [PATCH 060/127] Add release update Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/ucdDeploy.sh | 3 ++- Templates/Common-Backend-Scripts/ucdPackaging.sh | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/ucdDeploy.sh b/Templates/Common-Backend-Scripts/ucdDeploy.sh index 2fa2820a..d31a7640 100755 --- a/Templates/Common-Backend-Scripts/ucdDeploy.sh +++ b/Templates/Common-Backend-Scripts/ucdDeploy.sh @@ -27,6 +27,7 @@ # Date Who Vers Description # ---------- --- ---- -------------------------------------------------------------- # 2023/07/18 RBS 1.00 Initial Release +# 2025/03/13 DB 1.10 Locate groovy scripts in file system #=================================================================================== Help() { echo $PGM" - Deploy UCD Component " @@ -85,7 +86,7 @@ deployScript="${SCRIPT_HOME}/../../../Pipeline/DeployUCDComponentVersion/ucd-dep #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) diff --git a/Templates/Common-Backend-Scripts/ucdPackaging.sh b/Templates/Common-Backend-Scripts/ucdPackaging.sh index d6f201e2..74878e12 100755 --- a/Templates/Common-Backend-Scripts/ucdPackaging.sh +++ b/Templates/Common-Backend-Scripts/ucdPackaging.sh @@ -28,6 +28,11 @@ # # BuzTool - Location of the UCD buztool.sh # +# Maintenance Log +# Date Who Vers Description +# ---------- --- ---- -------------------------------------------------------------- +# 2023/07/18 RBS 1.00 Initial Release +# 2025/03/13 DB 1.10 Locate groovy scripts in file system #=================================================================================== Help() { echo $PGM" - Invoke DBB Build ("$PGMVERS") " @@ -107,7 +112,7 @@ ucdPackagingScript="${SCRIPT_HOME}/../../../Pipeline/CreateUCDComponentVersion/d #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) From ef71fc4bac490f0afc669e6446a7a0c77e7dd582 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:52:21 +0100 Subject: [PATCH 061/127] remote testing bits Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 8a45a0cd..2044cb32 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -10,7 +10,6 @@ import java.net.http.HttpResponse.BodyHandler import java.util.concurrent.CompletableFuture import java.nio.file.Paths -import java.nio.file.Path /** Very basic script to upload/download from an artifact repository server @@ -48,7 +47,6 @@ run(args) // public methods def upload(String url, String fileName, String user, String password, boolean verbose, String httpClientVersion) throws IOException { System.setProperty("jdk.httpclient.allowRestrictedHeaders", "Connection") - Path testing = Paths.get(fileName) println( "** ArtifactRepositoryHelper started for upload of $fileName to $url" ); // create http client From 4b55eb8d94f035d5eee568e28a47cde2e520a9c8 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:55:31 +0100 Subject: [PATCH 062/127] Add release update wdg Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/wazideploy-generate.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index ae6fcee6..87b3011b 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -27,6 +27,8 @@ # Date Who Vers Description # ---------- ---- ---- -------------------------------------------------------------- # 2023/10/19 MDLB 1.00 Initial Release +# 2025/03/13 DB 1.10 Allow pipelines to compute the artifact location to download +# packages via wazideploy-generate #=================================================================================== Help() { echo $PGM" - Generate Wazi Deploy Deployment Plan " @@ -142,7 +144,7 @@ computePackageUrlUtil="${SCRIPT_HOME}/utilities/computePackageUrl.sh" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) From 27d008e9108b3bd751f7d05a91f0f10c42f9de33 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 11:57:02 +0100 Subject: [PATCH 063/127] fix merge conflict Signed-off-by: Dennis Behm --- Templates/GitlabCIPipeline/.gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Templates/GitlabCIPipeline/.gitlab-ci.yml b/Templates/GitlabCIPipeline/.gitlab-ci.yml index d0ac3beb..335fdf99 100644 --- a/Templates/GitlabCIPipeline/.gitlab-ci.yml +++ b/Templates/GitlabCIPipeline/.gitlab-ci.yml @@ -569,11 +569,7 @@ Packaging: wdEvidencesDir: ${wdEvidencesRoot}/${application}/production artifacts: -<<<<<<< HEAD - name: productionDeploymentReport-${CI_PIPELINE_ID}" -======= name: "productionDeploymentReport-${CI_PIPELINE_ID}" ->>>>>>> bb3af91 ( remove odd char) when: always paths: - "${productionReportDir}/deployment-report.html" From 226e7f590ed41e30427cf45a1a999503779514df Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 12:04:18 +0100 Subject: [PATCH 064/127] remove function Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 2 +- .../pipelineBackend.config | 21 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 971fe24c..2745ad37 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -489,7 +489,7 @@ CLI parameter | Description **Artifact Upload options** -u | Flag to enable upload of outputs to the configured artifact repository. -a `` | **Application name** leveraged to define the artifact repository name. See function `computeArtifactRepositoryName()` in the pipelineBackend.config file. Ex.: `MortgageApplication-repo-local`. --b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. See function `computeArtifactRepositoryDirectory()` in the pipelineBackend.config file. +-b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. -v `` | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. -s `""` | (Optional) Name and email of the SBOM author enclosed with double quotes. Ex: "Build Engineer \" diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 48e47f41..cfdbf99b 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -200,27 +200,6 @@ computeArtifactRepositoryName() { echo "${App}-repo-local" } -# Method to return the directory path in the repository to store the build -# the default path in the repository is the branch name -# e.q.: $Branch -# If the type of pipeline is specified, we are appending it to the directory where -# the build is stored -computeArtifactRepositoryDirectory() { - if [ -z "${PipelineType}" ]; then - echo "${Branch}" - else - case $Branch in - main) - echo "${Branch}/${PipelineType}" - ;; - *) - echo "${Branch}" - ;; - esac - fi - -} - # Path of a property file containing application specific artifact repository details. # application specific artifactRepositoryPropertyFile # e.q.: application-conf/packageBuildOutputs.properties From 839fd3a064d3fa13d3dd3fadc62c602962b8874a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 12:05:22 +0100 Subject: [PATCH 065/127] remove function from config Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/pipelineBackend.config | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index cfdbf99b..b79fc151 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -190,16 +190,6 @@ artifactRepositoryUser=admin # e.q.: artifactRepositoryPassword=xxxxx artifactRepositoryPassword=artifactoryadmin -##################################################################################################### -#### Methods enforcing naming conventions ######################################################## -##################################################################################################### - -# Method to return the pattern for the artifact repository name to store the build -# e.q.: $App-repo-local -computeArtifactRepositoryName() { - echo "${App}-repo-local" -} - # Path of a property file containing application specific artifact repository details. # application specific artifactRepositoryPropertyFile # e.q.: application-conf/packageBuildOutputs.properties From eedd6b6860ef3c03d4015e5404c4cccd74733d34 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 12:07:28 +0100 Subject: [PATCH 066/127] update log outputs Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/ucdDeploy.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Templates/Common-Backend-Scripts/ucdDeploy.sh b/Templates/Common-Backend-Scripts/ucdDeploy.sh index d31a7640..f1c56aa2 100755 --- a/Templates/Common-Backend-Scripts/ucdDeploy.sh +++ b/Templates/Common-Backend-Scripts/ucdDeploy.sh @@ -291,13 +291,13 @@ fi if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Start UCD Component Deploy on HOST/USER: ${SYS}/${USER}" - echo $PGM": [INFO] ** Create UCD component version script:" ${deployScript} - echo $PGM": [INFO] ** UCD Application Name:" ${Application} - echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} - echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} - echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} - echo $PGM": [INFO] ** UCD Component Version:" ${Version} - echo $PGM": [INFO] ** UCD Application Process Name:" ${Process} + echo $PGM": [INFO] ** UCD deploy script :" ${deployScript} + echo $PGM": [INFO] ** UCD Application Name:" ${Application} + echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} + echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} + echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} + echo $PGM": [INFO] ** UCD Component Version:" ${Version} + echo $PGM": [INFO] * UCD Application Process Name:" ${Process} if [ ! -z "${Timeout}" ]; then echo $PGM": [INFO] ** UCD Timeout (seconds):" ${Timeout} From 676226d599d2a480bbe77450d0f3d6144cb2040e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 12:15:44 +0100 Subject: [PATCH 067/127] updated structure of AD for dependencies Signed-off-by: Dennis Behm --- .../utilities/applicationDescriptorUtils.groovy | 16 ++++++---------- .../utilities/fetchBuildDependencies.groovy | 14 +++++++------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index f3ddc76b..d5dda670 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -48,19 +48,15 @@ class Baseline { /* - name: "retirementCalculator" - reference: "release" - version: "1.2.3" - buildid: "875487" -- name: "GenApp" - reference: "build" - version: "feature/789-enhance-something" - buildid: "123456" + type: "release" + version: "rel-1.2.0" + buildid: "build-20241112.1" */ class DependencyDescriptor { String name - String reference + String type String version String buildid } @@ -206,7 +202,7 @@ def removeFileDefinition(ApplicationDescriptor applicationDescriptor, String sou * Method to add an application dependency */ -def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String reference, String version, String buildid) { +def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String type, String version, String buildid) { if (!applicationDescriptor.dependencies) { applicationDescriptor.dependencies = new ArrayList() } @@ -217,7 +213,7 @@ def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String if (!existingDependencies) { def dependency = new DependencyDescriptor() dependency.name = applicationDependency - dependency.reference = reference + dependency.type = type dependency.version = version dependency.buildid = buildid applicationDescriptor.dependencies.add(dependency) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 09c684c2..a444912d 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -86,7 +86,7 @@ if (applicationDescriptor.dependencies) { buildid: "123456"*/ // compute tar file name based on build type - if (dependency.reference.equalsIgnoreCase("release")) { + if (dependency.type.equalsIgnoreCase("release")) { assert dependency.version : "Missing dependency version in dependency record" assert dependency.buildid : "Missing buildid in dependency record" props.put("tarFileName","${dependency.name}-${dependency.version}-${dependency.buildid}.tar") @@ -95,7 +95,7 @@ if (applicationDescriptor.dependencies) { } props.put("versionName","${dependency.version}") // compute the version name being part of the path - props.put("artifactRepository.directory", "${dependency.reference}") // compute the main directory to classify builds + props.put("artifactRepository.directory", "${dependency.type}") // compute the main directory to classify builds props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) // The absolute url the package in artifact repo @@ -133,11 +133,11 @@ if (applicationDescriptor.dependencies) { p_version.value = dependency.version externalDependency.properties.add(p_version) - // reference - Property p_reference = new Property() - p_reference.key = "reference" - p_reference.value = dependency.reference - externalDependency.properties.add(p_reference) + // type + Property p_type = new Property() + p_type.key = "type" + p_type.value = dependency.type + externalDependency.properties.add(p_type) // buildid Property p_buildid = new Property() From 8d62c0de51e483d0d96fac78a5c0b18c4a8f0803 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 13 Mar 2025 16:43:13 +0100 Subject: [PATCH 068/127] align with new dependency attribute in application descriptor Signed-off-by: Dennis Behm --- .../applicationDescriptorUtils.groovy | 14 ++----- .../utilities/fetchBuildDependencies.groovy | 37 ++++++++----------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index d5dda670..ce4d4f3f 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -46,18 +46,10 @@ class Baseline { String baseline } -/* -- name: "retirementCalculator" - type: "release" - version: "rel-1.2.0" - buildid: "build-20241112.1" -*/ - - class DependencyDescriptor { String name String type - String version + String reference String buildid } @@ -202,7 +194,7 @@ def removeFileDefinition(ApplicationDescriptor applicationDescriptor, String sou * Method to add an application dependency */ -def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String type, String version, String buildid) { +def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String type, String reference, String buildid) { if (!applicationDescriptor.dependencies) { applicationDescriptor.dependencies = new ArrayList() } @@ -214,7 +206,7 @@ def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String def dependency = new DependencyDescriptor() dependency.name = applicationDependency dependency.type = type - dependency.version = version + dependency.reference = reference dependency.buildid = buildid applicationDescriptor.dependencies.add(dependency) applicationDescriptor.dependencies.sort { diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index a444912d..ef063032 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -76,25 +76,20 @@ if (applicationDescriptor.dependencies) { // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> -/* - name: "retirementCalculator" - reference: "release" - version: "1.2.3" - buildid: "875487" - - name: "GenApp" - reference: "build" - version: "feature/789-enhance-something" - buildid: "123456"*/ - + // validate dependency record + assert dependency.type : "Missing dependency type attribute in dependency record" + assert dependency.reference : "Missing dependency reference attribute in dependency record" + assert dependency.buildid : "Missing buildid attribute in dependency record" + assert dependency.name : "Missing name attribute in dependency record" + // compute tar file name based on build type if (dependency.type.equalsIgnoreCase("release")) { - assert dependency.version : "Missing dependency version in dependency record" - assert dependency.buildid : "Missing buildid in dependency record" - props.put("tarFileName","${dependency.name}-${dependency.version}-${dependency.buildid}.tar") + props.put("tarFileName","${dependency.name}-${dependency.reference}-${dependency.buildid}.tar") } else { props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") } - props.put("versionName","${dependency.version}") // compute the version name being part of the path + props.put("versionName","${dependency.reference}") // compute the version name being part of the path props.put("artifactRepository.directory", "${dependency.type}") // compute the main directory to classify builds props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) @@ -127,18 +122,18 @@ if (applicationDescriptor.dependencies) { p_uri.value = artifactUrl externalDependency.properties.add(p_uri) - // type - Property p_version = new Property() - p_version.key = "version" - p_version.value = dependency.version - externalDependency.properties.add(p_version) - - // type + // type - either build or release Property p_type = new Property() p_type.key = "type" p_type.value = dependency.type externalDependency.properties.add(p_type) - + + // reference - the release name or the branch + Property p_reference = new Property() + p_reference.key = "reference" + p_reference.value = dependency.reference + externalDependency.properties.add(p_reference) + // buildid Property p_buildid = new Property() p_buildid.key = "buildid" From 2d9990a27cbb51dbde0eaeefc8fa8aed85489b02 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 13:21:33 +0100 Subject: [PATCH 069/127] Doc updates Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 218 ++++++++++++--------- 1 file changed, 122 insertions(+), 96 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 2745ad37..a981e0fa 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -244,9 +244,7 @@ The Application Descriptor contains metadata about the application itself, but c Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. Fetching the dependencies is powered by the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script. -Packages will be stored at a cache location to improve performance. - -Fetched packages will be expanded to the pipeline working directory. +Packages will be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory called 'imports'. ### dbbBuild.sh for zAppBuild framework @@ -463,11 +461,30 @@ The [dbbzBuilderUtils](utilities/dbbzBuilderUtils.sh) script is a core utility s * the `mainBuildBranch` to configure feature branch pipelines to clone the corresponding DBB dependency metadata collections by generating a config.yaml that is passed into zBuilder. -## packageBuildOutputs.sh +## Packaging stage + +You can choose from either creating a package using the [PackageBuildOutputs](#packagebuildoutputssh) script that can be deployed with IBM Wazi Deploy or the [UCD packaging](#ucdpackagingsh) that creates the UCD shiplist and UCD component version + +### packageBuildOutputs.sh This script is to execute the `PackageBuildOutputs.groovy` that packages up the build outputs and optionally uploads it to an artifact repository to publish the artifacts created by a DBB build in the pipeline. -### Invocation +When uploading into the artifact repository, it implements strict conventions about the structure in the Artifact repository. The conventions are implemented in [utilities/packageUtils.sh](utilities/packageUtils.sh). The rules are: + +For any temporary, preliminary build that uses the `pipelineType=build`, the outputs are uploaded into the directory `build//-`: + +* **build** is defined for any builds, that are considered to some extend temporary and preliminary. +* **reference** is the branch name from which the builds originates from, so for instance `feature/123-update-mortgage-computation`, `main` or any hotfix and epic branches. +* **tarFile.tar** is computed by the application name and a unique build identifier (see argument `-i`), such as the pipeline build number that is passed in by the pipeline template. If it is not provided, it will use the current timestamp. + +For release builds that use the `pipelineType=release`, the outputs are uploaded into the directory `release//-`: + +* **release** is defined for release builds. +* **reference** is planned release name, so for instance `rel-1.2.3` (mandatory argument `-r`). +* **tarFile.tar** is computed by the application name, the release name (`-r`) and the unique build identifier (see argument `-i`), such as the pipeline build number that is passed in by the pipeline template. If it is not provided, it will use the current timestamp. + + +#### Invocation The `packageBuildOutputs.sh` script can be invoked as follows: @@ -491,26 +508,18 @@ CLI parameter | Description -a `` | **Application name** leveraged to define the artifact repository name. See function `computeArtifactRepositoryName()` in the pipelineBackend.config file. Ex.: `MortgageApplication-repo-local`. -b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. --v `` | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. +-r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. +-i `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number or a timestamp. +-v `` **removed** | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. This has been removed. Please switch to releaseIdentifier and buildIdentifier. -s `""` | (Optional) Name and email of the SBOM author enclosed with double quotes. Ex: "Build Engineer \" -### Script conventions - -**Directory Path within the artifact repo** - -For uploading, the backend script computes the directory path within the artifact repository : -* **Branch**/**artifactVersion** - -If it is the `main` branch, the pipeline type (-p) is evaluated to -* **Branch**/**pipelineType **/**artifactVersion** - -while **artifactVersion** is appended by the `PackageBuildOutputs.groovy` script. +#### Script conventions **SBOM Generation** The generation of an SBOM is controlled by the `generateSBOM` property defined in the [pipelineBackend.config](pipelineBackend.config) file. The default SBOM Author is also specified in the [pipelineBackend.config](pipelineBackend.config) file in the `sbomAuthor` property, but this property can be overridden with the `-s` parameter of this script. When the SBOM Author is provided as a parameter, it automatically enables the SBOM generation, even if set to `false` in the [pipelineBackend.config](pipelineBackend.config) file. -### Output +#### Output The section below contains the output that is produced by the `packageBuildOutputs.sh` script. @@ -588,11 +597,11 @@ rc=0 -## ucdPackaging.sh +### ucdPackaging.sh This script is to execute the `dbb-ucd-packaging.groovy` that invokes the Urban Code Deploy (UCD) buztool utility, to publish the artifacts created by the DBB Build from a pipeline. -### Invocation +#### Invocation The `ucdPackaging.sh` script can be invoked as follows: @@ -611,7 +620,7 @@ CLI parameter | Description -b `` | (Optional) Name of the **git branch**. -p `` | (Optional) URL to the pull request. -### Output +#### Output The section below contains the output that is produced by the `ucdPackaging.sh` script. @@ -643,80 +652,18 @@ ucdPackaging.sh: [INFO] groovyz /var/dbb/extensions/dbb20/Pipeline/CreateUCDCom -## ucdDeploy.sh - -This script is implementing the invocation of the `ucd-deploy.groovy` script to perform Urban Code Deploy (UCD) deployments. - - -### Invocation - -The `ucdDeploy.sh` script can be invoked as follows: - -``` -ucdDeploy.sh -a ucdApplicationName -p ucdApplicationProcessName -e ucdEnvironmentName -d ucdComponentName:ucdDeployVersion -``` - -CLI parameter | Description ----------- | ---------------------------------------------------------------------------------------- --a `` | **Application** name defined in UCD containing the component version to be deployed. --p `` | **Process** name in UCD associated with the application being deployed. --e `` | **Environment** name in UCD that the component version will be deployed to. --d `` | **Component name and version** to be deployed to the UCD environment. --t `` | (Optional) **Deployment timeout** value in seconds. Defaults to 300 seconds. --s ``| (Optional) **SSL protocol** to use. Default is TLSv1.2. --k | (Optional) Disable SSL verification flag. --v | (Optional) Verbose tracing flag. Used to produce additional tracing in the groovy script. - -### Output - -The section below contains the output that is produced by the `ucdDeploy.sh` script. +## Deployment Stage -
- Script Output - -``` -ucdDeploy.sh -a rbs-retirementCalculator -p rbs-retirementCalculator-process -e rbs-IntegrationTest -d rbs-retirementCalculator:latest -bash: [INFO] Deploy UCD Component. Version=1.00 -bash: [INFO] ************************************************************** -bash: [INFO] ** Start UCD Component Deploy on HOST/USER: z/OS ZT01 04.00 02 8561/BPXROOT -bash: [INFO] ** Location of ucd-deploy.groovy: /u/brice/groovyScripts -bash: [INFO] ** UCD Application Name: rbs-retirementCalculator -bash: [INFO] ** UCD Environment Name: rbs-IntegrationTest -bash: [INFO] ** UCD User Name: admin -bash: [INFO] ** UCD Server URL: http://10.3.20.231:8080/ -bash: [INFO] ** UCD Component Version: rbs-retirementCalculator:latest -bash: [INFO] ** UCD Application Process Name: rbs-retirementCalculator-process -bash: [INFO] ** Verbose: No -bash: [INFO] ** SSL Verification: Yes -bash: [INFO] ************************************************************** - -/usr/lpp/dbb/v2r0/bin/groovyz /u/brice/groovyScripts/ucd-deploy.groovy -a "rbs-retirementCalculator" -e "rbs-IntegrationTest" -U admin -P ******** -u http://10.3.20.231:8080/ -d "rbs-retirementCalculator:latest" -p rbs-retirementCalculator-process -** Request UCD Deployment start at 20230830.064205.042 -** Properties at startup: - application -> rbs-retirementCalculator - environment -> rbs-IntegrationTest - user -> admin - password -> xxxxxx - url -> http://10.3.20.231:8080/ - deployVersions -> rbs-retirementCalculator:latest - applicationProcess -> rbs-retirementCalculator-process -** Deploying component versions: rbs-retirementCalculator:latest -*** Starting deployment process 'rbs-retirementCalculator-process' of application 'rbs-retirementCalculator' in environment 'rbs-IntegrationTest' -*** SSL Verification disabled -*** Follow Process Request: https://ucd.server.com:8443/#applicationProcessRequest/184c812f-605f-5040-ad31-d3a31f87bb3c -Executing ...... -*** The deployment result is SUCCEEDED. See the UrbanCode Deploy deployment logs for details. -** Build finished -``` +Depending on the selected Deployment tool, select either from the scripts for IBM Wazi Deploy ([wazideploy-generate.sh](#wazideploy-generatesh), [wazideploy-deploy.sh](#wazideploy-deploysh) and [wazideploy-generate.sh](#wazideploy-evidencesh)) or [UCD deployment](#ucddeploysh) to submit a deployment request in IBM UrbanCode Deploy. -
-## wazideploy-generate.sh +### wazideploy-generate.sh This script invokes the Wazi Deploy Generate command to generate a Deployment Plan based on the content of a package. The package should be created with the `PackageBuildOutputs.groovy` script or through the `packageBuildOutputs.sh` script. +This script assesses the configuration option `publish` from the `pipelineBackend.config` file. In case the configuration has enabled the upload to the Artifact repository, the script computes the URL where the package is expected to be found, and passes the URL into the wazideploy-generate command. This means that wazideloy-generate will download the package from the Artifact repository and allows to restore the package on a different system. This step leverages the generic utility script computePackageUrl. It requires to pass in the additional arguments `-P`, `-R`, `-B` -### Invocation +#### Invocation The `wazideploy-generate.sh` script can be invoked as follows: @@ -728,18 +675,29 @@ Or by fully specifying the settings wazideploy-generate.sh -m deploymentMethod -p deploymentPlan -r deploymentPlanReport -i packageInputFile ``` +To enable the download based on build and release identifier +``` +wazideploy-generate.sh -w MortgageApplication/feature/15-fetch-application-dependencies/dbb-zappbuild.build_1234 -a MortgageApplication -P build -b feature/15-fetch-application-dependencies -I 1234 +``` + CLI parameter | Description ---------- | ---------------------------------------------------------------------------------------- -w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. Optional, if `deploymentPlan`, `deploymentPlanReport` and `packageOutputFile` are fully referenced. --i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to a TAR file on UNIX System Services, or the URL of the TAR file to retrieve (only Artifactory is supported). +-i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to a TAR file on UNIX System Services, or the URL of the TAR file to retrieve (only Artifactory is supported).

If the Common Backend scripts are configured to perform the upload, the scripts computes the location in the artifact repo and overrides the `-i` and `-o` argument and in that case the this arument is not required. -m `` | (Optional) Absolute path to the Wazi Deploy **Deployment Method** stored on UNIX System Services. If not specified, the deployment method file location is obtained from the `pipelineBackend.config`. -p `` | (Optional) Absolute or relative path to the **Deployment Plan** file, generated based on the content of the input package. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan location is obtained from the `pipelineBackend.config`. -r `` | (Optional) Absolute or relative path to the **Deployment Plan Report**. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan report location is obtained from the `pipelineBackend.config`. -o `` | (Optional) Absolute or relative path to the **Package Output File** that specifies the location where to store the downloaded tar file. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. Only required when wazideploy-generate is used to download the package. This is indicated when a URL is specified for the **Package Input File**. --c `` | (Optional) Absolute path to the **Configuration File** that contains information to connect to Artifactory. Only required when wazideploy-generate is used to download the package. This is indicated when a URL is specified for the **Package Input File**. + -d | (Optional) Debug tracing flag. Used to produce additional tracing with Wazi Deploy. +-- | - when retrieving the tar file from Artifact repo the below options are mandatory - +-b ``| Name of the **git branch** turning into a segment of the directory path for the location within the artifact repository. +-p `` | **Pipeline Type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules for release candidates). +-R `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. +-I `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number of the pipeline run. +-c `` | Absolute path to the Wazi Deploy **Configuration File** that contains information to connect to the artifact repository. See [IBM Wazi Deploy documentation](https://www.ibm.com/docs/en/developer-for-zos/17.0?topic=files-configuration-file) -### Output +#### Output The section below contains the output that is produced by the `wazideploy-generate.sh` script. @@ -785,11 +743,11 @@ wazideploy-generate.sh: [INFO] ************************************************* -## wazideploy-deploy.sh +### wazideploy-deploy.sh This script invokes the Wazi Deploy Deploy (with the Python Translator) command to deploy the content of a provided package with a Deployment Plan. -### Invocation +#### Invocation The `wazideploy-deploy.sh` script can be invoked as follows: @@ -875,11 +833,11 @@ wazideploy-deploy -wf /u/ado/workspace/MortgageApplication/main/build-20231019.1 -## wazideploy-evidence.sh +### wazideploy-evidence.sh This script invokes the Wazi Deploy Evidence command to generate a Deployment report from the Wazi Deploy Evidence YAML file created by the Wazi Deploy Deploy command. -### Invocation +#### Invocation The `wazideploy-evidence.sh` script can be invoked as follows: @@ -898,7 +856,7 @@ CLI parameter | Description -l `` | (Optional) Absolute or relative path to the **Evidence File** that contains the logs of all Wazi Deploy tasks. If not specified, evidence file location will be obtained from the `pipelineBackend.config`. -o `` | (Optional) Absolute or relative path to the **Output File** that will contain the Deployment Report. If not specified, evidence file location will be obtained from the `pipelineBackend.config`. -### Output +#### Output The section below contains the output that is produced by the `wazideploy-evidence.sh` script. @@ -930,6 +888,74 @@ wazideploy-evidence --index /u/ado/workspace/MortgageApplication/main/build-2023 1 record extracted to file '/u/ado/workspace/MortgageApplication/main/build-20231101.15/deploy/deployment-report.html' in 0.06 seconds + + +### ucdDeploy.sh + +This script is implementing the invocation of the `ucd-deploy.groovy` script to perform Urban Code Deploy (UCD) deployments. + + +#### Invocation + +The `ucdDeploy.sh` script can be invoked as follows: + +``` +ucdDeploy.sh -a ucdApplicationName -p ucdApplicationProcessName -e ucdEnvironmentName -d ucdComponentName:ucdDeployVersion +``` + +CLI parameter | Description +---------- | ---------------------------------------------------------------------------------------- +-a `` | **Application** name defined in UCD containing the component version to be deployed. +-p `` | **Process** name in UCD associated with the application being deployed. +-e `` | **Environment** name in UCD that the component version will be deployed to. +-d `` | **Component name and version** to be deployed to the UCD environment. +-t `` | (Optional) **Deployment timeout** value in seconds. Defaults to 300 seconds. +-s ``| (Optional) **SSL protocol** to use. Default is TLSv1.2. +-k | (Optional) Disable SSL verification flag. +-v | (Optional) Verbose tracing flag. Used to produce additional tracing in the groovy script. + +#### Output + +The section below contains the output that is produced by the `ucdDeploy.sh` script. + +
+ Script Output + +``` +ucdDeploy.sh -a rbs-retirementCalculator -p rbs-retirementCalculator-process -e rbs-IntegrationTest -d rbs-retirementCalculator:latest +bash: [INFO] Deploy UCD Component. Version=1.00 +bash: [INFO] ************************************************************** +bash: [INFO] ** Start UCD Component Deploy on HOST/USER: z/OS ZT01 04.00 02 8561/BPXROOT +bash: [INFO] ** Location of ucd-deploy.groovy: /u/brice/groovyScripts +bash: [INFO] ** UCD Application Name: rbs-retirementCalculator +bash: [INFO] ** UCD Environment Name: rbs-IntegrationTest +bash: [INFO] ** UCD User Name: admin +bash: [INFO] ** UCD Server URL: http://10.3.20.231:8080/ +bash: [INFO] ** UCD Component Version: rbs-retirementCalculator:latest +bash: [INFO] ** UCD Application Process Name: rbs-retirementCalculator-process +bash: [INFO] ** Verbose: No +bash: [INFO] ** SSL Verification: Yes +bash: [INFO] ************************************************************** + +/usr/lpp/dbb/v2r0/bin/groovyz /u/brice/groovyScripts/ucd-deploy.groovy -a "rbs-retirementCalculator" -e "rbs-IntegrationTest" -U admin -P ******** -u http://10.3.20.231:8080/ -d "rbs-retirementCalculator:latest" -p rbs-retirementCalculator-process +** Request UCD Deployment start at 20230830.064205.042 +** Properties at startup: + application -> rbs-retirementCalculator + environment -> rbs-IntegrationTest + user -> admin + password -> xxxxxx + url -> http://10.3.20.231:8080/ + deployVersions -> rbs-retirementCalculator:latest + applicationProcess -> rbs-retirementCalculator-process +** Deploying component versions: rbs-retirementCalculator:latest +*** Starting deployment process 'rbs-retirementCalculator-process' of application 'rbs-retirementCalculator' in environment 'rbs-IntegrationTest' +*** SSL Verification disabled +*** Follow Process Request: https://ucd.server.com:8443/#applicationProcessRequest/184c812f-605f-5040-ad31-d3a31f87bb3c +Executing ...... +*** The deployment result is SUCCEEDED. See the UrbanCode Deploy deployment logs for details. +** Build finished +``` +
## prepareLogs.sh From 89b8f8c46e858c948f9b17d57e88fb933bc646b3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 14:14:31 +0100 Subject: [PATCH 070/127] Doc updates Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 239 +++++++++++---------- 1 file changed, 123 insertions(+), 116 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index a981e0fa..d0cabf3a 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -242,9 +242,8 @@ Both implementations for the build stage are enabled to optionally pull external The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository, which are necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. -Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. Fetching the dependencies is powered by the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script. - -Packages will be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory called 'imports'. +Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. For +fetching the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The external packages can be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory 'imports'. ### dbbBuild.sh for zAppBuild framework @@ -495,7 +494,7 @@ packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar ``` - Package and Upload ``` -packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar -a MortgageApplication -b main -u -p release -v rel-1.0.0.2023-09-22-08.55.20 +packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar -a MortgageApplication -b main -u -p release -r rel-1.0.0 -i 4657 ``` CLI parameter | Description @@ -958,11 +957,17 @@ Executing ...... -## prepareLogs.sh +## Generic stages -Script to obtain the logs that were produced as part of the pipeline steps in the *logs* directory. +When your pipeline setup uses a remote agents that runs outside of the z/OS build machine (like with an agent on an x86 environment), you can benefit from the [prepareLogs.sh](#preparelogssh) script to create an archive of the logs, before transferring it to the remote agent for attaching them to the pipeline run. -### Invocation +Once the pipeline has completed all its tasks, you can use the [deleteWorkspace.sh](#deleteworkspacesh) to cleanup the created workspace on z/OS Unix Services and optionally the [generateCleanupCommands.sh](#generatecleanupcommandssh) for housekeeping activities of the DBB metadataststore. + +### prepareLogs.sh + +Script to obtain the logs that were produced as part of the pipeline step in the *logs* directory. + +#### Invocation The `prepareLogs.sh` script can be invoked as follows: @@ -979,7 +984,7 @@ On successful completion, the script writes a message to indicate the output dir Logs successfully stored at /var/dbb/pipelineBackend/workspace/MortApp/feature/setmainbuildbranch/build-1/logs.tar ``` -### Script output +#### Script output The section below contains the output that is produced by the `prepareLogs.sh` script. @@ -1013,7 +1018,7 @@ prepareLogs.sh: [INFO] Logs successfully stored at /var/dbb/pipelineBackend/work -### Download logs to non-z/OS runner/agent environment +#### Download logs to non-z/OS runner/agent environment While the script `prepareLogs.sh` only creates the TAR file on the workspace directory, the next step is to download the TAR file to the non-z/OS runner/agent environment, in order to attach it to the pipeline results. @@ -1038,116 +1043,11 @@ It _greps_ the information and invokes a download action. fi ``` -## generateCleanupCommands.sh - -Script to generate and run the necessary cleanup steps of DBB Metadatastore collections and build groups (build results), and the deletion of the build datasets using the [DeletePDS.groovy](../../Utilities/DeletePDS/README.md) utility. - -The script lists all the existing DBB collections obtained by applying a filter based on the zAppBuild naming conventions. It checks if Git branches corresponding to the provided application name exist in the Git repository. If one or more branches are found, it generates the necessary command files that contain the removal statements. The generated scripts can be can automatically executed, if the `-p` flag is passed to the script. - -### Invocation - -The `generateCleanupCommands.sh` script can be invoked as follows: - -``` -generateCleanupCommands.sh -w MortApp/main/build-1 -a MortApp -p -``` - -CLI parameter | Description ----------- | ---------------------------------------------------------------------------------------- --w `` | **Workspace directory** - an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. --a `` | **Application name** to be analyzed for stale DBB Metadatastore objects and build datasets. --p | Flag to control if the generated commands files should be executed by the pipeline. If the commands are not executed by the script, it is recommended to publish the generated files to the pipeline orchestrator, where an administrator can review and eventually execute them manually. - -### Additional notes - -This script can be embedded into a pipeline execution, but can also be used in a standalone setup. For a pipeline implementation, this task can be included in the release process to facilitate the cleanup of stale DBB collections and DBB build groups, and to delete the build datasets as well. - -For the standalone implementation, use the following process: -1. Have the Common Backend Scripts installed to z/OS Unix System Services and have them configured. -2. Clone the application repository including all remote references. -3. Execute the `generateCleanupCommands.sh` script like in the above sample. The user executing the script needs proper permissions on the DBB Metadatastore. - -Please note that the script leverages the [utilities/dbbBuildUtils.sh](utilities/dbbBuildUtils.sh) to compute the build high-level qualifier (HLQ). - -### Script output - -The section below contains the output that is produced by the `generateCleanupCommands.sh` script. - -
- Script Output - -``` -". ./.profile && generateCleanupCommands.sh -w /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -a MortgageApplication -p" -generateCleanupCommands.sh: [INFO] Generate Cleanup Command File. Version=1.0.0 -generateCleanupCommands.sh: [INFO] Creating output directory. /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds -generateCleanupCommands.sh: [INFO] ************************************************************** -generateCleanupCommands.sh: [INFO] ** Start Gen Cleanup Cmds on HOST/USER: z/OS ZT01 05.00 02 8561/GITHUB -generateCleanupCommands.sh: [INFO] ** Workspace: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -generateCleanupCommands.sh: [INFO] ** Application: MortgageApplication -generateCleanupCommands.sh: [INFO] ** AppDir: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/MortgageApplication -generateCleanupCommands.sh: [INFO] ** Cmd obsolete collections: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd -generateCleanupCommands.sh: [INFO] ** Cmd obsolete build groups: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd -generateCleanupCommands.sh: [INFO] ** Cmd obsolete build datasets: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd -generateCleanupCommands.sh: [INFO] ** DBB Metadastore Config: --type file --location /u/github/ -generateCleanupCommands.sh: [INFO] ** Process Cleanup Scripts: true -generateCleanupCommands.sh: [INFO] ************************************************************** - -generateCleanupCommands.sh: [STAGE] Retrieve all collections with application qualifier MortgageApplication -generateCleanupCommands.sh: [STAGE] Verifying Git references -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-main a corresponding branch (main) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-main-outputs a corresponding branch (main) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting a corresponding branch (feature/johnpipelinetesting) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting-outputs a corresponding branch (feature/johnpipelinetesting) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing has a corresponding active Git branch -generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing-outputs has a corresponding active Git branch -generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing-outputs does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing a corresponding branch (feature/pipeline-trigger-testing) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing-outputs a corresponding branch (feature/pipeline-trigger-testing) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale collections for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd created. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build groups for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd created. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build datasets for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd created. -generateCleanupCommands.sh: [STAGE] Executing Cleanup of DBB Metadatastore Objects -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd -BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing" -BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing-outputs" -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd -BGZTK0195I Successfully deleted group "MortgageApplication-feature/20-some-more-testing" -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd -** Deleting all datasets filtered with HLQ 'GITHUB.MORTGAGE.F20' -*** Deleting 'GITHUB.MORTGAGE.F20.COBOL' -*** Deleting 'GITHUB.MORTGAGE.F20.COPY' -*** Deleting 'GITHUB.MORTGAGE.F20.DBRM' -*** Deleting 'GITHUB.MORTGAGE.F20.LOAD' -*** Deleting 'GITHUB.MORTGAGE.F20.OBJ' -** Deleted 5 entries. -** Build finished -generateCleanupCommands.sh: [INFO] Generate Cleanup Cmds Complete. rc=0 - -``` - -
- - - -## deleteWorkspace.sh +### deleteWorkspace.sh Script delete the workspace and all empty directories in the working tree. -### Invocation +#### Invocation The `deleteWorkspace.sh` script can be invoked as follows: @@ -1257,6 +1157,113 @@ deleteWorkspace.sh: [INFO] Workspace directory successfully deleted. +### generateCleanupCommands.sh + +Script to generate and run the necessary cleanup steps of DBB Metadatastore collections and build groups (build results), and the deletion of the build datasets using the [DeletePDS.groovy](../../Utilities/DeletePDS/README.md) utility. + +The script lists all the existing DBB collections obtained by applying a filter based on the zAppBuild naming conventions. It checks if Git branches corresponding to the provided application name exist in the Git repository. If one or more branches are found, it generates the necessary command files that contain the removal statements. The generated scripts can be can automatically executed, if the `-p` flag is passed to the script. + +#### Invocation + +The `generateCleanupCommands.sh` script can be invoked as follows: + +``` +generateCleanupCommands.sh -w MortApp/main/build-1 -a MortApp -p +``` + +CLI parameter | Description +---------- | ---------------------------------------------------------------------------------------- +-w `` | **Workspace directory** - an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. +-a `` | **Application name** to be analyzed for stale DBB Metadatastore objects and build datasets. +-p | Flag to control if the generated commands files should be executed by the pipeline. If the commands are not executed by the script, it is recommended to publish the generated files to the pipeline orchestrator, where an administrator can review and eventually execute them manually. + +#### Additional notes + +This script can be embedded into a pipeline execution, but can also be used in a standalone setup. For a pipeline implementation, this task can be included in the release process to facilitate the cleanup of stale DBB collections and DBB build groups, and to delete the build datasets as well. + +For the standalone implementation, use the following process: +1. Have the Common Backend Scripts installed to z/OS Unix System Services and have them configured. +2. Clone the application repository including all remote references. +3. Execute the `generateCleanupCommands.sh` script like in the above sample. The user executing the script needs proper permissions on the DBB Metadatastore. + +Please note that the script leverages the [utilities/dbbBuildUtils.sh](utilities/dbbBuildUtils.sh) to compute the build high-level qualifier (HLQ). + +### Script output + +The section below contains the output that is produced by the `generateCleanupCommands.sh` script. + +
+ Script Output + +``` +". ./.profile && generateCleanupCommands.sh -w /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -a MortgageApplication -p" +generateCleanupCommands.sh: [INFO] Generate Cleanup Command File. Version=1.0.0 +generateCleanupCommands.sh: [INFO] Creating output directory. /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds +generateCleanupCommands.sh: [INFO] ************************************************************** +generateCleanupCommands.sh: [INFO] ** Start Gen Cleanup Cmds on HOST/USER: z/OS ZT01 05.00 02 8561/GITHUB +generateCleanupCommands.sh: [INFO] ** Workspace: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 +generateCleanupCommands.sh: [INFO] ** Application: MortgageApplication +generateCleanupCommands.sh: [INFO] ** AppDir: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/MortgageApplication +generateCleanupCommands.sh: [INFO] ** Cmd obsolete collections: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd +generateCleanupCommands.sh: [INFO] ** Cmd obsolete build groups: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd +generateCleanupCommands.sh: [INFO] ** Cmd obsolete build datasets: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd +generateCleanupCommands.sh: [INFO] ** DBB Metadastore Config: --type file --location /u/github/ +generateCleanupCommands.sh: [INFO] ** Process Cleanup Scripts: true +generateCleanupCommands.sh: [INFO] ************************************************************** + +generateCleanupCommands.sh: [STAGE] Retrieve all collections with application qualifier MortgageApplication +generateCleanupCommands.sh: [STAGE] Verifying Git references +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-main a corresponding branch (main) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-main-outputs a corresponding branch (main) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting a corresponding branch (feature/johnpipelinetesting) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting-outputs a corresponding branch (feature/johnpipelinetesting) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing has a corresponding active Git branch +generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing-outputs has a corresponding active Git branch +generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing-outputs does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing a corresponding branch (feature/pipeline-trigger-testing) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing-outputs a corresponding branch (feature/pipeline-trigger-testing) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale collections for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd created. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build groups for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd created. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build datasets for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd created. +generateCleanupCommands.sh: [STAGE] Executing Cleanup of DBB Metadatastore Objects +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd +BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing" +BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing-outputs" +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd +BGZTK0195I Successfully deleted group "MortgageApplication-feature/20-some-more-testing" +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd +** Deleting all datasets filtered with HLQ 'GITHUB.MORTGAGE.F20' +*** Deleting 'GITHUB.MORTGAGE.F20.COBOL' +*** Deleting 'GITHUB.MORTGAGE.F20.COPY' +*** Deleting 'GITHUB.MORTGAGE.F20.DBRM' +*** Deleting 'GITHUB.MORTGAGE.F20.LOAD' +*** Deleting 'GITHUB.MORTGAGE.F20.OBJ' +** Deleted 5 entries. +** Build finished +generateCleanupCommands.sh: [INFO] Generate Cleanup Cmds Complete. rc=0 + +``` + +
+ + + + + # Disclaimer THIS SAMPLE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From d55517f62d97a849008ac168893012a7f4aecd07 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 14:18:18 +0100 Subject: [PATCH 071/127] remove slurper Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index ef063032..09f70307 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -5,7 +5,6 @@ import com.ibm.dbb.build.* import groovy.transform.* import com.ibm.dbb.build.report.* import com.ibm.dbb.build.report.records.* -import groovy.yaml.YamlSlurper import groovy.yaml.YamlBuilder import groovy.lang.GroovyShell import groovy.cli.commons.* @@ -63,8 +62,6 @@ importFolder.mkdirs() tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean() && props.packageCacheLocation) ? new File(props.packageCacheLocation) : new File("$props.workspace/imports_download/") if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() -def yamlSlurper = new groovy.yaml.YamlSlurper() - // Parse the application descriptor and application configurations based on the defined schema applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) @@ -95,7 +92,6 @@ if (applicationDescriptor.dependencies) { // The absolute url the package in artifact repo artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) - println artifactUrl // Construct the path within the Artifact repo repositoryName="${dependency.name}-${props.artifactRepositoryNameSuffix}" From 6b56de83d741796281eeb93d4ad44fb9dae0c80e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 14:21:35 +0100 Subject: [PATCH 072/127] Use artifact name suffix Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 09f70307..33bba938 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -86,15 +86,17 @@ if (applicationDescriptor.dependencies) { props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") } + // Construct the path within the Artifact repo + repositoryName="${dependency.name}-${props.artifactRepositoryNameSuffix}" + + // Prepare for url computation using method in ArtifactRepoHelpers props.put("versionName","${dependency.reference}") // compute the version name being part of the path props.put("artifactRepository.directory", "${dependency.type}") // compute the main directory to classify builds - props.put("artifactRepository.repo", "${dependency.name}-repo-local") // Artifact repository name (hard-coded again) + props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name // The absolute url the package in artifact repo artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) - // Construct the path within the Artifact repo - repositoryName="${dependency.name}-${props.artifactRepositoryNameSuffix}" // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") From 49368895a8d63cf497206c8d07f762436140eaf9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 14:24:55 +0100 Subject: [PATCH 073/127] Change to blanks instead of tab Signed-off-by: Dennis Behm --- .../utilities/applicationDescriptorUtils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index ce4d4f3f..429b2bc2 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -48,9 +48,9 @@ class Baseline { class DependencyDescriptor { String name - String type + String type String reference - String buildid + String buildid } /** From 70c7551dd30b1010440df95efb3cdde81ae1e5bd Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 14:53:23 +0100 Subject: [PATCH 074/127] Cleanup packaging scripts Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 94 ++++++++++++------- .../utilities/packageUtils.sh | 2 +- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 8cef9df6..0bd423e6 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -64,20 +64,10 @@ Help() { echo " " echo " Ex: MortgageApplication " echo " " - echo " -p - Type of the pipeline to " - echo " control in which directory builds " - echo " are stored in the artifact repo " - echo " Accepted values: " - echo " build - " - echo " development builds " - echo " release - " - echo " builds with options for " - echo " performance optimized " - echo " executables for production env " - echo " " - echo " -b - Name of the git branch. " + echo " -t - Name of the package tar file " + echo " (Optional) " echo " " - echo " Ex: main " + echo " Ex: package.tar " echo " " echo " -v " echo " - Name of the artifactVersion " @@ -92,8 +82,34 @@ Help() { echo " enclosed with double quotes " echo " (Optional) " echo " " - echo " Ex: \"Build Engineer \" " + echo " Ex: \"Build Engineer \" " + echo " " + echo " " + echo " Mandatory arguments when publishing to artifact repo " echo " " + echo " -I - A unique build identifier " + echo " typically the buildID of the " + echo " pipeline " + echo " Ex: 6756 " + echo " " + echo " -R - The release identifier for " + echo " release pipeline builds " + echo " Ex: rel-1.2.3 " + echo " " + echo " -p - Type of the pipeline to " + echo " control in which directory builds " + echo " are stored in the artifact repo " + echo " Accepted values: " + echo " build - " + echo " development builds " + echo " release - " + echo " builds with options for " + echo " performance optimized " + echo " executables for production env " + echo " " + echo " -b - Name of the git branch. " + echo " " + echo " Ex: main " echo " -h - Display this Help. " echo " " exit 0 @@ -192,7 +208,7 @@ fi # # Get Options if [ $rc -eq 0 ]; then - while getopts ":h:w:a:b:i:r:v:p:us:" opt; do + while getopts ":h:w:a:b:t:i:r:v:p:us:" opt; do case $opt in h) Help @@ -219,6 +235,17 @@ if [ $rc -eq 0 ]; then fi App="$argument" ;; + t) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + tarFileName="$argument" + ;; i) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -317,16 +344,10 @@ validateOptions() { fi fi - if [ -z "${buildIdentifier}" ]; then - ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" - echo $ERRMSG - buildIdentifier=$(date +%Y%m%d_%H%M%S) - fi - # Validate Packaging script if [ ! -f "${packagingScript}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${packagingScript}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${packagingScript}. rc="$rc echo $ERRMSG fi @@ -334,11 +355,17 @@ validateOptions() { if [ ! -z "${PkgPropFile}" ]; then if [ ! -f "${PkgPropFile}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${PkgPropFile}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${PkgPropFile}. rc="$rc echo $ERRMSG fi fi + if [ -z "${buildIdentifier}" ] && [ "$publish" == "true" ]; then + ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" + echo $ERRMSG + buildIdentifier=$(date +%Y%m%d_%H%M%S) + fi + } # function to validate publishing options @@ -359,7 +386,7 @@ validatePublishingOptions() { if [ ! -z "${artifactRepositoryPropertyFile}" ]; then if [ ! -f "${artifactRepositoryPropertyFile}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${artifactRepositoryPropertyFile}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${artifactRepositoryPropertyFile}. rc="$rc echo $ERRMSG fi else @@ -407,12 +434,12 @@ validatePublishingOptions() { ;; "preview") rc=4 - ERRMSG=$PGM": [WARN] Default Pipeline Type : ${PipelineType} not supported for packaging." + ERRMSG=$PGM": [WARNING] Default Pipeline Type : ${PipelineType} not supported for packaging." echo $ERRMSG ;; *) rc=4 - ERRMSG=$PGM": [WARN] Inavlid Pipeline Type : ${PipelineType} specified." + ERRMSG=$PGM": [WARNING] Invalid Pipeline Type : ${PipelineType} specified." echo $ERRMSG ;; esac @@ -421,22 +448,21 @@ validatePublishingOptions() { fi } -# Call validate input options -if [ $rc -eq 0 ]; then - validateOptions -fi - -# compute packaging parameters -if [ $rc -eq 0 ]; then +# compute packaging parameters and validate publishing options +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then # invoke function in packageUtils computePackageInformation fi -# Call validate publishing options if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then validatePublishingOptions fi +# Call validate input options +if [ $rc -eq 0 ]; then + validateOptions +fi + # # Ready to go if [ $rc -eq 0 ]; then diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh index 36eda29d..ea380653 100755 --- a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/packageUtils.sh @@ -18,7 +18,7 @@ computePackageInformation() { ############################################# # configuration variable defining the Artifactory repository name pattern - artifactRepositoryRepoPattern="${App}-repo-local" + artifactRepositoryRepoPattern="${App}-${artifactRepositoryNameSuffix}" branchConvention=(${Branch//// }) From d5d0c15a0b74004423c3e6343465f8efd741b312 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 17 Mar 2025 19:01:49 +0100 Subject: [PATCH 075/127] additional documentation Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 4 +- .../utilities/README.md | 67 +++++++++++++++++++ .../utilities/computePackageUrl.sh | 6 +- 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 Templates/Common-Backend-Scripts/utilities/README.md diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index d0cabf3a..62f55080 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -504,7 +504,7 @@ CLI parameter | Description -t `` | (Optional) Name of the **tar file** to create. **Artifact Upload options** -u | Flag to enable upload of outputs to the configured artifact repository. --a `` | **Application name** leveraged to define the artifact repository name. See function `computeArtifactRepositoryName()` in the pipelineBackend.config file. Ex.: `MortgageApplication-repo-local`. +-a `` | **Application name** leveraged to define the artifact repository name. -b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. -r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. @@ -512,6 +512,8 @@ CLI parameter | Description -v `` **removed** | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. This has been removed. Please switch to releaseIdentifier and buildIdentifier. -s `""` | (Optional) Name and email of the SBOM author enclosed with double quotes. Ex: "Build Engineer \" +Check out the pipelineBackend.config to define the `artifactRepositoryNameSuffix` that is appended to the application name to set the repository name in the artifact repository. + #### Script conventions **SBOM Generation** diff --git a/Templates/Common-Backend-Scripts/utilities/README.md b/Templates/Common-Backend-Scripts/utilities/README.md new file mode 100644 index 00000000..7c66382a --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/README.md @@ -0,0 +1,67 @@ +# Contents + +This folder contains utility scripts, that are sourced by the primary common backend scripts. + +There is one exception that can be used standalone + +## computePackageUrl.sh + +Script to compute the url where a package got stored in the Artifact repository. + +### Invocation + +The `computePackageUrl.sh` script can be invoked as follows: + +``` +utilities/computePackageUrl.sh -w MortgageApplication/feature/15-update-mortgage-application/build_7645 -a MortgageApplication -b feature/15-update-mortgage-application -p build -i 7645 +``` + +CLI parameter | Description +---------- | ---------------------------------------------------------------------------------------- +-w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. +-a `` | **Application name** leveraged to define the artifact repository name. +-b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. +-p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. +-r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. +-i `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number or a timestamp. + +On successful completion the below message it displayed, and a file containing the information is stored the workspace log directory: +``` +computePackageUrl.sh: [INFO] Compute Package Url completed. Results stored in /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt. rc=0 +``` +packageVersionConfig.txt: +``` +packageUrl=http://10.3.20.231:8081/artifactory/MortgageApplication-repo-local/build/feature/15-update-mortgage-application/MortgageApplication-7645.tar +``` + + +### Script output + +The section below contains the output that is produced by the `utilities/computePackageUrl.sh` script. + +
+ Script Output + +``` +/u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh -w MortgageApplication/feature/15-update-mortgage-application/build_7645 -a MortgageApplication -b feature/15-update-mortgage-application -p build -i 7645 +computePackageUrl.sh: [INFO] Compute Package Url. Version=1.00 +computePackageUrl.sh: [INFO] ************************************************************** +computePackageUrl.sh: [INFO] ** Started - Compute Package Url on HOST/USER: z/OS ZT01 05.00 02 8561/BPXROOT +computePackageUrl.sh: [INFO] ** Application: MortgageApplication +computePackageUrl.sh: [INFO] ** Branch: feature/15-update-mortgage-application +computePackageUrl.sh: [INFO] ** Type of pipeline: build +computePackageUrl.sh: [INFO] ** Tar file Name: MortgageApplication-7645.tar +computePackageUrl.sh: [INFO] ** Artifact Repository Helpers: /u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +computePackageUrl.sh: [INFO] ** ArtifactRepo Url: http://10.3.20.231:8081/artifactory +computePackageUrl.sh: [INFO] ** ArtifactRepo Repo name: MortgageApplication-repo-local +computePackageUrl.sh: [INFO] ** ArtifactRepo Repo Dir: build +computePackageUrl.sh: [INFO] ** Output file: /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt +computePackageUrl.sh: [INFO] ** DBB_HOME: /usr/lpp/dbb/v3r0 +computePackageUrl.sh: [INFO] ************************************************************** + +computePackageUrl.sh: [INFO] Invoking the Package Build Outputs script to compute Package Url. +computePackageUrl.sh: [INFO] /usr/lpp/dbb/v3r0/bin/groovyz /u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy --computePackageUrl --tarFileName MortgageApplication-7645.tar --versionName feature/15-update-mortgage-application --artifactRepositoryUrl "http://10.3.20.231:8081/artifactory" --artifactRepositoryName MortgageApplication-repo-local --artifactRepositoryDirectory build +computePackageUrl.sh: [INFO] Compute Package Url completed. Results stored in /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt. rc=0 +``` + +
\ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh index fe6979d9..5942c1a2 100755 --- a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh +++ b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh @@ -344,7 +344,7 @@ validatePackagingOptions() { ;; *) rc=4 - ERRMSG=$PGM": [WARN] Inavlid Pipeline Type : ${PipelineType} specified." + ERRMSG=$PGM": [WARN] Invalid Pipeline Type : ${PipelineType} specified." echo $ERRMSG ;; esac @@ -372,7 +372,7 @@ fi # Ready to go if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" - echo $PGM": [INFO] ** Started - Package Build Outputs on HOST/USER: ${SYS}/${USER}" + echo $PGM": [INFO] ** Started - Compute Package Url on HOST/USER: ${SYS}/${USER}" if [ ! -z "${App}" ]; then echo $PGM": [INFO] ** Application:" ${App} fi @@ -408,7 +408,7 @@ fi # # Invoke the Package Build Outputs script if [ $rc -eq 0 ]; then - echo $PGM": [INFO] Invoking the Package Build Outputs script to compute Package Url." + echo $PGM": [INFO] Invoking the ArtifactRepositoryHelper groovy script to compute Package Url." CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computePackageUrl" From 716c8f38c97ed20a730d715685a783dec44a564c Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 19 Mar 2025 12:10:15 +0100 Subject: [PATCH 076/127] Align deployment step with download approach Signed-off-by: Dennis Behm --- .../utilities/computePackageUrl.sh | 2 +- .../wazideploy-deploy.sh | 17 ++++++++++ .../wazideploy-generate.sh | 31 ++++++++++--------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh index 5942c1a2..290f5311 100755 --- a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh +++ b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh @@ -274,7 +274,7 @@ validateOptions() { echo $ERRMSG fi - outputFile=$(getLogDir)/${tempVersionFile} + outputFile=$(getLogDir)/${wdPackageVersionFile} fi if [ -z "${buildIdentifier}" ]; then diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index ba18bcd2..da7e60af 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -306,6 +306,7 @@ validateOptions() { echo $ERRMSG fi + # validate package input file if [ -z "${PackageInputFile}" ]; then rc=8 @@ -337,6 +338,22 @@ validateOptions() { fi } + +# When publishing is enabled, try reading the wdPackageVersionFile + if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then + echo $PGM": [INFO] ** Found Wazi Deploy Package configuration file $(getLogDir)/${wdPackageVersionFile}. Import configuration." + if [ ! -z "${PackageInputFile}" ]; then + echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced by the Wazi Deploy configuration setting from $(getLogDir)/${wdPackageVersionFile}." + fi + source "$(getLogDir)/${wdPackageVersionFile}" + else + rc=4 + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc + echo $ERRMSG + fi +fi + # Call validate Options if [ $rc -eq 0 ]; then validateOptions diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 87b3011b..b7131851 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -438,15 +438,6 @@ validateOptions() { fi - # When a the packageUrl environment variable found in the file - if [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - PackageInputFile="${packageUrl}" - ## Take the last segment of the URL to define the tarFileName - tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(getLogDir)/${tarFileName}" - fi - # validate package input file if [ -z "${PackageInputFile}" ]; then rc=8 @@ -484,7 +475,7 @@ validateOptions() { fi } -# When publishing is enabled, try reading the tempVersionFile +# When publishing is enabled, try reading the wdPackageVersionFile # that needs to be computed before this step. if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; then checkWorkspace @@ -509,15 +500,27 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t ${CMD} rc=$? if [ $rc -eq 0 ]; then - if [ -f "$(getLogDir)/${tempVersionFile}" ]; then - echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${tempVersionFile}" - source "$(getLogDir)/${tempVersionFile}" + if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then + echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${wdPackageVersionFile}" + source "$(getLogDir)/${wdPackageVersionFile}" else rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${tempVersionFile} was not found. Check previous console output. rc="$rc + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc echo $ERRMSG fi fi + + # When a the packageUrl environment variable found in the file continue to compute + if [ $rc -eq 0 ] && [ ! -z "${packageUrl}" ]; then + echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + PackageInputFile="${packageUrl}" + ## Take the last segment of the URL to define the tarFileName + tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') + PackageOutputFile="$(getLogDir)/${tarFileName}" + echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." + echo "PackageInputFile=${PackageOutputFile}" >> $(getLogDir)/${wdPackageVersionFile} + fi + fi fi From 0e261b1f1bed0d13fe261414d0a1c48b83cc9274 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 19 Mar 2025 13:05:52 +0100 Subject: [PATCH 077/127] Pass applicationFolderPath to publish interfaces Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 0bd423e6..3a4b169f 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -142,6 +142,7 @@ rc=0 ERRMSG="" Workspace="" App="" +AppDir="" tarFileName="" PkgPropFile="" PipelineType="" @@ -365,6 +366,28 @@ validateOptions() { echo $ERRMSG buildIdentifier=$(date +%Y%m%d_%H%M%S) fi + + if [ -z "${App}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc + echo $ERRMSG + else + + AppDir=$(getApplicationDir) + + # Check if application directory contains + if [ -d "${AppDir}/${App}" ]; then + echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." + echo $PGM": [INFO] Assuming this as the new application location." + AppDir="${AppDir}/${App}" + fi + + if [ ! -d "${AppDir}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Application Directory (${AppDir}) was not found. rc="$rc + echo $ERRMSG + fi + fi } @@ -475,6 +498,11 @@ if [ $rc -eq 0 ]; then if [ ! -z "${Branch}" ]; then echo $PGM": [INFO] ** Branch:" ${Branch} fi + + if [ ! -z "${AppDir}" ]; then + echo $PGM": [INFO] ** Application directory:" ${AppDir} + fi + if [ ! -z "${PipelineType}" ]; then echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} fi @@ -490,6 +518,8 @@ if [ $rc -eq 0 ]; then if [ ! -z "${packageBuildIdentifier}" ]; then echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} fi + + echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} if [ "$publish" == "true" ]; then @@ -547,6 +577,11 @@ if [ $rc -eq 0 ]; then if [ ! -z "${Branch}" ]; then CMD="${CMD} --branch ${Branch}" fi + + # application directory + if [ ! -z "${AppDir}" ]; then + CMD="${CMD} --applicationFolderPath ${AppDir}" + fi # packaging properties file if [ ! -z "${PkgPropFile}" ]; then From 1bf7cd9fe87d19d12e5430aeb4ca52b7c16e4b60 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 19 Mar 2025 14:41:10 +0100 Subject: [PATCH 078/127] add check for tarfile Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 3a4b169f..9839610c 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -474,6 +474,11 @@ validatePublishingOptions() { # compute packaging parameters and validate publishing options if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then # invoke function in packageUtils + + if [ ! -z "${tarFileName}" ]; then + echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier." + fi + computePackageInformation fi From 9552aa1e518a6b76b26c6749bd956097e06509b6 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 19 Mar 2025 14:43:18 +0100 Subject: [PATCH 079/127] update readme Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 62f55080..e1ebf303 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -551,7 +551,7 @@ packageBuildOutputs.sh: [INFO] ** ArtifactRepo Repo Dir: main/build packageBuildOutputs.sh: [INFO] ** DBB_HOME: /usr/lpp/dbb/v2r0 packageBuildOutputs.sh: [INFO] ************************************************************** -packageBuildOutputs.sh: [INFO] Invoking the Package Build Outputs script. +packageBuildOutputs.sh: [INFO] Invoking the ArtifactRepositoryHelper groovy script. packageBuildOutputs.sh: [INFO] groovyz /var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy --workDir /var/dbb/pipelineBackend/workspace/MortApp/main/build-1/logs --tarFileName package.tar --packagingPropertiesFile /var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties --addExtension --publish --artifactRepositoryUrl "http://10.3.20.231:8081/artifactory" --versionName MortgageApplication.2023-09-22_13-55-20 --artifactRepositoryUser admin --artifactRepositoryPassword artifactoryadmin --artifactRepositoryName MortgageApplication-repo-local --artifactRepositoryDirectory main/build ** PackageBuildOutputs start at 20230922.125616.056 ** Properties at startup: From 0b5921efb7f3e9ea4920fadea4133f821b532257 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 19 Mar 2025 14:45:47 +0100 Subject: [PATCH 080/127] apply format Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 62 +++++++-------- .../wazideploy-deploy.sh | 28 ++++--- .../wazideploy-generate.sh | 76 +++++++++---------- 3 files changed, 79 insertions(+), 87 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 9839610c..919de524 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -77,7 +77,7 @@ Help() { echo " " echo " Ex: Pipeline Build ID (Build.buildid.tar) " echo " " - echo " -s "" - Name and email of " + echo " -s " " - Name and email of " echo " the SBOM author " echo " enclosed with double quotes " echo " (Optional) " @@ -95,7 +95,7 @@ Help() { echo " -R - The release identifier for " echo " release pipeline builds " echo " Ex: rel-1.2.3 " - echo " " + echo " " echo " -p - Type of the pipeline to " echo " control in which directory builds " echo " are stored in the artifact repo " @@ -142,7 +142,7 @@ rc=0 ERRMSG="" Workspace="" App="" -AppDir="" +AppDir="" tarFileName="" PkgPropFile="" PipelineType="" @@ -205,7 +205,6 @@ if [ $rc -eq 0 ]; then fi fi - # # Get Options if [ $rc -eq 0 ]; then @@ -246,7 +245,7 @@ if [ $rc -eq 0 ]; then break fi tarFileName="$argument" - ;; + ;; i) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -306,7 +305,7 @@ if [ $rc -eq 0 ]; then fi releaseIdentifier="$argument" ;; - v) + v) argument="$OPTARG" rc=4 ERRMSG=$PGM": [WARNING] The argument (-v) for naming the version is no longer supported. Please switch to supply the build identifier argument (-i) and for release pipelines the release identifier argument (-r). rc="$rc @@ -366,28 +365,28 @@ validateOptions() { echo $ERRMSG buildIdentifier=$(date +%Y%m%d_%H%M%S) fi - + if [ -z "${App}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc - echo $ERRMSG + rc=8 + ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc + echo $ERRMSG else AppDir=$(getApplicationDir) - + # Check if application directory contains if [ -d "${AppDir}/${App}" ]; then - echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." - echo $PGM": [INFO] Assuming this as the new application location." - AppDir="${AppDir}/${App}" + echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." + echo $PGM": [INFO] Assuming this as the new application location." + AppDir="${AppDir}/${App}" fi - + if [ ! -d "${AppDir}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Application Directory (${AppDir}) was not found. rc="$rc - echo $ERRMSG + rc=8 + ERRMSG=$PGM": [ERROR] Application Directory (${AppDir}) was not found. rc="$rc + echo $ERRMSG fi - fi + fi } @@ -474,11 +473,11 @@ validatePublishingOptions() { # compute packaging parameters and validate publishing options if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then # invoke function in packageUtils - + if [ ! -z "${tarFileName}" ]; then echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier." fi - + computePackageInformation fi @@ -503,11 +502,11 @@ if [ $rc -eq 0 ]; then if [ ! -z "${Branch}" ]; then echo $PGM": [INFO] ** Branch:" ${Branch} fi - + if [ ! -z "${AppDir}" ]; then echo $PGM": [INFO] ** Application directory:" ${AppDir} fi - + if [ ! -z "${PipelineType}" ]; then echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} fi @@ -523,8 +522,6 @@ if [ $rc -eq 0 ]; then if [ ! -z "${packageBuildIdentifier}" ]; then echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} fi - - echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} if [ "$publish" == "true" ]; then @@ -561,9 +558,9 @@ fi # Invoke the Package Build Outputs script if [ $rc -eq 0 ]; then echo $PGM": [INFO] Invoking the Package Build Outputs script." - + if [ ! -z "${cycloneDXlibraries}" ]; then - cycloneDXlibraries="-cp ${cycloneDXlibraries}" + cycloneDXlibraries="-cp ${cycloneDXlibraries}" fi CMD="$DBB_HOME/bin/groovyz ${log4j2} ${cycloneDXlibraries} ${packagingScript} --workDir ${logDir}" @@ -582,7 +579,7 @@ if [ $rc -eq 0 ]; then if [ ! -z "${Branch}" ]; then CMD="${CMD} --branch ${Branch}" fi - + # application directory if [ ! -z "${AppDir}" ]; then CMD="${CMD} --applicationFolderPath ${AppDir}" @@ -602,7 +599,7 @@ if [ $rc -eq 0 ]; then if [ ! -z "${artifactVersionName}" ]; then CMD="${CMD} --versionName ${artifactVersionName}" fi - + # Wazi Deploy build identifier if [ ! -z "${packageBuildIdentifier}" ]; then CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" @@ -637,12 +634,11 @@ if [ $rc -eq 0 ]; then # SBOM options if [ "$generateSBOM" == "true" ]; then CMD="${CMD} --sbom" - if [ ! -z "${sbomAuthor}" ]; then - CMD="${CMD} --sbomAuthor \"${sbomAuthor}\"" - fi + if [ ! -z "${sbomAuthor}" ]; then + CMD="${CMD} --sbomAuthor \"${sbomAuthor}\"" + fi fi - echo $PGM": [INFO] ${CMD}" ${CMD} rc=$? diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index da7e60af..9c79abad 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -279,7 +279,7 @@ validateOptions() { echo $ERRMSG fi fi - + # validate that deployment plan exists if [ ! -f "${DeploymentPlan}" ]; then rc=8 @@ -298,7 +298,7 @@ validateOptions() { EnvironmentFile="${wdEnvironmentConfigurations}/${EnvironmentFile}" fi fi - + # validate that environment file exists if [ ! -f "${EnvironmentFile}" ]; then rc=8 @@ -306,7 +306,6 @@ validateOptions() { echo $ERRMSG fi - # validate package input file if [ -z "${PackageInputFile}" ]; then rc=8 @@ -338,20 +337,19 @@ validateOptions() { fi } - # When publishing is enabled, try reading the wdPackageVersionFile - if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then - echo $PGM": [INFO] ** Found Wazi Deploy Package configuration file $(getLogDir)/${wdPackageVersionFile}. Import configuration." - if [ ! -z "${PackageInputFile}" ]; then - echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced by the Wazi Deploy configuration setting from $(getLogDir)/${wdPackageVersionFile}." - fi - source "$(getLogDir)/${wdPackageVersionFile}" - else - rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc - echo $ERRMSG +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then + echo $PGM": [INFO] ** Found Wazi Deploy Package configuration file $(getLogDir)/${wdPackageVersionFile}. Import configuration." + if [ ! -z "${PackageInputFile}" ]; then + echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced by the Wazi Deploy configuration setting from $(getLogDir)/${wdPackageVersionFile}." fi + source "$(getLogDir)/${wdPackageVersionFile}" + else + rc=4 + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc + echo $ERRMSG + fi fi # Call validate Options diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index b7131851..cf2cc607 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -97,7 +97,7 @@ Help() { echo " See wdDeployArtifactoryConfig " echo " in pipelineBackend.config " echo " " - echo " -P - Type of the pipeline to " + echo " -P - Type of the pipeline to " echo " control in which directory builds " echo " are stored in the artifact repo " echo " Accepted values: " @@ -305,19 +305,19 @@ if [ $rc -eq 0 ]; then # Add command to produce debug output with Wazi Deploy Debug=" -d" ;; - + a) # Application argument - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc - echo $ERRMSG - break - fi - App="$argument" - ;; + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + App="$argument" + ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -437,7 +437,6 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi - # validate package input file if [ -z "${PackageInputFile}" ]; then rc=8 @@ -445,9 +444,9 @@ validateOptions() { echo $ERRMSG else # check for relative path - if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${packageUrl}" ] ; then - checkWorkspace - PackageInputFile="$(getLogDir)/${PackageInputFile}" + if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${packageUrl}" ]; then + checkWorkspace + PackageInputFile="$(getLogDir)/${PackageInputFile}" fi fi @@ -481,13 +480,12 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t checkWorkspace CMD="${computePackageUrlUtil} -w $Workspace -a $App -b $Branch -i $buildIdentifier" - if [ ! -z "${PipelineType}" ]; then CMD+=" -p ${PipelineType}" - else - rc=8 - ERRMSG=$PGM": [ERROR] To compute the Package Url to automatically download the tar file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc - echo $ERRMSG + else + rc=8 + ERRMSG=$PGM": [ERROR] To compute the Package Url to automatically download the tar file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc + echo $ERRMSG fi if [ ! -z "${releaseIdentifier}" ]; then @@ -500,27 +498,27 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t ${CMD} rc=$? if [ $rc -eq 0 ]; then - if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then - echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${wdPackageVersionFile}" - source "$(getLogDir)/${wdPackageVersionFile}" - else - rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc - echo $ERRMSG - fi + if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then + echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${wdPackageVersionFile}" + source "$(getLogDir)/${wdPackageVersionFile}" + else + rc=4 + ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc + echo $ERRMSG + fi fi - - # When a the packageUrl environment variable found in the file continue to compute + + # When a the packageUrl environment variable found in the file continue to compute if [ $rc -eq 0 ] && [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - PackageInputFile="${packageUrl}" - ## Take the last segment of the URL to define the tarFileName - tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(getLogDir)/${tarFileName}" - echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." - echo "PackageInputFile=${PackageOutputFile}" >> $(getLogDir)/${wdPackageVersionFile} + echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." + PackageInputFile="${packageUrl}" + ## Take the last segment of the URL to define the tarFileName + tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') + PackageOutputFile="$(getLogDir)/${tarFileName}" + echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." + echo "PackageInputFile=${PackageOutputFile}" >>$(getLogDir)/${wdPackageVersionFile} fi - + fi fi From 07ff7daa7f2c5c7bfc34fb508e4e2d0993e2495b Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 09:14:38 +0100 Subject: [PATCH 081/127] Ensure backward compatibilty passing versionName Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 919de524..6b4f4265 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -307,9 +307,16 @@ if [ $rc -eq 0 ]; then ;; v) argument="$OPTARG" - rc=4 - ERRMSG=$PGM": [WARNING] The argument (-v) for naming the version is no longer supported. Please switch to supply the build identifier argument (-i) and for release pipelines the release identifier argument (-r). rc="$rc + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + echo $ERRMSG + break + fi + ERRMSG=$PGM": [WARNING] The argument (-v) for naming the version is deprecated. Please switch to the new options and supply the build identifier argument (-i) and for release pipelines the release identifier argument (-r)." echo $ERRMSG + artifactVersionName="$argument" ;; \?) Help @@ -475,7 +482,11 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then # invoke function in packageUtils if [ ! -z "${tarFileName}" ]; then - echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier." + echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." + fi + + if [ ! -z "${artifactVersionName}" ]; then + echo $PGM": [INFO] ** Identified that artifactVersionName is passed into packageBuildOutputs.sh (${artifactVersionName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." fi computePackageInformation From f2e4b89664190aae5f5083078953d3e83b4bea06 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 11:28:27 +0100 Subject: [PATCH 082/127] store package in deployfolder Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/wazideploy-generate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index cf2cc607..76b3681c 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -514,9 +514,9 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t PackageInputFile="${packageUrl}" ## Take the last segment of the URL to define the tarFileName tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(getLogDir)/${tarFileName}" + PackageOutputFile="$(wdDeployPackageDir)/${tarFileName}" echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." - echo "PackageInputFile=${PackageOutputFile}" >>$(getLogDir)/${wdPackageVersionFile} + echo "PackageInputFile=${PackageOutputFile}" >>$(wdDeployPackageDir)/${wdPackageVersionFile} fi fi From 8e5e23bbdcf70b22191b96341d7d88400125e18d Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 13:27:39 +0100 Subject: [PATCH 083/127] store package in deployfolder fix path Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/wazideploy-generate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 76b3681c..3440e1c4 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -516,7 +516,7 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') PackageOutputFile="$(wdDeployPackageDir)/${tarFileName}" echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." - echo "PackageInputFile=${PackageOutputFile}" >>$(wdDeployPackageDir)/${wdPackageVersionFile} + echo "PackageInputFile=${PackageOutputFile}" >> $(getLogDir)/${wdPackageVersionFile} fi fi From 139a28c215a4f16e870acd53194300811c86c403 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 15:49:07 +0100 Subject: [PATCH 084/127] Expect application descriptor on the root level of the application directory Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 5 +++-- Templates/Common-Backend-Scripts/dbbBuild.sh | 2 +- .../utilities/fetchBuildDependenciesUtils.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index e1ebf303..ec64b1fc 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -242,8 +242,9 @@ Both implementations for the build stage are enabled to optionally pull external The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository, which are necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. -Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. For -fetching the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The external packages can be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory 'imports'. +The Application descriptor file `applicationDescriptor.yml` is expected to be on the root level of the application repository. + +Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. For fetching the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The external packages can be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory 'imports'. ### dbbBuild.sh for zAppBuild framework diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 7813df43..76d3f23f 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -438,7 +438,7 @@ fi # Setup build environment and pull external dependencies if an Application Descriptor is found -if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then +if [ $rc -eq 0 ]; then fetchBuildDependencies fi diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index 9a453768..a342bcdd 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -85,7 +85,7 @@ runFetchLogic() { fetchBuildDependencies() { # extracting external dependencies is based on the application descriptor - applicationDescriptor="${AppDir}/applicationDescriptor.yml" + applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" From 85ee4456340336b27e84d8235bd5c803c17df5b8 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 15:59:34 +0100 Subject: [PATCH 085/127] Validate configuration parm Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 4 ++-- .../utilities/fetchBuildDependenciesUtils.sh | 2 +- Templates/Common-Backend-Scripts/zBuilder.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 76d3f23f..f881f652 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -438,8 +438,8 @@ fi # Setup build environment and pull external dependencies if an Application Descriptor is found -if [ $rc -eq 0 ]; then - fetchBuildDependencies +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then + fetchBuildDependenciesMethod fi # diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index a342bcdd..4602efcb 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -82,7 +82,7 @@ runFetchLogic() { # # Validate to fetch external dependencies is based on the ApplicationDescriptor -fetchBuildDependencies() { +fetchBuildDependenciesMethod() { # extracting external dependencies is based on the application descriptor applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh index 6ab6c48f..f2e00bb1 100755 --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -450,8 +450,8 @@ fi # Setup build environment and pull external dependencies if an ApplicationDescriptor is found -if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" = true ]; then - fetchBuildDependencies +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then + fetchBuildDependenciesMethod fi # From 718a6ea4c84d5ba074acb9c6e0d921fb7cca532f Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 16:08:36 +0100 Subject: [PATCH 086/127] hide pwds Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 33bba938..288b94b3 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -24,7 +24,10 @@ parseArgs(args) // Print parms println("** Script configuration:") props.sort().each { k,v-> - println " $k -> $v" + if ( k == "artifactRepository.password" ) + println " $k -> xxxxxx " + else + println " $k -> $v" } // Load and verify helpers From 7549a5a3f381296686e824f4392aac982014f841 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 16:13:51 +0100 Subject: [PATCH 087/127] remove prefixing Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 2 +- Templates/Common-Backend-Scripts/zBuilder.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index f881f652..708ec37c 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -508,7 +508,7 @@ if [ $rc -eq 0 ]; then CMD="${CMD} ${Type}" # Append zAppBuild Build Type echo $PGM": [INFO] ${CMD}" - ${CMD} | awk '{print "dbb (groovyz) [zAppBuild] " $0}' #DEBUG: I commented this out for testing purposed + ${CMD} rc=$? #exit 0 diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh index f2e00bb1..67b015b8 100755 --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -511,7 +511,7 @@ if [ $rc -eq 0 ]; then fi echo $PGM": [INFO] ${CMD}" - ${CMD} | awk '{print "dbb [zBuilder] " $0}' # run build + ${CMD} rc=$? if [ $rc -eq 0 ]; then From ad135089777f926dcd1e6a3b746b85705d6868ce Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 20 Mar 2025 16:16:57 +0100 Subject: [PATCH 088/127] Align with required location of the application descriptor Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 6b4f4265..94e712c9 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -593,7 +593,7 @@ if [ $rc -eq 0 ]; then # application directory if [ ! -z "${AppDir}" ]; then - CMD="${CMD} --applicationFolderPath ${AppDir}" + CMD="${CMD} --applicationFolderPath $(getApplicationDir)" fi # packaging properties file From 5e521197b6f1721aaaf9c91541795b554e50a9bb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 21 Mar 2025 09:24:52 +0100 Subject: [PATCH 089/127] Document external dependencies in wazi deploy app manifest Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 63 ++++++++++++------- .../pipelineBackend.config | 16 +++-- .../utilities/fetchBuildDependenciesUtils.sh | 10 ++- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 94e712c9..29213acb 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -163,6 +163,8 @@ artifactRepositoryPassword="" # required if artifactRepositoryPropertyFile n artifactRepositoryName="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryDirectory="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryPropertyFile="" # alternative to above cli parms +externalDependenciesLogFile="" # document fetched build dependencies + HELP=$1 if [ "$HELP" = "?" ]; then @@ -395,6 +397,17 @@ validateOptions() { fi fi + # validate if external dependency log exists + if [ "${fetchBuildDependencies}" == "true" ] && [ ! -z "${externalDependenciesLogName}" ]; then + externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" + # Validate Properties file + if [ ! -f "${externalDependenciesLogFile}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Unable to locate ${externalDependenciesLogFile}. rc="$rc + echo $ERRMSG + fi + fi + } # function to validate publishing options @@ -508,59 +521,62 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Started - Package Build Outputs on HOST/USER: ${SYS}/${USER}" echo $PGM": [INFO] ** WorkDir:" $(getWorkDirectory) if [ ! -z "${App}" ]; then - echo $PGM": [INFO] ** Application:" ${App} + echo $PGM": [INFO] ** Application:" ${App} fi if [ ! -z "${Branch}" ]; then - echo $PGM": [INFO] ** Branch:" ${Branch} + echo $PGM": [INFO] ** Branch:" ${Branch} fi if [ ! -z "${AppDir}" ]; then - echo $PGM": [INFO] ** Application directory:" ${AppDir} + echo $PGM": [INFO] ** Application directory:" ${AppDir} fi if [ ! -z "${PipelineType}" ]; then - echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} + echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} fi if [ ! -z "${tarFileName}" ]; then - echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} + echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} fi - echo $PGM": [INFO] ** BuildReport Location:" ${logDir} - echo $PGM": [INFO] ** PackagingScript Path:" ${packagingScript} + echo $PGM": [INFO] ** BuildReport Location:" ${logDir} + echo $PGM": [INFO] ** PackagingScript Path:" ${packagingScript} if [ ! -z "${PkgPropFile}" ]; then - echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} + echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} fi if [ ! -z "${packageBuildIdentifier}" ]; then - echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} + echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} fi - + echo $PGM": [INFO] ** Generate SBOM:" ${generateSBOM} + if [ ! -z "${sbomAuthor}" ]; then + echo $PGM": [INFO] ** SBOM Author:" ${sbomAuthor} + fi + if [ ! -z "${externalDependenciesLogFile}" ]; then + echo $PGM": [INFO] ** External Dependencies log:" ${externalDependenciesLogFile} + fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} if [ "$publish" == "true" ]; then if [ ! -z "${artifactRepositoryPropertyFile}" ]; then - echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} + echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} fi if [ ! -z "${artifactRepositoryUrl}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} + echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} fi if [ ! -z "${artifactRepositoryUser}" ]; then - echo $PGM": [INFO] ** ArtifactRepo User:" ${artifactRepositoryUser} + echo $PGM": [INFO] ** ArtifactRepo User:" ${artifactRepositoryUser} fi if [ ! -z "${artifactRepositoryPassword}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Password: xxxxx" + echo $PGM": [INFO] ** ArtifactRepo Password: xxxxx" fi if [ ! -z "${artifactRepositoryName}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} + echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} fi if [ ! -z "${artifactRepositoryDirectory}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} + echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} fi fi - echo $PGM": [INFO] ** Generate SBOM:" ${generateSBOM} - if [ ! -z "${sbomAuthor}" ]; then - echo $PGM": [INFO] ** SBOM Author:" ${sbomAuthor} - fi - echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} + + echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} echo $PGM": [INFO] **************************************************************" echo "" fi @@ -616,6 +632,11 @@ if [ $rc -eq 0 ]; then CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" fi + # Pass information about externally fetched modules to packaging to document them + if [ ! -z "${externalDependenciesLogFile}" ]; then + CMD="${CMD} --externalDependenciesEvidences ${externalDependenciesLogFile}" + fi + # publishing options if [ "$publish" == "true" ]; then CMD="${CMD} --publish" diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index b79fc151..a578e5f7 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -125,11 +125,10 @@ enablePackageCache=true # Location for cachine packages on the build machine packageCacheLocation=$HOME/cbs/.packagecache -# Name conventions for the Artifact repository within the Artifact repository server -# script will prefix it with the application name -# default: artifactRepositoryNameSuffix=repo-local -# leading to application-repo-local -artifactRepositoryNameSuffix=repo-local +# Internal file name documenting the fetched external dependencies, that can be tracked in the +# Wazi Deploy application manifest file +# externalDependenciesLogName=externalDependenciesLog.yml +externalDependenciesLogName=externalDependenciesLog.yml ##################################################################################################### ## End of Fetch Build Dependencies based on Application Descriptor ################################# @@ -139,6 +138,13 @@ artifactRepositoryNameSuffix=repo-local ## packageBuildOutputs.sh parameters and configuration ############################################ ##################################################################################################### +# Suffix aligning to naming conventions for name of the Artifact repository within +# the Artifact repository server. Scripts will prefix it with the application name +# default: artifactRepositoryNameSuffix=repo-local +# leading to -repo-local +# ex. MortgageApplication-repo-local +artifactRepositoryNameSuffix=repo-local + # (Optional) Absolute path of a property file containing default application packaging details. # e.g. PkgPropFile="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties" # PkgPropFile="" diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index 4602efcb..af31ae63 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -60,8 +60,8 @@ runFetchLogic() { echo $PGM": [INFO] ** Fetch Application Dependencies from Artifact Repository" cmd="groovyz ${PIPELINE_SCRIPTS}/utilities/fetchBuildDependencies.groovy -w $(getWorkDirectory) -a ${applicationDescriptor} -p ${pipelineConfiguration} -b ${Branch}" # - if [ ! -z "${externalDependenciesLog}" ]; then - cmd="${cmd} -d ${externalDependenciesLog}" + if [ ! -z "${externalDependenciesLogFile}" ]; then + cmd="${cmd} -d ${externalDependenciesLogFile}" fi if [ ! -z "${packageCacheLocation}" ]; then @@ -88,7 +88,11 @@ fetchBuildDependenciesMethod() { applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) - externalDependenciesLog="$(getLogDir)/externalDependenciesLog.yaml" + if [ ! -z "${externalDependenciesLogName}" ]; then + externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" + fi + + # create the log dir mkdir -p "$(getLogDir)" # Set up to perform the clone of the Repo From c0cdf1b42a2fb7116c837649c7ff6c1f7c83b5eb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 21 Mar 2025 10:20:05 +0100 Subject: [PATCH 090/127] update file references Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 6 +++--- .../utilities/fetchBuildDependenciesUtils.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index a578e5f7..8dfd529b 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -167,8 +167,8 @@ sbomAuthor= cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar # File name that is stored in the logs directory containing -# the computed absolute package url -tempVersionFile=packageVersionConfig.txt +# the computed absolute package url and downloaded tarFileName +wdPackageVersionFile=waziDeployPackageConfig.txt ##################################################################################################### #### Artifact repository upload options ########################################################## @@ -280,7 +280,7 @@ wdDeployArtifactoryConfig="/var/WaziDeploy/config/WaziDeploy-ConfigFile.yml" # e.g. wdEnvironmentConfigurations="/var/wazideploy/environment-configurations" # This directory then contains the available environment definitions # like IntegrationTest.yaml, AcceptanceTest.yaml -wdEnvironmentConfigurations="" +wdEnvironmentConfigurations="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/external-repos/environment-conf/python" # # Default name of the wazideploy **evidence file** file diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index af31ae63..a17da163 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -42,7 +42,7 @@ runFetchLogic() { echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} echo $PGM": [INFO] ** Use Package Cache :" ${enablePackageCache} echo $PGM": [INFO] ** Package Cache Location :" ${packageCacheLocation} - echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLog} + echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLogFile} echo $PGM": [INFO] **************************************************************" echo "" fi From 09a7574654ff16921be67fbab0fdfa5d2b62e0d9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 09:17:25 +0100 Subject: [PATCH 091/127] Simplify package management Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 10 +- .../utilities/README.md | 66 +-- .../utilities/computePackageUrl.sh | 451 ------------------ ...{packageUtils.sh => packagingUtilities.sh} | 81 +++- .../wazideploy-deploy.sh | 14 +- .../wazideploy-generate.sh | 62 +-- 6 files changed, 96 insertions(+), 588 deletions(-) delete mode 100755 Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh rename Templates/Common-Backend-Scripts/utilities/{packageUtils.sh => packagingUtilities.sh} (59%) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 29213acb..5a4e9b8b 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -120,7 +120,7 @@ Help() { SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" packagingScript="${SCRIPT_HOME}/../../Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -packageUtilities="${SCRIPT_HOME}/utilities/packageUtils.sh" +packagingUtilities="${SCRIPT_HOME}/utilities/packagingUtilities.sh" # Path and File Name to the advanced debug options. #log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" @@ -198,12 +198,12 @@ fi # Source packaging helper if [ $rc -eq 0 ]; then - if [ ! -f "${packageUtilities}" ]; then + if [ ! -f "${packagingUtilities}" ]; then rc=8 - ERRMSG=$PGM": [ERROR] Packaging Utils (${packageUtilities}) was not found. rc="$rc + ERRMSG=$PGM": [ERROR] Packaging Utils (${packagingUtilities}) was not found. rc="$rc echo $ERRMSG else - source $packageUtilities + source $packagingUtilities fi fi @@ -492,7 +492,7 @@ validatePublishingOptions() { # compute packaging parameters and validate publishing options if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - # invoke function in packageUtils + # invoke function in packagingUtilities if [ ! -z "${tarFileName}" ]; then echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." diff --git a/Templates/Common-Backend-Scripts/utilities/README.md b/Templates/Common-Backend-Scripts/utilities/README.md index 7c66382a..98e42a7d 100644 --- a/Templates/Common-Backend-Scripts/utilities/README.md +++ b/Templates/Common-Backend-Scripts/utilities/README.md @@ -1,67 +1,3 @@ # Contents -This folder contains utility scripts, that are sourced by the primary common backend scripts. - -There is one exception that can be used standalone - -## computePackageUrl.sh - -Script to compute the url where a package got stored in the Artifact repository. - -### Invocation - -The `computePackageUrl.sh` script can be invoked as follows: - -``` -utilities/computePackageUrl.sh -w MortgageApplication/feature/15-update-mortgage-application/build_7645 -a MortgageApplication -b feature/15-update-mortgage-application -p build -i 7645 -``` - -CLI parameter | Description ----------- | ---------------------------------------------------------------------------------------- --w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. --a `` | **Application name** leveraged to define the artifact repository name. --b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. --p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. --r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. --i `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number or a timestamp. - -On successful completion the below message it displayed, and a file containing the information is stored the workspace log directory: -``` -computePackageUrl.sh: [INFO] Compute Package Url completed. Results stored in /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt. rc=0 -``` -packageVersionConfig.txt: -``` -packageUrl=http://10.3.20.231:8081/artifactory/MortgageApplication-repo-local/build/feature/15-update-mortgage-application/MortgageApplication-7645.tar -``` - - -### Script output - -The section below contains the output that is produced by the `utilities/computePackageUrl.sh` script. - -
- Script Output - -``` -/u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh -w MortgageApplication/feature/15-update-mortgage-application/build_7645 -a MortgageApplication -b feature/15-update-mortgage-application -p build -i 7645 -computePackageUrl.sh: [INFO] Compute Package Url. Version=1.00 -computePackageUrl.sh: [INFO] ************************************************************** -computePackageUrl.sh: [INFO] ** Started - Compute Package Url on HOST/USER: z/OS ZT01 05.00 02 8561/BPXROOT -computePackageUrl.sh: [INFO] ** Application: MortgageApplication -computePackageUrl.sh: [INFO] ** Branch: feature/15-update-mortgage-application -computePackageUrl.sh: [INFO] ** Type of pipeline: build -computePackageUrl.sh: [INFO] ** Tar file Name: MortgageApplication-7645.tar -computePackageUrl.sh: [INFO] ** Artifact Repository Helpers: /u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy -computePackageUrl.sh: [INFO] ** ArtifactRepo Url: http://10.3.20.231:8081/artifactory -computePackageUrl.sh: [INFO] ** ArtifactRepo Repo name: MortgageApplication-repo-local -computePackageUrl.sh: [INFO] ** ArtifactRepo Repo Dir: build -computePackageUrl.sh: [INFO] ** Output file: /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt -computePackageUrl.sh: [INFO] ** DBB_HOME: /usr/lpp/dbb/v3r0 -computePackageUrl.sh: [INFO] ************************************************************** - -computePackageUrl.sh: [INFO] Invoking the Package Build Outputs script to compute Package Url. -computePackageUrl.sh: [INFO] /usr/lpp/dbb/v3r0/bin/groovyz /u/ibmuser/git/dbb/Templates/Common-Backend-Scripts/utilities/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy --computePackageUrl --tarFileName MortgageApplication-7645.tar --versionName feature/15-update-mortgage-application --artifactRepositoryUrl "http://10.3.20.231:8081/artifactory" --artifactRepositoryName MortgageApplication-repo-local --artifactRepositoryDirectory build -computePackageUrl.sh: [INFO] Compute Package Url completed. Results stored in /u/ibmuser/git/workspace_test/MortgageApplication/feature/15-update-mortgage-application/build_7645/logs/packageVersionConfig.txt. rc=0 -``` - -
\ No newline at end of file +This folder contains utility scripts, that are sourced by the primary common backend scripts. \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh b/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh deleted file mode 100755 index 290f5311..00000000 --- a/Templates/Common-Backend-Scripts/utilities/computePackageUrl.sh +++ /dev/null @@ -1,451 +0,0 @@ -#!/bin/env bash -#=================================================================================== -# NAME: computePackageUrl.sh -# -# DESCRIPTION: The purpose of this script is to recompute the URL, -# where a package got published/stored. -# -# SYNTAX: See Help() section below for usage -# -# RETURNS: -# -# rc - Return Code -# -# RETURN CODES: -# -# 0 - Successful -# 4 - Warning message(s) issued. See Console messages -# 8 - Error encountered. See Console messages -# -# NOTE(S): -# -# 1. Review the common naming conventions where tar files get stored. -# -#=================================================================================== -Help() { - echo $PGM" - Invoke Compute Package URL ("$PGMVERS") " - echo " " - echo "DESCRIPTION: The purpose of this script is to compute " - echo "the URL where a package got published/stored " - echo " " - echo "Syntax: " - echo " " - echo " "$PGM" [Options] " - echo " " - echo "Options: " - echo " " - echo " Mandatory parameters " - echo " " - echo " " - echo " -w - Directory Path to a unique " - echo " working directory " - echo " Either an absolute path " - echo " or relative path. " - echo " If a relative path is provided, " - echo " buildRootDir and the workspace " - echo " path are combined " - echo " Default=None, Required. " - echo " " - echo " Ex: MortgageApplication/main/build-1 " - echo " " - echo " -a - Application name " - echo " Used to compute " - echo " Artifact repository name. " - echo " " - echo " Ex: MortgageApplication " - echo " " - echo " -p - Type of the pipeline to " - echo " control in which directory builds " - echo " are stored in the artifact repo " - echo " Accepted values: " - echo " build - " - echo " development builds " - echo " release - " - echo " builds with options for " - echo " performance optimized " - echo " executables for production env " - echo " " - echo " -b - Name of the git branch. " - echo " " - echo " Ex: main " - echo " " - echo " " - echo " -i - A unique build identifier " - echo " typically the buildID of the " - echo " pipeline " - echo " " - echo " -r - The release identifier for " - echo " release pipeline builds " - echo " (optional) " - echo " " - echo " " - echo " -h - Display this Help. " - - exit 0 -} - -# Customization -# Central configuration file leveraged by the backend scripts -SCRIPT_HOME="$(dirname "$0")" -pipelineConfiguration="${SCRIPT_HOME}/../pipelineBackend.config" -packageUtilities="${SCRIPT_HOME}/packageUtils.sh" -artifactRepositoryHelpersScript="${SCRIPT_HOME}/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy" - -# Path and File Name to the advanced debug options. -#log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" - -# -# Internal Variables -#set -x # Uncomment to enable shell script debug -#export BASH_XTRACEFD=1 # Write set -x trace to file descriptor - -PGM=$(basename "$0") -PGMVERS="1.00" -USER=$(whoami) -SYS=$(uname -Ia) - -# -# Set initialization -# -rc=0 -ERRMSG="" -App="" -tarFileName="" -PipelineType="" -Branch="" - -buildIdentifier="" -releaseIdentifier="" - -artifactVersionName="" # required for publishing to artifact repo -artifactRepositoryUrl="" # required if artifactRepositoryPropertyFile not specified -artifactRepositoryName="" # required if artifactRepositoryPropertyFile not specified -artifactRepositoryDirectory="" # required if artifactRepositoryPropertyFile not specified -artifactRepositoryPropertyFile="" # alternative to above cli parms - -outputFile="" - -HELP=$1 - -if [ "$HELP" = "?" ]; then - Help -fi - -# Validate Shell environment -currentShell=$(ps -p $$ | grep bash) -if [ -z "${currentShell}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] The scripts are designed to run in bash. You are running a different shell. rc=${rc}. \n. $(ps -p $$)." - echo $ERRMSG -fi -# - -# Script label -if [ $rc -eq 0 ]; then - echo $PGM": [INFO] Compute Package Url. Version="$PGMVERS -fi - -# Read and import pipeline configuration -if [ $rc -eq 0 ]; then - if [ ! -f "${pipelineConfiguration}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Pipeline Configuration File (${pipelineConfiguration}) was not found. rc="$rc - echo $ERRMSG - else - source $pipelineConfiguration - fi -fi - -# Source packaging helper -if [ $rc -eq 0 ]; then - if [ ! -f "${packageUtilities}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Packaging Utils (${packageUtilities}) was not found. rc="$rc - echo $ERRMSG - else - source $packageUtilities - fi -fi - -# -# Get Options -if [ $rc -eq 0 ]; then - while getopts ":h:w:a:b:i:r:p:" opt; do - case $opt in - h) - Help - ;; - w) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Build Workspace Folder Name is required. rc="$rc - echo $ERRMSG - break - fi - Workspace="$argument" - ;; - a) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc - echo $ERRMSG - break - fi - App="$argument" - ;; - b) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc - echo $ERRMSG - break - fi - Branch="$argument" - ;; - p) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc - echo $INFO - break - fi - PipelineType="$argument" - ;; - i) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc - echo $ERRMSG - break - fi - buildIdentifier="$argument" - ;; - r) - argument="$OPTARG" - nextchar="$(expr substr $argument 1 1)" - if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then - rc=4 - ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc - echo $ERRMSG - break - fi - releaseIdentifier="$argument" - ;; - \?) - Help - rc=1 - break - ;; - :) - rc=4 - ERRMSG=$PGM": [WARNING] Option -$OPTARG requires an argument. rc="$rc - echo $ERRMSG - break - ;; - esac - done -fi -# - -validateOptions() { - - if [ -z "${Workspace}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Unique Workspace parameter (-w) is required. rc="$rc - echo $ERRMSG - else - - # Compute the logDir parameter - logDir=$(getLogDir) - - if [ ! -d "$logDir" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Build Log Directory ($logDir) was not found. rc="$rc - echo $ERRMSG - fi - - outputFile=$(getLogDir)/${wdPackageVersionFile} - fi - - if [ -z "${buildIdentifier}" ]; then - ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" - echo $ERRMSG - buildIdentifier=$(date +%Y%m%d_%H%M%S) - fi - - if [ ! -f "${artifactRepositoryHelpersScript}" ]; then - rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${artifactRepositoryHelpersScript}. rc="$rc - echo $ERRMSG - fi - - - -} - -# function to validate publishing input options -validatePackagingOptions() { - - if [ -z "${App}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc - echo $ERRMSG - fi - - if [ -z "${Branch}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Branch Name parameter (-b) is required. rc="$rc - echo $ERRMSG - fi - - if [ -z "${artifactRepositoryUrl}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] URL to artifact repository (artifactRepositoryUrl) is required. rc="$rc - echo $ERRMSG - fi - - if [ -z "${artifactRepositoryName}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] artifact repository name to store the build (artifactRepositoryName) is required. rc="$rc - echo $ERRMSG - fi - - if [ -z "${artifactRepositoryDirectory}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Directory path in the repository to store the build (artifactRepositoryDirectory) is required. rc="$rc - echo $ERRMSG - fi - - # If pipeline type is specified, evaluate the value - if [ ! -z "${PipelineType}" ]; then - tmp1=$(echo $PipelineType | tr '[:upper:]' '[:lower:]') - - case $tmp1 in - "build") - PipelineType=$tmp1 - ;; - "release") - PipelineType=$tmp1 - ;; - "preview") - rc=4 - ERRMSG=$PGM": [WARN] Default Pipeline Type : ${PipelineType} not supported for packaging." - echo $ERRMSG - ;; - *) - rc=4 - ERRMSG=$PGM": [WARN] Invalid Pipeline Type : ${PipelineType} specified." - echo $ERRMSG - ;; - esac - - fi -} - -# Call validate input options -if [ $rc -eq 0 ]; then - validateOptions -fi - -# compute packaging parameters -if [ $rc -eq 0 ]; then - # invoke function in packageUtils - computePackageInformation -fi - -# Call validate packaging options -if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - validatePackagingOptions -fi - -# -# Ready to go -if [ $rc -eq 0 ]; then - echo $PGM": [INFO] **************************************************************" - echo $PGM": [INFO] ** Started - Compute Package Url on HOST/USER: ${SYS}/${USER}" - if [ ! -z "${App}" ]; then - echo $PGM": [INFO] ** Application:" ${App} - fi - if [ ! -z "${Branch}" ]; then - echo $PGM": [INFO] ** Branch:" ${Branch} - fi - if [ ! -z "${PipelineType}" ]; then - echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} - fi - if [ ! -z "${tarFileName}" ]; then - echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} - fi - echo $PGM": [INFO] ** Artifact Repository Helpers:" ${artifactRepositoryHelpersScript} - - if [ ! -z "${artifactRepositoryUrl}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} - fi - - if [ ! -z "${artifactRepositoryName}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} - fi - if [ ! -z "${artifactRepositoryDirectory}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} - fi - - echo $PGM": [INFO] ** Output file:" ${outputFile} - - echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} - echo $PGM": [INFO] **************************************************************" - echo "" -fi - -# -# Invoke the Package Build Outputs script -if [ $rc -eq 0 ]; then - echo $PGM": [INFO] Invoking the ArtifactRepositoryHelper groovy script to compute Package Url." - - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computePackageUrl" - - # add tarfile name - if [ ! -z "${tarFileName}" ]; then - CMD="${CMD} --tarFileName ${tarFileName}" - fi - - # artifactVersionName - if [ ! -z "${artifactVersionName}" ]; then - CMD="${CMD} --versionName ${artifactVersionName}" - fi - - # Artifact repo options - if [ ! -z "${artifactRepositoryUrl}" ]; then - CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" - fi - - if [ ! -z "${artifactRepositoryName}" ]; then - CMD="${CMD} --artifactRepositoryName ${artifactRepositoryName}" - fi - if [ ! -z "${artifactRepositoryDirectory}" ]; then - CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" - fi - - echo $PGM": [INFO] ${CMD}" - ${CMD} | grep "packageUrl=" > $outputFile - rc=$? - - if [ $rc -eq 0 ]; then - ERRMSG=$PGM": [INFO] Compute Package Url completed. Results stored in $outputFile. rc="$rc - echo $ERRMSG - else - ERRMSG=$PGM": [ERR] Compute Package Url failed. Check Console for details. rc="$rc - echo $ERRMSG - rc=12 - fi -fi - -exit $rc diff --git a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh similarity index 59% rename from Templates/Common-Backend-Scripts/utilities/packageUtils.sh rename to Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index ea380653..e66bec4a 100755 --- a/Templates/Common-Backend-Scripts/utilities/packageUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -3,23 +3,21 @@ mainBranchSegment="" secondBranchSegment="" rc=0 - - +# Method implementing the conventions in the CBS for packaging using the PackageBuildOutputs.groovy script computePackageInformation() { ############################################# # output environment variables ############################################# - artifactRepositoryName="" # identifier of the artifact repo - artifactRepositoryDirectory="" # root directory folder in repo - artifactVersionName="" # subfolder in repo path identifying version / origin branch - tarFileName="" # computed tarFileName how it is stored in the artifact repository - artifactRepositoryAbsoluteUrl="" # absolute URL - packageBuildIdentifier="" # Identifier for Wazi Deploy Application Manifest file + artifactRepositoryName="" # identifier of the artifact repo + artifactRepositoryDirectory="" # root directory folder in repo + artifactVersionName="" # subfolder in repo path identifying version / origin branch + tarFileName="" # computed tarFileName how it is stored in the artifact repository + packageBuildIdentifier="" # Identifier for Wazi Deploy Application Manifest file ############################################# - - # configuration variable defining the Artifactory repository name pattern + + # configuration variable defining the Artifactory repository name pattern artifactRepositoryRepoPattern="${App}-${artifactRepositoryNameSuffix}" - + branchConvention=(${Branch//// }) if [ $rc -eq 0 ]; then @@ -39,7 +37,7 @@ computePackageInformation() { # remove chars (. -) from the name mainBranchSegmentTrimmed=$(echo ${mainBranchSegment} | tr -d '.-' | tr '[:lower:]' '[:upper:]') - + artifactRepositoryName=$(echo "${artifactRepositoryRepoPattern}") # evaluate main segment @@ -51,13 +49,13 @@ computePackageInformation() { # ///--.tar # MortgageApplication-repo-local/release/1.2.3/MortgageApplication-1.2.3-1234567890.tar ############################################# - + # Release builds are captured in the release directory of the artifact repo artifactRepositoryDirectory="release" - + # artifactVersionName is second identifier in the folder structure and represents the artifactVersionName=${releaseIdentifier} - + # building up the tarFileName tarFileName="${App}-${releaseIdentifier}-${buildIdentifier}.tar" packageBuildIdentifier="${releaseIdentifier}-${buildIdentifier}" @@ -67,7 +65,7 @@ computePackageInformation() { # ///-.tar # Mortgage-repo-local/build/feature/123-enhance-something/Mortgage-123456.tar ############################################# - + artifactRepositoryDirectory="build" artifactVersionName=${Branch} tarFileName="${App}-${buildIdentifier}.tar" @@ -86,9 +84,55 @@ computePackageInformation() { esac ############################################# - ### Construct the absolute repository URL + ### Construct the absolute repository URL (required when downloading the package) ############################################# - artifactRepositoryAbsoluteUrl="${artifactRepositoryUrl}/${artifactRepositoryName}/${artifactRepositoryDirectory}/${artifactVersionName}/${tarFileName}" + + if [ "${computeArchiveUrl}" == "true" ]; then + + # + # Invoke the Package Build Outputs script + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] Invoking the ArtifactRepositoryHelper groovy script to compute Package Url." + + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computePackageUrl" + + # add tarfile name + if [ ! -z "${tarFileName}" ]; then + CMD="${CMD} --tarFileName ${tarFileName}" + fi + + # artifactVersionName + if [ ! -z "${artifactVersionName}" ]; then + CMD="${CMD} --versionName ${artifactVersionName}" + fi + + # Artifact repo options + if [ ! -z "${artifactRepositoryUrl}" ]; then + CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" + fi + + if [ ! -z "${artifactRepositoryName}" ]; then + CMD="${CMD} --artifactRepositoryName ${artifactRepositoryName}" + fi + if [ ! -z "${artifactRepositoryDirectory}" ]; then + CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" + fi + + echo $PGM": [INFO] ${CMD}" + artifactRepositoryAbsoluteUrl="$(${CMD} | grep packageUrl=)" + + if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then + ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc + echo $ERRMSG + else + rc=12 + ERRMSG=$PGM": [ERR] Computation of Archive Url failed. Check Console for details. rc="$rc + echo $ERRMSG + + fi + fi + + fi # unset internal variables mainBranchSegment="" @@ -97,4 +141,3 @@ computePackageInformation() { fi } - diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index 9c79abad..59e72808 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -337,18 +337,16 @@ validateOptions() { fi } -# When publishing is enabled, try reading the wdPackageVersionFile +# When publishing is enabled, check if the tarfile exists in the expected location if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then - echo $PGM": [INFO] ** Found Wazi Deploy Package configuration file $(getLogDir)/${wdPackageVersionFile}. Import configuration." + if [ -f "$(wdDeployPackageDir)/applicationPackage.tar" ]; then # shared convention with wazideploy-generate.sh + echo $PGM": [INFO] ** Package file was found in expected location at $(wdDeployPackageDir)/applicationPackage.tar ." if [ ! -z "${PackageInputFile}" ]; then - echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced by the Wazi Deploy configuration setting from $(getLogDir)/${wdPackageVersionFile}." + echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced with $(wdDeployPackageDir)/applicationPackage.tar ." fi - source "$(getLogDir)/${wdPackageVersionFile}" + PackageInputFile="$(wdDeployPackageDir)/applicationPackage.tar" else - rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc - echo $ERRMSG + echo $PGM": [INFO] ** The CBS can automatically compute the Url of the package. Wazi Deploy will then download it. Read more about the capabilities in the CBS readme." fi fi diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 3440e1c4..2116d2a4 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -135,7 +135,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" -computePackageUrlUtil="${SCRIPT_HOME}/utilities/computePackageUrl.sh" +packagingUtilities="${SCRIPT_HOME}/utilities/packagingUtilities.sh" # Customization - End # @@ -168,6 +168,10 @@ Branch="" # takes cli option b buildIdentifier="" # takes cli option I releaseIdentifier="" # takes cli option R +# +computeArchiveUrl="true" # enables the computation of the url +artifactRepositoryAbsoluteUrl="" # Used to store the computed Url + Debug="" HELP=$1 @@ -201,13 +205,15 @@ if [ $rc -eq 0 ]; then fi fi -# Read and import pipeline configuration +# Source packaging helper if [ $rc -eq 0 ]; then - if [ ! -f "${computePackageUrlUtil}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] The Package Url Util (${computePackageUrlUtil}) was not found. rc="$rc - echo $ERRMSG - fi + if [ ! -f "${packagingUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Packaging Utilities file (${packagingUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $packagingUtilities + fi fi # @@ -478,46 +484,22 @@ validateOptions() { # that needs to be computed before this step. if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; then checkWorkspace - CMD="${computePackageUrlUtil} -w $Workspace -a $App -b $Branch -i $buildIdentifier" - if [ ! -z "${PipelineType}" ]; then - CMD+=" -p ${PipelineType}" - else + # validate options + if [ -z "${PipelineType}" ]; then rc=8 - ERRMSG=$PGM": [ERROR] To compute the Package Url to automatically download the tar file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc + ERRMSG=$PGM": [ERROR] To compute the Url of the stored package to enable the download of the archive file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc echo $ERRMSG fi - if [ ! -z "${releaseIdentifier}" ]; then - CMD+=" -r ${releaseIdentifier}" - fi - if [ $rc -eq 0 ]; then - echo $PGM": [INFO] ** Compute Package Url based on existing conventions using command" - echo $PGM": [INFO] ** Invoking subtask ${CMD}" - ${CMD} - rc=$? - if [ $rc -eq 0 ]; then - if [ -f "$(getLogDir)/${wdPackageVersionFile}" ]; then - echo $PGM": [INFO] ** Read configuration file $(getLogDir)/${wdPackageVersionFile}" - source "$(getLogDir)/${wdPackageVersionFile}" - else - rc=4 - ERRMSG=$PGM": [ERROR] ** The configuration file $(getLogDir)/${wdPackageVersionFile} was not found. Check previous console output. rc="$rc - echo $ERRMSG - fi - fi - # When a the packageUrl environment variable found in the file continue to compute - if [ $rc -eq 0 ] && [ ! -z "${packageUrl}" ]; then - echo $PGM": [INFO] ** Package Url configuration file found. Package Input File will be set to ${packageUrl}. Package Output file will be computed." - PackageInputFile="${packageUrl}" - ## Take the last segment of the URL to define the tarFileName - tarFileName=$(echo $PackageInputFile | awk -F "/" '{print $NF}') - PackageOutputFile="$(wdDeployPackageDir)/${tarFileName}" - echo $PGM": [INFO] ** Package Output file information stored in $(getLogDir)/${wdPackageVersionFile}." - echo "PackageInputFile=${PackageOutputFile}" >> $(getLogDir)/${wdPackageVersionFile} - fi + # Call utilities method + computePackageInformation + + # Set Input and output files for Wazi Deploy + PackageInputFile="${artifactRepositoryAbsoluteUrl}" + PackageOutputFile="$(wdDeployPackageDir)/applicationPackage.tar" # shared convention with wazideploy-deploy.sh fi fi From c4aedd30697cab55eb638f51259020eef8bece39 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 09:22:53 +0100 Subject: [PATCH 092/127] Pass Helper location Signed-off-by: Dennis Behm --- .../utilities/packagingUtilities.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index e66bec4a..2d8f372b 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -89,6 +89,15 @@ computePackageInformation() { if [ "${computeArchiveUrl}" == "true" ]; then + artifactRepositoryHelpersScript="${SCRIPT_HOME}/../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy" + + # validate options + if [ ! -f "${artifactRepositoryHelpersScript}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Unable to locate ${artifactRepositoryHelpersScript}. rc="$rc + echo $ERRMSG + fi + # # Invoke the Package Build Outputs script if [ $rc -eq 0 ]; then From 70ce2f3e47aab4370de465c44d88535e5d763ae3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 09:29:40 +0100 Subject: [PATCH 093/127] extract the url Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/utilities/packagingUtilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index 2d8f372b..c4f80ddb 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -128,7 +128,7 @@ computePackageInformation() { fi echo $PGM": [INFO] ${CMD}" - artifactRepositoryAbsoluteUrl="$(${CMD} | grep packageUrl=)" + artifactRepositoryAbsoluteUrl="${CMD} | awk -F "packageUrl=" '{print $NF}') if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc From 654ae8927474ae8ea2228f3eed251b9b6bf38e89 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 09:33:12 +0100 Subject: [PATCH 094/127] extract the url Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/utilities/packagingUtilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index c4f80ddb..1f5a8563 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -128,7 +128,7 @@ computePackageInformation() { fi echo $PGM": [INFO] ${CMD}" - artifactRepositoryAbsoluteUrl="${CMD} | awk -F "packageUrl=" '{print $NF}') + artifactRepositoryAbsoluteUrl="$(${CMD} | awk -F \"packageUrl=\" '{print $NF}')" if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc From 945e234e87938f8c3dc877853f838f9a1e390bd3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 10:02:31 +0100 Subject: [PATCH 095/127] update passing variables Signed-off-by: Dennis Behm --- .../Common-Backend-Scripts/utilities/packagingUtilities.sh | 2 +- Templates/Common-Backend-Scripts/wazideploy-generate.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index 1f5a8563..ef9fea58 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -128,7 +128,7 @@ computePackageInformation() { fi echo $PGM": [INFO] ${CMD}" - artifactRepositoryAbsoluteUrl="$(${CMD} | awk -F \"packageUrl=\" '{print $NF}')" + artifactRepositoryAbsoluteUrl=$(${CMD} | grep "packageUrl=" | awk -F "=" ' { print $2 }') if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 2116d2a4..8deef10b 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -171,6 +171,7 @@ releaseIdentifier="" # takes cli option R # computeArchiveUrl="true" # enables the computation of the url artifactRepositoryAbsoluteUrl="" # Used to store the computed Url +usePackageUrl="" # Internal flag indicating if the package url was computed Debug="" HELP=$1 @@ -450,7 +451,7 @@ validateOptions() { echo $ERRMSG else # check for relative path - if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${packageUrl}" ]; then + if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${usePackageUrl}" ]; then checkWorkspace PackageInputFile="$(getLogDir)/${PackageInputFile}" fi @@ -500,7 +501,7 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t # Set Input and output files for Wazi Deploy PackageInputFile="${artifactRepositoryAbsoluteUrl}" PackageOutputFile="$(wdDeployPackageDir)/applicationPackage.tar" # shared convention with wazideploy-deploy.sh - + usePackageUrl="true" fi fi From 5c45d09ba880802c6921f472be78e27f0b47d3a4 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 24 Mar 2025 18:09:20 +0100 Subject: [PATCH 096/127] expand fetching baseline package Signed-off-by: Dennis Behm --- .../applicationDescriptorUtils.groovy | 30 +-- .../packageBuildOutputs.sh | 13 ++ .../utilities/fetchBuildDependencies.groovy | 195 +++++++++++------- 3 files changed, 134 insertions(+), 104 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 429b2bc2..5fd6cf47 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -43,7 +43,9 @@ class FileDef { class Baseline { String branch - String baseline + String type + String reference + String buildid } class DependencyDescriptor { @@ -370,29 +372,3 @@ def getFilesByTypeAndUsage(ApplicationDescriptor applicationDescriptor, String a } } } - -/** - * Method to add a baseline - * If an existing baseline for a given branch already exists, the method replaces it - */ - -def addBaseline(ApplicationDescriptor applicationDescriptor, String branch, String baseline) { - if (applicationDescriptor.baselines) { - def existingBaselines = applicationDescriptor.baselines.findAll() { baselineDefinition -> - baselineDefinition.branch.equals(branch) - } - existingBaselines.forEach() { existingBaseline -> - applicationDescriptor.baselines.remove(existingBaseline) - } - } else { - applicationDescriptor.baselines = new ArrayList() - } - - if (applicationDescriptor.baselines) { - applicationDescriptor.sources = new ArrayList() - } - Baseline newBaseline = new Baseline() - newBaseline.branch = branch - newBaseline.baseline = baseline - applicationDescriptor.baselines.add(newBaseline) -} diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 5a4e9b8b..a48e5c35 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -636,6 +636,19 @@ if [ $rc -eq 0 ]; then if [ ! -z "${externalDependenciesLogFile}" ]; then CMD="${CMD} --externalDependenciesEvidences ${externalDependenciesLogFile}" fi + + baselineDirectory="$(getWorkDirectory)/baseline" + if [ -d "${baselineDirectory}" ]; then + baselineTarName=$(ls "${baselineDirectory}") + baselineTarFile="${baselineDirectory}/${baselineTarName}" + if [ -f "$baselineTarFile" ]; then + CMD="${CMD} --baselinePackage ${baselineTarFile}" + else + echo $PGM": [INFO] Baseline Tar file $baselineTarFile not found." + fi + else + echo $PGM": [INFO] Baseline directory $baselineDirectory not found." + fi # publishing options if [ "$publish" == "true" ]; then diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 288b94b3..5ff7048f 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -58,6 +58,7 @@ if (!applicationDescriptorFile.exists()) { // setup import directory File importFolder = new File("$props.workspace/imports") +File baselineFolder = new File("$props.workspace/baseline") if (importFolder.exists()) importFolder.deleteDir() importFolder.mkdirs() @@ -81,12 +82,12 @@ if (applicationDescriptor.dependencies) { assert dependency.reference : "Missing dependency reference attribute in dependency record" assert dependency.buildid : "Missing buildid attribute in dependency record" assert dependency.name : "Missing name attribute in dependency record" - + // compute tar file name based on build type if (dependency.type.equalsIgnoreCase("release")) { props.put("tarFileName","${dependency.name}-${dependency.reference}-${dependency.buildid}.tar") } else { - props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") + props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") } // Construct the path within the Artifact repo @@ -99,14 +100,14 @@ if (applicationDescriptor.dependencies) { // The absolute url the package in artifact repo artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) - + // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") - + // File in cache / workspace tarFile="${tmpPackageDir}/${artifactRelPath}" tarFileDir=tarFile.replaceAll(props.tarFileName, "") - + println("*** Fetching package '${dependency.name}:${artifactUrl}' ") // Generating information for documentation in yaml file of retrieved dependencies @@ -122,7 +123,7 @@ if (applicationDescriptor.dependencies) { p_uri.key = "uri" p_uri.value = artifactUrl externalDependency.properties.add(p_uri) - + // type - either build or release Property p_type = new Property() p_type.key = "type" @@ -134,7 +135,7 @@ if (applicationDescriptor.dependencies) { p_reference.key = "reference" p_reference.value = dependency.reference externalDependency.properties.add(p_reference) - + // buildid Property p_buildid = new Property() p_buildid.key = "buildid" @@ -146,7 +147,7 @@ if (applicationDescriptor.dependencies) { } // download from artifact repo - + // foldername in workspace directory String includeFolder = "${importFolder}/${dependency.name}" @@ -197,75 +198,115 @@ if (applicationDescriptor.dependencies) { } } -// Do we actually need this or can this just be obtained from the dependencies? - -// Fetch baseline package for internal dependencies such as -// derived copybooks and object decks - -//baselineRecord = applicationDescriptor.baselines.find() { baseline -> -// baseline.branch.equals(props.branch) -//} -// -//if (baselineRecord){ -// version = matchingBaseline.baseline -// println("** Retrieving baseline package at version '${version}' for '${applicationDescriptor.application}'") -// -// baselineName=applicationDescriptor.application -// -// repositoryName="${props.artifactRepositoryNameSuffix}".replaceAll("§application§", baselineName) -// -// def String artifactUrl -// def String artifactReference -// def String artifactRelPath -// -// if (dependency.version.startsWith("rel-")){ -// artifactRelPath="${repositoryName}/main/release/${baseline.version}" -// artifactReference="${artifactRelPath}/${baselineName}.tar" -// artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" -// } else { -// artifactRelPath="${repositoryName}" -// artifactReference="${artifactRelPath}/${baselineName}.tar" -// artifactUrl="${props.artifactRepositoryUrl}/${artifactReference}" -// } -// println("*** Fetching baseline package '${baselineName}:${baseline.version}' ") -// -// String tarFile = "${tmpPackageDir}/${artifactReference}" -// String includeFolder = "${importFolder}/${baselineName}" -// -// if (new File(tarFile).exists()) { -// println("** Package was already found in package cache at '${tmpPackageDir}/${artifactRelPath}'") -// } else { -// String user = props.artifactRepositoryUser -// String password = props.artifactRepositoryPassword -// -// if (!(new File("${tmpPackageDir}/${artifactRelPath}").exists())) (new File("${tmpPackageDir}/${artifactRelPath}")).mkdirs() -// -// println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tmpPackageDir}/${artifactRelPath}.") -// def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile as String, user, password, true) -// println "download complete $rc" // TODO: Error handling in helper -// } -// -// // subdir for each dependency -// File includeFolderFile = new File(includeFolder) -// if (!includeFolderFile.exists()) { -// includeFolderFile.mkdirs() -// } -// -// // Expand tar file -// def processCmd = [ -// "/bin/sh", -// "-c", -// "tar -C $includeFolder -xvf $tarFile" -// ] -// -// def rc = runProcess(processCmd) -// if (rc != 0) { -// println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") -// System.exit(1) -// } -//} else { -// println("** [INFO] No baseline record found for application '${applicationDescriptor.application}'.") -//} + + +baselineRecord = applicationDescriptor.baselines.find() { baseline -> + baseline.branch.equals(props.branch) +} + +if (baselineRecord){ + println("*** Fetching baseline package") + + // validate dependency record + assert baselineRecord.type : "Missing baseline type attribute in baseline record" + assert baselineRecord.reference : "Missing baseline reference attribute in baseline record" + assert baselineRecord.buildid : "Missing buildid attribute in baseline record" + assert applicationDescriptor.application : "Missing application name in Application Descriptor" + def applicationName = applicationDescriptor.application + + // compute tar file name based on build type + if (baselineRecord.type.equalsIgnoreCase("release")) { + props.put("tarFileName","${applicationName}-${baselineRecord.reference}-${baselineRecord.buildid}.tar") + } else { + props.put("tarFileName","${applicationName}-${baselineRecord.buildid}.tar") + } + + // Construct the path within the Artifact repo + repositoryName="${applicationName}-${props.artifactRepositoryNameSuffix}" + + // Prepare for url computation using method in ArtifactRepoHelpers + props.put("versionName","${baselineRecord.reference}") // compute the version name being part of the path + props.put("artifactRepository.directory", "${baselineRecord.type}") // compute the main directory to classify builds + props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name + + // The absolute url the package in artifact repo + artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) + + // retrieve path without artifact url + artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") + + // File in cache / workspace + tarFile="${tmpPackageDir}/${artifactRelPath}" + tarFileDir=tarFile.replaceAll(props.tarFileName, "") + if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() // create tmpDownload + + println("*** Fetching baseline package '${applicationName}:${artifactUrl}' ") + + if (new File(tarFile).exists()) { + println("** Package was already found in package cache at '${tarFile}'") + } else { + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + + if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() + + println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tarFileDir}.") + def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) + + if (rc != 0) { + println "** Download of application package '$artifactUrl' failed. Process exists. Return code:"$rc + exitFetchDependencyProcess() + } else { + //println "* Download successful." + } + } + + // setup baseline directory + if (baselineFolder.exists()) baselineFolder.deleteDir() + baselineFolder.mkdirs() + + println("** Saving tar file '${tarFile}' into '$baselineFolder' ") + + def processCmd = [ + "/bin/sh", + "-c", + "cp ${tarFile} ${baselineFolder}/" + ] + + def rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to copy '$tarFile' to '$baselineFolder' with rc=$rc") + System.exit(1) + } + + // setup import folder + String includeFolder = "${importFolder}/${applicationName}" + File includeFolderFile = new File(includeFolder) + if (!includeFolderFile.exists()) { + includeFolderFile.mkdirs() + } + + println("** Expanding tar file '${tarFile}' to '$includeFolder' ") + + processCmd = [ + "/bin/sh", + "-c", + "tar -C $includeFolder -xvf ${tarFile}" + ] + + rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") + System.exit(1) + } + + // Delete temporary download location if cache is not used + if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) { + tmpPackageDir.deleteDir() + } +} else { + println("** [INFO] No baseline record found for application '${applicationDescriptor.application}'.") +} def yamlBuilder = new YamlBuilder() // write file From 90ded9438ae99808369d23b5b003e936331f3f3a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 09:19:16 +0100 Subject: [PATCH 097/127] manage baseline package during packaging step Signed-off-by: Dennis Behm --- .../packageBuildOutputs.sh | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index a48e5c35..943cbc8a 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -165,6 +165,10 @@ artifactRepositoryDirectory="" # required if artifactRepositoryPropertyFile n artifactRepositoryPropertyFile="" # alternative to above cli parms externalDependenciesLogFile="" # document fetched build dependencies +# managing baseline package +baselineTarFile="" # computed if a baseline package has been retrieved during the fetch phase +baselineFolder="baseline" # per convention this is the subdir into which the fetch script loads the baseline + HELP=$1 if [ "$HELP" = "?" ]; then @@ -402,9 +406,25 @@ validateOptions() { externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" # Validate Properties file if [ ! -f "${externalDependenciesLogFile}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Unable to locate ${externalDependenciesLogFile}. rc="$rc - echo $ERRMSG + rc=8 + ERRMSG=$PGM": [ERROR] Unable to locate ${externalDependenciesLogFile}. rc="$rc + echo $ERRMSG + fi + fi + + # validate baseline package + if [ "${fetchBuildDependencies}" == "true" ]; then + + # validate baseline directory and baseline package + baselineDirectory="$(getWorkDirectory)/${baselineFolder}" + if [ -d "${baselineDirectory}" ]; then + baselineTarName=$(ls "${baselineDirectory}") + baselineTarFile="${baselineDirectory}/${baselineTarName}" + if [ ! -f "$baselineTarFile" ]; then + echo $PGM": [INFO] The baseline package $baselineTarFile was not found." + fi + else + echo $PGM": [INFO] The directory for the baseline package $baselineDirectory was not found." fi fi @@ -497,7 +517,7 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then if [ ! -z "${tarFileName}" ]; then echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." fi - + if [ ! -z "${artifactVersionName}" ]; then echo $PGM": [INFO] ** Identified that artifactVersionName is passed into packageBuildOutputs.sh (${artifactVersionName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." fi @@ -552,7 +572,10 @@ if [ $rc -eq 0 ]; then fi if [ ! -z "${externalDependenciesLogFile}" ]; then echo $PGM": [INFO] ** External Dependencies log:" ${externalDependenciesLogFile} - fi + fi + if [ ! -f "$baselineTarFile" ]; then + echo $PGM": [INFO] ** Baseline package:" ${baselineTarFile} + fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} if [ "$publish" == "true" ]; then if [ ! -z "${artifactRepositoryPropertyFile}" ]; then @@ -635,19 +658,11 @@ if [ $rc -eq 0 ]; then # Pass information about externally fetched modules to packaging to document them if [ ! -z "${externalDependenciesLogFile}" ]; then CMD="${CMD} --externalDependenciesEvidences ${externalDependenciesLogFile}" - fi - - baselineDirectory="$(getWorkDirectory)/baseline" - if [ -d "${baselineDirectory}" ]; then - baselineTarName=$(ls "${baselineDirectory}") - baselineTarFile="${baselineDirectory}/${baselineTarName}" - if [ -f "$baselineTarFile" ]; then - CMD="${CMD} --baselinePackage ${baselineTarFile}" - else - echo $PGM": [INFO] Baseline Tar file $baselineTarFile not found." - fi - else - echo $PGM": [INFO] Baseline directory $baselineDirectory not found." + fi + + # Pass baseline package + if [ ! -f "$baselineTarFile" ]; then + CMD="${CMD} --baselinePackage ${baselineTarFile}" fi # publishing options From b05ebd821caaba510fc857f7a58c06c852d71ee5 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 13:11:07 +0100 Subject: [PATCH 098/127] remove scriptDir Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 2044cb32..36deae75 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -251,7 +251,6 @@ def run(String[] cliArgs) { assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" // load script - def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent packageUrl = computeAbsoluteRepositoryUrl(props) // the println is used in a script by the CBS to grep the packageUrl println "packageUrl=$packageUrl" From 6d73045c9b96885eef81b0fa2e4ebb60424757bb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 13:20:52 +0100 Subject: [PATCH 099/127] Apply suggestions from code review - primarily documentation Co-authored-by: Mathieu Dalbin --- Templates/Common-Backend-Scripts/README.md | 31 ++++++++++--------- .../utilities/fetchBuildDependencies.groovy | 4 +-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index ec64b1fc..9cbca0b5 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -238,13 +238,13 @@ Note that the location of the baselineReferences.config file can be customized i #### Fetching build dependencies -Both implementations for the build stage are enabled to optionally pull external dependencies into the build workspace based on the dependency definitions specified in the *Application Descriptor*. +The build stage is enabled to pull external dependencies into the build workspace, based on the dependencies definition specified in the Application Descriptor file. -The Application Descriptor contains metadata about the application itself, but can contain references to other application packages managed in an artifact repository, which are necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. +The Application Descriptor contains metadata about the application itself, but can contain references to other applications versions managed in an artifact repository, which contain necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. -The Application descriptor file `applicationDescriptor.yml` is expected to be on the root level of the application repository. +The Application Descriptor file, called `applicationDescriptor.yml`, is expected to be on the root level of the application's Git repository. -Each application package can export shared components such as public or shared include files and even build outputs, such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend scripts. For fetching the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The external packages can be stored at a cache location to improve performance. Fetched packages will be expanded to the pipeline working directory 'imports'. +Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend Scripts. To fetch the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. ### dbbBuild.sh for zAppBuild framework @@ -463,25 +463,26 @@ The [dbbzBuilderUtils](utilities/dbbzBuilderUtils.sh) script is a core utility s ## Packaging stage -You can choose from either creating a package using the [PackageBuildOutputs](#packagebuildoutputssh) script that can be deployed with IBM Wazi Deploy or the [UCD packaging](#ucdpackagingsh) that creates the UCD shiplist and UCD component version +Depending on the Deployment Manager tool you are using, you can choose from either creating a package with the [PackageBuildOutputs](#packagebuildoutputssh) script that can be used with IBM Wazi Deploy, or the [UCD packaging](#ucdpackagingsh) script that creates the UCD shiplist and UCD component version. ### packageBuildOutputs.sh This script is to execute the `PackageBuildOutputs.groovy` that packages up the build outputs and optionally uploads it to an artifact repository to publish the artifacts created by a DBB build in the pipeline. -When uploading into the artifact repository, it implements strict conventions about the structure in the Artifact repository. The conventions are implemented in [utilities/packageUtils.sh](utilities/packageUtils.sh). The rules are: +When uploading the archive to an artifact repository, this script implements naming conventions for the repository layout. The conventions are implemented in [utilities/packageUtils.sh](utilities/packageUtils.sh). +The rules for the naming conventions are detailed hereafter. -For any temporary, preliminary build that uses the `pipelineType=build`, the outputs are uploaded into the directory `build//-`: +For any preliminary build (that uses the `pipelineType=build`), the outputs are uploaded into the directory `build//-`: * **build** is defined for any builds, that are considered to some extend temporary and preliminary. -* **reference** is the branch name from which the builds originates from, so for instance `feature/123-update-mortgage-computation`, `main` or any hotfix and epic branches. -* **tarFile.tar** is computed by the application name and a unique build identifier (see argument `-i`), such as the pipeline build number that is passed in by the pipeline template. If it is not provided, it will use the current timestamp. +* **reference** is the name of the branch which the build originates from: for instance, `feature/123-update-mortgage-computation`, `main` or any hotfix and epic branches. +* The archive's file name is computed using the application's name and a unique build identifier (`-i` argument). This parameter is typically the pipeline build number that is passed by the pipeline orchestrator. If a build identifier is not provided, the current timestamp is used. -For release builds that use the `pipelineType=release`, the outputs are uploaded into the directory `release//-`: +For release builds (that use the `pipelineType=release`), the archive is uploaded to the directory `release//-`: * **release** is defined for release builds. -* **reference** is planned release name, so for instance `rel-1.2.3` (mandatory argument `-r`). -* **tarFile.tar** is computed by the application name, the release name (`-r`) and the unique build identifier (see argument `-i`), such as the pipeline build number that is passed in by the pipeline template. If it is not provided, it will use the current timestamp. +* **reference** is the release name: for instance, `rel-1.2.3` (provided through the mandatory `-r` argument). +The archive's file name is computed using the application's name, the release name (`-r` argument) and a unique build identifier (`-i` argument). This parameter is typically the pipeline build number that is passed by the pipeline orchestrator. If a build identifier is not provided, the current timestamp is used. #### Invocation @@ -509,8 +510,8 @@ CLI parameter | Description -b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. -r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. --i `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number or a timestamp. --v `` **removed** | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. This has been removed. Please switch to releaseIdentifier and buildIdentifier. +-i `` | **Build identifier** a unique value, typically the build number provided by the pipeline orchestrator or a timestamp. Used to help qualifying the archive file. This is a computed value provided by the pipeline templates. +-v `` **deprecated** | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. Deprecated - switch to `-r ` and `-i `. -s `""` | (Optional) Name and email of the SBOM author enclosed with double quotes. Ex: "Build Engineer \" Check out the pipelineBackend.config to define the `artifactRepositoryNameSuffix` that is appended to the application name to set the repository name in the artifact repository. @@ -685,7 +686,7 @@ wazideploy-generate.sh -w MortgageApplication/feature/15-fetch-application-depe CLI parameter | Description ---------- | ---------------------------------------------------------------------------------------- -w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. Optional, if `deploymentPlan`, `deploymentPlanReport` and `packageOutputFile` are fully referenced. --i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to a TAR file on UNIX System Services, or the URL of the TAR file to retrieve (only Artifactory is supported).

If the Common Backend scripts are configured to perform the upload, the scripts computes the location in the artifact repo and overrides the `-i` and `-o` argument and in that case the this arument is not required. +-i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to an archive (TAR file) on UNIX System Services, or the URL of the archive (TAR file) to retrieve (only Artifactory is supported).

If the Common Backend Scripts are configured to perform the upload, the scripts computes the location in the artifact repo and overrides the `-i` and `-o` argument and in that case the this arument is not required. -m `` | (Optional) Absolute path to the Wazi Deploy **Deployment Method** stored on UNIX System Services. If not specified, the deployment method file location is obtained from the `pipelineBackend.config`. -p `` | (Optional) Absolute or relative path to the **Deployment Plan** file, generated based on the content of the input package. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan location is obtained from the `pipelineBackend.config`. -r `` | (Optional) Absolute or relative path to the **Deployment Plan Report**. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan report location is obtained from the `pipelineBackend.config`. diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 5ff7048f..a6ccc2ee 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -159,11 +159,11 @@ if (applicationDescriptor.dependencies) { if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tarFileDir}.") + println("** Downloading archive '$artifactUrl' from Artifact Repository into '${tarFileDir}'.") def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { - println "** Download of application package '$artifactUrl' failed. Process exists. Return code:"$rc + println("*! [ERROR] Download of archive '$artifactUrl' failed. Exiting with return code:${rc}.") exitFetchDependencyProcess() } else { //println "* Download successful." From daec39d74b4388275f104e253b5ee155616922df Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:08:23 +0100 Subject: [PATCH 100/127] add description about baseline package Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 34 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 9cbca0b5..33cccadd 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -135,10 +135,10 @@ Artifact Name | Description | Script details [zBuilder.sh](zBuilder.sh) | Pipeline Shell script to invoke the zBuilder framework [zBuilder](https://www.ibm.com/docs/en/dbb/3.0?topic=building-zos-applications-zbuilder) | [script details](#zbuildersh-for-dbb-zbuilder) [packageBuildOutputs.sh](packageBuildOutputs.sh) | Pipeline Shell Script to create a Package using the [PackageBuildOutputs groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/PackageBuildOutputs) | [script details](#packagebuildoutputssh) [ucdPackage.sh](ucdPackaging.sh) | Pipeline Shell Script to publish to UCD Code Station binary repository using the [CreateUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/CreateUCDComponentVersion) | [script details](#ucdpackagingsh) -[ucdDeploy.sh](ucdDeploy.sh) | Pipeline Shell Script to trigger a UCD Deployment via its REST interface using the [DeployUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/DeployUCDComponentVersion) | [script details](#ucddeploysh) [wazideploy-generate.sh](wazideploy-generate.sh) | Pipeline Shell Script to generate a Deployment Plan to be used with Wazi Deploy | [script details](#wazideploy-generatesh) [wazideploy-deploy.sh](wazideploy-deploy.sh) | Pipeline Shell Script to trigger a deployment of a package based on Deployment Plan with Wazi Deploy | [script details](#wazideploy-deploysh) [wazideploy-evidence.sh](wazideploy-evidence.sh) | Pipeline Shell Script to query the Wazi Deploy Evidence YAML file and create a deployment report | [script details](#wazideploy-generatesh) +[ucdDeploy.sh](ucdDeploy.sh) | Pipeline Shell Script to trigger a UCD Deployment via its REST interface using the [DeployUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/DeployUCDComponentVersion) | [script details](#ucddeploysh) [prepareLogs.sh](prepareLogs.sh) | Pipeline Shell Script to prepare a TAR file containing log files that can then be retrieved. | [script details](#preparelogssh) [generateCleanupCommands.sh](generateCleanupCommands.sh) | Pipeline Shell Script to generate necessary DBB Metadatastore cleanup tasks including the deletion of the build datasets. | [script details](#generatecleanupcommandssh) [deleteWorkspace.sh](deleteWorkspace.sh) | Pipeline Shell Script to delete the working directory on Unix System Services. | [script details](#deleteworkspacesh) @@ -238,13 +238,37 @@ Note that the location of the baselineReferences.config file can be customized i #### Fetching build dependencies -The build stage is enabled to pull external dependencies into the build workspace, based on the dependencies definition specified in the Application Descriptor file. +The build stage can be enabled to pull external build dependencies into the build workspace based on the dependencies definition specified in the Application Descriptor file. -The Application Descriptor contains metadata about the application itself, but can contain references to other applications versions managed in an artifact repository, which contain necessary inputs to the build. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. +The Application Descriptor contains metadata about the application itself, but can contain the dependency configuration to other applications versions managed in an artifact repository, which contain necessary inputs to the build process. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. + +In the `dependencies` section in the Application Descriptor file, users can configure which application versions should be fetched into the build workspace. The below snippet references the release build of the Cards application with the reference to `rel-1.2.0` and the concrete buildid `build-20241112.1` + +```yaml +dependencies: +- name: ”Cards" + type: "release" + reference: "rel-1.2.0" + buildid: "build-20241112.1" +``` The Application Descriptor file, called `applicationDescriptor.yml`, is expected to be on the root level of the application's Git repository. -Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend Scripts. To fetch the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. +Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend Scripts. To fetch the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies into the working directory. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. + +#### Fetch baseline package + +(Prototype) Along with the fetching of external build dependencies, the fetch phase can retrieve the application's baseline package from the Artifact repository. This is configured through the `baselines` section of the Application Descriptor. Use the baseline if your application architecture uses static calls or required derived build outputs that is an mandatory input to subsequent builds. A good sample for derived build outputs are bms copybooks, that are inputs to CICS programs. Instead of storing the generated bms copybooks, it is made available through the baseline package. + +Baseline archives are defined similarly like external dependencies. Under the `baselines` section, the application team manages the references for the corresponding build branch: + +``` +baselines: +- branch: ”main" + type: "release" + reference: "rel-1.2.0" + buildid: "build-7656" +``` ### dbbBuild.sh for zAppBuild framework @@ -1066,7 +1090,7 @@ CLI parameter | Description Note that the script deletes all empty folders in the working tree. It supresses the message `EDC5136I Directory not empty.` and handles that as a INFO message. -### Script output +#### Script output The section below contains the output that is produced by the `deleteWorkspace.sh` script. From ae02e6910f5ab8f2fb894d808fbd8e6b58b10604 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:21:06 +0100 Subject: [PATCH 101/127] add description about baseline package Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 24 ++++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 33cccadd..6e15f41f 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -238,9 +238,11 @@ Note that the location of the baselineReferences.config file can be customized i #### Fetching build dependencies -The build stage can be enabled to pull external build dependencies into the build workspace based on the dependencies definition specified in the Application Descriptor file. +The build stage can be enabled to pull external build dependencies into the build workspace based on the dependencies definition specified in the Application Descriptor file. -The Application Descriptor contains metadata about the application itself, but can contain the dependency configuration to other applications versions managed in an artifact repository, which contain necessary inputs to the build process. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. +Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The application archive needs to be created with the [packageBuildOutputs.sh](#packagebuildoutputssh) script and be uploaded to the artifact repository based on the implemented conventions. + +The Application Descriptor contains metadata about the application itself, but can contain the dependency configuration to other applications versions managed in an artifact repository, which are necessary inputs to the build process. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. In the `dependencies` section in the Application Descriptor file, users can configure which application versions should be fetched into the build workspace. The below snippet references the release build of the Cards application with the reference to `rel-1.2.0` and the concrete buildid `build-20241112.1` @@ -254,11 +256,11 @@ dependencies: The Application Descriptor file, called `applicationDescriptor.yml`, is expected to be on the root level of the application's Git repository. -Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The package needs to be created with the [PackageBuildOutputs](../../Pipeline/PackageBuildOutputs/README.md) script and be uploaded to the artifact repository through the Common Backend Scripts. To fetch the dependencies, it uses the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) that is referenced by both dbbBuild.sh and zBuilder.sh. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies into the working directory. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. +To fetch the dependencies, the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) is used. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies into the working directory. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. -#### Fetch baseline package +**Fetch baseline package** -(Prototype) Along with the fetching of external build dependencies, the fetch phase can retrieve the application's baseline package from the Artifact repository. This is configured through the `baselines` section of the Application Descriptor. Use the baseline if your application architecture uses static calls or required derived build outputs that is an mandatory input to subsequent builds. A good sample for derived build outputs are bms copybooks, that are inputs to CICS programs. Instead of storing the generated bms copybooks, it is made available through the baseline package. +Along with the fetching of external build dependencies, the fetch phase can retrieve the application's baseline package from the Artifact repository. This is configured through the `baselines` section of the Application Descriptor. Use the baseline if your application architecture uses static calls or requires derived build outputs that is an mandatory input to subsequent builds. A good sample for derived build outputs are bms copybooks, that are inputs to CICS programs. Instead of including the generated bms copybooks via concatenation of pds libraries, it is made available through the baseline package. Baseline archives are defined similarly like external dependencies. Under the `baselines` section, the application team manages the references for the corresponding build branch: @@ -496,13 +498,17 @@ This script is to execute the `PackageBuildOutputs.groovy` that packages up the When uploading the archive to an artifact repository, this script implements naming conventions for the repository layout. The conventions are implemented in [utilities/packageUtils.sh](utilities/packageUtils.sh). The rules for the naming conventions are detailed hereafter. -For any preliminary build (that uses the `pipelineType=build`), the outputs are uploaded into the directory `build//-`: +For any preliminary build (that uses the `pipelineType=build`), the outputs are uploaded into the directory + +`build//-`: -* **build** is defined for any builds, that are considered to some extend temporary and preliminary. +* **build** is defined for any builds, that are considered to be preliminary. They cannot be deployed to production. * **reference** is the name of the branch which the build originates from: for instance, `feature/123-update-mortgage-computation`, `main` or any hotfix and epic branches. * The archive's file name is computed using the application's name and a unique build identifier (`-i` argument). This parameter is typically the pipeline build number that is passed by the pipeline orchestrator. If a build identifier is not provided, the current timestamp is used. -For release builds (that use the `pipelineType=release`), the archive is uploaded to the directory `release//-`: +For release builds (that use the `pipelineType=release`), the archive is uploaded to the directory + +`release//-`: * **release** is defined for release builds. * **reference** is the release name: for instance, `rel-1.2.3` (provided through the mandatory `-r` argument). @@ -529,7 +535,7 @@ CLI parameter | Description -w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. The `packageBuildOutputs.sh` script is evaluating the logs directory. -t `` | (Optional) Name of the **tar file** to create. **Artifact Upload options** --u | Flag to enable upload of outputs to the configured artifact repository. +-u | Flag to enable upload of outputs to the configured artifact repository. Also available as a general setting in `pipelineBackend.config`. -a `` | **Application name** leveraged to define the artifact repository name. -b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. From 75311e89f9f8c0c919e019c3b7f4e49a09b55944 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:28:42 +0100 Subject: [PATCH 102/127] Rename internal method Signed-off-by: Dennis Behm --- .../ArtifactRepositoryHelpers.groovy | 4 ++-- .../PackageBuildOutputs/PackageBuildOutputs.groovy | 4 ++-- .../utilities/fetchBuildDependencies.groovy | 13 ++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 36deae75..f06a3c95 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -162,7 +162,7 @@ def download(String url, String fileName, String user, String password, boolean } // Method directly accessed by PackageBuildOutputs and Common Backend script functionality -def computeAbsoluteRepositoryUrl(Properties props) { +def computePackageUrl(Properties props) { def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL return url @@ -251,7 +251,7 @@ def run(String[] cliArgs) { assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" // load script - packageUrl = computeAbsoluteRepositoryUrl(props) + packageUrl = computePackageUrl(props) // the println is used in a script by the CBS to grep the packageUrl println "packageUrl=$packageUrl" } else diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 26a0ca86..d0ce22bd 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -638,7 +638,7 @@ if (rc == 0) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") packageInfo.put("name", props.packageBuildIdentifier) - packageUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) + packageUrl = artifactRepositoryHelpers.computePackageUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } @@ -761,7 +761,7 @@ if (rc == 0) { //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def url = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) + def url = artifactRepositoryHelpers.computePackageUrl(props) def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index a6ccc2ee..346294bc 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -74,6 +74,8 @@ ArrayList externalDependencies = new ArrayList<>() if (applicationDescriptor.dependencies) { + println("*** Fetching external application packages") + // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> @@ -99,7 +101,7 @@ if (applicationDescriptor.dependencies) { props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name // The absolute url the package in artifact repo - artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) + artifactUrl = artifactRepositoryHelpers.computePackageUrl(props) // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") @@ -108,7 +110,7 @@ if (applicationDescriptor.dependencies) { tarFile="${tmpPackageDir}/${artifactRelPath}" tarFileDir=tarFile.replaceAll(props.tarFileName, "") - println("*** Fetching package '${dependency.name}:${artifactUrl}' ") + println("** Fetching package '${dependency.name}:${artifactUrl}' ") // Generating information for documentation in yaml file of retrieved dependencies if (dependency.name != applicationDescriptor.application) { @@ -230,7 +232,7 @@ if (baselineRecord){ props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name // The absolute url the package in artifact repo - artifactUrl = artifactRepositoryHelpers.computeAbsoluteRepositoryUrl(props) + artifactUrl = artifactRepositoryHelpers.computePackageUrl(props) // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") @@ -240,7 +242,7 @@ if (baselineRecord){ tarFileDir=tarFile.replaceAll(props.tarFileName, "") if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() // create tmpDownload - println("*** Fetching baseline package '${applicationName}:${artifactUrl}' ") + println("** Fetching baseline package '${applicationName}:${artifactUrl}' ") if (new File(tarFile).exists()) { println("** Package was already found in package cache at '${tarFile}'") @@ -402,9 +404,6 @@ def parseArgs(String[] args) { } } -/********************************************************************************** - * run process - **********************************************************************************/ def runProcess(ArrayList cmd){ StringBuffer response = new StringBuffer() StringBuffer error = new StringBuffer() From 1f5f65fb8556cdfe1716ffa53b6fb2e26b79a607 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:33:37 +0100 Subject: [PATCH 103/127] Reviewer feedback to rename variable Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 6 +++--- .../utilities/WaziDeployManifestGenerator.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index d0ce22bd..24eb9a3a 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -637,7 +637,7 @@ if (rc == 0) { if (props.publish && props.publish.toBoolean()) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") - packageInfo.put("name", props.packageBuildIdentifier) + packageInfo.put("name", props.buildIdentifier) packageUrl = artifactRepositoryHelpers.computePackageUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) @@ -944,7 +944,7 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') - cli.bi(longOpt:'packageBuildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier metadata stored in Wazi Deploys Application Manifest file.') + cli.bi(longOpt:'buildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier metadata stored in Wazi Deploys Application Manifest file.') cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') @@ -1018,7 +1018,7 @@ def parseInput(String[] cliArgs){ if (opts.il) props.includeLogs = opts.il if (opts.a) props.application = opts.a if (opts.b) props.branch = opts.b - if (opts.bi) props.packageBuildIdentifier = opts.bi + if (opts.bi) props.buildIdentifier = opts.bi // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : (props.generateWaziDeployAppManifest ? props.generateWaziDeployAppManifest : 'false') diff --git a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy index 47e50e18..a762c47f 100644 --- a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy @@ -29,7 +29,7 @@ def initWaziDeployManifestGenerator(Properties props) { } // Metadata information - wdManifest.metadata.version = (props.packageBuildIdentifier) ? props.packageBuildIdentifier : props.startTime + wdManifest.metadata.version = (props.buildIdentifier) ? props.buildIdentifier : props.startTime if (props.application) wdManifest.metadata.name = props.application // Annotations From 6de3d4cb7dcfdf4f18fd48168bc2307d992c88b3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:42:35 +0100 Subject: [PATCH 104/127] Reviewer feedback to rename outputs Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 943cbc8a..91aea571 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -316,7 +316,7 @@ if [ $rc -eq 0 ]; then nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + ERRMSG=$PGM": [WARNING] The name of the artifact version is required. rc="$rc echo $ERRMSG break fi @@ -374,7 +374,7 @@ validateOptions() { fi if [ -z "${buildIdentifier}" ] && [ "$publish" == "true" ]; then - ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) has been supplied. A unique name based on version and build id is recommended. Using timestamp" + ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) was supplied. Using timestamp." echo $ERRMSG buildIdentifier=$(date +%Y%m%d_%H%M%S) fi From b3cfe7479d3477f33993859de250bcf66b695142 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:43:28 +0100 Subject: [PATCH 105/127] remove unused variable Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 8dfd529b..7d39e788 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -166,10 +166,6 @@ sbomAuthor= # Example: /var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar -# File name that is stored in the logs directory containing -# the computed absolute package url and downloaded tarFileName -wdPackageVersionFile=waziDeployPackageConfig.txt - ##################################################################################################### #### Artifact repository upload options ########################################################## ##################################################################################################### From 942d4a21ce693bed1ab7253b5d3a5c80f529a3ce Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:48:08 +0100 Subject: [PATCH 106/127] remove odd exit path Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 346294bc..7163c424 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -166,9 +166,7 @@ if (applicationDescriptor.dependencies) { if (rc != 0) { println("*! [ERROR] Download of archive '$artifactUrl' failed. Exiting with return code:${rc}.") - exitFetchDependencyProcess() - } else { - //println "* Download successful." + System.exit(rc) } } @@ -257,9 +255,7 @@ if (baselineRecord){ if (rc != 0) { println "** Download of application package '$artifactUrl' failed. Process exists. Return code:"$rc - exitFetchDependencyProcess() - } else { - //println "* Download successful." + System.exit(rc) } } @@ -323,10 +319,6 @@ if (props.externalDependenciesFilePath) { } } -def exitFetchDependencyProcess(){ - println("** [ERROR] fetchBuildDependencies encountered a problem. Please review log. Exiting") - System.exit(1) -} /** * Parse CLI config From 824e6528f2773821160f34b995f787d156b2a984 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 17:54:22 +0100 Subject: [PATCH 107/127] call the utilities instead of sourcing it Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/dbbBuild.sh | 5 +-- .../utilities/fetchBuildDependenciesUtils.sh | 38 +++++++++---------- Templates/Common-Backend-Scripts/zBuilder.sh | 5 +-- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 708ec37c..4415c43f 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -214,8 +214,6 @@ if [ $rc -eq 0 ]; then rc=8 ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc echo $ERRMSG - else - source $fetchBuildDependenciesUtilities fi # @@ -439,7 +437,8 @@ fi # Setup build environment and pull external dependencies if an Application Descriptor is found if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then - fetchBuildDependenciesMethod + # call utilities script + . ${fetchBuildDependenciesUtilities} fi # diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index a17da163..aac8b5a5 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -81,26 +81,24 @@ runFetchLogic() { } # -# Validate to fetch external dependencies is based on the ApplicationDescriptor -fetchBuildDependenciesMethod() { +# Script Logic +# - # extracting external dependencies is based on the application descriptor - applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" - - # this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) - if [ ! -z "${externalDependenciesLogName}" ]; then - externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" - fi +# extracting external dependencies is based on the application descriptor +applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" - # create the log dir - mkdir -p "$(getLogDir)" +# this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) +if [ ! -z "${externalDependenciesLogName}" ]; then + externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" +fi - # Set up to perform the clone of the Repo - if [ ! -f "${applicationDescriptor}" ]; then - rc=8 - ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc="$rc - echo $ERRMSG - else - runFetchLogic - fi -} \ No newline at end of file +# create the log dir +mkdir -p "$(getLogDir)" + +# Set up to perform the clone of the Repo +if [ ! -f "${applicationDescriptor}" ]; then + ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc=" + echo $ERRMSG +else + runFetchLogic +fi diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh index 67b015b8..ace89d1c 100755 --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -211,8 +211,6 @@ if [ $rc -eq 0 ]; then rc=8 ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc echo $ERRMSG - else - source $fetchBuildDependenciesUtilities fi # @@ -451,7 +449,8 @@ fi # Setup build environment and pull external dependencies if an ApplicationDescriptor is found if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then - fetchBuildDependenciesMethod + # call utilities script + . ${fetchBuildDependenciesUtilities} fi # From ac0a7ccaed68e81bed7ee15b3a9de14018721d8d Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 18:13:16 +0100 Subject: [PATCH 108/127] pass correct cli to packaging script Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 91aea571..8a2f73cd 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -652,7 +652,7 @@ if [ $rc -eq 0 ]; then # Wazi Deploy build identifier if [ ! -z "${packageBuildIdentifier}" ]; then - CMD="${CMD} --packageBuildIdentifier ${packageBuildIdentifier}" + CMD="${CMD} --buildIdentifier ${packageBuildIdentifier}" fi # Pass information about externally fetched modules to packaging to document them From dea582affc2d92ea62d08a024841a3dad32d1809 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 18:15:32 +0100 Subject: [PATCH 109/127] fix baseline package reference Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/packageBuildOutputs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 8a2f73cd..2d02d284 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -573,7 +573,7 @@ if [ $rc -eq 0 ]; then if [ ! -z "${externalDependenciesLogFile}" ]; then echo $PGM": [INFO] ** External Dependencies log:" ${externalDependenciesLogFile} fi - if [ ! -f "$baselineTarFile" ]; then + if [ ! -z "$baselineTarFile" ]; then echo $PGM": [INFO] ** Baseline package:" ${baselineTarFile} fi echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} @@ -661,7 +661,7 @@ if [ $rc -eq 0 ]; then fi # Pass baseline package - if [ ! -f "$baselineTarFile" ]; then + if [ ! -z "$baselineTarFile" ]; then CMD="${CMD} --baselinePackage ${baselineTarFile}" fi From b1bf88e83cef860d9f1c3a60d1e49cb8c79a32d3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 26 Mar 2025 12:25:13 +0100 Subject: [PATCH 110/127] README updates Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 6e15f41f..3074c3a1 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -2,7 +2,7 @@ ## Overview -The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for Mainframe applications. +The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for Mainframe applications. They use and simplify the parameterization of existing scripts of this repository to perform build, packaging and deployment steps. Implementing a pipeline configuration, such as an Azure pipeline, a JenkinsFile, or the .gitlab-ci.yml file, requires accommodation of the selected development workflow with Git. To achieve consistency across various applications, rules must be implemented in pipeline code or configurations to address: * naming conventions of build datasets, @@ -23,20 +23,21 @@ This asset implements the rules and conventions of the Git-based workflow outlin ## Setup -The provided scripts of this asset are implemented as bash scripts and need to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks. +The provided scripts of this asset are implemented as bash scripts. The entire repository needs to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks, because as previously mentioned the scripts invoke individual pipeline scripts from the [Pipeline](../../Pipeline/) directory. ### Pre-requisites The following are required to use these scripts: -* DBB v2.x toolkit is installed. -* zAppBuild is set up on Unix Systems Services. +* DBB v2.x / 3.x toolkit is installed. +* zAppBuild or zBuilder is set up on Unix Systems Services. * Git repository which follows the Git-based workflow outlined in IBM's documentation `The Git-based workflow for Mainframe development`. -* Build dependency information is available before performing the build run. - ### Installation -* Copy/clone the Common Backend Scripts into z/OS UNIX System Services under a protected directory, e.g. `/usr/dbb/pipelineBackend`. - * Update the permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. +* Make this Git repository available in your internal git provider. Perform necessary site-specific [script configuration](#script-configuration). + +* Clone the repository that include the Common Backend Scripts into z/OS UNIX System Services under a protected directory. + + * Verify that permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. * The following environment variables need to be defined (for instance within the `.profile`) for the mainframe users who will execute the scripts on UNIX System Services: From 858591be16d82d61b47fdfcb3d5bda8fe4def777 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Mar 2025 14:15:45 +0100 Subject: [PATCH 111/127] Apply suggestions from code review Co-authored-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> --- Templates/Common-Backend-Scripts/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 3074c3a1..4317d788 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -2,7 +2,7 @@ ## Overview -The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for Mainframe applications. They use and simplify the parameterization of existing scripts of this repository to perform build, packaging and deployment steps. +The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for mainframe applications. They use and simplify the parameterization of existing scripts in this repository to perform build, packaging, and deployment steps. Implementing a pipeline configuration, such as an Azure pipeline, a JenkinsFile, or the .gitlab-ci.yml file, requires accommodation of the selected development workflow with Git. To achieve consistency across various applications, rules must be implemented in pipeline code or configurations to address: * naming conventions of build datasets, @@ -23,19 +23,19 @@ This asset implements the rules and conventions of the Git-based workflow outlin ## Setup -The provided scripts of this asset are implemented as bash scripts. The entire repository needs to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks, because as previously mentioned the scripts invoke individual pipeline scripts from the [Pipeline](../../Pipeline/) directory. +The provided scripts of this asset are implemented as bash scripts. The entire repository needs to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks, as the scripts invoke individual pipeline scripts from the [Pipeline](../../Pipeline/) directory. ### Pre-requisites The following are required to use these scripts: -* DBB v2.x / 3.x toolkit is installed. +* DBB 2.x or DBB 3.x toolkit is installed. * zAppBuild or zBuilder is set up on Unix Systems Services. * Git repository which follows the Git-based workflow outlined in IBM's documentation `The Git-based workflow for Mainframe development`. ### Installation -* Make this Git repository available in your internal git provider. Perform necessary site-specific [script configuration](#script-configuration). +1. Make this Git repository available in your enterprise Git provider. Perform necessary site-specific [script configurations](#script-configuration). -* Clone the repository that include the Common Backend Scripts into z/OS UNIX System Services under a protected directory. +2. Clone the repository customized in the previous step (including the Common Backend Scripts) into z/OS UNIX System Services under a protected directory. * Verify that permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. From 15504feaedb21ebffe1c9416232e7a2f94db0028 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Mar 2025 14:17:06 +0100 Subject: [PATCH 112/127] numbering in readme Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 4317d788..46ab6ec0 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -39,7 +39,7 @@ The following are required to use these scripts: * Verify that permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. -* The following environment variables need to be defined (for instance within the `.profile`) for the mainframe users who will execute the scripts on UNIX System Services: +3. The following environment variables need to be defined (for instance within the `.profile`) for the mainframe users who will execute the scripts on UNIX System Services: * `PIPELINE_SCRIPTS` - Environment variable to define the path to the Common Backend Scripts. From 1391d58a00d5ad5f1c286d28d559bd8bfb6e2d37 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 28 Mar 2025 14:19:48 +0100 Subject: [PATCH 113/127] Switch to UTF-8 output Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 7163c424..010a5585 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -314,7 +314,7 @@ if (props.externalDependenciesFilePath) { yamlBuilder externalDependencies externalDependencyFile = new File(props.externalDependenciesFilePath) - externalDependencyFile.withWriter("IBM-1047") { extDepWriter -> + externalDependencyFile.withWriter("UTF-8") { extDepWriter -> extDepWriter.write(yamlBuilder.toString()) } } From deeb9bbdb62396f6564c36c60bf5dc195e08f2c6 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 31 Mar 2025 17:58:06 +0200 Subject: [PATCH 114/127] Fix path in ucdPackaging Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/ucdPackaging.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/ucdPackaging.sh b/Templates/Common-Backend-Scripts/ucdPackaging.sh index 74878e12..d421728d 100755 --- a/Templates/Common-Backend-Scripts/ucdPackaging.sh +++ b/Templates/Common-Backend-Scripts/ucdPackaging.sh @@ -100,7 +100,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" -ucdPackagingScript="${SCRIPT_HOME}/../../../Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" +ucdPackagingScript="${SCRIPT_HOME}/../../Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" # Customization - End # Path and File Name to the advanced debug options. From e4e41f2f6950efc8ff14c9d5f994bd52cf653b4b Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 1 Apr 2025 11:44:24 +0200 Subject: [PATCH 115/127] Remove configuration Signed-off-by: Dennis Behm --- .../pipelineBackend.config | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 7d39e788..dbd91fcf 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -41,12 +41,9 @@ logsDir="logs" # Path to main build script build.groovy. zAppBuild="$HOME/dbb-zappbuild" -# DBEHM: MyCustomization for Path to build.groovy. -zAppBuild="/var/dbb/dbb-zappbuild_300" # (Optional) Additional property files to customize the build framework zAppBuildPropFiles="" -zAppBuildPropFiles="/var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties,/var/dbb/dbb-zappbuild-config/buildFolder.properties" # High-Level Data Set Qualifier used for the DBB Build. # This prefix value will be used along with the @@ -127,7 +124,7 @@ packageCacheLocation=$HOME/cbs/.packagecache # Internal file name documenting the fetched external dependencies, that can be tracked in the # Wazi Deploy application manifest file -# externalDependenciesLogName=externalDependenciesLog.yml +# default: externalDependenciesLogName=externalDependenciesLog.yml externalDependenciesLogName=externalDependenciesLog.yml ##################################################################################################### @@ -178,19 +175,20 @@ cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/v # Flag to indicate package upload to the provided Artifactory server. # Set to true to enable package upload # can be overridden by cli parm -p -publish=true +# default: publish=false +publish=false # URL to the Artifact repository server # e.q.: artifactRepositoryUrl=http://artifactoryserver:8081/artifactory -artifactRepositoryUrl=http://10.3.20.231:8081/artifactory +artifactRepositoryUrl= # User to connect to the Artifact repository server # e.q.: artifactRepositoryUser=admin -artifactRepositoryUser=admin +artifactRepositoryUser="" # Password to connect to the Artifact repository server # e.q.: artifactRepositoryPassword=xxxxx -artifactRepositoryPassword=artifactoryadmin +artifactRepositoryPassword="" # Path of a property file containing application specific artifact repository details. # application specific artifactRepositoryPropertyFile @@ -209,7 +207,7 @@ artifactRepositoryPropertyFile= # Path to UCD buztool.sh script # e.g. BuzTool="/var/ucd-agent/bin/buztool.sh" -BuzTool="/var/ucd-agent/bin/buztool.sh" +BuzTool="" ##################################################################################################### ## End of ucdPackaging.sh parameters ############################################################# @@ -252,7 +250,7 @@ wdWorkDir=waziDeployDir # # wazideploy deployment method file # wdDeploymentMethod= -wdDeploymentMethod="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/external-repos/deployment-method/deployment-method.yml" +wdDeploymentMethod="" # # Default name of the wazideploy **Deployment Plan** file @@ -268,7 +266,7 @@ wdDeploymentPlanReportName="deploymentPlanReport.html" # # (Optional) Only necessary when wazideploy-generate is used to download the package # wdDeployArtifactoryConfig= -wdDeployArtifactoryConfig="/var/WaziDeploy/config/WaziDeploy-ConfigFile.yml" +wdDeployArtifactoryConfig="" # # Path to the environment configurations for Wazi Deploy @@ -276,7 +274,7 @@ wdDeployArtifactoryConfig="/var/WaziDeploy/config/WaziDeploy-ConfigFile.yml" # e.g. wdEnvironmentConfigurations="/var/wazideploy/environment-configurations" # This directory then contains the available environment definitions # like IntegrationTest.yaml, AcceptanceTest.yaml -wdEnvironmentConfigurations="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/external-repos/environment-conf/python" +wdEnvironmentConfigurations= # # Default name of the wazideploy **evidence file** file @@ -290,12 +288,12 @@ wdIndexFolder="index" # # wazideploy evidence search template # wdSearchTemplate= -wdSearchTemplate="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/evidences/templates/full-report.yml" +wdSearchTemplate= # # wazideploy evidence report renderer # wdReportRenderer= -wdReportRenderer="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/evidences/renderers/full-report.html" +wdReportRenderer= # # Default path of wazideploy evidence output file From 275a52667f76b55f142d155e4b6a06af3a0d99ec Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 1 Apr 2025 17:15:40 +0200 Subject: [PATCH 116/127] Apply suggestions from code review Co-authored-by: Mathieu Dalbin --- .../PackageBuildOutputs.groovy | 2 +- .../packageBuildOutputs.sh | 6 ++--- .../utilities/fetchBuildDependencies.groovy | 26 +++++++++---------- .../utilities/packagingUtilities.sh | 10 +++---- .../wazideploy-deploy.sh | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 24eb9a3a..bd1f969b 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -944,7 +944,7 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') - cli.bi(longOpt:'buildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier metadata stored in Wazi Deploys Application Manifest file.') + cli.bi(longOpt:'buildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier stored in Wazi Deploy Application Manifest file.') cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 2d02d284..2479dddc 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -257,7 +257,7 @@ if [ $rc -eq 0 ]; then nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + ERRMSG=$PGM": [WARNING] The build identifier is required. rc="$rc echo $ERRMSG break fi @@ -305,7 +305,7 @@ if [ $rc -eq 0 ]; then nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + ERRMSG=$PGM": [WARNING] The release identifier is required. rc="$rc echo $ERRMSG break fi @@ -316,7 +316,7 @@ if [ $rc -eq 0 ]; then nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the artifact version is required. rc="$rc + ERRMSG=$PGM": [WARNING] The artifact version is required. rc="$rc echo $ERRMSG break fi diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 010a5585..24af0c39 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -110,7 +110,7 @@ if (applicationDescriptor.dependencies) { tarFile="${tmpPackageDir}/${artifactRelPath}" tarFileDir=tarFile.replaceAll(props.tarFileName, "") - println("** Fetching package '${dependency.name}:${artifactUrl}' ") + println("** Fetching archive for application '${dependency.name}' from '${artifactUrl}'") // Generating information for documentation in yaml file of retrieved dependencies if (dependency.name != applicationDescriptor.application) { @@ -151,10 +151,10 @@ if (applicationDescriptor.dependencies) { // download from artifact repo // foldername in workspace directory - String includeFolder = "${importFolder}/${dependency.name}" + String importFolder= "${importFolder}/${dependency.name}" if (new File(tarFile).exists()) { - println("** Package was already found in package cache at '${tarFile}'") + println("** Archive was already found in archive cache at '${tarFile}'") } else { String user = props.artifactRepositoryUser String password = props.artifactRepositoryPassword @@ -171,9 +171,9 @@ if (applicationDescriptor.dependencies) { } - File includeFolderFile = new File(includeFolder) - if (!includeFolderFile.exists()) { - includeFolderFile.mkdirs() + File importFolderFile = new File(importFolder) + if (!importFolderFile.exists()) { + importFolderFile.mkdirs() } @@ -182,12 +182,12 @@ if (applicationDescriptor.dependencies) { def processCmd = [ "/bin/sh", "-c", - "tar -C $includeFolder -xvf ${tarFile}" + "tar -C $importFolder -xvf ${tarFile}" ] def rc = runProcess(processCmd) if (rc != 0) { - println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") + println("** [ERROR] Failed to untar '$tarFile' to '$importFolder' with rc=$rc") System.exit(1) } @@ -205,7 +205,7 @@ baselineRecord = applicationDescriptor.baselines.find() { baseline -> } if (baselineRecord){ - println("*** Fetching baseline package") + println("*** Fetching baseline archive") // validate dependency record assert baselineRecord.type : "Missing baseline type attribute in baseline record" @@ -243,18 +243,18 @@ if (baselineRecord){ println("** Fetching baseline package '${applicationName}:${artifactUrl}' ") if (new File(tarFile).exists()) { - println("** Package was already found in package cache at '${tarFile}'") + println("** Archive was already found in archive cache at '${tarFile}'") } else { String user = props.artifactRepositoryUser String password = props.artifactRepositoryPassword if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() - println("** Downloading application package '$artifactUrl' from Artifact Repository into ${tarFileDir}.") + println("** Downloading archive with '$artifactUrl' from Artifact Repository into ${tarFileDir}.") def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { - println "** Download of application package '$artifactUrl' failed. Process exists. Return code:"$rc + println("** Download of archive '$artifactUrl' failed. Process exists. Return code:"$rc) System.exit(rc) } } @@ -263,7 +263,7 @@ if (baselineRecord){ if (baselineFolder.exists()) baselineFolder.deleteDir() baselineFolder.mkdirs() - println("** Saving tar file '${tarFile}' into '$baselineFolder' ") + println("** Saving tar file '${tarFile}' into '${baselineFolder}'") def processCmd = [ "/bin/sh", diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index ef9fea58..ae259686 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -4,7 +4,7 @@ secondBranchSegment="" rc=0 # Method implementing the conventions in the CBS for packaging using the PackageBuildOutputs.groovy script -computePackageInformation() { +computeArchiveInformation() { ############################################# # output environment variables ############################################# @@ -12,7 +12,7 @@ computePackageInformation() { artifactRepositoryDirectory="" # root directory folder in repo artifactVersionName="" # subfolder in repo path identifying version / origin branch tarFileName="" # computed tarFileName how it is stored in the artifact repository - packageBuildIdentifier="" # Identifier for Wazi Deploy Application Manifest file + archiveIdentifier="" # Identifier for Wazi Deploy Application Manifest file ############################################# # configuration variable defining the Artifactory repository name pattern @@ -58,7 +58,7 @@ computePackageInformation() { # building up the tarFileName tarFileName="${App}-${releaseIdentifier}-${buildIdentifier}.tar" - packageBuildIdentifier="${releaseIdentifier}-${buildIdentifier}" + archiveIdentifier="${releaseIdentifier}-${buildIdentifier}" else ############################################# # Conventions for snapshot builds: @@ -69,7 +69,7 @@ computePackageInformation() { artifactRepositoryDirectory="build" artifactVersionName=${Branch} tarFileName="${App}-${buildIdentifier}.tar" - packageBuildIdentifier="${buildIdentifier}" + archiveIdentifier="${buildIdentifier}" fi ;; *) @@ -79,7 +79,7 @@ computePackageInformation() { artifactRepositoryDirectory="build" artifactVersionName=${Branch} tarFileName="${App}-${buildIdentifier}.tar" - packageBuildIdentifier="${buildIdentifier}" + archiveIdentifier="${buildIdentifier}" ;; esac diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index 59e72808..ef1830de 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -340,7 +340,7 @@ validateOptions() { # When publishing is enabled, check if the tarfile exists in the expected location if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then if [ -f "$(wdDeployPackageDir)/applicationPackage.tar" ]; then # shared convention with wazideploy-generate.sh - echo $PGM": [INFO] ** Package file was found in expected location at $(wdDeployPackageDir)/applicationPackage.tar ." + echo $PGM": [INFO] ** Archive was found at location '${wdDeployPackageDir}/applicationArchive.tar'." if [ ! -z "${PackageInputFile}" ]; then echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced with $(wdDeployPackageDir)/applicationPackage.tar ." fi From 51a0da223178684e8675894a654e2c2e3843e2cd Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 1 Apr 2025 17:32:08 +0200 Subject: [PATCH 117/127] Use Archive instead of Package Signed-off-by: Dennis Behm --- .../ArtifactRepositoryHelpers.groovy | 14 +- .../PackageBuildOutputs.groovy | 4 +- .../applicationDescriptorUtils.groovy | 350 +++++++++--------- Templates/Common-Backend-Scripts/README.md | 2 +- .../packageBuildOutputs.sh | 2 +- .../pipelineBackend.config | 25 +- .../utilities/fetchBuildDependencies.groovy | 4 +- .../utilities/packagingUtilities.sh | 2 +- .../wazideploy-deploy.sh | 8 +- .../wazideploy-generate.sh | 4 +- 10 files changed, 209 insertions(+), 206 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index f06a3c95..72ef3846 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -162,7 +162,7 @@ def download(String url, String fileName, String user, String password, boolean } // Method directly accessed by PackageBuildOutputs and Common Backend script functionality -def computePackageUrl(Properties props) { +def computeArchiveUrl(Properties props) { def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL return url @@ -199,7 +199,7 @@ def run(String[] cliArgs) { // recompute options // Compute Flag to recompute url - cli.c(longOpt:'computePackageUrl', 'Action Flag to identify to recompute the uri of a given package') + cli.c(longOpt:'computeArchiveUrl', 'Action Flag to identify to recompute the uri of a given package') cli.t(longOpt:'tarFileName', args:1, argName:'filename', 'Name of the package tar file. (Optional unless using --buildReportOrder or --buildReportOrderFile)') cli.aRU(longOpt:'artifactRepositoryUrl', args:1, 'Artifact repository url') cli.aRN(longOpt:'artifactRepositoryName', args:1, 'Artifact repository name') @@ -213,7 +213,7 @@ def run(String[] cliArgs) { System.exit(1) } - if (opts.c) props.computePackageUrl = true + if (opts.c) props.computeArchiveUrl = true if (opts.t) props.tarFileName = opts.t if (opts.aRU) props.put('artifactRepository.url', opts.aRU) if (opts.aRN) props.put('artifactRepository.repo', opts.aRN) @@ -239,10 +239,10 @@ def run(String[] cliArgs) { assert opts.U : "Missing option: Artifact repository user id or token" assert opts.P : "Missing option: Artifactory password" download(opts.u, opts.fD, opts.U, opts.P, opts.v) - } else if (props.computePackageUrl){ + } else if (props.computeArchiveUrl){ // invoke processing - if (props.computePackageUrl && props.computePackageUrl.toBoolean()) { + if (props.computeArchiveUrl && props.computeArchiveUrl.toBoolean()) { // check requires cli arguments for this operation assert props.tarFileName : "Missing option tarFileName (--tarFileName)" assert props.versionName : "Missing option versionName (--versionName)" @@ -251,11 +251,11 @@ def run(String[] cliArgs) { assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" // load script - packageUrl = computePackageUrl(props) + packageUrl = computeArchiveUrl(props) // the println is used in a script by the CBS to grep the packageUrl println "packageUrl=$packageUrl" } else - println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computePackageUrl') "); + println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computeArchiveUrl') "); } } diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index bd1f969b..f9e72be6 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -638,7 +638,7 @@ if (rc == 0) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") packageInfo.put("name", props.buildIdentifier) - packageUrl = artifactRepositoryHelpers.computePackageUrl(props) + packageUrl = artifactRepositoryHelpers.computeArchiveUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } @@ -761,7 +761,7 @@ if (rc == 0) { //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def url = artifactRepositoryHelpers.computePackageUrl(props) + def url = artifactRepositoryHelpers.computeArchiveUrl(props) def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 5fd6cf47..d584a415 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -12,161 +12,164 @@ import groovy.util.* import java.nio.file.* /** - * Utilities to read, update or export existing ApplicationDescriptor from/to YAML + * Utilities to read, update or export existing ApplicationDescriptor from/to YAML */ class ApplicationDescriptor { - String application - String description - String owner - ArrayList sources - ArrayList baselines - ArrayList dependencies - ArrayList consumers + String application + String description + String owner + ArrayList sources + ArrayList baselines + ArrayList dependencies + ArrayList consumers } class Source { - String name - String repositoryPath - String language - String languageProcessor - String fileExtension - String artifactsType - ArrayList files + String name + String repositoryPath + String language + String languageProcessor + String fileExtension + String artifactsType + ArrayList files } class FileDef { - String name - String type - String usage + String name + String type + String usage } class Baseline { - String branch - String type - String reference - String buildid + String branch + String type + String reference + String buildid } class DependencyDescriptor { - String name - String type - String reference - String buildid + String name + String type + String reference + String buildid } /** - * + * * Reads an existing application descriptor YAML * returns an ApplicationDescriptor Object - * + * */ def readApplicationDescriptor(File yamlFile) { - // Internal objects - def yamlSlurper = new groovy.yaml.YamlSlurper() - ApplicationDescriptor applicationDescriptor = yamlSlurper.parse(yamlFile) - return applicationDescriptor + // Internal objects + def yamlSlurper = new groovy.yaml.YamlSlurper() + ApplicationDescriptor applicationDescriptor = yamlSlurper.parse(yamlFile) + return applicationDescriptor } /** * Write an ApplicationDescriptor Object into a YAML file */ def writeApplicationDescriptor(File yamlFile, ApplicationDescriptor applicationDescriptor) { - // Sort source groups and files by name before writing to YAML file - if (applicationDescriptor.sources) { - applicationDescriptor.sources.sort { - it.name - } - applicationDescriptor.sources.each() { source -> - source.files.sort { - it.name - } - } - } - - def yamlBuilder = new YamlBuilder() - // build updated application descriptor - - yamlBuilder { - application applicationDescriptor.application - description applicationDescriptor.description - owner applicationDescriptor.owner - sources (applicationDescriptor.sources) - baselines (applicationDescriptor.baselines) - if (applicationDescriptor.dependencies) { - dependencies applicationDescriptor.dependencies - } - if (applicationDescriptor.consumers) { - consumers applicationDescriptor.consumers - } - } - - // write file - yamlFile.withWriter("IBM-1047") { writer -> - writer.write(yamlBuilder.toString()) - } + // Sort source groups and files by name before writing to YAML file + if (applicationDescriptor.sources) { + applicationDescriptor.sources.sort { + it.name + } + applicationDescriptor.sources.each() { source -> + source.files.sort { + it.name + } + } + } + + def yamlBuilder = new YamlBuilder() + // build updated application descriptor + + yamlBuilder { + application applicationDescriptor.application + description applicationDescriptor.description + owner applicationDescriptor.owner + sources (applicationDescriptor.sources) + baselines (applicationDescriptor.baselines) + if (applicationDescriptor.dependencies) { + dependencies applicationDescriptor.dependencies + } + if (applicationDescriptor.consumers) { + consumers applicationDescriptor.consumers + } + } + + // write file + yamlFile.withWriter("IBM-1047") { writer -> + writer.write(yamlBuilder.toString()) + } Process process = "chtag -tc IBM-1047 ${yamlFile.getAbsolutePath()}".execute() - process.waitFor() + process.waitFor() } /** * Method to update the Application Descriptor - * + * * Appends to an existing source sourceGroupName, if it exists. * If the sourceGroupName cannot be found, it creates a new sourceGroup - * + * */ def appendFileDefinition(ApplicationDescriptor applicationDescriptor, String sourceGroupName, - String language, String languageProcessor, String artifactsType, - String fileExtension, String repositoryPath, String name, - String type, String usage) { - - def sourceGroupRecord - - def fileRecord = new FileDef() - fileRecord.name = name - fileRecord.type = type - fileRecord.usage = usage - - if (!applicationDescriptor.sources) { - applicationDescriptor.sources = new ArrayList() - } - - existingSourceGroup = applicationDescriptor.sources.find(){ source -> - source.name == sourceGroupName - } - - if (existingSourceGroup) { // append file record definition to existing sourceGroup - sourceGroupRecord = existingSourceGroup - - // check if the fileRecord already exists, and this is an update - - existingFileRecord = sourceGroupRecord.files.find(){ file -> - file.name == fileRecord.name - } - - if (existingFileRecord) { // update existing file record - existingFileRecord.type = type - existingFileRecord.usage = usage - } else { // add a new record - sourceGroupRecord.files.add(fileRecord) - } + String language, String languageProcessor, String artifactsType, + String fileExtension, String repositoryPath, String name, + String type, String usage) { + + def sourceGroupRecord + + def fileRecord = new FileDef() + fileRecord.name = name + fileRecord.type = type + fileRecord.usage = usage + + if (!applicationDescriptor.sources) { + applicationDescriptor.sources = new ArrayList() + } + + existingSourceGroup = applicationDescriptor.sources.find(){ source -> + source.name == sourceGroupName + } + + if (existingSourceGroup) { + // append file record definition to existing sourceGroup + sourceGroupRecord = existingSourceGroup + + // check if the fileRecord already exists, and this is an update + + existingFileRecord = sourceGroupRecord.files.find(){ file -> + file.name == fileRecord.name + } + + if (existingFileRecord) { + // update existing file record + existingFileRecord.type = type + existingFileRecord.usage = usage + } else { + // add a new record + sourceGroupRecord.files.add(fileRecord) + } } else { - // create a new source group entry - sourceGroupRecord = new Source() - sourceGroupRecord.name = sourceGroupName - sourceGroupRecord.language = language - sourceGroupRecord.languageProcessor = languageProcessor - sourceGroupRecord.fileExtension = fileExtension - sourceGroupRecord.artifactsType = artifactsType - sourceGroupRecord.repositoryPath = repositoryPath - - sourceGroupRecord.files = new ArrayList() - // append file record - sourceGroupRecord.files.add(fileRecord) - applicationDescriptor.sources.add(sourceGroupRecord) - } + // create a new source group entry + sourceGroupRecord = new Source() + sourceGroupRecord.name = sourceGroupName + sourceGroupRecord.language = language + sourceGroupRecord.languageProcessor = languageProcessor + sourceGroupRecord.fileExtension = fileExtension + sourceGroupRecord.artifactsType = artifactsType + sourceGroupRecord.repositoryPath = repositoryPath + + sourceGroupRecord.files = new ArrayList() + // append file record + sourceGroupRecord.files.add(fileRecord) + applicationDescriptor.sources.add(sourceGroupRecord) + } } @@ -176,45 +179,46 @@ def appendFileDefinition(ApplicationDescriptor applicationDescriptor, String sou def removeFileDefinition(ApplicationDescriptor applicationDescriptor, String sourceGroupName, String name) { - if (applicationDescriptor.sources) { - def existingSourceGroup = applicationDescriptor.sources.find() { source -> - source.name == sourceGroupName - } - if (existingSourceGroup) { // Found an existing Source Group that matches - def existingFileDef = existingSourceGroup.files.find { file -> - file.name.equals(name) - } - if (existingFileDef) { -// println "Found matching file ${existingFileDef.name}" - existingSourceGroup.files.remove(existingFileDef) - } - } - } + if (applicationDescriptor.sources) { + def existingSourceGroup = applicationDescriptor.sources.find() { source -> + source.name == sourceGroupName + } + if (existingSourceGroup) { + // Found an existing Source Group that matches + def existingFileDef = existingSourceGroup.files.find { file -> + file.name.equals(name) + } + if (existingFileDef) { + // println "Found matching file ${existingFileDef.name}" + existingSourceGroup.files.remove(existingFileDef) + } + } + } } /** - * Method to add an application dependency + * Method to add an application dependency */ def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String type, String reference, String buildid) { - if (!applicationDescriptor.dependencies) { - applicationDescriptor.dependencies = new ArrayList() - } + if (!applicationDescriptor.dependencies) { + applicationDescriptor.dependencies = new ArrayList() + } // skip readding same/similar entries - def existingDependencies = applicationDescriptor.dependencies.findAll() { - it.name.equals(applicationDependency) - } - if (!existingDependencies) { - def dependency = new DependencyDescriptor() - dependency.name = applicationDependency + def existingDependencies = applicationDescriptor.dependencies.findAll() { + it.name.equals(applicationDependency) + } + if (!existingDependencies) { + def dependency = new DependencyDescriptor() + dependency.name = applicationDependency dependency.type = type - dependency.reference = reference - dependency.buildid = buildid - applicationDescriptor.dependencies.add(dependency) - applicationDescriptor.dependencies.sort { - it.name - } - } + dependency.reference = reference + dependency.buildid = buildid + applicationDescriptor.dependencies.add(dependency) + applicationDescriptor.dependencies.sort { + it.name + } + } } /** @@ -222,20 +226,20 @@ def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String */ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String consumingApplication) { - - if (!applicationDescriptor.consumers) { - applicationDescriptor.consumers = new ArrayList() - } - // don't add the "owning" application - if (applicationDescriptor.application != consumingApplication) { - def existingConsumers = applicationDescriptor.consumers.findAll() { - it.equals(consumingApplication) - } - if (!existingConsumers) { - applicationDescriptor.consumers.add(consumingApplication) - applicationDescriptor.consumers.sort() - } - } + + if (!applicationDescriptor.consumers) { + applicationDescriptor.consumers = new ArrayList() + } + // don't add the "owning" application + if (applicationDescriptor.application != consumingApplication) { + def existingConsumers = applicationDescriptor.consumers.findAll() { + it.equals(consumingApplication) + } + if (!existingConsumers) { + applicationDescriptor.consumers.add(consumingApplication) + applicationDescriptor.consumers.sort() + } + } } /** @@ -243,7 +247,7 @@ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String c */ def resetAllSourceGroups(ApplicationDescriptor applicationDescriptor) { - applicationDescriptor.sources = new ArrayList() + applicationDescriptor.sources = new ArrayList() } /** @@ -251,18 +255,18 @@ def resetAllSourceGroups(ApplicationDescriptor applicationDescriptor) { */ def resetConsumersAndDependencies(ApplicationDescriptor applicationDescriptor) { - applicationDescriptor.consumers = new ArrayList() - applicationDescriptor.dependencies = new ArrayList() + applicationDescriptor.consumers = new ArrayList() + applicationDescriptor.dependencies = new ArrayList() } /** - * Method to create an empty application descriptor object + * Method to create an empty application descriptor object */ def createEmptyApplicationDescriptor() { - ApplicationDescriptor applicationDescriptor = new ApplicationDescriptor() - applicationDescriptor.sources = new ArrayList() - applicationDescriptor.baselines = new ArrayList() - return applicationDescriptor + ApplicationDescriptor applicationDescriptor = new ApplicationDescriptor() + applicationDescriptor.sources = new ArrayList() + applicationDescriptor.baselines = new ArrayList() + return applicationDescriptor } /** @@ -290,7 +294,7 @@ def getFileUsage(ApplicationDescriptor applicationDescriptor, String sourceGroup } else { println("*! [WARNING] No file found matching '${name}'. Skipping search.") return null - } + } } else { return null } @@ -328,12 +332,12 @@ def getFileUsageByType(ApplicationDescriptor applicationDescriptor, String artif if (allMatchingFiles.size() == 1) { return allMatchingFiles[0].usage } else if (allMatchingFiles.size() > 1) { - println("*! [WARNING] Multiple files found matching '${name}'. Skipping search.") + println("*! [WARNING] Multiple files found matching '${name}'. Skipping search.") return null } else { - println("*! [WARNING] No file found matching '${name}'. Skipping search.") + println("*! [WARNING] No file found matching '${name}'. Skipping search.") return null - } + } } else { return null } diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index 46ab6ec0..a029a43d 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -695,7 +695,7 @@ Depending on the selected Deployment tool, select either from the scripts for IB This script invokes the Wazi Deploy Generate command to generate a Deployment Plan based on the content of a package. The package should be created with the `PackageBuildOutputs.groovy` script or through the `packageBuildOutputs.sh` script. -This script assesses the configuration option `publish` from the `pipelineBackend.config` file. In case the configuration has enabled the upload to the Artifact repository, the script computes the URL where the package is expected to be found, and passes the URL into the wazideploy-generate command. This means that wazideloy-generate will download the package from the Artifact repository and allows to restore the package on a different system. This step leverages the generic utility script computePackageUrl. It requires to pass in the additional arguments `-P`, `-R`, `-B` +This script assesses the configuration option `publish` from the `pipelineBackend.config` file. In case the configuration has enabled the upload to the Artifact repository, the script computes the URL where the package is expected to be found, and passes the URL into the wazideploy-generate command. This means that wazideloy-generate will download the package from the Artifact repository and allows to restore the package on a different system. It requires to pass in the additional arguments `-P`, `-R`, `-B` #### Invocation diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 2479dddc..de4b78b9 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -522,7 +522,7 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then echo $PGM": [INFO] ** Identified that artifactVersionName is passed into packageBuildOutputs.sh (${artifactVersionName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." fi - computePackageInformation + computeArchiveInformation fi if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index dbd91fcf..639b65f4 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -156,7 +156,7 @@ generateSBOM=false # (Optional) Name and email of the SBOM author enclosed with double quotes # Example: "Build Engineer " -sbomAuthor= +sbomAuthor="" # (Required if SBOM generation is performed) # Paths to the cycloneDX Java libraries (and their dependencies) to be added to the JAVA CLASSPATH @@ -180,7 +180,7 @@ publish=false # URL to the Artifact repository server # e.q.: artifactRepositoryUrl=http://artifactoryserver:8081/artifactory -artifactRepositoryUrl= +artifactRepositoryUrl="" # User to connect to the Artifact repository server # e.q.: artifactRepositoryUser=admin @@ -195,7 +195,7 @@ artifactRepositoryPassword="" # e.q.: application-conf/packageBuildOutputs.properties # see template: # https://github.com/IBM/dbb/blob/main/Pipeline/PackageBuildOutputs/appArtifactRepository.properties -artifactRepositoryPropertyFile= +artifactRepositoryPropertyFile="" ##################################################################################################### ## End of packageBuildOutputs.sh parameters ###################################################### @@ -244,12 +244,11 @@ ucdPassword="" # # Work directory for Wazi Deploy -# default: +# default: waziDeployDir wdWorkDir=waziDeployDir # # wazideploy deployment method file -# wdDeploymentMethod= wdDeploymentMethod="" # @@ -265,7 +264,7 @@ wdDeploymentPlanReportName="deploymentPlanReport.html" # to connect to Artifactory # # (Optional) Only necessary when wazideploy-generate is used to download the package -# wdDeployArtifactoryConfig= +# wdDeployArtifactoryConfig="" wdDeployArtifactoryConfig="" # @@ -274,7 +273,7 @@ wdDeployArtifactoryConfig="" # e.g. wdEnvironmentConfigurations="/var/wazideploy/environment-configurations" # This directory then contains the available environment definitions # like IntegrationTest.yaml, AcceptanceTest.yaml -wdEnvironmentConfigurations= +wdEnvironmentConfigurations="" # # Default name of the wazideploy **evidence file** file @@ -282,22 +281,22 @@ wdEvidenceFileName="deploy/evidences/evidence.yaml" # # Default path of wazideploy evidence index folder -# wdIndexFolder= +# wdIndexFolder="" wdIndexFolder="index" # # wazideploy evidence search template -# wdSearchTemplate= -wdSearchTemplate= +# wdSearchTemplate="" +wdSearchTemplate="" # # wazideploy evidence report renderer -# wdReportRenderer= -wdReportRenderer= +# wdReportRenderer="" +wdReportRenderer="" # # Default path of wazideploy evidence output file -# wdOutputFile= +# wdOutputFile="" wdOutputFile="deploy/deployment-report.html" # diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 24af0c39..7f425f7c 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -101,7 +101,7 @@ if (applicationDescriptor.dependencies) { props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name // The absolute url the package in artifact repo - artifactUrl = artifactRepositoryHelpers.computePackageUrl(props) + artifactUrl = artifactRepositoryHelpers.computeArchiveUrl(props) // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") @@ -230,7 +230,7 @@ if (baselineRecord){ props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name // The absolute url the package in artifact repo - artifactUrl = artifactRepositoryHelpers.computePackageUrl(props) + artifactUrl = artifactRepositoryHelpers.computeArchiveUrl(props) // retrieve path without artifact url artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index ae259686..7a2a22f5 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -103,7 +103,7 @@ computeArchiveInformation() { if [ $rc -eq 0 ]; then echo $PGM": [INFO] Invoking the ArtifactRepositoryHelper groovy script to compute Package Url." - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computePackageUrl" + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computeArchiveUrl" # add tarfile name if [ ! -z "${tarFileName}" ]; then diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index ef1830de..9c91d14b 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -339,14 +339,14 @@ validateOptions() { # When publishing is enabled, check if the tarfile exists in the expected location if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then - if [ -f "$(wdDeployPackageDir)/applicationPackage.tar" ]; then # shared convention with wazideploy-generate.sh + if [ -f "$(wdDeployPackageDir)/applicationArchive.tar" ]; then # shared convention with wazideploy-generate.sh echo $PGM": [INFO] ** Archive was found at location '${wdDeployPackageDir}/applicationArchive.tar'." if [ ! -z "${PackageInputFile}" ]; then - echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced with $(wdDeployPackageDir)/applicationPackage.tar ." + echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced with $(wdDeployPackageDir)/applicationArchive.tar ." fi - PackageInputFile="$(wdDeployPackageDir)/applicationPackage.tar" + PackageInputFile="$(wdDeployPackageDir)/applicationArchive.tar" else - echo $PGM": [INFO] ** The CBS can automatically compute the Url of the package. Wazi Deploy will then download it. Read more about the capabilities in the CBS readme." + echo $PGM": [INFO] ** The CBS can automatically compute the Url of the archive. Wazi Deploy will then download it. Read more about the capabilities in the CBS readme." fi fi diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index 8deef10b..02a78f11 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -496,11 +496,11 @@ if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; t if [ $rc -eq 0 ]; then # Call utilities method - computePackageInformation + computeArchiveInformation # Set Input and output files for Wazi Deploy PackageInputFile="${artifactRepositoryAbsoluteUrl}" - PackageOutputFile="$(wdDeployPackageDir)/applicationPackage.tar" # shared convention with wazideploy-deploy.sh + PackageOutputFile="$(wdDeployPackageDir)/applicationArchive.tar" # shared convention with wazideploy-deploy.sh usePackageUrl="true" fi fi From 9de0df984dc733533339c19b3a8c13c43228e8a9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 1 Apr 2025 17:51:05 +0200 Subject: [PATCH 118/127] Fix renamed object names Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 7f425f7c..c729a57f 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -151,7 +151,7 @@ if (applicationDescriptor.dependencies) { // download from artifact repo // foldername in workspace directory - String importFolder= "${importFolder}/${dependency.name}" + String importSubFolder= "${importFolder}/${dependency.name}" if (new File(tarFile).exists()) { println("** Archive was already found in archive cache at '${tarFile}'") @@ -171,23 +171,23 @@ if (applicationDescriptor.dependencies) { } - File importFolderFile = new File(importFolder) - if (!importFolderFile.exists()) { - importFolderFile.mkdirs() + File importSubFolderFile = new File(importSubFolder) + if (!importSubFolderFile.exists()) { + importSubFolderFile.mkdirs() } - println("** Expanding tar file '${tarFile}' to '$includeFolder' ") + println("** Expanding tar file '${tarFile}' to '$importSubFolder' ") def processCmd = [ "/bin/sh", "-c", - "tar -C $importFolder -xvf ${tarFile}" + "tar -C $importSubFolder -xvf ${tarFile}" ] def rc = runProcess(processCmd) if (rc != 0) { - println("** [ERROR] Failed to untar '$tarFile' to '$importFolder' with rc=$rc") + println("** [ERROR] Failed to untar '$tarFile' to '$importSubFolder' with rc=$rc") System.exit(1) } @@ -254,7 +254,7 @@ if (baselineRecord){ def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { - println("** Download of archive '$artifactUrl' failed. Process exists. Return code:"$rc) + println("** Download of archive '$artifactUrl' failed. Process exists. Return code:$rc") System.exit(rc) } } @@ -278,23 +278,23 @@ if (baselineRecord){ } // setup import folder - String includeFolder = "${importFolder}/${applicationName}" - File includeFolderFile = new File(includeFolder) - if (!includeFolderFile.exists()) { - includeFolderFile.mkdirs() + String importSubFolder = "${importFolder}/${applicationName}" + File importSubFolderFile = new File(importSubFolder) + if (!importSubFolderFile.exists()) { + importSubFolderFile.mkdirs() } - println("** Expanding tar file '${tarFile}' to '$includeFolder' ") + println("** Expanding tar file '${tarFile}' to '$importSubFolder' ") processCmd = [ "/bin/sh", "-c", - "tar -C $includeFolder -xvf ${tarFile}" + "tar -C $importSubFolder -xvf ${tarFile}" ] rc = runProcess(processCmd) if (rc != 0) { - println("** [ERROR] Failed to untar '$tarFile' to '$includeFolder' with rc=$rc") + println("** [ERROR] Failed to untar '$tarFile' to '$importSubFolder' with rc=$rc") System.exit(1) } From eaca96367cf5e888fa1c3f654ef708a636655ccc Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 2 Apr 2025 08:19:54 +0200 Subject: [PATCH 119/127] sync with latest AD schema Signed-off-by: Dennis Behm --- .../applicationDescriptorUtils.groovy | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index d584a415..9b7e6e03 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -17,12 +17,13 @@ import java.nio.file.* class ApplicationDescriptor { String application + String schemaVersion = "applicationDescriptor/0.11.0" String description String owner ArrayList sources ArrayList baselines ArrayList dependencies - ArrayList consumers + ArrayList consumers } class Source { @@ -55,6 +56,10 @@ class DependencyDescriptor { String buildid } +class Consumer { + String name +} + /** * * Reads an existing application descriptor YAML @@ -89,6 +94,7 @@ def writeApplicationDescriptor(File yamlFile, ApplicationDescriptor applicationD yamlBuilder { application applicationDescriptor.application + schemaVersion applicationDescriptor.schemaVersion description applicationDescriptor.description owner applicationDescriptor.owner sources (applicationDescriptor.sources) @@ -233,10 +239,12 @@ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String c // don't add the "owning" application if (applicationDescriptor.application != consumingApplication) { def existingConsumers = applicationDescriptor.consumers.findAll() { - it.equals(consumingApplication) + it.name.equals(consumingApplication) } if (!existingConsumers) { - applicationDescriptor.consumers.add(consumingApplication) + Consumer consumer = new Consumer() + consumer.name = consumingApplication + applicationDescriptor.consumers.add(consumer) applicationDescriptor.consumers.sort() } } @@ -376,3 +384,26 @@ def getFilesByTypeAndUsage(ApplicationDescriptor applicationDescriptor, String a } } } + +/** + * Method to add a baseline + * If an existing baseline for a given branch already exists, the method replaces it + */ + +def addBaseline(ApplicationDescriptor applicationDescriptor, String branch, String baseline) { + if (applicationDescriptor.baselines) { + def existingBaselines = applicationDescriptor.baselines.findAll() { baselineDefinition -> + baselineDefinition.branch.equals(branch) + } + existingBaselines.forEach() { existingBaseline -> + applicationDescriptor.baselines.remove(existingBaseline) + } + } else { + applicationDescriptor.baselines = new ArrayList() + } + + Baseline newBaseline = new Baseline() + newBaseline.branch = branch + newBaseline.baseline = baseline + applicationDescriptor.baselines.add(newBaseline) +} \ No newline at end of file From 5da0453f6fb9acb89bf92a20d4758192c0e93a1e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 2 Apr 2025 08:23:27 +0200 Subject: [PATCH 120/127] code formatting Signed-off-by: Dennis Behm --- .../applicationDescriptorUtils.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 9b7e6e03..3290cbe3 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -17,13 +17,13 @@ import java.nio.file.* class ApplicationDescriptor { String application - String schemaVersion = "applicationDescriptor/0.11.0" + String schemaVersion = "applicationDescriptor/0.11.0" String description String owner ArrayList sources ArrayList baselines ArrayList dependencies - ArrayList consumers + ArrayList consumers } class Source { @@ -57,7 +57,7 @@ class DependencyDescriptor { } class Consumer { - String name + String name } /** @@ -94,7 +94,7 @@ def writeApplicationDescriptor(File yamlFile, ApplicationDescriptor applicationD yamlBuilder { application applicationDescriptor.application - schemaVersion applicationDescriptor.schemaVersion + schemaVersion applicationDescriptor.schemaVersion description applicationDescriptor.description owner applicationDescriptor.owner sources (applicationDescriptor.sources) @@ -239,12 +239,12 @@ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String c // don't add the "owning" application if (applicationDescriptor.application != consumingApplication) { def existingConsumers = applicationDescriptor.consumers.findAll() { - it.name.equals(consumingApplication) + it.name.equals(consumingApplication) } if (!existingConsumers) { - Consumer consumer = new Consumer() - consumer.name = consumingApplication - applicationDescriptor.consumers.add(consumer) + Consumer consumer = new Consumer() + consumer.name = consumingApplication + applicationDescriptor.consumers.add(consumer) applicationDescriptor.consumers.sort() } } @@ -386,7 +386,7 @@ def getFilesByTypeAndUsage(ApplicationDescriptor applicationDescriptor, String a } /** - * Method to add a baseline + * Method to add a baseline * If an existing baseline for a given branch already exists, the method replaces it */ @@ -397,7 +397,7 @@ def addBaseline(ApplicationDescriptor applicationDescriptor, String branch, Stri } existingBaselines.forEach() { existingBaseline -> applicationDescriptor.baselines.remove(existingBaseline) - } + } } else { applicationDescriptor.baselines = new ArrayList() } From f03c4120c04d011d6663a57584b944e325b67eb7 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 2 Apr 2025 08:32:44 +0200 Subject: [PATCH 121/127] Updating messages for more consistency Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index c729a57f..6b61c707 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -74,7 +74,7 @@ ArrayList externalDependencies = new ArrayList<>() if (applicationDescriptor.dependencies) { - println("*** Fetching external application packages") + println("*** Fetching dependent application archives") // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> @@ -161,7 +161,7 @@ if (applicationDescriptor.dependencies) { if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() - println("** Downloading archive '$artifactUrl' from Artifact Repository into '${tarFileDir}'.") + println("** Downloading archive '$artifactUrl' from artifact repository into '${tarFileDir}'.") def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { @@ -240,7 +240,7 @@ if (baselineRecord){ tarFileDir=tarFile.replaceAll(props.tarFileName, "") if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() // create tmpDownload - println("** Fetching baseline package '${applicationName}:${artifactUrl}' ") + println("** Fetching baseline archive for '${applicationName}' from '${artifactUrl}' ") if (new File(tarFile).exists()) { println("** Archive was already found in archive cache at '${tarFile}'") From 3739497552d5c8b900c5136d808029711e6c3549 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 2 Apr 2025 08:35:37 +0200 Subject: [PATCH 122/127] rename to archiveCache Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 2 +- .../utilities/fetchBuildDependencies.groovy | 6 +++--- .../utilities/fetchBuildDependenciesUtils.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 639b65f4..6d7b3f1a 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -120,7 +120,7 @@ fetchBuildDependencies=true enablePackageCache=true # Location for cachine packages on the build machine -packageCacheLocation=$HOME/cbs/.packagecache +archiveCache=$HOME/cbs/.archiveCache # Internal file name documenting the fetched external dependencies, that can be tracked in the # Wazi Deploy application manifest file diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 6b61c707..62a0bad6 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -63,7 +63,7 @@ if (importFolder.exists()) importFolder.deleteDir() importFolder.mkdirs() // setup package cache -tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean() && props.packageCacheLocation) ? new File(props.packageCacheLocation) : new File("$props.workspace/imports_download/") +tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean() && props.archiveCache) ? new File(props.archiveCache) : new File("$props.workspace/imports_download/") if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() // Parse the application descriptor and application configurations based on the defined schema @@ -334,7 +334,7 @@ def parseArgs(String[] args) { cli.d(longOpt:'externalDependenciesFilePath', args:1, 'Absolute path to the external dependencies file') cli.p(longOpt:'pipelineBackendConfigFilePath', args:1, 'Absolute path to the pipelineBackend.config file') cli.b(longOpt:'branch', args:1, 'Current branch of the application') - cli.c(longOpt:'packageCacheLocation', args:1, 'Location of the Package cache') + cli.c(longOpt:'archiveCache', args:1, 'Location of the Package cache') def opts = cli.parse(args) @@ -371,7 +371,7 @@ def parseArgs(String[] args) { } if(opts.c){ - props.packageCacheLocation = opts.c + props.archiveCache = opts.c } if (opts.p) { diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh index aac8b5a5..b52351f5 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -41,7 +41,7 @@ runFetchLogic() { echo $PGM": [INFO] ** Branch:" ${Branch} echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} echo $PGM": [INFO] ** Use Package Cache :" ${enablePackageCache} - echo $PGM": [INFO] ** Package Cache Location :" ${packageCacheLocation} + echo $PGM": [INFO] ** Archive Cache Location :" ${archiveCache} echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLogFile} echo $PGM": [INFO] **************************************************************" echo "" @@ -64,8 +64,8 @@ runFetchLogic() { cmd="${cmd} -d ${externalDependenciesLogFile}" fi - if [ ! -z "${packageCacheLocation}" ]; then - cmd="${cmd} -c ${packageCacheLocation}" + if [ ! -z "${archiveCache}" ]; then + cmd="${cmd} -c ${archiveCache}" fi echo $PGM": [INFO] ** CMD : ${cmd}" ${cmd} From 6134ce3f6f6edb6cce7aa349efde012e4761386a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 2 Apr 2025 08:41:19 +0200 Subject: [PATCH 123/127] disable new feature Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 6d7b3f1a..e9778a48 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -111,7 +111,7 @@ featureBranchBuildBehaviour=merge-base # this leverages the configuration for the publishing script PackageBuildOutputs # to retrieve build dependencies from the Artifact repo. # default: false -fetchBuildDependencies=true +fetchBuildDependencies=false # Settings for caching packages on build machine From be55b7aed87038edc45e6abf6b67be2c2211d286 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 7 Apr 2025 14:21:06 +0200 Subject: [PATCH 124/127] rename to just url Signed-off-by: Dennis Behm --- .../PackageBuildOutputs/ArtifactRepositoryHelpers.groovy | 6 +++--- .../Common-Backend-Scripts/utilities/packagingUtilities.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 72ef3846..6473ad15 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -251,9 +251,9 @@ def run(String[] cliArgs) { assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" // load script - packageUrl = computeArchiveUrl(props) - // the println is used in a script by the CBS to grep the packageUrl - println "packageUrl=$packageUrl" + archiveUrl = computeArchiveUrl(props) + // the println is used in a script by the CBS to grep the url + println "url=$archiveUrl" } else println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computeArchiveUrl') "); } diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh index 7a2a22f5..e982a860 100755 --- a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -128,7 +128,7 @@ computeArchiveInformation() { fi echo $PGM": [INFO] ${CMD}" - artifactRepositoryAbsoluteUrl=$(${CMD} | grep "packageUrl=" | awk -F "=" ' { print $2 }') + artifactRepositoryAbsoluteUrl=$(${CMD} | grep "url=" | awk -F "=" ' { print $2 }') if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc From 5d73f63d6ff0377f7216997f79fcbdb97f860633 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 7 Apr 2025 14:42:49 +0200 Subject: [PATCH 125/127] manage missing AD definitions Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 69 ++++++++++++++++--- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index 62a0bad6..e0ccd860 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -77,13 +77,38 @@ if (applicationDescriptor.dependencies) { println("*** Fetching dependent application archives") // Loop through all dependencies found in AD - applicationDescriptor.dependencies.each { dependency -> + applicationDescriptor.dependencies.each { + dependency -> // validate dependency record - assert dependency.type : "Missing dependency type attribute in dependency record" - assert dependency.reference : "Missing dependency reference attribute in dependency record" - assert dependency.buildid : "Missing buildid attribute in dependency record" - assert dependency.name : "Missing name attribute in dependency record" + if { + !dependency.name + } { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'name' attribute. rc=$rc") + } + if (!dependency.type) { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'type' attribute. rc=$rc") + } + if { + !dependency.reference + } { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") + } + if { + !dependency.buildid + } { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'buildid' attribute. rc=$rc") + } + + if (rc != 0) { + println("*! Fetching external dependencies exits after failing the validation of the dependency record.") + System.exit(1) + } + // compute tar file name based on build type if (dependency.type.equalsIgnoreCase("release")) { @@ -207,11 +232,35 @@ baselineRecord = applicationDescriptor.baselines.find() { baseline -> if (baselineRecord){ println("*** Fetching baseline archive") - // validate dependency record - assert baselineRecord.type : "Missing baseline type attribute in baseline record" - assert baselineRecord.reference : "Missing baseline reference attribute in baseline record" - assert baselineRecord.buildid : "Missing buildid attribute in baseline record" - assert applicationDescriptor.application : "Missing application name in Application Descriptor" + // validate baseline record + if { + !applicationDescriptor.application + } { + rc=1 + println("*! [ERROR] Application Descriptor missing the 'application' name attribute. rc=$rc") + } + if (!baselineRecord.type) { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the 'type' attribute. rc=$rc") + } + if { + !baselineRecord.reference + } { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") + } + if { + !baselineRecord.buildid + } { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the 'buildid' attribute. rc=$rc") + } + if (rc != 0) { + // exit after validation + println("*! Fetching external dependencies exits after failing the validation of the baseline record.") + System.exit(1) + } + def applicationName = applicationDescriptor.application // compute tar file name based on build type From 28b88c42d878cfe2e10f94c499bf9a17a8c08df3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 7 Apr 2025 14:54:58 +0200 Subject: [PATCH 126/127] fix syntax error Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index e0ccd860..ca1fb43b 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -77,13 +77,10 @@ if (applicationDescriptor.dependencies) { println("*** Fetching dependent application archives") // Loop through all dependencies found in AD - applicationDescriptor.dependencies.each { - dependency -> + applicationDescriptor.dependencies.each { dependency -> // validate dependency record - if { - !dependency.name - } { + if (!dependency.name) { rc=1 println("*! [ERROR] Dependency record in Application Descriptor missing the 'name' attribute. rc=$rc") } @@ -91,15 +88,11 @@ if (applicationDescriptor.dependencies) { rc=1 println("*! [ERROR] Dependency record in Application Descriptor missing the 'type' attribute. rc=$rc") } - if { - !dependency.reference - } { + if (!dependency.reference) { rc=1 println("*! [ERROR] Dependency record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") } - if { - !dependency.buildid - } { + if (!dependency.buildid) { rc=1 println("*! [ERROR] Dependency record in Application Descriptor missing the 'buildid' attribute. rc=$rc") } @@ -233,9 +226,7 @@ if (baselineRecord){ println("*** Fetching baseline archive") // validate baseline record - if { - !applicationDescriptor.application - } { + if (!applicationDescriptor.application) { rc=1 println("*! [ERROR] Application Descriptor missing the 'application' name attribute. rc=$rc") } @@ -243,15 +234,11 @@ if (baselineRecord){ rc=1 println("*! [ERROR] Baseline record in Application Descriptor missing the 'type' attribute. rc=$rc") } - if { - !baselineRecord.reference - } { + if (!baselineRecord.reference) { rc=1 println("*! [ERROR] Baseline record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") } - if { - !baselineRecord.buildid - } { + if (!baselineRecord.buildid) { rc=1 println("*! [ERROR] Baseline record in Application Descriptor missing the 'buildid' attribute. rc=$rc") } From 837e847c04ef8b7691f0d7760332eb42f420b4c4 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 7 Apr 2025 15:02:40 +0200 Subject: [PATCH 127/127] fix syntax error Signed-off-by: Dennis Behm --- .../utilities/fetchBuildDependencies.groovy | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy index ca1fb43b..16a41990 100644 --- a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -79,6 +79,8 @@ if (applicationDescriptor.dependencies) { // Loop through all dependencies found in AD applicationDescriptor.dependencies.each { dependency -> + def rc=0 + // validate dependency record if (!dependency.name) { rc=1 @@ -180,7 +182,7 @@ if (applicationDescriptor.dependencies) { if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() println("** Downloading archive '$artifactUrl' from artifact repository into '${tarFileDir}'.") - def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) + rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { println("*! [ERROR] Download of archive '$artifactUrl' failed. Exiting with return code:${rc}.") @@ -203,7 +205,7 @@ if (applicationDescriptor.dependencies) { "tar -C $importSubFolder -xvf ${tarFile}" ] - def rc = runProcess(processCmd) + rc = runProcess(processCmd) if (rc != 0) { println("** [ERROR] Failed to untar '$tarFile' to '$importSubFolder' with rc=$rc") System.exit(1) @@ -225,6 +227,8 @@ baselineRecord = applicationDescriptor.baselines.find() { baseline -> if (baselineRecord){ println("*** Fetching baseline archive") + def rc=0 + // validate baseline record if (!applicationDescriptor.application) { rc=1 @@ -287,7 +291,7 @@ if (baselineRecord){ if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() println("** Downloading archive with '$artifactUrl' from Artifact Repository into ${tarFileDir}.") - def rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) + rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) if (rc != 0) { println("** Download of archive '$artifactUrl' failed. Process exists. Return code:$rc") @@ -307,7 +311,7 @@ if (baselineRecord){ "cp ${tarFile} ${baselineFolder}/" ] - def rc = runProcess(processCmd) + rc = runProcess(processCmd) if (rc != 0) { println("** [ERROR] Failed to copy '$tarFile' to '$baselineFolder' with rc=$rc") System.exit(1)