From 89cbfea19479894b98dcdf9a8783b2e85829de9e Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Sun, 22 Jan 2023 15:25:00 +0200 Subject: [PATCH 01/10] add jenkinsfile --- Jenkinsfile | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..b37e1b5f04 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,37 @@ +@Library('ni-utils') _ + +//service name is extrapolated from repository name check +def svcName = currentBuild.rawBuild.project.parent.displayName + +// Define pod +def pod = libraryResource 'com/naturalint/kafka-agent-gradle.yaml' +print pod + +// Define sharedLibrary +def sharedLibrary = new com.naturalint.kafkaConnectGradle() + +// Custom maven build command +// def mavenBuildCommand="" + +// Custom unit test command + +// Set slack channel +def slackChannel = "kafka-connect-cicd" + +// Args for pipeline +def compileData = [run: true] //, maven_build_cmd: mavenBuildCommand] +def testData = [run: false] // Default is: python -m unittest +def artifactData = [run: true] +def intTestData = [run: false] +def deploymentData = [run: false] +def buildCommands = [ + compileData: compileData, + testData: testData, + artifactData: artifactData, + intTestData: intTestData, + deploymentData: deploymentData +] + +timestamps { + commonPipeline(sharedLibrary, svcName, buildCommands, pod, slackChannel) +} From 7884ca362634ef6667276805932de69fdf94807c Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Sun, 22 Jan 2023 16:50:27 +0200 Subject: [PATCH 02/10] support openLineage --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b37e1b5f04..72611fa5c5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('ni-utils') _ +@Library('ni-utils@support-openLineage') _ //service name is extrapolated from repository name check def svcName = currentBuild.rawBuild.project.parent.displayName @@ -19,12 +19,15 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() def slackChannel = "kafka-connect-cicd" // Args for pipeline +def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage" newAirflowEdition: true] + def compileData = [run: true] //, maven_build_cmd: mavenBuildCommand] def testData = [run: false] // Default is: python -m unittest def artifactData = [run: true] def intTestData = [run: false] def deploymentData = [run: false] def buildCommands = [ + initiateData: initiateData, compileData: compileData, testData: testData, artifactData: artifactData, From dedeb3c9a116f568bbdf13e1c1c3fb579d32f472 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Sun, 22 Jan 2023 16:54:28 +0200 Subject: [PATCH 03/10] support openLineage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 72611fa5c5..0c6e135646 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() def slackChannel = "kafka-connect-cicd" // Args for pipeline -def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage" newAirflowEdition: true] +def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage"] def compileData = [run: true] //, maven_build_cmd: mavenBuildCommand] def testData = [run: false] // Default is: python -m unittest From 277f320c4124142299be3fcdac4c824b5b1168a7 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Sun, 22 Jan 2023 17:08:16 +0200 Subject: [PATCH 04/10] support openLineage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0c6e135646..07f7d98aca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() def slackChannel = "kafka-connect-cicd" // Args for pipeline -def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage"] +def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage", grepVersion: "version"] def compileData = [run: true] //, maven_build_cmd: mavenBuildCommand] def testData = [run: false] // Default is: python -m unittest From 5adf5211b8dfed3af6158373750dae9c0a649815 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Mon, 23 Jan 2023 10:58:22 +0200 Subject: [PATCH 05/10] support openLineage --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07f7d98aca..ca6dd6d0a0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,11 +17,11 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() // Set slack channel def slackChannel = "kafka-connect-cicd" +def mavenBuildCommand = "./gradlew build :spotlessApply" // Args for pipeline -def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "OpenLineage", grepVersion: "version"] - -def compileData = [run: true] //, maven_build_cmd: mavenBuildCommand] +def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "openlineage", grepVersion: "version"] +def compileData = [run: true, maven_build_cmd: mavenBuildCommand] def testData = [run: false] // Default is: python -m unittest def artifactData = [run: true] def intTestData = [run: false] From 8b81a742f1bd30c9cd20ab640ba2aae0c604ba31 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Mon, 23 Jan 2023 12:05:05 +0200 Subject: [PATCH 06/10] support openLineage --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca6dd6d0a0..e5f3801677 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,11 +17,12 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() // Set slack channel def slackChannel = "kafka-connect-cicd" -def mavenBuildCommand = "./gradlew build :spotlessApply" +#def mavenBuildCommand = "./gradlew build :spotlessApply" +def mavenBuildCommand = "./gradlew publishToMavenLocal" // Args for pipeline def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "openlineage", grepVersion: "version"] -def compileData = [run: true, maven_build_cmd: mavenBuildCommand] +def compileData = [run: true, maven_build_cmd: mavenBuildCommand, projectToCompile: ["client/java","integration/spark" ]] def testData = [run: false] // Default is: python -m unittest def artifactData = [run: true] def intTestData = [run: false] From 2f0d6eda67c17ed479b6151ccbbd9693baed0b6e Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Mon, 23 Jan 2023 12:05:57 +0200 Subject: [PATCH 07/10] support openLineage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e5f3801677..d3bfb0bbf7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,7 @@ def sharedLibrary = new com.naturalint.kafkaConnectGradle() // Set slack channel def slackChannel = "kafka-connect-cicd" -#def mavenBuildCommand = "./gradlew build :spotlessApply" +//def mavenBuildCommand = "./gradlew build :spotlessApply" def mavenBuildCommand = "./gradlew publishToMavenLocal" // Args for pipeline From 21c67203152cc4818e60854fd401b9ec614810a0 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Mon, 23 Jan 2023 15:55:14 +0200 Subject: [PATCH 08/10] support openLineage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d3bfb0bbf7..62f74e2612 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,7 @@ def slackChannel = "kafka-connect-cicd" def mavenBuildCommand = "./gradlew publishToMavenLocal" // Args for pipeline -def initiateData = [projectPath: "client/java", projectArtifact: "jar", projectBucketName: "openlineage", grepVersion: "version"] +def initiateData = [projectPathVersion: "client/java", projectArtifact: "jar", projectBucketName: "openlineage", grepVersion: "version"] def compileData = [run: true, maven_build_cmd: mavenBuildCommand, projectToCompile: ["client/java","integration/spark" ]] def testData = [run: false] // Default is: python -m unittest def artifactData = [run: true] From 5f162f797ae8d53120798e65e305620d9504c831 Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Tue, 24 Jan 2023 15:24:20 +0200 Subject: [PATCH 09/10] add java version 8 --- Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 62f74e2612..dcb8deb5a5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,8 +3,12 @@ //service name is extrapolated from repository name check def svcName = currentBuild.rawBuild.project.parent.displayName -// Define pod +//get pod template definition def pod = libraryResource 'com/naturalint/kafka-agent-gradle.yaml' +def template_vars = [ + 'java_version': '8' +] +pod = renderTemplate(pod, template_vars) print pod // Define sharedLibrary From 42d67e2b652db04f2972dc23080cc8e8131ca49e Mon Sep 17 00:00:00 2001 From: lidor ettinger Date: Wed, 25 Jan 2023 17:16:13 +0200 Subject: [PATCH 10/10] update awscli installation --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index dcb8deb5a5..71f4bd1253 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('ni-utils@support-openLineage') _ +@Library('ni-utils') _ //service name is extrapolated from repository name check def svcName = currentBuild.rawBuild.project.parent.displayName