diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index bcbae10b7cc1..18f4f7ddebf6 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -58,6 +58,7 @@ dependencies { runtime "ca.cutterslade.gradle:gradle-dependency-analyze:1.4.2" // Enable dep analysis runtime "gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11" // Enable dep vulnerability analysis runtime "org.checkerframework:checkerframework-gradle-plugin:0.5.11" // Enable enhanced static checking plugin + runtime "org.jfrog.buildinfo:build-info-extractor-gradle:4.24.14" // Enable submission to jfrog artifactory } // Because buildSrc is built and tested automatically _before_ gradle diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 000000000000..cdfce963718b --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Plugins for configuring _this build_ of the module +plugins { + `java-gradle-plugin` + groovy + id("com.diffplug.spotless") version "5.6.1" +} + +// Define the set of repositories required to fetch and enable plugins. +repositories { + jcenter() + maven { url = uri("https://plugins.gradle.org/m2/") } + maven { + url = uri("https://repo.spring.io/plugins-release/") + content { includeGroup("io.spring.gradle") } + } +} + +// Dependencies on other plugins used when this plugin is invoked +dependencies { + compile(gradleApi()) + compile(localGroovy()) + compile("com.github.jengelman.gradle.plugins:shadow:6.1.0") + compile("gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.5.0") + + runtime("net.ltgt.gradle:gradle-apt-plugin:0.21") // Enable a Java annotation processor + runtime("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation + runtime("io.spring.gradle:propdeps-plugin:0.0.9.RELEASE") // Enable provided and optional configurations + runtime("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0") // Enable Avro code generation + runtime("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin + runtime("gradle.plugin.com.github.blindpirate:gogradle:0.11.4") // Enable Go code compilation + runtime("gradle.plugin.com.palantir.gradle.docker:gradle-docker:0.22.0") // Enable building Docker containers + runtime("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree + runtime("com.github.jengelman.gradle.plugins:shadow:6.1.0") // Enable shading Java dependencies + runtime("ca.coglinc:javacc-gradle-plugin:2.4.0") // Enable the JavaCC parser generator + runtime("net.linguica.gradle:maven-settings-plugin:0.5") + runtime("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository + runtime("net.ltgt.gradle:gradle-errorprone-plugin:1.2.1") // Enable errorprone Java static analysis + runtime("org.ajoberstar.grgit:grgit-gradle:4.0.2") // Enable website git publish to asf-site branch + runtime("com.avast.gradle:gradle-docker-compose-plugin:0.13.2") // Enable docker compose tasks + runtime("ca.cutterslade.gradle:gradle-dependency-analyze:1.4.3") // Enable dep analysis + runtime("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis + runtime("org.checkerframework:checkerframework-gradle-plugin:0.5.16") // Enable enhanced static checking plugin + runtime("org.jfrog.buildinfo:build-info-extractor-gradle:4.24.14") // Enable submission to jfrog artifactory +} + +// Because buildSrc is built and tested automatically _before_ gradle +// does anything else, it is not possible to spotlessApply because +// spotlessCheck fails before that. So this hack allows disabling +// the check for the moment of application. +// +// ./gradlew :buildSrc:spotlessApply -PdisableSpotlessCheck=true +val disableSpotlessCheck: String by project +val isSpotlessCheckDisabled = project.hasProperty("disableSpotlessCheck") && + disableSpotlessCheck == "true" + +spotless { + isEnforceCheck = !isSpotlessCheckDisabled + groovy { + excludeJava() + greclipse().configFile("greclipse.properties") + } + groovyGradle { + greclipse().configFile("greclipse.properties") + } +} + +gradlePlugin { + plugins { + create("beamModule") { + id = "org.apache.beam.module" + implementationClass = "org.apache.beam.gradle.BeamModulePlugin" + } + create("vendorJava") { + id = "org.apache.beam.vendor-java" + implementationClass = "org.apache.beam.gradle.VendorJavaPlugin" + } + create("beamJenkins") { + id = "org.apache.beam.jenkins" + implementationClass = "org.apache.beam.gradle.BeamJenkinsPlugin" + } + } +} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 13ddfc4a3be9..7f11fcd01422 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -346,6 +346,10 @@ class BeamModulePlugin implements Plugin { return project.hasProperty('isRelease') } + def isLinkedin(Project project) { + return project.hasProperty('isLinkedin') + } + def defaultArchivesBaseName(Project p) { return 'beam' + p.path.replace(':', '-') } @@ -355,15 +359,17 @@ class BeamModulePlugin implements Plugin { /** ***********************************************************************************************/ // Apply common properties/repositories and tasks to all projects. - //project.group = 'com.linkedin.beam' - project.ext.mavenGroupId = 'com.linkedin.beam' + project.ext.mavenGroupId = 'org.apache.beam' // Automatically use the official release version if we are performing a release // otherwise append '-SNAPSHOT' - project.version = '3.2260.0' + project.version = '3.2260.2' if (!isRelease(project)) { project.version += '-SNAPSHOT' } + if (isLinkedin(project)) { + project.version += '-LI' + } // Default to dash-separated directories for artifact base name, // which will also be the default artifactId for maven publications @@ -654,22 +660,14 @@ class BeamModulePlugin implements Plugin { url "file://${project.rootProject.projectDir}/testPublication/" } maven { - url(project.properties['distMgmtSnapshotsUrl'] ?: isRelease(project) - ? 'https://repository.apache.org/service/local/staging/deploy/maven2' - : 'https://repository.apache.org/content/repositories/snapshots') - name(project.properties['distMgmtServerId'] ?: isRelease(project) - ? 'apache.releases.https' : 'apache.snapshots.https') + name "linkedin.jfrog.https" + url "https://linkedin.jfrog.io/artifactory/beam/" // The maven settings plugin will load credentials from ~/.m2/settings.xml file that a user // has configured with the Apache release and snapshot staging credentials. // // // - // apache.releases.https - // USER_TOKEN - // PASS_TOKEN - // - // - // apache.snapshots.https + // linkedin.jfrog.https // USER_TOKEN // PASS_TOKEN // @@ -1156,6 +1154,7 @@ class BeamModulePlugin implements Plugin { if ((isRelease(project) || project.hasProperty('publishing')) && configuration.publish) { project.apply plugin: "maven-publish" + project.apply plugin: 'com.jfrog.artifactory' // Create a task which emulates the maven-archiver plugin in generating a // pom.properties file. diff --git a/runners/samza/build.gradle b/runners/samza/build.gradle index afa2991eadd2..1abcd56cd566 100644 --- a/runners/samza/build.gradle +++ b/runners/samza/build.gradle @@ -34,20 +34,6 @@ description = "Apache Beam :: Runners :: Samza" evaluationDependsOn(":sdks:java:core") repositories { - maven { - url "https://artifactory.corp.linkedin.com:8083/artifactory/DDS/" - metadataSources { - mavenPom() - artifact() - } - } - maven { - url "https://artifactory.corp.linkedin.com:8083/artifactory/mintdev-publish-repo/" - metadataSources { - mavenPom() - artifact() - } - } maven { url "file://" + System.getProperty("user.home") + "/local-repo" metadataSources { @@ -61,17 +47,7 @@ configurations { validatesRunner } -def get_samza_version = { -> - def path = file("${project.rootDir}" + "/../build/dependency-spec.json").getCanonicalPath() - def stdout = new ByteArrayOutputStream() - exec { - commandLine "bash", "-c", "grep 'com.linkedin.samza-li:samza-api' ${path} | awk -F ':' '{print \$4}' | tr -d \"\\\",\"" - standardOutput = stdout - } - return stdout.toString().trim() -} - -def samza_version = get_samza_version() +def samza_version = "1.6.0" dependencies { compile library.java.vendored_guava_26_0_jre @@ -91,14 +67,13 @@ dependencies { compile "org.rocksdb:rocksdbjni:6.15.2" compile "org.apache.commons:commons-collections4:4.0" compile "org.scala-lang:scala-library:2.11.8" - compile "com.linkedin.samza-li:samza-api:$samza_version" - compile "com.linkedin.samza-li:samza-core_2.11:$samza_version" - runtimeOnly "com.linkedin.samza-li:samza-kafka_2.11:$samza_version" - compile "com.linkedin.samza-li:samza-kv_2.11:$samza_version" - compile "com.linkedin.samza-li:samza-kv-rocksdb_2.11:$samza_version" - compile "com.linkedin.samza-li:samza-kv-inmemory_2.11:$samza_version" - compile "com.linkedin.samza-li:samza-li-core-adapters:$samza_version" - compile "com.linkedin.samza-li:samza-yarn_2.11:$samza_version" + compile "org.apache.samza:samza-api:$samza_version" + compile "org.apache.samza:samza-core_2.11:$samza_version" + runtimeOnly "org.apache.samza:samza-kafka_2.11:$samza_version" + compile "org.apache.samza:samza-kv_2.11:$samza_version" + compile "org.apache.samza:samza-kv-rocksdb_2.11:$samza_version" + compile "org.apache.samza:samza-kv-inmemory_2.11:$samza_version" + compile "org.apache.samza:samza-yarn_2.11:$samza_version" runtimeOnly "org.apache.kafka:kafka-clients:2.0.1" compile library.java.vendored_grpc_1_26_0 compile project(path: ":model:fn-execution", configuration: "shadow") diff --git a/runners/samza/job-server/build.gradle b/runners/samza/job-server/build.gradle index 46c1d66c6190..d5ced4e6b65a 100644 --- a/runners/samza/job-server/build.gradle +++ b/runners/samza/job-server/build.gradle @@ -24,20 +24,6 @@ apply plugin: 'application' mainClassName = "org.apache.beam.runners.samza.SamzaJobServerDriver" repositories { - maven { - url "https://artifactory.corp.linkedin.com:8083/artifactory/DDS/" - metadataSources { - mavenPom() - artifact() - } - } - maven { - url "https://artifactory.corp.linkedin.com:8083/artifactory/mintdev-publish-repo/" - metadataSources { - mavenPom() - artifact() - } - } maven { url "file://" + System.getProperty("user.home") + "/local-repo" metadataSources { diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java index 82c5ace32635..533828a25074 100644 --- a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java +++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaStoreStateInternalsTest.java @@ -63,7 +63,6 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets; import org.apache.samza.context.ContainerContext; import org.apache.samza.context.JobContext; -import org.apache.samza.job.model.TaskModel; import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.serializers.Serde; import org.apache.samza.storage.StorageEngineFactory; @@ -200,7 +199,6 @@ public static class TestStorageEngine extends InMemoryKeyValueStorageEngineFacto @Override public KeyValueStore getKVStore( String storeName, - TaskModel taskModel, File storeDir, MetricsRegistry registry, SystemStreamPartition changeLogSystemStreamPartition,