Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ dependencies {
runtime "ca.cutterslade.gradle:gradle-dependency-analyze:1.4.2" // Enable dep analysis
runtime "gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11" // Enable dep vulnerability analysis
runtime "org.checkerframework:checkerframework-gradle-plugin:0.5.11" // Enable enhanced static checking plugin
runtime "org.jfrog.buildinfo:build-info-extractor-gradle:4.24.14" // Enable submission to jfrog artifactory
}

// Because buildSrc is built and tested automatically _before_ gradle
Expand Down
100 changes: 100 additions & 0 deletions buildSrc/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// Plugins for configuring _this build_ of the module
plugins {
`java-gradle-plugin`
groovy
id("com.diffplug.spotless") version "5.6.1"
}

// Define the set of repositories required to fetch and enable plugins.
repositories {
jcenter()
maven { url = uri("https://plugins.gradle.org/m2/") }
maven {
url = uri("https://repo.spring.io/plugins-release/")
content { includeGroup("io.spring.gradle") }
}
}

// Dependencies on other plugins used when this plugin is invoked
dependencies {
compile(gradleApi())
compile(localGroovy())
compile("com.github.jengelman.gradle.plugins:shadow:6.1.0")
compile("gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.5.0")

runtime("net.ltgt.gradle:gradle-apt-plugin:0.21") // Enable a Java annotation processor
runtime("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation
runtime("io.spring.gradle:propdeps-plugin:0.0.9.RELEASE") // Enable provided and optional configurations
runtime("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0") // Enable Avro code generation
runtime("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin
runtime("gradle.plugin.com.github.blindpirate:gogradle:0.11.4") // Enable Go code compilation
runtime("gradle.plugin.com.palantir.gradle.docker:gradle-docker:0.22.0") // Enable building Docker containers
runtime("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree
runtime("com.github.jengelman.gradle.plugins:shadow:6.1.0") // Enable shading Java dependencies
runtime("ca.coglinc:javacc-gradle-plugin:2.4.0") // Enable the JavaCC parser generator
runtime("net.linguica.gradle:maven-settings-plugin:0.5")
runtime("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository
runtime("net.ltgt.gradle:gradle-errorprone-plugin:1.2.1") // Enable errorprone Java static analysis
runtime("org.ajoberstar.grgit:grgit-gradle:4.0.2") // Enable website git publish to asf-site branch
runtime("com.avast.gradle:gradle-docker-compose-plugin:0.13.2") // Enable docker compose tasks
runtime("ca.cutterslade.gradle:gradle-dependency-analyze:1.4.3") // Enable dep analysis
runtime("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis
runtime("org.checkerframework:checkerframework-gradle-plugin:0.5.16") // Enable enhanced static checking plugin
runtime("org.jfrog.buildinfo:build-info-extractor-gradle:4.24.14") // Enable submission to jfrog artifactory
}

// Because buildSrc is built and tested automatically _before_ gradle
// does anything else, it is not possible to spotlessApply because
// spotlessCheck fails before that. So this hack allows disabling
// the check for the moment of application.
//
// ./gradlew :buildSrc:spotlessApply -PdisableSpotlessCheck=true
val disableSpotlessCheck: String by project
val isSpotlessCheckDisabled = project.hasProperty("disableSpotlessCheck") &&
disableSpotlessCheck == "true"

spotless {
isEnforceCheck = !isSpotlessCheckDisabled
groovy {
excludeJava()
greclipse().configFile("greclipse.properties")
}
groovyGradle {
greclipse().configFile("greclipse.properties")
}
}

gradlePlugin {
plugins {
create("beamModule") {
id = "org.apache.beam.module"
implementationClass = "org.apache.beam.gradle.BeamModulePlugin"
}
create("vendorJava") {
id = "org.apache.beam.vendor-java"
implementationClass = "org.apache.beam.gradle.VendorJavaPlugin"
}
create("beamJenkins") {
id = "org.apache.beam.jenkins"
implementationClass = "org.apache.beam.gradle.BeamJenkinsPlugin"
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,10 @@ class BeamModulePlugin implements Plugin<Project> {
return project.hasProperty('isRelease')
}

def isLinkedin(Project project) {
return project.hasProperty('isLinkedin')
}

def defaultArchivesBaseName(Project p) {
return 'beam' + p.path.replace(':', '-')
}
Expand All @@ -355,15 +359,17 @@ class BeamModulePlugin implements Plugin<Project> {
/** ***********************************************************************************************/
// Apply common properties/repositories and tasks to all projects.

//project.group = 'com.linkedin.beam'
project.ext.mavenGroupId = 'com.linkedin.beam'
project.ext.mavenGroupId = 'org.apache.beam'

// Automatically use the official release version if we are performing a release
// otherwise append '-SNAPSHOT'
project.version = '3.2260.0'
project.version = '3.2260.2'
if (!isRelease(project)) {
project.version += '-SNAPSHOT'
}
if (isLinkedin(project)) {
project.version += '-LI'
}

// Default to dash-separated directories for artifact base name,
// which will also be the default artifactId for maven publications
Expand Down Expand Up @@ -654,22 +660,14 @@ class BeamModulePlugin implements Plugin<Project> {
url "file://${project.rootProject.projectDir}/testPublication/"
}
maven {
url(project.properties['distMgmtSnapshotsUrl'] ?: isRelease(project)
? 'https://repository.apache.org/service/local/staging/deploy/maven2'
: 'https://repository.apache.org/content/repositories/snapshots')
name(project.properties['distMgmtServerId'] ?: isRelease(project)
? 'apache.releases.https' : 'apache.snapshots.https')
name "linkedin.jfrog.https"
url "https://linkedin.jfrog.io/artifactory/beam/"
// The maven settings plugin will load credentials from ~/.m2/settings.xml file that a user
// has configured with the Apache release and snapshot staging credentials.
// <settings>
// <servers>
// <server>
// <id>apache.releases.https</id>
// <username>USER_TOKEN</username>
// <password>PASS_TOKEN</password>
// </server>
// <server>
// <id>apache.snapshots.https</id>
// <id>linkedin.jfrog.https</id>
// <username>USER_TOKEN</username>
// <password>PASS_TOKEN</password>
// </server>
Expand Down Expand Up @@ -1156,6 +1154,7 @@ class BeamModulePlugin implements Plugin<Project> {
if ((isRelease(project) || project.hasProperty('publishing')) &&
configuration.publish) {
project.apply plugin: "maven-publish"
project.apply plugin: 'com.jfrog.artifactory'

// Create a task which emulates the maven-archiver plugin in generating a
// pom.properties file.
Expand Down
41 changes: 8 additions & 33 deletions runners/samza/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,6 @@ description = "Apache Beam :: Runners :: Samza"
evaluationDependsOn(":sdks:java:core")

repositories {
maven {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIRC, we need to update runners/samza/job-server/build.gradle as well

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also remove there as well, thanks for the reminder

url "https://artifactory.corp.linkedin.com:8083/artifactory/DDS/"
metadataSources {
mavenPom()
artifact()
}
}
maven {
url "https://artifactory.corp.linkedin.com:8083/artifactory/mintdev-publish-repo/"
metadataSources {
mavenPom()
artifact()
}
}
maven {
url "file://" + System.getProperty("user.home") + "/local-repo"
metadataSources {
Expand All @@ -61,17 +47,7 @@ configurations {
validatesRunner
}

def get_samza_version = { ->
def path = file("${project.rootDir}" + "/../build/dependency-spec.json").getCanonicalPath()
def stdout = new ByteArrayOutputStream()
exec {
commandLine "bash", "-c", "grep 'com.linkedin.samza-li:samza-api' ${path} | awk -F ':' '{print \$4}' | tr -d \"\\\",\""
standardOutput = stdout
}
return stdout.toString().trim()
}

def samza_version = get_samza_version()
def samza_version = "1.6.0"

dependencies {
compile library.java.vendored_guava_26_0_jre
Expand All @@ -91,14 +67,13 @@ dependencies {
compile "org.rocksdb:rocksdbjni:6.15.2"
compile "org.apache.commons:commons-collections4:4.0"
compile "org.scala-lang:scala-library:2.11.8"
compile "com.linkedin.samza-li:samza-api:$samza_version"
compile "com.linkedin.samza-li:samza-core_2.11:$samza_version"
runtimeOnly "com.linkedin.samza-li:samza-kafka_2.11:$samza_version"
compile "com.linkedin.samza-li:samza-kv_2.11:$samza_version"
compile "com.linkedin.samza-li:samza-kv-rocksdb_2.11:$samza_version"
compile "com.linkedin.samza-li:samza-kv-inmemory_2.11:$samza_version"
compile "com.linkedin.samza-li:samza-li-core-adapters:$samza_version"
compile "com.linkedin.samza-li:samza-yarn_2.11:$samza_version"
compile "org.apache.samza:samza-api:$samza_version"
compile "org.apache.samza:samza-core_2.11:$samza_version"
runtimeOnly "org.apache.samza:samza-kafka_2.11:$samza_version"
compile "org.apache.samza:samza-kv_2.11:$samza_version"
compile "org.apache.samza:samza-kv-rocksdb_2.11:$samza_version"
compile "org.apache.samza:samza-kv-inmemory_2.11:$samza_version"
compile "org.apache.samza:samza-yarn_2.11:$samza_version"
runtimeOnly "org.apache.kafka:kafka-clients:2.0.1"
compile library.java.vendored_grpc_1_26_0
compile project(path: ":model:fn-execution", configuration: "shadow")
Expand Down
14 changes: 0 additions & 14 deletions runners/samza/job-server/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,6 @@ apply plugin: 'application'
mainClassName = "org.apache.beam.runners.samza.SamzaJobServerDriver"

repositories {
maven {
url "https://artifactory.corp.linkedin.com:8083/artifactory/DDS/"
metadataSources {
mavenPom()
artifact()
}
}
maven {
url "https://artifactory.corp.linkedin.com:8083/artifactory/mintdev-publish-repo/"
metadataSources {
mavenPom()
artifact()
}
}
maven {
url "file://" + System.getProperty("user.home") + "/local-repo"
metadataSources {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets;
import org.apache.samza.context.ContainerContext;
import org.apache.samza.context.JobContext;
import org.apache.samza.job.model.TaskModel;
import org.apache.samza.metrics.MetricsRegistry;
import org.apache.samza.serializers.Serde;
import org.apache.samza.storage.StorageEngineFactory;
Expand Down Expand Up @@ -200,7 +199,6 @@ public static class TestStorageEngine extends InMemoryKeyValueStorageEngineFacto
@Override
public KeyValueStore<byte[], byte[]> getKVStore(
String storeName,
TaskModel taskModel,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We won't need this if we are on Samza 1.6

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is only available in samza 1.7 which is not yet released

File storeDir,
MetricsRegistry registry,
SystemStreamPartition changeLogSystemStreamPartition,
Expand Down