From b20cf63373d664155e6c4a7ade61ebc46e7d1949 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 5 Aug 2021 17:02:45 -0500 Subject: [PATCH 1/2] [BEAM-10955] Update flink minor versions and enable testSavepointRestoreLegacy flaky test [BEAM-10955] Fix spotlessApply [BEAM-10955] Add a notice to CHANGES.md of upgraded Flink versions [BEAM-10955] Only run legacy savepoint test on Flink >1.12. --- CHANGES.md | 1 + runners/flink/1.11/build.gradle | 2 +- runners/flink/1.12/build.gradle | 2 +- runners/flink/1.13/build.gradle | 2 +- .../org/apache/beam/runners/flink/FlinkSavepointTest.java | 6 ++++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5f130443dfc1..11b58d7cb671 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -88,6 +88,7 @@ * Reading from JDBC source by partitions (Java) ([BEAM-12456](https://issues.apache.org/jira/browse/BEAM-12456)). * PubsubIO can now write to a dead-letter topic after a parsing error (Java)([BEAM-12474](https://issues.apache.org/jira/browse/BEAM-12474)). * New append-only option for Elasticsearch sink (Java) [BEAM-12601](https://issues.apache.org/jira/browse/BEAM-12601) +* Upgrade Flink runner to Flink versions 1.13.2, 1.12.5 and 1.11.4 ([BEAM-10955](https://issues.apache.org/jira/browse/BEAM-10955)). ## Breaking Changes diff --git a/runners/flink/1.11/build.gradle b/runners/flink/1.11/build.gradle index 7124e2e6b59d..c3736edfc614 100644 --- a/runners/flink/1.11/build.gradle +++ b/runners/flink/1.11/build.gradle @@ -20,7 +20,7 @@ def basePath = '..' /* All properties required for loading the Flink build script */ project.ext { // Set the version of all Flink-related dependencies here. - flink_version = '1.11.3' + flink_version = '1.11.4' // Version specific code overrides. main_source_overrides = ['./src/main/java'] test_source_overrides = ['./src/test/java'] diff --git a/runners/flink/1.12/build.gradle b/runners/flink/1.12/build.gradle index 99dd015baf24..31878af92ddc 100644 --- a/runners/flink/1.12/build.gradle +++ b/runners/flink/1.12/build.gradle @@ -20,7 +20,7 @@ def basePath = '..' /* All properties required for loading the Flink build script */ project.ext { // Set the version of all Flink-related dependencies here. - flink_version = '1.12.4' + flink_version = '1.12.5' // Version specific code overrides. main_source_overrides = ["${basePath}/1.11/src/main/java", './src/main/java'] test_source_overrides = ["${basePath}/1.11/src/test/java", './src/test/java'] diff --git a/runners/flink/1.13/build.gradle b/runners/flink/1.13/build.gradle index 067933f9b0c4..d6d04eb83bec 100644 --- a/runners/flink/1.13/build.gradle +++ b/runners/flink/1.13/build.gradle @@ -20,7 +20,7 @@ def basePath = '..' /* All properties required for loading the Flink build script */ project.ext { // Set the version of all Flink-related dependencies here. - flink_version = '1.13.1' + flink_version = '1.13.2' // Version specific code overrides. main_source_overrides = ["${basePath}/1.11/src/main/java", "${basePath}/1.12/src/main/java", './src/main/java'] test_source_overrides = ["${basePath}/1.11/src/test/java", "${basePath}/1.12/src/test/java", './src/test/java'] diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java index 66159b6eca8c..e973079a38af 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java @@ -18,6 +18,7 @@ package org.apache.beam.runners.flink; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assume.assumeFalse; import java.io.Serializable; import java.net.URI; @@ -61,6 +62,7 @@ import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; +import org.apache.flink.runtime.util.EnvironmentInformation; import org.hamcrest.Matchers; import org.hamcrest.core.IsIterableContaining; import org.joda.time.Instant; @@ -68,7 +70,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -142,9 +143,10 @@ public void afterTest() throws Exception { ensureNoJobRunning(); } - @Ignore("https://issues.apache.org/jira/projects/BEAM/issues/BEAM-10955") @Test public void testSavepointRestoreLegacy() throws Exception { + // Don't run on Flink 1.11. https://issues.apache.org/jira/browse/BEAM-10955 + assumeFalse(EnvironmentInformation.getVersion().startsWith("1.11")); runSavepointAndRestore(false); } From 416fbe2a88a0562d9b29af04ab24b1df1c41b845 Mon Sep 17 00:00:00 2001 From: Kyle Weaver Date: Tue, 17 Aug 2021 10:26:54 -0700 Subject: [PATCH 2/2] Move change notice to new 2.33.0 section, since 2.32.0 is already cut. --- CHANGES.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 11b58d7cb671..6da170d9af35 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,33 @@ * Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). --> +# [2.33.X] - Unreleased + +## Highlights + +* New highly anticipated feature X added to Python SDK ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* New highly anticipated feature Y added to Java SDK ([BEAM-Y](https://issues.apache.org/jira/browse/BEAM-Y)). + +## I/Os + +* Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## New Features / Improvements + +* X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). +* Upgrade Flink runner to Flink versions 1.13.2, 1.12.5 and 1.11.4 ([BEAM-10955](https://issues.apache.org/jira/browse/BEAM-10955)). + +## Breaking Changes + +* X behavior was changed ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Deprecations + +* X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). + +## Known Issues + +* Fixed X (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). # [2.32.X] - Unreleased @@ -88,7 +115,6 @@ * Reading from JDBC source by partitions (Java) ([BEAM-12456](https://issues.apache.org/jira/browse/BEAM-12456)). * PubsubIO can now write to a dead-letter topic after a parsing error (Java)([BEAM-12474](https://issues.apache.org/jira/browse/BEAM-12474)). * New append-only option for Elasticsearch sink (Java) [BEAM-12601](https://issues.apache.org/jira/browse/BEAM-12601) -* Upgrade Flink runner to Flink versions 1.13.2, 1.12.5 and 1.11.4 ([BEAM-10955](https://issues.apache.org/jira/browse/BEAM-10955)). ## Breaking Changes