From 4daf0ece245a2dc640be217cf4ad481ea430f996 Mon Sep 17 00:00:00 2001 From: Mario Molina Date: Mon, 10 Sep 2018 09:48:13 -0500 Subject: [PATCH 1/4] Removing Optional Spark Java API --- .../org/apache/spark/api/java/Optional.java | 188 ------------------ .../apache/spark/api/java/JavaPairRDD.scala | 2 +- .../apache/spark/api/java/JavaRDDLike.scala | 2 +- .../spark/api/java/JavaSparkContext.scala | 2 +- .../org/apache/spark/api/java/JavaUtils.scala | 1 + .../apache/spark/api/java/OptionalSuite.java | 94 --------- .../org/apache/spark/Java8RDDAPISuite.java | 1 - .../test/org/apache/spark/JavaAPISuite.java | 6 +- .../JavaStatefulNetworkWordCount.java | 2 +- .../apache/spark/streaming/StateSpec.scala | 4 +- .../streaming/api/java/JavaPairDStream.scala | 4 +- .../streaming/JavaMapWithStateSuite.java | 2 +- .../apache/spark/streaming/Java8APISuite.java | 1 - .../apache/spark/streaming/JavaAPISuite.java | 1 - 14 files changed, 14 insertions(+), 296 deletions(-) delete mode 100644 core/src/main/java/org/apache/spark/api/java/Optional.java delete mode 100644 core/src/test/java/org/apache/spark/api/java/OptionalSuite.java diff --git a/core/src/main/java/org/apache/spark/api/java/Optional.java b/core/src/main/java/org/apache/spark/api/java/Optional.java deleted file mode 100644 index fd0f495ca29da..0000000000000 --- a/core/src/main/java/org/apache/spark/api/java/Optional.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.api.java; - -import java.io.Serializable; -import java.util.Objects; - -import com.google.common.base.Preconditions; - -/** - *

Like {@code java.util.Optional} in Java 8, {@code scala.Option} in Scala, and - * {@code com.google.common.base.Optional} in Google Guava, this class represents a - * value of a given type that may or may not exist. It is used in methods that wish - * to optionally return a value, in preference to returning {@code null}.

- * - *

In fact, the class here is a reimplementation of the essential API of both - * {@code java.util.Optional} and {@code com.google.common.base.Optional}. From - * {@code java.util.Optional}, it implements:

- * - * - * - *

From {@code com.google.common.base.Optional} it implements:

- * - * - * - *

{@code java.util.Optional} itself was not used because at the time, the - * project did not require Java 8. Using {@code com.google.common.base.Optional} - * has in the past caused serious library version conflicts with Guava that can't - * be resolved by shading. Hence this work-alike clone.

- * - * @param type of value held inside - */ -public final class Optional implements Serializable { - - private static final Optional EMPTY = new Optional<>(); - - private final T value; - - private Optional() { - this.value = null; - } - - private Optional(T value) { - Preconditions.checkNotNull(value); - this.value = value; - } - - // java.util.Optional API (subset) - - /** - * @return an empty {@code Optional} - */ - public static Optional empty() { - @SuppressWarnings("unchecked") - Optional t = (Optional) EMPTY; - return t; - } - - /** - * @param value non-null value to wrap - * @return {@code Optional} wrapping this value - * @throws NullPointerException if value is null - */ - public static Optional of(T value) { - return new Optional<>(value); - } - - /** - * @param value value to wrap, which may be null - * @return {@code Optional} wrapping this value, which may be empty - */ - public static Optional ofNullable(T value) { - if (value == null) { - return empty(); - } else { - return of(value); - } - } - - /** - * @return the value wrapped by this {@code Optional} - * @throws NullPointerException if this is empty (contains no value) - */ - public T get() { - Preconditions.checkNotNull(value); - return value; - } - - /** - * @param other value to return if this is empty - * @return this {@code Optional}'s value if present, or else the given value - */ - public T orElse(T other) { - return value != null ? value : other; - } - - /** - * @return true iff this {@code Optional} contains a value (non-empty) - */ - public boolean isPresent() { - return value != null; - } - - // Guava API (subset) - // of(), get() and isPresent() are identically present in the Guava API - - /** - * @return an empty {@code Optional} - */ - public static Optional absent() { - return empty(); - } - - /** - * @param value value to wrap, which may be null - * @return {@code Optional} wrapping this value, which may be empty - */ - public static Optional fromNullable(T value) { - return ofNullable(value); - } - - /** - * @param other value to return if this is empty - * @return this {@code Optional}'s value if present, or else the given value - */ - public T or(T other) { - return value != null ? value : other; - } - - /** - * @return this {@code Optional}'s value if present, or else null - */ - public T orNull() { - return value; - } - - // Common methods - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof Optional)) { - return false; - } - Optional other = (Optional) obj; - return Objects.equals(value, other.value); - } - - @Override - public int hashCode() { - return value == null ? 0 : value.hashCode(); - } - - @Override - public String toString() { - return value == null ? "Optional.empty" : String.format("Optional[%s]", value); - } - -} diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala index 9544475ff0428..c675b597e969d 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala @@ -19,7 +19,7 @@ package org.apache.spark.api.java import java.{lang => jl} import java.lang.{Iterable => JIterable} -import java.util.{Comparator, List => JList} +import java.util.{Comparator, List => JList, Optional} import scala.collection.JavaConverters._ import scala.language.implicitConversions diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala index 91ae1002abd21..de55c47bbc73b 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala @@ -19,7 +19,7 @@ package org.apache.spark.api.java import java.{lang => jl} import java.lang.{Iterable => JIterable} -import java.util.{Comparator, Iterator => JIterator, List => JList, Map => JMap} +import java.util.{Comparator, Iterator => JIterator, List => JList, Map => JMap, Optional} import scala.collection.JavaConverters._ import scala.reflect.ClassTag diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 09c83849e26b2..b6ce950acc8f3 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -19,7 +19,7 @@ package org.apache.spark.api.java import java.io.Closeable import java.util -import java.util.{Map => JMap} +import java.util.{Map => JMap, Optional} import scala.collection.JavaConverters._ import scala.language.implicitConversions diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala index fd96052f95d3f..16730e89df696 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala @@ -19,6 +19,7 @@ package org.apache.spark.api.java import java.{util => ju} import java.util.Map.Entry +import java.util.Optional import scala.collection.mutable diff --git a/core/src/test/java/org/apache/spark/api/java/OptionalSuite.java b/core/src/test/java/org/apache/spark/api/java/OptionalSuite.java deleted file mode 100644 index 4b97c18198c1a..0000000000000 --- a/core/src/test/java/org/apache/spark/api/java/OptionalSuite.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.api.java; - -import org.junit.Assert; -import org.junit.Test; - -/** - * Tests {@link Optional}. - */ -public class OptionalSuite { - - @Test - public void testEmpty() { - Assert.assertFalse(Optional.empty().isPresent()); - Assert.assertNull(Optional.empty().orNull()); - Assert.assertEquals("foo", Optional.empty().or("foo")); - Assert.assertEquals("foo", Optional.empty().orElse("foo")); - } - - @Test(expected = NullPointerException.class) - public void testEmptyGet() { - Optional.empty().get(); - } - - @Test - public void testAbsent() { - Assert.assertFalse(Optional.absent().isPresent()); - Assert.assertNull(Optional.absent().orNull()); - Assert.assertEquals("foo", Optional.absent().or("foo")); - Assert.assertEquals("foo", Optional.absent().orElse("foo")); - } - - @Test(expected = NullPointerException.class) - public void testAbsentGet() { - Optional.absent().get(); - } - - @Test - public void testOf() { - Assert.assertTrue(Optional.of(1).isPresent()); - Assert.assertNotNull(Optional.of(1).orNull()); - Assert.assertEquals(Integer.valueOf(1), Optional.of(1).get()); - Assert.assertEquals(Integer.valueOf(1), Optional.of(1).or(2)); - Assert.assertEquals(Integer.valueOf(1), Optional.of(1).orElse(2)); - } - - @Test(expected = NullPointerException.class) - public void testOfWithNull() { - Optional.of(null); - } - - @Test - public void testOfNullable() { - Assert.assertTrue(Optional.ofNullable(1).isPresent()); - Assert.assertNotNull(Optional.ofNullable(1).orNull()); - Assert.assertEquals(Integer.valueOf(1), Optional.ofNullable(1).get()); - Assert.assertEquals(Integer.valueOf(1), Optional.ofNullable(1).or(2)); - Assert.assertEquals(Integer.valueOf(1), Optional.ofNullable(1).orElse(2)); - Assert.assertFalse(Optional.ofNullable(null).isPresent()); - Assert.assertNull(Optional.ofNullable(null).orNull()); - Assert.assertEquals(Integer.valueOf(2), Optional.ofNullable(null).or(2)); - Assert.assertEquals(Integer.valueOf(2), Optional.ofNullable(null).orElse(2)); - } - - @Test - public void testFromNullable() { - Assert.assertTrue(Optional.fromNullable(1).isPresent()); - Assert.assertNotNull(Optional.fromNullable(1).orNull()); - Assert.assertEquals(Integer.valueOf(1), Optional.fromNullable(1).get()); - Assert.assertEquals(Integer.valueOf(1), Optional.fromNullable(1).or(2)); - Assert.assertEquals(Integer.valueOf(1), Optional.fromNullable(1).orElse(2)); - Assert.assertFalse(Optional.fromNullable(null).isPresent()); - Assert.assertNull(Optional.fromNullable(null).orNull()); - Assert.assertEquals(Integer.valueOf(2), Optional.fromNullable(null).or(2)); - Assert.assertEquals(Integer.valueOf(2), Optional.fromNullable(null).orElse(2)); - } - -} diff --git a/core/src/test/java/test/org/apache/spark/Java8RDDAPISuite.java b/core/src/test/java/test/org/apache/spark/Java8RDDAPISuite.java index 1d2b05ebc2503..737e8a30eb4d8 100644 --- a/core/src/test/java/test/org/apache/spark/Java8RDDAPISuite.java +++ b/core/src/test/java/test/org/apache/spark/Java8RDDAPISuite.java @@ -37,7 +37,6 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.function.*; import org.apache.spark.util.Utils; diff --git a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java index 01b5fb7b46684..db40ea04d6863 100644 --- a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java @@ -31,6 +31,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.*; import org.apache.spark.Accumulator; @@ -67,7 +68,6 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.function.*; import org.apache.spark.input.PortableDataStream; import org.apache.spark.partial.BoundedDouble; @@ -476,10 +476,10 @@ public void leftOuterJoin() { new Tuple2<>(2, 'z'), new Tuple2<>(4, 'w') )); - List>>> joined = + List>>> joined = rdd1.leftOuterJoin(rdd2).collect(); assertEquals(5, joined.size()); - Tuple2>> firstUnmatched = + Tuple2>> firstUnmatched = rdd1.leftOuterJoin(rdd2).filter(tup -> !tup._2()._2().isPresent()).first(); assertEquals(3, firstUnmatched._1().intValue()); } diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java index 9d8bd7fd11ebd..16f77b38eaa84 100644 --- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java +++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import scala.Tuple2; @@ -26,7 +27,6 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.*; import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.StorageLevels; import org.apache.spark.streaming.Durations; import org.apache.spark.streaming.State; diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala b/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala index dcd698c860d8b..b245cb0127941 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala @@ -17,9 +17,11 @@ package org.apache.spark.streaming +import java.util.Optional + import org.apache.spark.{HashPartitioner, Partitioner} import org.apache.spark.annotation.Experimental -import org.apache.spark.api.java.{JavaPairRDD, JavaUtils, Optional} +import org.apache.spark.api.java.{JavaPairRDD, JavaUtils} import org.apache.spark.api.java.function.{Function3 => JFunction3, Function4 => JFunction4} import org.apache.spark.rdd.RDD import org.apache.spark.util.ClosureCleaner diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala index 2ec907c8cfd5f..0971ca969068f 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala @@ -19,7 +19,7 @@ package org.apache.spark.streaming.api.java import java.{lang => jl} import java.lang.{Iterable => JIterable} -import java.util.{List => JList} +import java.util.{List => JList, Optional} import scala.collection.JavaConverters._ import scala.language.implicitConversions @@ -31,7 +31,7 @@ import org.apache.hadoop.mapreduce.{OutputFormat => NewOutputFormat} import org.apache.spark.Partitioner import org.apache.spark.annotation.Experimental -import org.apache.spark.api.java.{JavaPairRDD, JavaSparkContext, JavaUtils, Optional} +import org.apache.spark.api.java.{JavaPairRDD, JavaSparkContext, JavaUtils} import org.apache.spark.api.java.JavaPairRDD._ import org.apache.spark.api.java.JavaSparkContext.fakeClassTag import org.apache.spark.api.java.function.{Function => JFunction, Function2 => JFunction2} diff --git a/streaming/src/test/java/org/apache/spark/streaming/JavaMapWithStateSuite.java b/streaming/src/test/java/org/apache/spark/streaming/JavaMapWithStateSuite.java index b1367b8f2aed2..738d9324def80 100644 --- a/streaming/src/test/java/org/apache/spark/streaming/JavaMapWithStateSuite.java +++ b/streaming/src/test/java/org/apache/spark/streaming/JavaMapWithStateSuite.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; import scala.Tuple2; @@ -34,7 +35,6 @@ import org.apache.spark.HashPartitioner; import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.function.Function3; import org.apache.spark.api.java.function.Function4; import org.apache.spark.streaming.api.java.JavaPairDStream; diff --git a/streaming/src/test/java/test/org/apache/spark/streaming/Java8APISuite.java b/streaming/src/test/java/test/org/apache/spark/streaming/Java8APISuite.java index 90d1f8c5035b3..e6ed62de3d003 100644 --- a/streaming/src/test/java/test/org/apache/spark/streaming/Java8APISuite.java +++ b/streaming/src/test/java/test/org/apache/spark/streaming/Java8APISuite.java @@ -36,7 +36,6 @@ import org.junit.Test; import org.apache.spark.HashPartitioner; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.PairFunction; diff --git a/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java b/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java index 6c86cacec8279..af31469515e92 100644 --- a/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java +++ b/streaming/src/test/java/test/org/apache/spark/streaming/JavaAPISuite.java @@ -47,7 +47,6 @@ import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.Optional; import org.apache.spark.api.java.function.*; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.api.java.*; From 054c79a9f2940440346bca631bd926a6a4e434b6 Mon Sep 17 00:00:00 2001 From: Mario Molina Date: Thu, 11 Oct 2018 08:34:19 -0500 Subject: [PATCH 2/4] Removing whitespaces --- core/src/test/java/test/org/apache/spark/JavaAPISuite.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java index fe02985102549..f20ea34e70d2e 100644 --- a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java @@ -476,10 +476,10 @@ public void leftOuterJoin() { new Tuple2<>(2, 'z'), new Tuple2<>(4, 'w') )); - List>>> joined = + List>>> joined = rdd1.leftOuterJoin(rdd2).collect(); assertEquals(5, joined.size()); - Tuple2>> firstUnmatched = + Tuple2>> firstUnmatched = rdd1.leftOuterJoin(rdd2).filter(tup -> !tup._2()._2().isPresent()).first(); assertEquals(3, firstUnmatched._1().intValue()); } From 0348a00dcc6e8aae5065e77f9e87b8d10ff5531b Mon Sep 17 00:00:00 2001 From: Mario Molina Date: Thu, 11 Oct 2018 11:40:21 -0500 Subject: [PATCH 3/4] Class exclude for 3.0 version --- project/MimaExcludes.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 0b074fbf64eda..c16f1636ecf6d 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -36,6 +36,8 @@ object MimaExcludes { // Exclude rules for 3.0.x lazy val v30excludes = v24excludes ++ Seq( + // [SPARK-25362][JavaAPI] Replace Spark Optional class with Java Optional + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.java.Optional") ) // Exclude rules for 2.4.x From f1882433071ccb9661a5e41d0ff3358d3ea6d5a3 Mon Sep 17 00:00:00 2001 From: Mario Molina Date: Sun, 14 Oct 2018 22:03:07 -0500 Subject: [PATCH 4/4] Updating mima excludes --- project/MimaExcludes.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index df3a751c19e38..2f3b421629bab 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -36,11 +36,15 @@ object MimaExcludes { // Exclude rules for 3.0.x lazy val v30excludes = v24excludes ++ Seq( - // [SPARK-25362][JavaAPI] Replace Spark Optional class with Java Optional - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.java.Optional") + // [SPARK-19287] JavaPairRDD flatMapValues requires function returning Iterable, not Iterator ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.java.JavaPairRDD.flatMapValues"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.streaming.api.java.JavaPairDStream.flatMapValues") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.streaming.api.java.JavaPairDStream.flatMapValues"), + + // [SPARK-24109] Remove class SnappyOutputStreamWrapper ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.io.SnappyCompressionCodec.version"), + + // [SPARK-25362][JavaAPI] Replace Spark Optional class with Java Optional + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.java.Optional") ) // Exclude rules for 2.4.x