From 21c4d5cdf94b7c918f64bd6ad415ec6def91d9ed Mon Sep 17 00:00:00 2001 From: Elva Liu Date: Thu, 5 Dec 2019 14:52:30 -0800 Subject: [PATCH] remove toPythonIterator for Spark 3.0 --- .../Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs | 6 +++++- src/csharp/Microsoft.Spark/Sql/DataFrame.cs | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs index f6f3e87d1..11467c5f2 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs @@ -487,7 +487,11 @@ public void TestSignaturesV2_3_X() Assert.IsType(_df.Collect().ToArray()); - Assert.IsType(_df.ToLocalIterator().ToArray()); + if (SparkSettings.Version < new Version(Versions.V3_0_0)) + { + // The following APIs are removed in Spark 3.0. + Assert.IsType(_df.ToLocalIterator().ToArray()); + } Assert.IsType(_df.Count()); diff --git a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs index 53bf85c7f..84e745eb2 100644 --- a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs +++ b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs @@ -714,7 +714,11 @@ public IEnumerable Collect() /// Returns an iterator that contains all of the rows in this `DataFrame`. /// The iterator will consume as much memory as the largest partition in this `DataFrame`. /// + /// + /// This API is removed in Spark 3.0. + /// /// Row objects + [Removed(Versions.V3_0_0)] public IEnumerable ToLocalIterator() { return GetRows("toPythonIterator");