From 98feb920dd86ec8e4a3ced37b4dc799663fa1f4c Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 21 Nov 2019 23:34:00 -0800 Subject: [PATCH 01/11] initial checkin --- azure-pipelines.yml | 11 +++++++++++ script/download-spark-distros.cmd | 1 + src/csharp/Microsoft.Spark.E2ETest/SparkSettings.cs | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 94896d2e7..9c1519e33 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -156,6 +156,17 @@ jobs: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64 +- task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 3.0.0-preview' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-preview-bin-hadoop2.7 + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64 + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 displayName: Stage .NET artifacts diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index a10118085..0e712cdd5 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -22,5 +22,6 @@ curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/ curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz +curl -k -L -o spark-3.0.0-preview.tgz https://archive.apache.org/dist/spark/spark-3.0.0-preview/spark-3.0.0-preview-bin-hadoop2.7.tgz && tar xzvf spark-3.0.0-preview.tgz endlocal \ No newline at end of file diff --git a/src/csharp/Microsoft.Spark.E2ETest/SparkSettings.cs b/src/csharp/Microsoft.Spark.E2ETest/SparkSettings.cs index 2bbb383b6..a568586d3 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/SparkSettings.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/SparkSettings.cs @@ -35,7 +35,13 @@ private static void InitVersion() // Spark 2.3.2 built for Hadoop 2.7.3 string firstLine = File.ReadLines($"{SparkHome}{Path.DirectorySeparatorChar}RELEASE").First(); - Version = new Version(firstLine.Split(' ')[1]); + + // Grab "2.3.2" from "Spark 2.3.2 built for Hadoop 2.7.3" + string versionStr = firstLine.Split(' ')[1]; + + // Strip anything below version number. + // For example, "3.0.0-preview" should become "3.0.0". + Version = new Version(versionStr.Split('-')[0]); } } } From 2c443a074e7ef5278bf3410f726b9757dc084fe9 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 21 Nov 2019 23:44:25 -0800 Subject: [PATCH 02/11] fix indentation --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9c1519e33..e7efe4ff4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -156,7 +156,7 @@ jobs: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64 -- task: DotNetCoreCLI@2 + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 3.0.0-preview' inputs: command: test From 9abaca6368a3896c8fb392b3ff78fdecc2357ed1 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sat, 5 Sep 2020 22:09:44 -0700 Subject: [PATCH 03/11] fix conflict merge --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e0b2fc135..ea7464a54 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,7 +76,6 @@ stages: projects: '**/*UnitTest/*.csproj' arguments: '--configuration $(buildConfiguration)' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 displayName: Stage Maven build jars inputs: From 2c01babcc891200949e1b5ff0136256758ceb580 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sat, 5 Sep 2020 23:01:13 -0700 Subject: [PATCH 04/11] Use official Spark 3.0 distro --- azure-pipelines.yml | 12 ++++++------ script/download-spark-distros.cmd | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea7464a54..eee21570d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -285,13 +285,13 @@ stages: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.6-bin-hadoop2.7 - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 3.0.0-preview' + displayName: 'E2E tests for Spark 3.0.0' inputs: command: test projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-preview-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7 - stage: ForwardCompatibility displayName: E2E Forward Compatibility Tests @@ -418,13 +418,13 @@ stages: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 3.0.0-preview' + displayName: 'E2E tests for Spark 3.0.0' inputs: command: test projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-preview-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7 - stage: BackwardCompatibility displayName: E2E Backward Compatibility Tests @@ -565,11 +565,11 @@ stages: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.6-bin-hadoop2.7 - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 3.0.0-preview' + displayName: 'E2E tests for Spark 3.0.0' inputs: command: test projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-preview-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7 diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index 6296d41db..c0f3e5064 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -24,6 +24,6 @@ curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/ curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz curl -k -L -o spark-2.4.6.tgz https://archive.apache.org/dist/spark/spark-2.4.6/spark-2.4.6-bin-hadoop2.7.tgz && tar xzvf spark-2.4.6.tgz -curl -k -L -o spark-3.0.0-preview.tgz https://archive.apache.org/dist/spark/spark-3.0.0-preview/spark-3.0.0-preview-bin-hadoop2.7.tgz && tar xzvf spark-3.0.0-preview.tgz +curl -k -L -o spark-3.0.0.tgz https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && tar xzvf spark-3.0.0.tgz endlocal From 892e4a9d63718bce7f590167f364ab0c3eea263c Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sat, 5 Sep 2020 23:18:53 -0700 Subject: [PATCH 05/11] Remove 3.0 for backward compatibility tests --- azure-pipelines.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eee21570d..c3aca811d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -417,15 +417,6 @@ stages: env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 3.0.0' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7 - - stage: BackwardCompatibility displayName: E2E Backward Compatibility Tests dependsOn: Build From a1286fae3447dff27296e25773dd75be04cb9db0 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 13:50:58 -0700 Subject: [PATCH 06/11] fixes --- .../DeltaFixture.cs | 10 ++++++- .../DeltaTableTests.cs | 4 +-- .../HyperspaceFixture.cs | 10 ++++++- .../HyperspaceTests.cs | 4 +-- .../IpcTests/BroadcastTests.cs | 7 ++--- .../IpcTests/Sql/DataFrameTests.cs | 8 +++--- .../Utils/VersionBasedFacts.cs | 27 ++++++++++++++++++- src/scala/microsoft-spark-3.0.x/pom.xml | 2 +- src/scala/pom.xml | 1 + 9 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaFixture.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaFixture.cs index 9ca3851f0..d78b6d68d 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaFixture.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaFixture.cs @@ -14,9 +14,17 @@ public class DeltaFixture public DeltaFixture() { + Version sparkVersion = SparkSettings.Version; + string deltaVersion = sparkVersion.Major switch + { + 2 => "delta-core_2.11:0.6.1", + 3 => "delta-core_2.12:0.7.0", + _ => throw new NotSupportedException($"Spark {sparkVersion} not supported.") + }; + Environment.SetEnvironmentVariable( SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs, - "--packages io.delta:delta-core_2.11:0.6.1 " + + $"--packages io.delta:{deltaVersion} " + "--conf spark.databricks.delta.snapshotPartitions=2 " + "--conf spark.sql.sources.parallelPartitionDiscovery.parallelism=5"); SparkFixture = new SparkFixture(); diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs index fab7c74dc..d13c76c36 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs @@ -30,7 +30,7 @@ public DeltaTableTests(DeltaFixture fixture) /// Run the end-to-end scenario from the Delta Quickstart tutorial. /// /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_2)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_0_0)] public void TestTutorialScenario() { using var tempDirectory = new TemporaryDirectory(); @@ -223,7 +223,7 @@ void testWrapper( /// /// Test that methods return the expected signature. /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_2)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_0_0)] public void TestSignatures() { using var tempDirectory = new TemporaryDirectory(); diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs index 8578c77f0..6d33ee66f 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceFixture.cs @@ -12,9 +12,17 @@ public class HyperspaceFixture { public HyperspaceFixture() { + Version sparkVersion = SparkSettings.Version; + string hyperspaceVersion = sparkVersion.Major switch + { + 2 => "hyperspace-core_2.11:0.2.0", + 3 => "hyperspace-core_2.12:0.2.0", + _ => throw new NotSupportedException($"Spark {sparkVersion} not supported.") + }; + Environment.SetEnvironmentVariable( SparkFixture.EnvironmentVariableNames.ExtraSparkSubmitArgs, - "--packages com.microsoft.hyperspace:hyperspace-core_2.11:0.1.0"); + $"--packages com.microsoft.hyperspace:{hyperspaceVersion}"); SparkFixture = new SparkFixture(); } diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs index 12e8bca60..4d9cd1242 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs @@ -53,7 +53,7 @@ public void Dispose() /// /// Test the method signatures for all Hyperspace APIs. /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_0, Versions.V3_0_0)] public void TestSignatures() { // Indexes API. @@ -116,7 +116,7 @@ public void TestIndexCreateAndDelete() /// /// Test that the explain API generates the expected string. /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_0, Versions.V3_0_0)] public void TestExplainAPI() { // Run a query that hits the index. diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/BroadcastTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/BroadcastTests.cs index 511f5a122..ef6ea71b3 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/BroadcastTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/BroadcastTests.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using Microsoft.Spark.E2ETest.Utils; using Microsoft.Spark.Sql; using Xunit; using static Microsoft.Spark.Sql.Functions; @@ -34,7 +35,7 @@ public BroadcastTests(SparkFixture fixture) /// /// Test Broadcast support by using multiple broadcast variables in a UDF. /// - [Fact] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestMultipleBroadcastWithoutEncryption() { var obj1 = new TestBroadcastVariable(1, "first"); @@ -55,7 +56,7 @@ public void TestMultipleBroadcastWithoutEncryption() /// Test Broadcast.Destroy() that destroys all data and metadata related to the broadcast /// variable and makes it inaccessible from workers. /// - [Fact] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestDestroy() { var obj1 = new TestBroadcastVariable(5, "destroy"); @@ -96,7 +97,7 @@ public void TestDestroy() /// Test Broadcast.Unpersist() deletes cached copies of the broadcast on the executors. If /// the broadcast is used after unpersist is called, it is re-sent to the executors. /// - [Fact] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestUnpersist() { var obj = new TestBroadcastVariable(1, "unpersist"); diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs index f036ad346..e04ef1087 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs @@ -156,7 +156,7 @@ public void TestUDF() } } - [Fact] + [SkipIfSparkVersionIsLessThan(Versions.V3_0_0)] public void TestVectorUdf() { Func udf1Func = @@ -224,7 +224,7 @@ public void TestVectorUdf() } } - [Fact] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestDataFrameVectorUdf() { Func udf1Func = @@ -290,7 +290,7 @@ public void TestDataFrameVectorUdf() } } - [Fact] + [SkipIfSparkVersionIsLessThan(Versions.V3_0_0)] public void TestGroupedMapUdf() { DataFrame df = _spark @@ -369,7 +369,7 @@ private static RecordBatch ArrowBasedCountCharacters(RecordBatch records) } - [Fact] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestDataFrameGroupedMapUdf() { DataFrame df = _spark diff --git a/src/csharp/Microsoft.Spark.E2ETest/Utils/VersionBasedFacts.cs b/src/csharp/Microsoft.Spark.E2ETest/Utils/VersionBasedFacts.cs index fa2be4a6b..137dbd1b1 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/Utils/VersionBasedFacts.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/Utils/VersionBasedFacts.cs @@ -13,7 +13,32 @@ public SkipIfSparkVersionIsLessThan(string version) { if (SparkSettings.Version < new Version(version)) { - Skip = $"Ignore on Spark version ({SparkSettings.Version}) <= {version}"; + Skip = $"Ignore on Spark version ({SparkSettings.Version}) < {version}"; + } + } + } + + public sealed class SkipIfSparkVersionIsGreaterOrEqualTo : FactAttribute + { + public SkipIfSparkVersionIsGreaterOrEqualTo(string version) + { + if (SparkSettings.Version >= new Version(version)) + { + Skip = $"Ignore on Spark version ({SparkSettings.Version}) >= {version}"; + } + } + } + + // Skip if the spark version is not in range [minVersion, maxVersion). + public sealed class SkipIfSparkVersionIsNotInRange : FactAttribute + { + public SkipIfSparkVersionIsNotInRange(string minInclusive, string maxExclusive) + { + if (SparkSettings.Version < new Version(minInclusive) || + SparkSettings.Version >= new Version(maxExclusive)) + { + Skip = $"Ignore on Spark version ({SparkSettings.Version}) not in range of " + + $"[{minInclusive}, {maxExclusive})"; } } } diff --git a/src/scala/microsoft-spark-3.0.x/pom.xml b/src/scala/microsoft-spark-3.0.x/pom.xml index 4eeaeb764..179239b67 100644 --- a/src/scala/microsoft-spark-3.0.x/pom.xml +++ b/src/scala/microsoft-spark-3.0.x/pom.xml @@ -12,7 +12,7 @@ UTF-8 2.12.8 2.12 - 3.0.0-preview + 3.0.0 diff --git a/src/scala/pom.xml b/src/scala/pom.xml index 035221cd4..89254a8c4 100644 --- a/src/scala/pom.xml +++ b/src/scala/pom.xml @@ -13,6 +13,7 @@ microsoft-spark-2.3.x microsoft-spark-2.4.x + microsoft-spark-3.0.x From 719ba7533d0c1e7a5092e956688256cb114950b6 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 13:52:30 -0700 Subject: [PATCH 07/11] minor fix --- .../Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs index e04ef1087..f154f3380 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs @@ -290,7 +290,7 @@ public void TestDataFrameVectorUdf() } } - [SkipIfSparkVersionIsLessThan(Versions.V3_0_0)] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestGroupedMapUdf() { DataFrame df = _spark From bee8af5743bd5b34d02cba85dfb89d23a061c229 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 13:57:17 -0700 Subject: [PATCH 08/11] fix --- .../Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs index f154f3380..e830a7e42 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameTests.cs @@ -156,7 +156,7 @@ public void TestUDF() } } - [SkipIfSparkVersionIsLessThan(Versions.V3_0_0)] + [SkipIfSparkVersionIsGreaterOrEqualTo(Versions.V3_0_0)] public void TestVectorUdf() { Func udf1Func = From 9c04728f3013e446e686f72572d51b4f0b2c534a Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 14:56:56 -0700 Subject: [PATCH 09/11] disable hyperspace tests --- .../HyperspaceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs index 4d9cd1242..41619cc68 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs @@ -86,7 +86,7 @@ public void TestSignatures() /// /// Test E2E functionality of index CRUD APIs. /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_0, Versions.V3_0_0)] public void TestIndexCreateAndDelete() { // Should be one active index. From d4509f9ac1bebe3e30d50565b79eb154499ecf75 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 16:02:58 -0700 Subject: [PATCH 10/11] filter for backward compat tests for Spark 3.0 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4df5c1b48..0c2d64843 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -563,7 +563,7 @@ stages: inputs: command: test projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' + arguments: '--configuration $(buildConfiguration)' --filter $(TestsToFilterOut)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7 From 0dad47298c1c708fb8f0c93fe21567a5855a1b1d Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 6 Sep 2020 16:38:44 -0700 Subject: [PATCH 11/11] fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0c2d64843..d4ee2238c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -563,7 +563,7 @@ stages: inputs: command: test projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' --filter $(TestsToFilterOut)' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7