diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 7a3e601553dc0..21c1e0fee1ddf 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -36,11 +36,6 @@ hadoop-cloud - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - @@ -83,33 +78,6 @@ hadoop-aws ${hadoop.version} ${hadoop.deps.scope} - - - org.apache.hadoop - hadoop-common - - - commons-logging - commons-logging - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - - com.amazonaws - aws-java-sdk - - com.google.cloud.bigdataoss @@ -166,153 +134,79 @@ hadoop-azure ${hadoop.version} ${hadoop.deps.scope} + + + + org.apache.hadoop + hadoop-cloud-storage + ${hadoop.version} + ${hadoop.deps.scope} - org.apache.hadoop - hadoop-common - - - com.fasterxml.jackson.core - jackson-core + + org.jacoco + org.jacoco.agent + - com.google.guava - guava + org.apache.hadoop + hadoop-cos + + + org.eclipse.jetty + jetty-util + ${hadoop.deps.scope} + + + org.eclipse.jetty + jetty-util-ajax + ${jetty.version} + ${hadoop.deps.scope} + - - - - hadoop-3 - - true - - - src/hadoop-3/main/java - src/hadoop-3/test/java - src/hadoop-3/main/scala - src/hadoop-3/test/scala - + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes - - - - org.scalatest - scalatest-maven-plugin - - - test - test - - test - - - org.apache.spark.internal.io.cloud.IntegrationTestSuite - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-extra-sources - generate-sources - - add-source - - - - ${extra.java.source.dir} - ${extra.scala.source.dir} - - - - - add-extra-test-sources - generate-test-sources - - add-test-source - - - - ${extra.java.testsource.dir} - ${extra.scala.testsource.dir} - - - - - - - - - - - org.apache.hadoop - hadoop-cloud-storage - ${hadoop.version} - ${hadoop.deps.scope} - - - org.apache.hadoop - hadoop-common - - - com.fasterxml.jackson.core - jackson-core - - - com.google.guava - guava - - - - org.jacoco - org.jacoco.agent - - - - org.apache.hadoop - hadoop-cos - - - - - - org.eclipse.jetty - jetty-util - ${hadoop.deps.scope} - - - org.eclipse.jetty - jetty-util-ajax - ${jetty.version} - ${hadoop.deps.scope} - - - + + + org.scalatest + scalatest-maven-plugin + + + test + test + + test + + + org.apache.spark.internal.io.cloud.IntegrationTestSuite + + + + + + + integration-test diff --git a/hadoop-cloud/src/hadoop-3/test/resources/log4j2.properties b/hadoop-cloud/src/hadoop-3/test/resources/log4j2.properties deleted file mode 100644 index 01a9cafafa85e..0000000000000 --- a/hadoop-cloud/src/hadoop-3/test/resources/log4j2.properties +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the file target/unit-tests.log -rootLogger.level = info -rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} - -appender.file.type = File -appender.file.name = File -appender.file.fileName = target/unit-tests.log -appender.file.append = true -appender.file.layout.type = PatternLayout -appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex - -# Tests that launch java subprocesses can set the "test.appender" system property to -# "console" to avoid having the child process's logs overwrite the unit test's -# log file. -appender.console.type = Console -appender.console.name = STDERR -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %t: %m%n%ex - -# Ignore messages below warning level from Jetty, because it's a bit verbose -logger.jetty.name = org.spark_project.jetty -logger.jetty.level = warn diff --git a/hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala b/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala rename to hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala diff --git a/hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/BindingParquetOutputCommitter.scala b/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/BindingParquetOutputCommitter.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/BindingParquetOutputCommitter.scala rename to hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/BindingParquetOutputCommitter.scala diff --git a/hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/PathOutputCommitProtocol.scala b/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/PathOutputCommitProtocol.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/main/scala/org/apache/spark/internal/io/cloud/PathOutputCommitProtocol.scala rename to hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/PathOutputCommitProtocol.scala diff --git a/hadoop-cloud/src/hadoop-3/test/java/org/apache/spark/internal/io/cloud/abortable/AbortableFileSystem.java b/hadoop-cloud/src/test/java/org/apache/spark/internal/io/cloud/abortable/AbortableFileSystem.java similarity index 100% rename from hadoop-cloud/src/hadoop-3/test/java/org/apache/spark/internal/io/cloud/abortable/AbortableFileSystem.java rename to hadoop-cloud/src/test/java/org/apache/spark/internal/io/cloud/abortable/AbortableFileSystem.java diff --git a/hadoop-cloud/src/hadoop-3/test/java/org/apache/spark/internal/io/cloud/abortable/AbstractAbortableFileSystem.java b/hadoop-cloud/src/test/java/org/apache/spark/internal/io/cloud/abortable/AbstractAbortableFileSystem.java similarity index 100% rename from hadoop-cloud/src/hadoop-3/test/java/org/apache/spark/internal/io/cloud/abortable/AbstractAbortableFileSystem.java rename to hadoop-cloud/src/test/java/org/apache/spark/internal/io/cloud/abortable/AbstractAbortableFileSystem.java diff --git a/hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala rename to hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala diff --git a/hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala rename to hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala diff --git a/hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/StubPathOutputCommitter.scala b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/StubPathOutputCommitter.scala similarity index 100% rename from hadoop-cloud/src/hadoop-3/test/scala/org/apache/spark/internal/io/cloud/StubPathOutputCommitter.scala rename to hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/StubPathOutputCommitter.scala