From 106cefe5ad881d5b086589a8683921f6501975eb Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Tue, 26 Apr 2016 11:43:26 -0700 Subject: [PATCH 1/4] [BEAM-151] Move over some more Dataflow specific classes. Note that users should use proto ByteString instead of RandomAccessData since it provides a safer version of the same functionality. I hoped that I would be able to move over more of the *Cloud* classes and their helpers but they are embedded part of coders. Nothing more can be done here until there is an official Beam representation of a coder decoupled from Dataflow CloudKnownTypes. --- runners/google-cloud-dataflow-java/pom.xml | 6 ++++++ .../beam/sdk/runners/worker/IsmFormat.java | 0 .../beam/sdk/runners/worker/package-info.java | 0 .../org/apache/beam/sdk/util/DoFnInfo.java | 0 .../apache/beam/sdk/util/OutputReference.java | 0 .../beam/sdk/util/RandomAccessData.java | 0 .../org/apache/beam/sdk/util/TimeUtil.java | 0 .../beam/sdk/util/RandomAccessDataTest.java | 0 .../apache/beam/sdk/util/TimeUtilTest.java | 0 .../sdk/util/common/worker/package-info.java | 19 ------------------- 10 files changed, 6 insertions(+), 19 deletions(-) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/util/OutputReference.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/main/java/org/apache/beam/sdk/util/TimeUtil.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java (100%) rename {sdks/java/core => runners/google-cloud-dataflow-java}/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java (100%) delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/worker/package-info.java diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml index beb340c44201..23b7f5fc881b 100644 --- a/runners/google-cloud-dataflow-java/pom.xml +++ b/runners/google-cloud-dataflow-java/pom.xml @@ -524,6 +524,12 @@ 1.0-rc2 true + + com.google.auto.value + auto-value + 1.1 + provided + diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/OutputReference.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/OutputReference.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/OutputReference.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/OutputReference.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimeUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/TimeUtil.java similarity index 100% rename from sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimeUtil.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/TimeUtil.java diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java similarity index 100% rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java similarity index 100% rename from sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/worker/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/worker/package-info.java deleted file mode 100644 index f295419cc386..000000000000 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/common/worker/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** Defines utilities used to implement the harness that runs user code. **/ -package org.apache.beam.sdk.util.common.worker; From 28b58426e8b22f2047998239a03b85dbda364ef6 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Wed, 27 Apr 2016 10:35:33 -0700 Subject: [PATCH 2/4] [BEAM-151] Rebase onto apache/master updating packages to new structure --- .../runners/dataflow/DataflowPipelineJob.java | 2 +- .../dataflow/DataflowPipelineRunner.java | 8 +++--- .../dataflow/DataflowPipelineTranslator.java | 4 +-- .../dataflow/internal}/IsmFormat.java | 4 +-- .../dataflow}/util/DoFnInfo.java | 3 ++- .../runners/dataflow/util/MonitoringUtil.java | 3 +-- .../dataflow}/util/OutputReference.java | 2 +- .../dataflow}/util/RandomAccessData.java | 3 ++- .../dataflow}/util/TimeUtil.java | 2 +- .../beam/sdk/runners/worker/package-info.java | 25 ------------------- .../dataflow/DataflowPipelineRunnerTest.java | 8 +++--- .../DataflowPipelineTranslatorTest.java | 2 +- .../TestDataflowPipelineRunnerTest.java | 2 +- .../dataflow/util/MonitoringUtilTest.java | 1 - .../dataflow}/util/RandomAccessDataTest.java | 4 +-- .../dataflow}/util/TimeUtilTest.java | 10 ++++---- 16 files changed, 29 insertions(+), 54 deletions(-) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/{sdk/runners/worker => runners/dataflow/internal}/IsmFormat.java (99%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/{sdk => runners/dataflow}/util/DoFnInfo.java (95%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/{sdk => runners/dataflow}/util/OutputReference.java (96%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/{sdk => runners/dataflow}/util/RandomAccessData.java (99%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/{sdk => runners/dataflow}/util/TimeUtil.java (99%) delete mode 100644 runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/{sdk => runners/dataflow}/util/RandomAccessDataTest.java (98%) rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/{sdk => runners/dataflow}/util/TimeUtilTest.java (89%) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java index 19df0a194be5..69565ac6573e 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow; -import static org.apache.beam.sdk.util.TimeUtil.fromCloudTime; +import static org.apache.beam.runners.dataflow.util.TimeUtil.fromCloudTime; import org.apache.beam.runners.dataflow.internal.DataflowAggregatorTransforms; import org.apache.beam.runners.dataflow.internal.DataflowMetricUpdateExtractor; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java index ec4a60c305be..2f01101a62f2 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java @@ -29,6 +29,10 @@ import org.apache.beam.runners.dataflow.DataflowPipelineTranslator.TranslationContext; import org.apache.beam.runners.dataflow.internal.AssignWindows; import org.apache.beam.runners.dataflow.internal.DataflowAggregatorTransforms; +import org.apache.beam.runners.dataflow.internal.IsmFormat; +import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecord; +import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecordCoder; +import org.apache.beam.runners.dataflow.internal.IsmFormat.MetadataKeyCoder; import org.apache.beam.runners.dataflow.internal.PubsubIOTranslator; import org.apache.beam.runners.dataflow.internal.ReadTranslator; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; @@ -70,10 +74,6 @@ import org.apache.beam.sdk.runners.AggregatorPipelineExtractor; import org.apache.beam.sdk.runners.PipelineRunner; import org.apache.beam.sdk.runners.TransformTreeNode; -import org.apache.beam.sdk.runners.worker.IsmFormat; -import org.apache.beam.sdk.runners.worker.IsmFormat.IsmRecord; -import org.apache.beam.sdk.runners.worker.IsmFormat.IsmRecordCoder; -import org.apache.beam.sdk.runners.worker.IsmFormat.MetadataKeyCoder; import org.apache.beam.sdk.transforms.Aggregator; import org.apache.beam.sdk.transforms.Combine; import org.apache.beam.sdk.transforms.Combine.CombineFn; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java index 0f2d325a6982..4ef1bdb9878d 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java @@ -35,6 +35,8 @@ import org.apache.beam.runners.dataflow.internal.PubsubIOTranslator; import org.apache.beam.runners.dataflow.internal.ReadTranslator; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.runners.dataflow.util.DoFnInfo; +import org.apache.beam.runners.dataflow.util.OutputReference; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.Pipeline.PipelineVisitor; import org.apache.beam.sdk.coders.Coder; @@ -57,8 +59,6 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.util.AppliedCombineFn; import org.apache.beam.sdk.util.CloudObject; -import org.apache.beam.sdk.util.DoFnInfo; -import org.apache.beam.sdk.util.OutputReference; import org.apache.beam.sdk.util.PropertyNames; import org.apache.beam.sdk.util.SerializableUtils; import org.apache.beam.sdk.util.WindowedValue; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java similarity index 99% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java index 8df46dd4149a..1969cfb79b49 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/IsmFormat.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.runners.worker; +package org.apache.beam.runners.dataflow.internal; import static org.apache.beam.sdk.util.Structs.addLong; @@ -23,6 +23,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import org.apache.beam.runners.dataflow.util.RandomAccessData; import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.coders.Coder; @@ -34,7 +35,6 @@ import org.apache.beam.sdk.coders.VarLongCoder; import org.apache.beam.sdk.util.CloudObject; import org.apache.beam.sdk.util.PropertyNames; -import org.apache.beam.sdk.util.RandomAccessData; import org.apache.beam.sdk.util.VarInt; import org.apache.beam.sdk.values.PCollection; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java similarity index 95% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java index ae19a171c80f..f83acbcb8754 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/DoFnInfo.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.util.WindowingStrategy; import org.apache.beam.sdk.values.PCollectionView; import java.io.Serializable; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java index 2eec9ccf9aa8..67cdfa691e11 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java @@ -17,11 +17,10 @@ */ package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.sdk.util.TimeUtil.fromCloudTime; +import static org.apache.beam.runners.dataflow.util.TimeUtil.fromCloudTime; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.sdk.util.TimeUtil; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Messages; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/OutputReference.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java similarity index 96% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/OutputReference.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java index 5e3017223d11..1b525ac4ef89 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/OutputReference.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/OutputReference.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; import static com.google.api.client.util.Preconditions.checkNotNull; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java similarity index 99% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java index 2d902f49f53e..9e102422b7db 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/RandomAccessData.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -24,6 +24,7 @@ import org.apache.beam.sdk.coders.ByteArrayCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.util.VarInt; import com.google.common.base.MoreObjects; import com.google.common.io.ByteStreams; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/TimeUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java similarity index 99% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/TimeUtil.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java index db5c760858a2..434c83f7da22 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/util/TimeUtil.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/TimeUtil.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; import org.joda.time.DateTime; import org.joda.time.Duration; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java deleted file mode 100644 index 61331480986a..000000000000 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/sdk/runners/worker/package-info.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Implementation of the harness that runs on each Google Compute Engine instance to coordinate - * execution of Pipeline code. - */ -@ParametersAreNonnullByDefault -package org.apache.beam.sdk.runners.worker; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java index 79e281e70f73..d4d4b3b73186 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java @@ -42,6 +42,10 @@ import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsMap; import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsMultimap; import org.apache.beam.runners.dataflow.DataflowPipelineRunner.TransformedMap; +import org.apache.beam.runners.dataflow.internal.IsmFormat; +import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecord; +import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecordCoder; +import org.apache.beam.runners.dataflow.internal.IsmFormat.MetadataKeyCoder; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; @@ -60,10 +64,6 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.runners.TransformTreeNode; import org.apache.beam.sdk.runners.dataflow.TestCountingSource; -import org.apache.beam.sdk.runners.worker.IsmFormat; -import org.apache.beam.sdk.runners.worker.IsmFormat.IsmRecord; -import org.apache.beam.sdk.runners.worker.IsmFormat.IsmRecordCoder; -import org.apache.beam.sdk.runners.worker.IsmFormat.MetadataKeyCoder; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.DoFnTester; import org.apache.beam.sdk.transforms.PTransform; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java index 3a39e41e2b7b..7d3d579d3a36 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java @@ -38,6 +38,7 @@ import org.apache.beam.runners.dataflow.DataflowPipelineTranslator.TranslationContext; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; +import org.apache.beam.runners.dataflow.util.OutputReference; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StringUtf8Coder; @@ -55,7 +56,6 @@ import org.apache.beam.sdk.transforms.View; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.util.GcsUtil; -import org.apache.beam.sdk.util.OutputReference; import org.apache.beam.sdk.util.PropertyNames; import org.apache.beam.sdk.util.Structs; import org.apache.beam.sdk.util.TestCredential; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java index d6de50158c55..a45284c9afa9 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java @@ -34,6 +34,7 @@ import org.apache.beam.runners.dataflow.DataflowPipelineRunner; import org.apache.beam.runners.dataflow.util.MonitoringUtil; import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler; +import org.apache.beam.runners.dataflow.util.TimeUtil; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult.State; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -43,7 +44,6 @@ import org.apache.beam.sdk.util.GcsUtil; import org.apache.beam.sdk.util.NoopPathValidator; import org.apache.beam.sdk.util.TestCredential; -import org.apache.beam.sdk.util.TimeUtil; import org.apache.beam.sdk.util.Transport; import org.apache.beam.sdk.values.PCollection; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java index ee1532da80e3..4b0ab2f504b9 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/MonitoringUtilTest.java @@ -26,7 +26,6 @@ import org.apache.beam.sdk.PipelineResult.State; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.TestCredential; -import org.apache.beam.sdk.util.TimeUtil; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.JobMessage; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java similarity index 98% rename from runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java index b9902124ae9b..c5c77cfaab16 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/RandomAccessDataTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/RandomAccessDataTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -23,10 +23,10 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import org.apache.beam.runners.dataflow.util.RandomAccessData.RandomAccessDataCoder; import org.apache.beam.sdk.coders.Coder.Context; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.testing.CoderProperties; -import org.apache.beam.sdk.util.RandomAccessData.RandomAccessDataCoder; import com.google.common.primitives.UnsignedBytes; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java similarity index 89% rename from runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java index b318dee05d2f..c22c223e2747 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/sdk/util/TimeUtilTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/TimeUtilTest.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.util; +package org.apache.beam.runners.dataflow.util; -import static org.apache.beam.sdk.util.TimeUtil.fromCloudDuration; -import static org.apache.beam.sdk.util.TimeUtil.fromCloudTime; -import static org.apache.beam.sdk.util.TimeUtil.toCloudDuration; -import static org.apache.beam.sdk.util.TimeUtil.toCloudTime; +import static org.apache.beam.runners.dataflow.util.TimeUtil.fromCloudDuration; +import static org.apache.beam.runners.dataflow.util.TimeUtil.fromCloudTime; +import static org.apache.beam.runners.dataflow.util.TimeUtil.toCloudDuration; +import static org.apache.beam.runners.dataflow.util.TimeUtil.toCloudTime; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; From 06fe253f55b84fe6732ed3a9d65ba65784ac5836 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Wed, 27 Apr 2016 16:55:56 -0700 Subject: [PATCH 3/4] [BEAM-151] Update worker image name that corresponds to class moves. --- .../apache/beam/runners/dataflow/DataflowPipelineRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java index 2f01101a62f2..41b4df7717b2 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java @@ -217,9 +217,9 @@ public class DataflowPipelineRunner extends PipelineRunner // Default Docker container images that execute Dataflow worker harness, residing in Google // Container Registry, separately for Batch and Streaming. public static final String BATCH_WORKER_HARNESS_CONTAINER_IMAGE - = "dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20160426"; + = "dataflow.gcr.io/v1beta3/beam-java-batch:beam-master-20160427"; public static final String STREAMING_WORKER_HARNESS_CONTAINER_IMAGE - = "dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20160426"; + = "dataflow.gcr.io/v1beta3/beam-java-streaming:beam-master-20160427"; // The limit of CreateJob request size. private static final int CREATE_JOB_REQUEST_LIMIT_BYTES = 10 * 1024 * 1024; From 243ed435924422024b8499801f253c1a4b83487d Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Wed, 27 Apr 2016 17:00:30 -0700 Subject: [PATCH 4/4] [BEAM-151] !fixup Fix whitespace in pom files. --- runners/google-cloud-dataflow-java/pom.xml | 1 + sdks/java/core/pom.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml index 23b7f5fc881b..3dd9cb8499db 100644 --- a/runners/google-cloud-dataflow-java/pom.xml +++ b/runners/google-cloud-dataflow-java/pom.xml @@ -524,6 +524,7 @@ 1.0-rc2 true + com.google.auto.value auto-value diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml index 07d2fce78cef..0d530e1ae3b4 100644 --- a/sdks/java/core/pom.xml +++ b/sdks/java/core/pom.xml @@ -697,6 +697,7 @@ 1.0-rc2 true + com.google.auto.value auto-value