From c9d555a3f4e3eb21eefe0868f4a0578058156c05 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 12:56:34 -0700 Subject: [PATCH 01/17] Isort the libraries --- sdks/python/apache_beam/coders/coder_impl.py | 4 +- sdks/python/apache_beam/coders/coders.py | 4 +- sdks/python/apache_beam/coders/coders_test.py | 2 +- .../apache_beam/coders/coders_test_common.py | 12 +++-- .../apache_beam/coders/fast_coders_test.py | 2 - .../apache_beam/coders/observable_test.py | 1 - .../coders/proto2_coder_test_messages_pb2.py | 4 +- .../apache_beam/coders/slow_coders_test.py | 2 - .../coders/standard_coders_test.py | 7 ++- sdks/python/apache_beam/coders/stream_test.py | 1 - sdks/python/apache_beam/coders/typecoders.py | 1 - .../apache_beam/coders/typecoders_test.py | 3 +- .../examples/complete/autocomplete.py | 6 +-- .../examples/complete/autocomplete_test.py | 3 +- .../examples/complete/estimate_pi.py | 8 +--- .../examples/complete/estimate_pi_test.py | 3 +- .../examples/complete/game/game_stats.py | 10 ++--- .../examples/complete/game/game_stats_test.py | 3 +- .../complete/game/hourly_team_score.py | 8 ++-- .../complete/game/hourly_team_score_test.py | 3 +- .../examples/complete/game/leader_board.py | 10 ++--- .../complete/game/leader_board_test.py | 3 +- .../examples/complete/game/user_score_test.py | 3 +- .../juliaset/juliaset/juliaset_test.py | 1 - .../complete/juliaset/juliaset_main.py | 2 - .../examples/complete/juliaset/setup.py | 2 +- .../apache_beam/examples/complete/tfidf.py | 6 +-- .../examples/complete/tfidf_test.py | 5 +-- .../complete/top_wikipedia_sessions.py | 12 ++--- .../complete/top_wikipedia_sessions_test.py | 4 +- .../examples/cookbook/bigquery_side_input.py | 7 +-- .../cookbook/bigquery_side_input_test.py | 3 +- .../cookbook/bigquery_tornadoes_it_test.py | 2 +- .../cookbook/bigquery_tornadoes_test.py | 3 +- .../apache_beam/examples/cookbook/coders.py | 6 +-- .../examples/cookbook/coders_test.py | 3 +- .../examples/cookbook/combiners_test.py | 3 +- .../examples/cookbook/custom_ptransform.py | 4 +- .../cookbook/custom_ptransform_test.py | 3 +- .../examples/cookbook/datastore_wordcount.py | 15 +++---- .../examples/cookbook/filters_test.py | 3 +- .../examples/cookbook/group_with_coder.py | 6 +-- .../cookbook/group_with_coder_test.py | 1 - .../examples/cookbook/mergecontacts.py | 9 ++-- .../cookbook/multiple_output_pardo.py | 6 +-- .../apache_beam/examples/snippets/snippets.py | 12 +++-- .../examples/snippets/snippets_test.py | 12 ++--- .../examples/streaming_wordcount.py | 5 +-- .../examples/windowed_wordcount.py | 1 - sdks/python/apache_beam/examples/wordcount.py | 6 +-- .../examples/wordcount_debugging.py | 9 ++-- .../apache_beam/examples/wordcount_it_test.py | 6 +-- .../apache_beam/examples/wordcount_minimal.py | 6 +-- sdks/python/apache_beam/internal/gcp/auth.py | 4 +- .../apache_beam/internal/gcp/json_value.py | 1 - .../internal/gcp/json_value_test.py | 8 ++-- .../apache_beam/internal/pickler_test.py | 3 +- sdks/python/apache_beam/internal/util.py | 2 +- sdks/python/apache_beam/internal/util_test.py | 6 +-- sdks/python/apache_beam/io/avroio.py | 7 +-- sdks/python/apache_beam/io/avroio_test.py | 27 +++++------ .../apache_beam/io/concat_source_test.py | 8 +--- sdks/python/apache_beam/io/filebasedsink.py | 8 ++-- .../apache_beam/io/filebasedsink_test.py | 3 +- sdks/python/apache_beam/io/filebasedsource.py | 17 +++---- .../apache_beam/io/filebasedsource_test.py | 17 +++---- sdks/python/apache_beam/io/filesystem.py | 4 +- sdks/python/apache_beam/io/filesystem_test.py | 2 +- sdks/python/apache_beam/io/filesystems.py | 5 +-- .../python/apache_beam/io/filesystems_test.py | 4 +- sdks/python/apache_beam/io/gcp/bigquery.py | 11 ++--- .../apache_beam/io/gcp/bigquery_test.py | 9 ++-- .../datastore/v1/adaptive_throttler_test.py | 1 + .../io/gcp/datastore/v1/datastoreio.py | 20 +++------ .../io/gcp/datastore/v1/datastoreio_test.py | 11 +++-- .../apache_beam/io/gcp/datastore/v1/helper.py | 7 +-- .../io/gcp/datastore/v1/helper_test.py | 6 +-- .../gcp/datastore/v1/query_splitter_test.py | 6 +-- .../apache_beam/io/gcp/gcsfilesystem.py | 9 ++-- .../apache_beam/io/gcp/gcsfilesystem_test.py | 4 +- sdks/python/apache_beam/io/gcp/gcsio.py | 1 + .../clients/bigquery/bigquery_v2_client.py | 3 +- .../clients/bigquery/bigquery_v2_messages.py | 6 +-- .../clients/storage/storage_v1_client.py | 3 +- .../clients/storage/storage_v1_messages.py | 6 +-- sdks/python/apache_beam/io/gcp/pubsub.py | 9 +--- sdks/python/apache_beam/io/gcp/pubsub_test.py | 8 ++-- .../apache_beam/io/gcp/tests/utils_test.py | 1 + sdks/python/apache_beam/io/iobase.py | 16 +++---- sdks/python/apache_beam/io/localfilesystem.py | 9 ++-- .../apache_beam/io/localfilesystem_test.py | 4 +- .../apache_beam/io/source_test_utils.py | 4 +- .../apache_beam/io/source_test_utils_test.py | 2 +- sdks/python/apache_beam/io/sources_test.py | 7 +-- sdks/python/apache_beam/io/textio.py | 10 ++--- sdks/python/apache_beam/io/textio_test.py | 23 +++------- sdks/python/apache_beam/io/tfrecordio.py | 9 ++-- sdks/python/apache_beam/io/tfrecordio_test.py | 15 +++---- sdks/python/apache_beam/metrics/cells.py | 3 +- sdks/python/apache_beam/metrics/cells_test.py | 6 +-- sdks/python/apache_beam/metrics/execution.py | 2 +- .../apache_beam/metrics/execution_test.py | 7 ++- sdks/python/apache_beam/metrics/metric.py | 3 +- .../python/apache_beam/metrics/metric_test.py | 9 ++-- .../apache_beam/options/pipeline_options.py | 7 ++- .../options/pipeline_options_test.py | 7 +-- .../options/pipeline_options_validator.py | 12 +++-- .../pipeline_options_validator_test.py | 6 ++- .../apache_beam/options/value_provider.py | 1 - .../options/value_provider_test.py | 4 +- sdks/python/apache_beam/pipeline.py | 18 +++----- sdks/python/apache_beam/pipeline_test.py | 29 +++++------- sdks/python/apache_beam/pvalue.py | 1 - sdks/python/apache_beam/runners/common.py | 5 +-- .../python/apache_beam/runners/common_test.py | 2 +- .../runners/dataflow/dataflow_metrics.py | 8 ++-- .../runners/dataflow/dataflow_metrics_test.py | 6 +-- .../runners/dataflow/dataflow_runner.py | 25 +++++------ .../runners/dataflow/dataflow_runner_test.py | 20 ++++----- .../runners/dataflow/internal/apiclient.py | 17 ++++--- .../dataflow/internal/apiclient_test.py | 1 - .../clients/dataflow/dataflow_v1b3_client.py | 3 +- .../dataflow/dataflow_v1b3_messages.py | 4 +- .../clients/dataflow/message_matchers.py | 1 - .../clients/dataflow/message_matchers_test.py | 6 ++- .../runners/dataflow/internal/dependency.py | 4 +- .../dataflow/internal/dependency_test.py | 9 ++-- .../runners/dataflow/native_io/iobase_test.py | 26 +++++------ .../dataflow/native_io/streaming_create.py | 6 +-- .../runners/dataflow/template_runner_test.py | 2 +- .../runners/dataflow/test_dataflow_runner.py | 3 +- ...consumer_tracking_pipeline_visitor_test.py | 14 +++--- .../runners/direct/direct_metrics.py | 8 ++-- .../runners/direct/direct_metrics_test.py | 7 +-- .../runners/direct/direct_runner.py | 15 +++---- .../runners/direct/evaluation_context.py | 6 +-- .../apache_beam/runners/direct/executor.py | 4 +- .../runners/direct/helper_transforms.py | 2 +- .../runners/direct/transform_evaluator.py | 45 ++++++++----------- .../runners/direct/watermark_manager.py | 8 ++-- .../python_rpc_direct_runner.py | 4 +- .../experimental/python_rpc_direct/server.py | 7 ++- sdks/python/apache_beam/runners/job/utils.py | 3 +- .../apache_beam/runners/pipeline_context.py | 7 +-- .../runners/portability/fn_api_runner.py | 24 ++++------ .../runners/portability/fn_api_runner_test.py | 7 ++- .../portability/maptask_executor_runner.py | 18 ++++---- .../maptask_executor_runner_test.py | 11 ++--- sdks/python/apache_beam/runners/runner.py | 1 - .../python/apache_beam/runners/runner_test.py | 14 +++--- .../runners/worker/bundle_processor.py | 19 +++----- .../apache_beam/runners/worker/data_plane.py | 7 ++- .../runners/worker/data_plane_test.py | 6 +-- .../apache_beam/runners/worker/log_handler.py | 3 +- .../runners/worker/log_handler_test.py | 2 +- .../apache_beam/runners/worker/opcounters.py | 1 + .../runners/worker/opcounters_test.py | 1 - .../apache_beam/runners/worker/operations.py | 17 +++---- .../apache_beam/runners/worker/sdk_worker.py | 7 +-- .../runners/worker/sdk_worker_test.py | 9 ++-- .../apache_beam/testing/pipeline_verifiers.py | 1 - .../testing/pipeline_verifiers_test.py | 5 +-- .../apache_beam/testing/test_pipeline.py | 6 +-- .../apache_beam/testing/test_pipeline_test.py | 4 +- .../python/apache_beam/testing/test_stream.py | 13 ++---- .../apache_beam/testing/test_stream_test.py | 13 +++--- sdks/python/apache_beam/testing/test_utils.py | 2 +- .../apache_beam/testing/test_utils_test.py | 1 + sdks/python/apache_beam/testing/util.py | 7 +-- .../apache_beam/transforms/combiners.py | 16 ++----- .../apache_beam/transforms/combiners_test.py | 6 +-- sdks/python/apache_beam/transforms/core.py | 40 ++++++----------- .../apache_beam/transforms/create_test.py | 3 +- sdks/python/apache_beam/transforms/display.py | 3 +- .../apache_beam/transforms/display_test.py | 7 ++- .../apache_beam/transforms/ptransform.py | 18 +++----- .../apache_beam/transforms/ptransform_test.py | 12 +++-- .../python/apache_beam/transforms/timeutil.py | 4 +- sdks/python/apache_beam/transforms/trigger.py | 19 +++----- .../apache_beam/transforms/trigger_test.py | 29 +++++------- sdks/python/apache_beam/transforms/util.py | 9 +--- sdks/python/apache_beam/transforms/window.py | 17 +++---- .../apache_beam/transforms/window_test.py | 30 ++++--------- .../transforms/write_ptransform_test.py | 1 - .../apache_beam/typehints/decorators.py | 12 +++-- .../typehints/native_type_compatibility.py | 1 + .../native_type_compatibility_test.py | 6 +-- sdks/python/apache_beam/typehints/opcodes.py | 2 +- .../typehints/trivial_inference.py | 3 +- .../typehints/trivial_inference_test.py | 4 +- .../python/apache_beam/typehints/typecheck.py | 13 +++--- .../typehints/typed_pipeline_test.py | 6 +-- .../python/apache_beam/typehints/typehints.py | 1 - .../apache_beam/typehints/typehints_test.py | 21 ++++----- sdks/python/apache_beam/utils/annotations.py | 3 +- .../apache_beam/utils/annotations_test.py | 4 +- sdks/python/apache_beam/utils/counters.py | 3 +- .../apache_beam/utils/processes_test.py | 1 - sdks/python/apache_beam/utils/proto_utils.py | 3 +- sdks/python/apache_beam/utils/retry_test.py | 3 +- .../apache_beam/utils/timestamp_test.py | 3 +- sdks/python/apache_beam/utils/urns.py | 1 - .../apache_beam/utils/windowed_value.py | 4 +- sdks/python/run_pylint.sh | 4 ++ 204 files changed, 562 insertions(+), 954 deletions(-) diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py index 2670250c36b6..bea3d55caf5f 100644 --- a/sdks/python/apache_beam/coders/coder_impl.py +++ b/sdks/python/apache_beam/coders/coder_impl.py @@ -29,10 +29,8 @@ from types import NoneType from apache_beam.coders import observable -from apache_beam.utils.timestamp import Timestamp -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP from apache_beam.utils import windowed_value +from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP, Timestamp # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 10fb07b6a34f..70676f78b53f 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -22,12 +22,12 @@ import base64 import cPickle as pickle + import google.protobuf from apache_beam.coders import coder_impl from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.utils import urns -from apache_beam.utils import proto_utils +from apache_beam.utils import proto_utils, urns # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py index c89e81028b8a..705de8920d52 100644 --- a/sdks/python/apache_beam/coders/coders_test.py +++ b/sdks/python/apache_beam/coders/coders_test.py @@ -20,8 +20,8 @@ import logging import unittest -from apache_beam.coders import coders from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.coders import coders from apache_beam.coders.typecoders import registry as coders_registry diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py index 8b0353d922e7..5547b05bb384 100644 --- a/sdks/python/apache_beam/coders/coders_test_common.py +++ b/sdks/python/apache_beam/coders/coders_test_common.py @@ -23,16 +23,14 @@ import dill -from apache_beam.transforms.window import GlobalWindow -from apache_beam.utils.timestamp import MIN_TIMESTAMP import observable +from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.coders import coders from apache_beam.runners import pipeline_context from apache_beam.transforms import window -from apache_beam.utils import timestamp -from apache_beam.utils import windowed_value - -from apache_beam.coders import coders -from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.transforms.window import GlobalWindow +from apache_beam.utils import timestamp, windowed_value +from apache_beam.utils.timestamp import MIN_TIMESTAMP # Defined out of line for picklability. diff --git a/sdks/python/apache_beam/coders/fast_coders_test.py b/sdks/python/apache_beam/coders/fast_coders_test.py index a13334a2c26f..d060117a147e 100644 --- a/sdks/python/apache_beam/coders/fast_coders_test.py +++ b/sdks/python/apache_beam/coders/fast_coders_test.py @@ -20,8 +20,6 @@ import logging import unittest - -# Run all the standard coder test cases. from apache_beam.coders.coders_test_common import * diff --git a/sdks/python/apache_beam/coders/observable_test.py b/sdks/python/apache_beam/coders/observable_test.py index eaf1aec9e600..09ca3041c298 100644 --- a/sdks/python/apache_beam/coders/observable_test.py +++ b/sdks/python/apache_beam/coders/observable_test.py @@ -20,7 +20,6 @@ import logging import unittest - from apache_beam.coders import observable diff --git a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py index 16b1b4d6b4b3..98667c8404c4 100644 --- a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py +++ b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py @@ -19,12 +19,14 @@ # source: sdks/java/core/src/main/proto/proto2_coder_test_messages.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/sdks/python/apache_beam/coders/slow_coders_test.py b/sdks/python/apache_beam/coders/slow_coders_test.py index 97aa39ca094f..7032e081658a 100644 --- a/sdks/python/apache_beam/coders/slow_coders_test.py +++ b/sdks/python/apache_beam/coders/slow_coders_test.py @@ -20,8 +20,6 @@ import logging import unittest - -# Run all the standard coder test cases. from apache_beam.coders.coders_test_common import * diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py index 5f9845564477..0028a0bca4b5 100644 --- a/sdks/python/apache_beam/coders/standard_coders_test.py +++ b/sdks/python/apache_beam/coders/standard_coders_test.py @@ -26,12 +26,11 @@ import yaml -from apache_beam.coders import coders -from apache_beam.coders import coder_impl +from apache_beam.coders import coder_impl, coders +from apache_beam.transforms import window +from apache_beam.transforms.window import IntervalWindow from apache_beam.utils import windowed_value from apache_beam.utils.timestamp import Timestamp -from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms import window STANDARD_CODERS_YAML = os.path.join( os.path.dirname(__file__), '..', 'testing', 'data', 'standard_coders.yaml') diff --git a/sdks/python/apache_beam/coders/stream_test.py b/sdks/python/apache_beam/coders/stream_test.py index e6108b68aee9..15bc5eb9ba93 100644 --- a/sdks/python/apache_beam/coders/stream_test.py +++ b/sdks/python/apache_beam/coders/stream_test.py @@ -21,7 +21,6 @@ import math import unittest - from apache_beam.coders import slow_stream diff --git a/sdks/python/apache_beam/coders/typecoders.py b/sdks/python/apache_beam/coders/typecoders.py index 3894bb588172..797aee5815f5 100644 --- a/sdks/python/apache_beam/coders/typecoders.py +++ b/sdks/python/apache_beam/coders/typecoders.py @@ -70,7 +70,6 @@ def MakeXyzs(v): from apache_beam.coders import coders from apache_beam.typehints import typehints - __all__ = ['registry'] diff --git a/sdks/python/apache_beam/coders/typecoders_test.py b/sdks/python/apache_beam/coders/typecoders_test.py index 2b6aa7a51298..552d185f88da 100644 --- a/sdks/python/apache_beam/coders/typecoders_test.py +++ b/sdks/python/apache_beam/coders/typecoders_test.py @@ -19,8 +19,7 @@ import unittest -from apache_beam.coders import coders -from apache_beam.coders import typecoders +from apache_beam.coders import coders, typecoders from apache_beam.internal import pickler from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py index ab3397cfe335..13c7e3b73d32 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete.py @@ -24,10 +24,8 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions def run(argv=None): diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py index e2c84d68d3d4..050d35bb5d0e 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py @@ -22,8 +22,7 @@ import apache_beam as beam from apache_beam.examples.complete import autocomplete from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class AutocompleteTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py index 7e3c4cd35a27..7b05f385f33c 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py @@ -31,14 +31,10 @@ import logging import random - import apache_beam as beam from apache_beam.io import WriteToText -from apache_beam.typehints import Any -from apache_beam.typehints import Iterable -from apache_beam.typehints import Tuple -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.typehints import Any, Iterable, Tuple @beam.typehints.with_output_types(Tuple[int, int, int]) diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py index f1cbb0a24d56..94399da32499 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py @@ -22,8 +22,7 @@ from apache_beam.examples.complete import estimate_pi from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import BeamAssertException +from apache_beam.testing.util import BeamAssertException, assert_that def in_between(lower, upper): diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py index 41813235e70e..895e042aa830 100644 --- a/sdks/python/apache_beam/examples/complete/game/game_stats.py +++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py @@ -77,8 +77,7 @@ -------------------------------------------------------------------------------- """ -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function import argparse import csv @@ -89,10 +88,9 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + PipelineOptions, SetupOptions, + StandardOptions) def timestamp2str(t, fmt='%Y-%m-%d %H:%M:%S.000'): diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py index 971f9dc9e299..249373d97be7 100644 --- a/sdks/python/apache_beam/examples/complete/game/game_stats_test.py +++ b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.complete.game import game_stats from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class GameStatsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py index 9dd8b05a5c5c..fbd7b5d0d057 100644 --- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py +++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py @@ -64,8 +64,7 @@ --temp_location gs://$BUCKET/user_score/temp """ -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function import argparse import csv @@ -76,9 +75,8 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + PipelineOptions, SetupOptions) def str2timestamp(s, fmt='%Y-%m-%d-%H-%M'): diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py index 9c30127d7416..39fa72cb46d4 100644 --- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py +++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.complete.game import hourly_team_score from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class HourlyTeamScoreTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py index 2936bc96a31a..42c864e25fc0 100644 --- a/sdks/python/apache_beam/examples/complete/game/leader_board.py +++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py @@ -84,8 +84,7 @@ -------------------------------------------------------------------------------- """ -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function import argparse import csv @@ -96,10 +95,9 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + PipelineOptions, SetupOptions, + StandardOptions) from apache_beam.transforms import trigger diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py index aece2649b898..1a8445810f80 100644 --- a/sdks/python/apache_beam/examples/complete/game/leader_board_test.py +++ b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.complete.game import leader_board from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class LeaderBoardTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/user_score_test.py b/sdks/python/apache_beam/examples/complete/game/user_score_test.py index 59903d985460..e2880213bf3c 100644 --- a/sdks/python/apache_beam/examples/complete/game/user_score_test.py +++ b/sdks/python/apache_beam/examples/complete/game/user_score_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.complete.game import user_score from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class UserScoreTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py index 91c75aa57927..e4986271e940 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py @@ -23,7 +23,6 @@ import tempfile import unittest - from apache_beam.examples.complete.juliaset.juliaset import juliaset from apache_beam.testing.util import open_shards diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py index 0db5431e31c3..1d521bec2e4b 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py @@ -49,10 +49,8 @@ import logging - from apache_beam.examples.complete.juliaset.juliaset import juliaset - if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) juliaset.run() diff --git a/sdks/python/apache_beam/examples/complete/juliaset/setup.py b/sdks/python/apache_beam/examples/complete/juliaset/setup.py index 589e47c83059..2062e2a0cd81 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/setup.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/setup.py @@ -25,8 +25,8 @@ when running the workflow for remote execution. """ -from distutils.command.build import build as _build import subprocess +from distutils.command.build import build as _build import setuptools diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py index a88ff827766c..8f0c0040c49a 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf.py +++ b/sdks/python/apache_beam/examples/complete/tfidf.py @@ -29,11 +29,9 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions from apache_beam.pvalue import AsSingleton -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions def read_documents(pipeline, uris): diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py index b6f88255887c..1e32ce0dbabe 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf_test.py +++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py @@ -26,10 +26,7 @@ import apache_beam as beam from apache_beam.examples.complete import tfidf from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to -from apache_beam.testing.util import open_shards - +from apache_beam.testing.util import assert_that, equal_to, open_shards EXPECTED_RESULTS = set([ ('ghi', '1.txt', 0.3662040962227032), diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py index 9a9ad7865c2c..0813bd01b521 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py @@ -47,14 +47,10 @@ import apache_beam as beam from apache_beam import combiners -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.transforms.window import FixedWindows -from apache_beam.transforms.window import Sessions -from apache_beam.transforms.window import TimestampedValue -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions - +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.transforms.window import (FixedWindows, Sessions, + TimestampedValue) ONE_HOUR_IN_SECONDS = 3600 THIRTY_DAYS_IN_SECONDS = 30 * 24 * ONE_HOUR_IN_SECONDS diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py index ced8a44af13c..bdf65be9f9c1 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py @@ -20,12 +20,10 @@ import json import unittest - import apache_beam as beam from apache_beam.examples.complete import top_wikipedia_sessions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class ComputeTopSessionsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py index 9911a6716bd8..29735dc71c1d 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py @@ -32,12 +32,9 @@ from random import randrange import apache_beam as beam - from apache_beam.io import WriteToText -from apache_beam.pvalue import AsList -from apache_beam.pvalue import AsSingleton -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.pvalue import AsList, AsSingleton def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py index 964b35b3f08f..2be7abaf21ff 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.cookbook import bigquery_side_input from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class BigQuerySideInputTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py index 05ee3c53f48f..961284908046 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py @@ -25,8 +25,8 @@ from nose.plugins.attrib import attr from apache_beam.examples.cookbook import bigquery_tornadoes -from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher from apache_beam.io.gcp.tests import utils +from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py index 45dcabaf853c..2d12db5050f5 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.cookbook import bigquery_tornadoes from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class BigQueryTornadoesTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py index f97b0f2d9c9a..b9e1a6896aa8 100644 --- a/sdks/python/apache_beam/examples/cookbook/coders.py +++ b/sdks/python/apache_beam/examples/cookbook/coders.py @@ -35,10 +35,8 @@ import logging import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions class JsonCoder(object): diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py index 988d3c9d25e1..9bad505cea32 100644 --- a/sdks/python/apache_beam/examples/cookbook/coders_test.py +++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.cookbook import coders from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class CodersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/combiners_test.py b/sdks/python/apache_beam/examples/cookbook/combiners_test.py index ee1fb7765d47..26c9f60b256b 100644 --- a/sdks/python/apache_beam/examples/cookbook/combiners_test.py +++ b/sdks/python/apache_beam/examples/cookbook/combiners_test.py @@ -28,8 +28,7 @@ import apache_beam as beam from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class CombinersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py index aee69d23ff65..d274d019fa27 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py @@ -26,11 +26,9 @@ import logging import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText +from apache_beam.io import ReadFromText, WriteToText from apache_beam.options.pipeline_options import PipelineOptions - # pylint doesn't understand our pipeline syntax: # pylint:disable=expression-not-assigned diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py index 7aaccb4ac1c0..88d83fe2a6ea 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.cookbook import custom_ptransform from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class CustomCountTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py index 7161cff1c255..b76b4bfe3110 100644 --- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py +++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py @@ -66,19 +66,18 @@ import re import uuid -from google.cloud.proto.datastore.v1 import entity_pb2 -from google.cloud.proto.datastore.v1 import query_pb2 -from googledatastore import helper as datastore_helper, PropertyFilter +from google.cloud.proto.datastore.v1 import entity_pb2, query_pb2 +from googledatastore import helper as datastore_helper +from googledatastore import PropertyFilter import apache_beam as beam from apache_beam.io import ReadFromText -from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore -from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore +from apache_beam.io.gcp.datastore.v1.datastoreio import (ReadFromDatastore, + WriteToDatastore) from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + PipelineOptions, SetupOptions) class WordExtractingDoFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/cookbook/filters_test.py b/sdks/python/apache_beam/examples/cookbook/filters_test.py index fd49f938742c..ff70caf7cba7 100644 --- a/sdks/python/apache_beam/examples/cookbook/filters_test.py +++ b/sdks/python/apache_beam/examples/cookbook/filters_test.py @@ -23,8 +23,7 @@ import apache_beam as beam from apache_beam.examples.cookbook import filters from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class FiltersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py index 9c0d04b816ac..8ca22b3a8ee2 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py @@ -33,12 +33,10 @@ import apache_beam as beam from apache_beam import coders -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions from apache_beam.typehints import typehints from apache_beam.typehints.decorators import with_output_types -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions class Player(object): diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py index fb630ba465f7..ed38b5d8a4ed 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py @@ -24,7 +24,6 @@ from apache_beam.examples.cookbook import group_with_coder from apache_beam.testing.util import open_shards - # Patch group_with_coder.PlayerCoder.decode(). To test that the PlayerCoder was # used, we do not strip the prepended 'x:' string when decoding a Player object. group_with_coder.PlayerCoder.decode = lambda self, s: group_with_coder.Player(s) diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py index 9acdd9073478..86e98a8d53ed 100644 --- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py +++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py @@ -36,12 +36,9 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.testing.util import assert_that, equal_to def run(argv=None, assert_results=None): diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py index 2316c6611c06..aaa1e0363032 100644 --- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py +++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py @@ -57,10 +57,8 @@ import apache_beam as beam from apache_beam import pvalue -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions class SplitLinesToWordsFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index 27b81208fcd8..64f21cf1d0b0 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -31,10 +31,13 @@ """ import apache_beam as beam +from apache_beam.io import iobase +from apache_beam.io.range_trackers import OffsetRangeTracker from apache_beam.metrics import Metrics +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to +from apache_beam.transforms.core import PTransform # Quiet some pylint warnings that happen because of the somewhat special # format for the code snippets. @@ -655,11 +658,6 @@ def process(self, an_int): result.wait_until_finish() -import apache_beam as beam -from apache_beam.io import iobase -from apache_beam.io.range_trackers import OffsetRangeTracker -from apache_beam.transforms.core import PTransform -from apache_beam.options.pipeline_options import PipelineOptions # Defining a new source. diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index 9183d0dfea19..3f6743ce8957 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -26,20 +26,16 @@ import uuid import apache_beam as beam -from apache_beam import coders -from apache_beam import pvalue -from apache_beam import typehints +from apache_beam import coders, pvalue, typehints from apache_beam.coders.coders import ToStringCoder -from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.examples.snippets import snippets from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to -from apache_beam.utils.windowed_value import WindowedValue - +from apache_beam.options.pipeline_options import PipelineOptions # pylint: disable=expression-not-assigned from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that, equal_to +from apache_beam.utils.windowed_value import WindowedValue # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py index 7696d7789323..d0effbf64bd1 100644 --- a/sdks/python/apache_beam/examples/streaming_wordcount.py +++ b/sdks/python/apache_beam/examples/streaming_wordcount.py @@ -26,11 +26,10 @@ import argparse import logging - import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import StandardOptions import apache_beam.transforms.window as window +from apache_beam.options.pipeline_options import (PipelineOptions, + StandardOptions) def split_fn(lines): diff --git a/sdks/python/apache_beam/examples/windowed_wordcount.py b/sdks/python/apache_beam/examples/windowed_wordcount.py index bd57847c67f9..508f18d00097 100644 --- a/sdks/python/apache_beam/examples/windowed_wordcount.py +++ b/sdks/python/apache_beam/examples/windowed_wordcount.py @@ -26,7 +26,6 @@ import argparse import logging - import apache_beam as beam import apache_beam.transforms.window as window diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py index 34dedb2b819a..1c4e2d728c8a 100644 --- a/sdks/python/apache_beam/examples/wordcount.py +++ b/sdks/python/apache_beam/examples/wordcount.py @@ -24,12 +24,10 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText +from apache_beam.io import ReadFromText, WriteToText from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions class WordExtractingDoFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py index c0ffd356364c..cef76446f55e 100644 --- a/sdks/python/apache_beam/examples/wordcount_debugging.py +++ b/sdks/python/apache_beam/examples/wordcount_debugging.py @@ -46,13 +46,10 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText +from apache_beam.io import ReadFromText, WriteToText from apache_beam.metrics import Metrics -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.testing.util import assert_that, equal_to class FilterTextFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index 8d2e73e28f6b..dc8e8482c80c 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -25,10 +25,10 @@ from nose.plugins.attrib import attr from apache_beam.examples import wordcount -from apache_beam.testing.test_utils import delete_files -from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher -from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher +from apache_beam.testing.pipeline_verifiers import (FileChecksumMatcher, + PipelineStateMatcher) from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.test_utils import delete_files class WordCountIT(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/wordcount_minimal.py b/sdks/python/apache_beam/examples/wordcount_minimal.py index 76b0a221df7a..794f170a840c 100644 --- a/sdks/python/apache_beam/examples/wordcount_minimal.py +++ b/sdks/python/apache_beam/examples/wordcount_minimal.py @@ -51,10 +51,8 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText -from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.io import ReadFromText, WriteToText +from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions def run(argv=None): diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 9f32092f4166..5956b3ec6dbe 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -23,12 +23,10 @@ import os import urllib2 -from oauth2client.client import GoogleCredentials -from oauth2client.client import OAuth2Credentials +from oauth2client.client import GoogleCredentials, OAuth2Credentials from apache_beam.utils import retry - # When we are running in GCE, we can authenticate with VM credentials. is_running_in_gce = False diff --git a/sdks/python/apache_beam/internal/gcp/json_value.py b/sdks/python/apache_beam/internal/gcp/json_value.py index 167b173b1f84..7a5dc543eefa 100644 --- a/sdks/python/apache_beam/internal/gcp/json_value.py +++ b/sdks/python/apache_beam/internal/gcp/json_value.py @@ -27,7 +27,6 @@ from apache_beam.options.value_provider import ValueProvider - _MAXINT64 = (1 << 63) - 1 _MININT64 = - (1 << 63) diff --git a/sdks/python/apache_beam/internal/gcp/json_value_test.py b/sdks/python/apache_beam/internal/gcp/json_value_test.py index b1fd63f048e9..ab79e5e283eb 100644 --- a/sdks/python/apache_beam/internal/gcp/json_value_test.py +++ b/sdks/python/apache_beam/internal/gcp/json_value_test.py @@ -19,11 +19,9 @@ import unittest -from apache_beam.internal.gcp.json_value import from_json_value -from apache_beam.internal.gcp.json_value import to_json_value -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import RuntimeValueProvider - +from apache_beam.internal.gcp.json_value import from_json_value, to_json_value +from apache_beam.options.value_provider import (RuntimeValueProvider, + StaticValueProvider) # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/internal/pickler_test.py b/sdks/python/apache_beam/internal/pickler_test.py index 05062d282da1..d859b4ed62d5 100644 --- a/sdks/python/apache_beam/internal/pickler_test.py +++ b/sdks/python/apache_beam/internal/pickler_test.py @@ -20,8 +20,7 @@ import unittest from apache_beam.internal import module_test -from apache_beam.internal.pickler import dumps -from apache_beam.internal.pickler import loads +from apache_beam.internal.pickler import dumps, loads class PicklerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/internal/util.py b/sdks/python/apache_beam/internal/util.py index dbbeafcc1e75..3e943b0db1e3 100644 --- a/sdks/python/apache_beam/internal/util.py +++ b/sdks/python/apache_beam/internal/util.py @@ -21,9 +21,9 @@ """ import logging -from multiprocessing.pool import ThreadPool import threading import weakref +from multiprocessing.pool import ThreadPool class ArgumentPlaceholder(object): diff --git a/sdks/python/apache_beam/internal/util_test.py b/sdks/python/apache_beam/internal/util_test.py index 9a2e3977a708..374307ac09bb 100644 --- a/sdks/python/apache_beam/internal/util_test.py +++ b/sdks/python/apache_beam/internal/util_test.py @@ -19,9 +19,9 @@ import unittest -from apache_beam.internal.util import ArgumentPlaceholder -from apache_beam.internal.util import insert_values_in_args -from apache_beam.internal.util import remove_objects_from_args +from apache_beam.internal.util import (ArgumentPlaceholder, + insert_values_in_args, + remove_objects_from_args) class UtilTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py index cb14c6542176..e67c7df1390e 100644 --- a/sdks/python/apache_beam/io/avroio.py +++ b/sdks/python/apache_beam/io/avroio.py @@ -48,14 +48,11 @@ from functools import partial import avro -from avro import datafile from avro import io as avroio -from avro import schema +from avro import datafile, schema import apache_beam as beam -from apache_beam.io import filebasedsource -from apache_beam.io import filebasedsink -from apache_beam.io import iobase +from apache_beam.io import filebasedsink, filebasedsource, iobase from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.iobase import Read from apache_beam.transforms import PTransform diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index 969f44050854..048c412230f8 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -21,29 +21,22 @@ import tempfile import unittest +import avro.datafile +import avro.schema +import hamcrest as hc +from avro.datafile import DataFileWriter +from avro.io import DatumWriter + import apache_beam as beam from apache_beam import Create -from apache_beam.io import iobase -from apache_beam.io import avroio -from apache_beam.io import filebasedsource -from apache_beam.io import source_test_utils +from apache_beam.io import avroio, filebasedsource, iobase, source_test_utils +from apache_beam.io.avroio import _AvroSink as AvroSink +from apache_beam.io.avroio import _AvroSource as AvroSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -# Importing following private class for testing purposes. -from apache_beam.io.avroio import _AvroSource as AvroSource -from apache_beam.io.avroio import _AvroSink as AvroSink - -import avro.datafile -from avro.datafile import DataFileWriter -from avro.io import DatumWriter -import avro.schema -import hamcrest as hc - - # Import snappy optionally; some tests will be skipped when import fails. try: import snappy # pylint: disable=import-error diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py index 4a8f5193dee2..ec027e9f98df 100644 --- a/sdks/python/apache_beam/io/concat_source_test.py +++ b/sdks/python/apache_beam/io/concat_source_test.py @@ -21,14 +21,10 @@ import unittest import apache_beam as beam - -from apache_beam.io import iobase -from apache_beam.io import range_trackers -from apache_beam.io import source_test_utils +from apache_beam.io import iobase, range_trackers, source_test_utils from apache_beam.io.concat_source import ConcatSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class RangeSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/io/filebasedsink.py b/sdks/python/apache_beam/io/filebasedsink.py index eb99d0849017..fdad50efbce8 100644 --- a/sdks/python/apache_beam/io/filebasedsink.py +++ b/sdks/python/apache_beam/io/filebasedsink.py @@ -27,13 +27,11 @@ from apache_beam.internal import util from apache_beam.io import iobase -from apache_beam.io.filesystem import BeamIOError -from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystem import BeamIOError, CompressionTypes from apache_beam.io.filesystems import FileSystems +from apache_beam.options.value_provider import (StaticValueProvider, + ValueProvider, check_accessible) from apache_beam.transforms.display import DisplayDataItem -from apache_beam.options.value_provider import ValueProvider -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import check_accessible DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN' diff --git a/sdks/python/apache_beam/io/filebasedsink_test.py b/sdks/python/apache_beam/io/filebasedsink_test.py index 7c8ddb4072ef..156251a09123 100644 --- a/sdks/python/apache_beam/io/filebasedsink_test.py +++ b/sdks/python/apache_beam/io/filebasedsink_test.py @@ -31,12 +31,11 @@ import apache_beam as beam from apache_beam.coders import coders from apache_beam.io import filebasedsink +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.options.value_provider import StaticValueProvider - # TODO: Refactor code so all io tests are using same library # TestCaseWithTempDirCleanup class. diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py index 649693076c5f..49498f5c41d8 100644 --- a/sdks/python/apache_beam/io/filebasedsource.py +++ b/sdks/python/apache_beam/io/filebasedsource.py @@ -27,23 +27,16 @@ """ import uuid -from apache_beam.transforms.core import DoFn -from apache_beam.transforms.core import ParDo -from apache_beam.transforms.core import GroupByKey -from apache_beam.transforms.core import PTransform -from apache_beam.transforms.core import FlatMap -from apache_beam.transforms.core import Map from apache_beam.internal import pickler -from apache_beam.io import concat_source -from apache_beam.io import iobase -from apache_beam.io import range_trackers +from apache_beam.io import concat_source, iobase, range_trackers from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems from apache_beam.io.range_trackers import OffsetRange +from apache_beam.options.value_provider import (StaticValueProvider, + ValueProvider, check_accessible) +from apache_beam.transforms.core import (DoFn, FlatMap, GroupByKey, Map, ParDo, + PTransform) from apache_beam.transforms.display import DisplayDataItem -from apache_beam.options.value_provider import ValueProvider -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import check_accessible from apache_beam.transforms.trigger import DefaultTrigger MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25 diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index afb340d7608b..456d9daf1b52 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -20,29 +20,24 @@ import gzip import logging import math -import random import os +import random import tempfile import unittest import hamcrest as hc import apache_beam as beam -from apache_beam.io import filebasedsource -from apache_beam.io import iobase -from apache_beam.io import range_trackers -from apache_beam.io.filesystem import CompressionTypes - +from apache_beam.io import filebasedsource, iobase, range_trackers # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource - from apache_beam.io.filebasedsource import FileBasedSource -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.io.filesystem import CompressionTypes +from apache_beam.options.value_provider import (RuntimeValueProvider, + StaticValueProvider) from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py index 5804d0048af0..69049ae67a45 100644 --- a/sdks/python/apache_beam/io/filesystem.py +++ b/sdks/python/apache_beam/io/filesystem.py @@ -21,10 +21,10 @@ import abc import bz2 import cStringIO -import os -import zlib import logging +import os import time +import zlib from apache_beam.utils.plugin import BeamPlugin diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py index 607393d3a555..53cd4ef3451f 100644 --- a/sdks/python/apache_beam/io/filesystem_test.py +++ b/sdks/python/apache_beam/io/filesystem_test.py @@ -20,8 +20,8 @@ import bz2 import gzip import os -import unittest import tempfile +import unittest from StringIO import StringIO from apache_beam.io.filesystem import CompressedFile, CompressionTypes diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py index e03968628d59..6ac54876cf2e 100644 --- a/sdks/python/apache_beam/io/filesystems.py +++ b/sdks/python/apache_beam/io/filesystems.py @@ -19,10 +19,7 @@ import re -from apache_beam.io.filesystem import BeamIOError -from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.filesystem import FileSystem - +from apache_beam.io.filesystem import BeamIOError, CompressionTypes, FileSystem # All filesystem implements should be added here # pylint: disable=wrong-import-position, unused-import from apache_beam.io.localfilesystem import LocalFileSystem diff --git a/sdks/python/apache_beam/io/filesystems_test.py b/sdks/python/apache_beam/io/filesystems_test.py index 9a6f013ff479..eaaa40f9f7eb 100644 --- a/sdks/python/apache_beam/io/filesystems_test.py +++ b/sdks/python/apache_beam/io/filesystems_test.py @@ -18,12 +18,12 @@ """Unit tests for LocalFileSystem.""" -import unittest - import filecmp import os import shutil import tempfile +import unittest + import mock from apache_beam.io import localfilesystem diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 33d67bf7c53a..88c3fc36e1eb 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -112,16 +112,13 @@ from apache_beam import coders from apache_beam.internal.gcp import auth -from apache_beam.internal.gcp.json_value import from_json_value -from apache_beam.internal.gcp.json_value import to_json_value +from apache_beam.internal.gcp.json_value import from_json_value, to_json_value +from apache_beam.io.gcp.internal.clients import bigquery +from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.runners.dataflow.native_io import iobase as dataflow_io -from apache_beam.transforms import DoFn -from apache_beam.transforms import ParDo -from apache_beam.transforms import PTransform +from apache_beam.transforms import DoFn, ParDo, PTransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.utils import retry -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.io.gcp.internal.clients import bigquery # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index bfd06ace6400..5c33e3773977 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -27,14 +27,13 @@ import mock import apache_beam as beam -from apache_beam.io.gcp.bigquery import RowAsDictJsonCoder -from apache_beam.io.gcp.bigquery import TableRowJsonCoder -from apache_beam.io.gcp.bigquery import parse_table_schema_from_json -from apache_beam.io.gcp.internal.clients import bigquery from apache_beam.internal.gcp.json_value import to_json_value +from apache_beam.io.gcp.bigquery import (RowAsDictJsonCoder, TableRowJsonCoder, + parse_table_schema_from_json) +from apache_beam.io.gcp.internal.clients import bigquery +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.options.pipeline_options import PipelineOptions # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py index 93b91ad7cfab..1ac23930f65f 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py @@ -16,6 +16,7 @@ # import unittest + from mock import patch from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py index 3cfba93d2e8c..c498933788d0 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py @@ -20,6 +20,13 @@ import logging import time +from apache_beam.io.gcp.datastore.v1 import helper, query_splitter, util +from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler +from apache_beam.metrics.metric import Metrics +from apache_beam.transforms import (Create, DoFn, FlatMap, GroupByKey, Map, + ParDo, PTransform) +from apache_beam.transforms.util import Values + # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: @@ -29,19 +36,6 @@ pass # pylint: enable=wrong-import-order, wrong-import-position -from apache_beam.io.gcp.datastore.v1 import helper -from apache_beam.io.gcp.datastore.v1 import query_splitter -from apache_beam.io.gcp.datastore.v1 import util -from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler -from apache_beam.transforms import Create -from apache_beam.transforms import DoFn -from apache_beam.transforms import FlatMap -from apache_beam.transforms import GroupByKey -from apache_beam.transforms import Map -from apache_beam.transforms import PTransform -from apache_beam.transforms import ParDo -from apache_beam.transforms.util import Values -from apache_beam.metrics.metric import Metrics __all__ = ['ReadFromDatastore', 'WriteToDatastore', 'DeleteFromDatastore'] diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py index 72c4c8cc0c13..d0ba7eb8af28 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py @@ -19,12 +19,11 @@ from mock import MagicMock, call, patch -from apache_beam.io.gcp.datastore.v1 import fake_datastore -from apache_beam.io.gcp.datastore.v1 import helper -from apache_beam.io.gcp.datastore.v1 import query_splitter -from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate -from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore -from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore +from apache_beam.io.gcp.datastore.v1 import (fake_datastore, helper, + query_splitter) +from apache_beam.io.gcp.datastore.v1.datastoreio import (ReadFromDatastore, + WriteToDatastore, + _Mutate) # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py index 5cde25540da4..84bf1eac7810 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py @@ -21,10 +21,13 @@ """ import errno -from socket import error as SocketError import logging import sys import time +from socket import error as SocketError + +from apache_beam.internal.gcp import auth +from apache_beam.utils import retry # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position @@ -41,8 +44,6 @@ pass # pylint: enable=wrong-import-order, wrong-import-position -from apache_beam.internal.gcp import auth -from apache_beam.utils import retry def key_comparator(k1, k2): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py index a8b1bb11231f..cde75a03f9bd 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py @@ -18,17 +18,15 @@ """Tests for datastore helper.""" import errno import random -from socket import error as SocketError import sys import unittest +from socket import error as SocketError from mock import MagicMock -from apache_beam.io.gcp.datastore.v1 import fake_datastore -from apache_beam.io.gcp.datastore.v1 import helper +from apache_beam.io.gcp.datastore.v1 import fake_datastore, helper from apache_beam.testing.test_utils import patch_retry - # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py index 52f25facd058..80ca2277c740 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py @@ -19,11 +19,9 @@ import unittest -from mock import MagicMock -from mock import call +from mock import MagicMock, call -from apache_beam.io.gcp.datastore.v1 import fake_datastore -from apache_beam.io.gcp.datastore.v1 import query_splitter +from apache_beam.io.gcp.datastore.v1 import fake_datastore, query_splitter # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py index ce8b5e6e4244..e63e36ab35b3 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py @@ -18,12 +18,9 @@ from __future__ import absolute_import -from apache_beam.io.filesystem import BeamIOError -from apache_beam.io.filesystem import CompressedFile -from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.filesystem import FileMetadata -from apache_beam.io.filesystem import FileSystem -from apache_beam.io.filesystem import MatchResult +from apache_beam.io.filesystem import (BeamIOError, CompressedFile, + CompressionTypes, FileMetadata, + FileSystem, MatchResult) from apache_beam.io.gcp import gcsio __all__ = ['GCSFileSystem'] diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py index 923fc7dd4b68..6d216fc4cb82 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py @@ -21,8 +21,8 @@ import unittest import mock -from apache_beam.io.filesystem import BeamIOError -from apache_beam.io.filesystem import FileMetadata + +from apache_beam.io.filesystem import BeamIOError, FileMetadata # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index b18a2875aa0c..0db4ba5dc27c 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -31,6 +31,7 @@ import threading import time import traceback + import httplib2 from apache_beam.utils import retry diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py index 201a1830b878..b32fc5638f7d 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py @@ -19,7 +19,8 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_messages as messages +from apache_beam.io.gcp.internal.clients.bigquery import \ + bigquery_v2_messages as messages class BigqueryV2(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py index 404542863f3c..4f3bae3d6abe 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py @@ -22,9 +22,7 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding -from apitools.base.py import extra_types - +from apitools.base.py import encoding, extra_types package = 'bigquery' @@ -1906,5 +1904,3 @@ class ViewDefinition(_messages.Message): query = _messages.StringField(1) userDefinedFunctionResources = _messages.MessageField('UserDefinedFunctionResource', 2, repeated=True) - - diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py index 1b46d917f143..72b453dab19c 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py @@ -19,7 +19,8 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.io.gcp.internal.clients.storage import storage_v1_messages as messages +from apache_beam.io.gcp.internal.clients.storage import \ + storage_v1_messages as messages class StorageV1(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py index dc9e5e692554..d1d226881adf 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py @@ -23,9 +23,7 @@ from apitools.base.protorpclite import message_types as _message_types from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding -from apitools.base.py import extra_types - +from apitools.base.py import encoding, extra_types package = 'storage' @@ -1916,5 +1914,3 @@ class ProjectionValueValuesEnum(_messages.Enum): prefix = _messages.StringField(6) projection = _messages.EnumField('ProjectionValueValuesEnum', 7) versions = _messages.BooleanField(8) - - diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py index 7d1f3553f4fa..130e145f2415 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub.py +++ b/sdks/python/apache_beam/io/gcp/pubsub.py @@ -27,16 +27,11 @@ import re from apache_beam import coders -from apache_beam.io.iobase import Read -from apache_beam.io.iobase import Write +from apache_beam.io.iobase import Read, Write from apache_beam.runners.dataflow.native_io import iobase as dataflow_io -from apache_beam.transforms import core -from apache_beam.transforms import PTransform -from apache_beam.transforms import Map -from apache_beam.transforms import window +from apache_beam.transforms import Map, PTransform, core, window from apache_beam.transforms.display import DisplayDataItem - __all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub'] diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py index 0dcc3c39ab5f..c49da0302179 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py @@ -22,15 +22,13 @@ import hamcrest as hc -from apache_beam.io.gcp.pubsub import _PubSubPayloadSink -from apache_beam.io.gcp.pubsub import _PubSubPayloadSource -from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub -from apache_beam.io.gcp.pubsub import WriteStringsToPubSub +from apache_beam.io.gcp.pubsub import (ReadStringsFromPubSub, + WriteStringsToPubSub, _PubSubPayloadSink, + _PubSubPayloadSource) from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher - # Protect against environments where the PubSub library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/io/gcp/tests/utils_test.py b/sdks/python/apache_beam/io/gcp/tests/utils_test.py index 270750a25217..702c43b4955d 100644 --- a/sdks/python/apache_beam/io/gcp/tests/utils_test.py +++ b/sdks/python/apache_beam/io/gcp/tests/utils_test.py @@ -19,6 +19,7 @@ import logging import unittest + from mock import Mock, patch from apache_beam.io.gcp.tests import utils diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index db75fe38baa9..d88eebcf6ca1 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -29,22 +29,16 @@ the sink. """ -from collections import namedtuple - import logging import random import uuid +from collections import namedtuple -from apache_beam import pvalue -from apache_beam import coders +from apache_beam import coders, pvalue from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.pvalue import AsIter -from apache_beam.pvalue import AsSingleton -from apache_beam.transforms import core -from apache_beam.transforms import ptransform -from apache_beam.transforms import window -from apache_beam.transforms.display import HasDisplayData -from apache_beam.transforms.display import DisplayDataItem +from apache_beam.pvalue import AsIter, AsSingleton +from apache_beam.transforms import core, ptransform, window +from apache_beam.transforms.display import DisplayDataItem, HasDisplayData from apache_beam.utils import urns from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py index b08ac494846e..d784ba349e62 100644 --- a/sdks/python/apache_beam/io/localfilesystem.py +++ b/sdks/python/apache_beam/io/localfilesystem.py @@ -22,12 +22,9 @@ import os import shutil -from apache_beam.io.filesystem import BeamIOError -from apache_beam.io.filesystem import CompressedFile -from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.filesystem import FileMetadata -from apache_beam.io.filesystem import FileSystem -from apache_beam.io.filesystem import MatchResult +from apache_beam.io.filesystem import (BeamIOError, CompressedFile, + CompressionTypes, FileMetadata, + FileSystem, MatchResult) __all__ = ['LocalFileSystem'] diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py b/sdks/python/apache_beam/io/localfilesystem_test.py index 04cf5b7c6f4d..8c34ecdf0d36 100644 --- a/sdks/python/apache_beam/io/localfilesystem_test.py +++ b/sdks/python/apache_beam/io/localfilesystem_test.py @@ -18,12 +18,12 @@ """Unit tests for LocalFileSystem.""" -import unittest - import filecmp import os import shutil import tempfile +import unittest + import mock from apache_beam.io import localfilesystem diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py index bea970850838..712049b99f1e 100644 --- a/sdks/python/apache_beam/io/source_test_utils.py +++ b/sdks/python/apache_beam/io/source_test_utils.py @@ -44,12 +44,12 @@ * apache_beam.io.avroio_test.py """ -from collections import namedtuple import logging import threading import weakref - +from collections import namedtuple from multiprocessing.pool import ThreadPool + from apache_beam.io import iobase __all__ = ['read_from_source', 'assert_sources_equal_reference_source', diff --git a/sdks/python/apache_beam/io/source_test_utils_test.py b/sdks/python/apache_beam/io/source_test_utils_test.py index 00522c9fd3f5..af2d4b87d6a7 100644 --- a/sdks/python/apache_beam/io/source_test_utils_test.py +++ b/sdks/python/apache_beam/io/source_test_utils_test.py @@ -19,8 +19,8 @@ import tempfile import unittest -from apache_beam.io.filebasedsource_test import LineSource import apache_beam.io.source_test_utils as source_test_utils +from apache_beam.io.filebasedsource_test import LineSource class SourceTestUtilsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py index 10d401b8d2a2..76df095c0c34 100644 --- a/sdks/python/apache_beam/io/sources_test.py +++ b/sdks/python/apache_beam/io/sources_test.py @@ -23,13 +23,10 @@ import unittest import apache_beam as beam - from apache_beam import coders -from apache_beam.io import iobase -from apache_beam.io import range_trackers +from apache_beam.io import iobase, range_trackers from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.testing.util import assert_that, equal_to class LineSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py index 9708df7c753c..98ca49554034 100644 --- a/sdks/python/apache_beam/io/textio.py +++ b/sdks/python/apache_beam/io/textio.py @@ -19,17 +19,15 @@ from __future__ import absolute_import -from functools import partial + import logging +from functools import partial from apache_beam.coders import coders -from apache_beam.io import filebasedsource -from apache_beam.io import filebasedsink -from apache_beam.io import iobase +from apache_beam.io import filebasedsink, filebasedsource, iobase from apache_beam.io.filebasedsource import ReadAllFiles from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.iobase import Read -from apache_beam.io.iobase import Write +from apache_beam.io.iobase import Read, Write from apache_beam.transforms import PTransform from apache_beam.transforms.display import DisplayDataItem diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py index b29ca5a97137..9fe000bb3a1a 100644 --- a/sdks/python/apache_beam/io/textio_test.py +++ b/sdks/python/apache_beam/io/textio_test.py @@ -27,26 +27,17 @@ import unittest import apache_beam as beam -from apache_beam.io import iobase, ReadAllFromText import apache_beam.io.source_test_utils as source_test_utils - -# Importing following private classes for testing. -from apache_beam.io.textio import _TextSink as TextSink -from apache_beam.io.textio import _TextSource as TextSource - -from apache_beam.io.textio import ReadFromText -from apache_beam.io.textio import WriteToText - from apache_beam import coders -from apache_beam.io.filebasedsource_test import EOL -from apache_beam.io.filebasedsource_test import write_data -from apache_beam.io.filebasedsource_test import write_pattern +from apache_beam.io import ReadAllFromText, iobase +from apache_beam.io.filebasedsource_test import EOL, write_data, write_pattern from apache_beam.io.filesystem import CompressionTypes - +from apache_beam.io.textio import _TextSink as TextSink +from apache_beam.io.textio import _TextSource as TextSource +# Importing following private classes for testing. +from apache_beam.io.textio import ReadFromText, WriteToText from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to - +from apache_beam.testing.util import assert_that, equal_to from apache_beam.transforms.core import Create diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py index d7eb932cd655..6119deeeff87 100644 --- a/sdks/python/apache_beam/io/tfrecordio.py +++ b/sdks/python/apache_beam/io/tfrecordio.py @@ -21,14 +21,13 @@ import logging import struct +import crcmod + from apache_beam import coders -from apache_beam.io import filebasedsource -from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsink, filebasedsource from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.iobase import Read -from apache_beam.io.iobase import Write +from apache_beam.io.iobase import Read, Write from apache_beam.transforms import PTransform -import crcmod __all__ = ['ReadFromTFRecord', 'WriteToTFRecord'] diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py index 3c70ade3dbe3..e48b78de9ef2 100644 --- a/sdks/python/apache_beam/io/tfrecordio_test.py +++ b/sdks/python/apache_beam/io/tfrecordio_test.py @@ -27,19 +27,16 @@ import tempfile import unittest +import crcmod + import apache_beam as beam from apache_beam import coders from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.tfrecordio import _TFRecordSink -from apache_beam.io.tfrecordio import _TFRecordSource -from apache_beam.io.tfrecordio import _TFRecordUtil -from apache_beam.io.tfrecordio import ReadFromTFRecord -from apache_beam.io.tfrecordio import WriteToTFRecord +from apache_beam.io.tfrecordio import (ReadFromTFRecord, WriteToTFRecord, + _TFRecordSink, _TFRecordSource, + _TFRecordUtil) from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to -import crcmod - +from apache_beam.testing.util import assert_that, equal_to try: import tensorflow as tf # pylint: disable=import-error diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py index ba840f752b18..5507cce0c70f 100644 --- a/sdks/python/apache_beam/metrics/cells.py +++ b/sdks/python/apache_beam/metrics/cells.py @@ -26,8 +26,7 @@ import threading -from apache_beam.metrics.metricbase import Counter -from apache_beam.metrics.metricbase import Distribution +from apache_beam.metrics.metricbase import Counter, Distribution __all__ = ['DistributionResult'] diff --git a/sdks/python/apache_beam/metrics/cells_test.py b/sdks/python/apache_beam/metrics/cells_test.py index a4c8a43705fb..ef3774498c14 100644 --- a/sdks/python/apache_beam/metrics/cells_test.py +++ b/sdks/python/apache_beam/metrics/cells_test.py @@ -18,10 +18,8 @@ import threading import unittest -from apache_beam.metrics.cells import CounterCell -from apache_beam.metrics.cells import DistributionCell -from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import CellCommitState +from apache_beam.metrics.cells import (CellCommitState, CounterCell, + DistributionCell, DistributionData) class TestCounterCell(unittest.TestCase): diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py index 675e49c1234c..fe672241cf7a 100644 --- a/sdks/python/apache_beam/metrics/execution.py +++ b/sdks/python/apache_beam/metrics/execution.py @@ -29,8 +29,8 @@ - MetricsContainer - Holds the metrics of a single step and a single unit-of-commit (bundle). """ -from collections import defaultdict import threading +from collections import defaultdict from apache_beam.metrics.cells import CounterCell, DistributionCell diff --git a/sdks/python/apache_beam/metrics/execution_test.py b/sdks/python/apache_beam/metrics/execution_test.py index 54569c10e99d..b25e0e5638f3 100644 --- a/sdks/python/apache_beam/metrics/execution_test.py +++ b/sdks/python/apache_beam/metrics/execution_test.py @@ -18,10 +18,9 @@ import unittest from apache_beam.metrics.cells import CellCommitState -from apache_beam.metrics.execution import MetricsContainer -from apache_beam.metrics.execution import ScopedMetricsContainer -from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import (MetricKey, MetricsContainer, + MetricsEnvironment, + ScopedMetricsContainer) from apache_beam.metrics.metric import Metrics from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py index f99c0c415d56..2283b64f8367 100644 --- a/sdks/python/apache_beam/metrics/metric.py +++ b/sdks/python/apache_beam/metrics/metric.py @@ -27,8 +27,7 @@ import inspect from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.metricbase import Counter, Distribution -from apache_beam.metrics.metricbase import MetricName +from apache_beam.metrics.metricbase import Counter, Distribution, MetricName __all__ = ['Metrics', 'MetricsFilter'] diff --git a/sdks/python/apache_beam/metrics/metric_test.py b/sdks/python/apache_beam/metrics/metric_test.py index 56b74680a97d..e962074d8948 100644 --- a/sdks/python/apache_beam/metrics/metric_test.py +++ b/sdks/python/apache_beam/metrics/metric_test.py @@ -18,12 +18,9 @@ import unittest from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricsContainer -from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.metric import Metrics -from apache_beam.metrics.metric import MetricsFilter -from apache_beam.metrics.metric import MetricResults +from apache_beam.metrics.execution import (MetricKey, MetricsContainer, + MetricsEnvironment) +from apache_beam.metrics.metric import MetricResults, Metrics, MetricsFilter from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index db65b3c65837..2285af1047da 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -19,11 +19,10 @@ import argparse +from apache_beam.options.value_provider import (RuntimeValueProvider, + StaticValueProvider, + ValueProvider) from apache_beam.transforms.display import HasDisplayData -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import RuntimeValueProvider -from apache_beam.options.value_provider import ValueProvider - __all__ = [ 'PipelineOptions', diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index f4dd4d92b788..cc42ae9f7910 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -21,11 +21,12 @@ import unittest import hamcrest as hc + +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.value_provider import (RuntimeValueProvider, + StaticValueProvider) from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.value_provider import StaticValueProvider -from apache_beam.options.value_provider import RuntimeValueProvider class PipelineOptionsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/options/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py index 24d2e55eadbe..df0b561a0474 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator.py @@ -22,13 +22,11 @@ import re from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import DebugOptions -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import TestOptions -from apache_beam.options.pipeline_options import TypeOptions -from apache_beam.options.pipeline_options import WorkerOptions +from apache_beam.options.pipeline_options import (DebugOptions, + GoogleCloudOptions, + SetupOptions, StandardOptions, + TestOptions, TypeOptions, + WorkerOptions) class PipelineOptionsValidator(object): diff --git a/sdks/python/apache_beam/options/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py index 97834cc4821b..f7e576ec11f5 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py @@ -20,10 +20,12 @@ import logging import unittest +from hamcrest.core.base_matcher import BaseMatcher + from apache_beam.internal import pickler from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator -from hamcrest.core.base_matcher import BaseMatcher +from apache_beam.options.pipeline_options_validator import \ + PipelineOptionsValidator # Mock runners to use for validations. diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py index f7630bd8d326..fe4614dc37da 100644 --- a/sdks/python/apache_beam/options/value_provider.py +++ b/sdks/python/apache_beam/options/value_provider.py @@ -23,7 +23,6 @@ from apache_beam import error - __all__ = [ 'ValueProvider', 'StaticValueProvider', diff --git a/sdks/python/apache_beam/options/value_provider_test.py b/sdks/python/apache_beam/options/value_provider_test.py index 17e9590d2a36..60d10a171359 100644 --- a/sdks/python/apache_beam/options/value_provider_test.py +++ b/sdks/python/apache_beam/options/value_provider_test.py @@ -20,8 +20,8 @@ import unittest from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.value_provider import RuntimeValueProvider -from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import (RuntimeValueProvider, + StaticValueProvider) # TODO(BEAM-1319): Require unique names only within a test. diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 1ade6c067c5f..d29026b56b5f 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -55,20 +55,16 @@ from apache_beam import pvalue from apache_beam.internal import pickler +from apache_beam.options.pipeline_options import (PipelineOptions, SetupOptions, + StandardOptions, TypeOptions) +from apache_beam.options.pipeline_options_validator import \ + PipelineOptionsValidator from apache_beam.pvalue import PCollection -from apache_beam.runners import create_runner -from apache_beam.runners import PipelineRunner +from apache_beam.runners import PipelineRunner, create_runner from apache_beam.transforms import ptransform -from apache_beam.typehints import typehints -from apache_beam.typehints import TypeCheckError -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import TypeOptions -from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator -from apache_beam.utils.annotations import deprecated +from apache_beam.typehints import TypeCheckError, typehints from apache_beam.utils import urns - +from apache_beam.utils.annotations import deprecated __all__ = ['Pipeline'] diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index b3ac100780fe..83ada1eae6d0 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -21,34 +21,25 @@ import platform import unittest -# TODO(BEAM-1555): Test is failing on the service, with FakeSource. -# from nose.plugins.attrib import attr - import apache_beam as beam from apache_beam.io import Read from apache_beam.metrics import Metrics -from apache_beam.pipeline import Pipeline -from apache_beam.pipeline import PTransformOverride -from apache_beam.pipeline import PipelineOptions -from apache_beam.pipeline import PipelineVisitor +from apache_beam.pipeline import (Pipeline, PipelineOptions, PipelineVisitor, + PTransformOverride) from apache_beam.pvalue import AsSingleton from apache_beam.runners import DirectRunner from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to -from apache_beam.transforms import CombineGlobally -from apache_beam.transforms import Create -from apache_beam.transforms import FlatMap -from apache_beam.transforms import Map -from apache_beam.transforms import DoFn -from apache_beam.transforms import ParDo -from apache_beam.transforms import PTransform -from apache_beam.transforms import WindowInto -from apache_beam.transforms.window import SlidingWindows -from apache_beam.transforms.window import TimestampedValue +from apache_beam.testing.util import assert_that, equal_to +from apache_beam.transforms import (CombineGlobally, Create, DoFn, FlatMap, Map, + ParDo, PTransform, WindowInto) +from apache_beam.transforms.window import SlidingWindows, TimestampedValue from apache_beam.utils.timestamp import MIN_TIMESTAMP +# TODO(BEAM-1555): Test is failing on the service, with FakeSource. +# from nose.plugins.attrib import attr + + class FakeSource(NativeSource): """Fake source returning a fixed list of values.""" diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 35686f7fa381..53a6121eef34 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -30,7 +30,6 @@ from apache_beam import typehints - __all__ = [ 'PCollection', 'TaggedOutput', diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index 84535692806b..8f1ecb5695ec 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -29,9 +29,8 @@ from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.pvalue import TaggedOutput from apache_beam.transforms import core -from apache_beam.transforms.window import TimestampedValue -from apache_beam.transforms.window import WindowFn -from apache_beam.transforms.window import GlobalWindow +from apache_beam.transforms.window import (GlobalWindow, TimestampedValue, + WindowFn) from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/common_test.py b/sdks/python/apache_beam/runners/common_test.py index 62a6955f6ce4..e0f628c71ee1 100644 --- a/sdks/python/apache_beam/runners/common_test.py +++ b/sdks/python/apache_beam/runners/common_test.py @@ -17,8 +17,8 @@ import unittest -from apache_beam.transforms.core import DoFn from apache_beam.runners.common import DoFnSignature +from apache_beam.transforms.core import DoFn class DoFnSignatureTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py index 24916fd49278..ec224909d157 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py @@ -21,13 +21,11 @@ service. """ -from collections import defaultdict import numbers +from collections import defaultdict -from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import DistributionResult -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.cells import DistributionData, DistributionResult +from apache_beam.metrics.execution import MetricKey, MetricResult from apache_beam.metrics.metric import MetricResults from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py index dd3cbe1156a4..c077ea393328 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py @@ -23,10 +23,8 @@ import mock -from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import DistributionResult -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.cells import DistributionData, DistributionResult +from apache_beam.metrics.execution import MetricKey, MetricResult from apache_beam.metrics.metricbase import MetricName from apache_beam.runners.dataflow import dataflow_metrics diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 56cc46c815a8..042c1f16c3a3 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -21,37 +21,32 @@ to the Dataflow Service for remote execution by a worker. """ -from collections import defaultdict import logging import threading import time import traceback import urllib +from collections import defaultdict import apache_beam as beam -from apache_beam import error -from apache_beam import coders -from apache_beam import pvalue +from apache_beam import coders, error, pvalue from apache_beam.internal import pickler from apache_beam.internal.gcp import json_value +from apache_beam.options.pipeline_options import (SetupOptions, StandardOptions, + TestOptions) from apache_beam.pvalue import AsSideInput from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics from apache_beam.runners.dataflow.internal import names -from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api -from apache_beam.runners.dataflow.internal.names import PropertyNames -from apache_beam.runners.dataflow.internal.names import TransformNames -from apache_beam.runners.runner import PValueCache -from apache_beam.runners.runner import PipelineResult -from apache_beam.runners.runner import PipelineRunner -from apache_beam.runners.runner import PipelineState +from apache_beam.runners.dataflow.internal.clients import \ + dataflow as dataflow_api +from apache_beam.runners.dataflow.internal.names import (PropertyNames, + TransformNames) +from apache_beam.runners.runner import (PipelineResult, PipelineRunner, + PipelineState, PValueCache) from apache_beam.transforms.display import DisplayData from apache_beam.typehints import typehints -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import TestOptions from apache_beam.utils.plugin import BeamPlugin - __all__ = ['DataflowRunner'] diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index 8e708e6fde29..e77350c16831 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -25,22 +25,20 @@ import apache_beam as beam import apache_beam.transforms as ptransform - from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.pipeline import Pipeline, AppliedPTransform +from apache_beam.pipeline import AppliedPTransform, Pipeline from apache_beam.pvalue import PCollection -from apache_beam.runners import create_runner -from apache_beam.runners import DataflowRunner -from apache_beam.runners import TestDataflowRunner -from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult -from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException -from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api +from apache_beam.runners import (DataflowRunner, TestDataflowRunner, + create_runner) +from apache_beam.runners.dataflow.dataflow_runner import (DataflowPipelineResult, + DataflowRuntimeException) +from apache_beam.runners.dataflow.internal.clients import \ + dataflow as dataflow_api from apache_beam.runners.runner import PipelineState from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.transforms.display import DisplayDataItem -from apache_beam.transforms.core import _GroupByKeyOnly -from apache_beam.transforms.core import Windowing from apache_beam.transforms import window +from apache_beam.transforms.core import Windowing, _GroupByKeyOnly +from apache_beam.transforms.display import DisplayDataItem from apache_beam.typehints import typehints # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index a1f9301b19ba..a0c1be43f94e 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -26,28 +26,27 @@ import os import re import time -from StringIO import StringIO from datetime import datetime +from StringIO import StringIO -from apitools.base.py import encoding -from apitools.base.py import exceptions +from apitools.base.py import encoding, exceptions from apache_beam.internal.gcp.auth import get_service_credentials from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.io.filesystems import FileSystems from apache_beam.io.gcp.internal.clients import storage +from apache_beam.options.pipeline_options import (DebugOptions, + GoogleCloudOptions, + StandardOptions, + WorkerOptions) from apache_beam.runners.dataflow.internal import dependency from apache_beam.runners.dataflow.internal.clients import dataflow -from apache_beam.runners.dataflow.internal.dependency import get_sdk_name_and_version +from apache_beam.runners.dataflow.internal.dependency import \ + get_sdk_name_and_version from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.transforms import cy_combiners from apache_beam.transforms.display import DisplayData from apache_beam.utils import retry -from apache_beam.options.pipeline_options import DebugOptions -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import WorkerOptions - # Environment version information. It is passed to the service during a # a job submission and is used by the service to establish what features diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 407ffcf2ad72..e7cced7b4f3f 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -21,7 +21,6 @@ from apache_beam.metrics.cells import DistributionData from apache_beam.options.pipeline_options import PipelineOptions - from apache_beam.runners.dataflow.internal.clients import dataflow # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py index 61d02730ab54..977a4c03c038 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py @@ -19,7 +19,8 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.runners.dataflow.internal.clients.dataflow import dataflow_v1b3_messages as messages +from apache_beam.runners.dataflow.internal.clients.dataflow import \ + dataflow_v1b3_messages as messages class DataflowV1b3(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py index e81329fca80f..d3fca37a7586 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py @@ -23,9 +23,7 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding -from apitools.base.py import extra_types - +from apitools.base.py import encoding, extra_types package = 'dataflow' diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py index 4dda47ad9e05..805473a8838c 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py @@ -17,7 +17,6 @@ from hamcrest.core.base_matcher import BaseMatcher - IGNORED = object() diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py index 3163c9b25125..22b13f511efa 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py @@ -15,11 +15,13 @@ # limitations under the License. # import unittest + import hamcrest as hc -import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow +import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow from apache_beam.internal.gcp.json_value import to_json_value -from apache_beam.runners.dataflow.internal.clients.dataflow import message_matchers +from apache_beam.runners.dataflow.internal.clients.dataflow import \ + message_matchers # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index f5dd2bd35732..d5e61241439b 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -64,10 +64,10 @@ from apache_beam import version as beam_version from apache_beam.internal import pickler from apache_beam.io.filesystems import FileSystems +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + SetupOptions) from apache_beam.runners.dataflow.internal import names from apache_beam.utils import processes -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import SetupOptions # All constants are for internal use only; no backwards-compatibility # guarantees. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py index e555b69f15ad..643de6af10b3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py @@ -24,12 +24,9 @@ import unittest from apache_beam.io.filesystems import FileSystems -from apache_beam.runners.dataflow.internal import dependency -from apache_beam.runners.dataflow.internal import names -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions - +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + PipelineOptions, SetupOptions) +from apache_beam.runners.dataflow.internal import dependency, names # Protect against environments where GCS library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index 3d8c24f5651c..fdee2ee2c15f 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -20,21 +20,17 @@ import unittest -from apache_beam import Create -from apache_beam import error -from apache_beam import pvalue -from apache_beam.runners.dataflow.native_io.iobase import ( - _dict_printable_fields, - _NativeWrite, - ConcatPosition, - DynamicSplitRequest, - DynamicSplitResultWithPosition, - NativeSink, - NativeSinkWriter, - NativeSource, - ReaderPosition, - ReaderProgress -) +from apache_beam import Create, error, pvalue +from apache_beam.runners.dataflow.native_io.iobase import (ConcatPosition, + DynamicSplitRequest, + DynamicSplitResultWithPosition, + NativeSink, + NativeSinkWriter, + NativeSource, + ReaderPosition, + ReaderProgress, + _dict_printable_fields, + _NativeWrite) from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py index 8c6c8d6d5299..929da81ed8a5 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py @@ -17,11 +17,7 @@ """Create transform for streaming.""" -from apache_beam import pvalue -from apache_beam import DoFn -from apache_beam import ParDo -from apache_beam import PTransform -from apache_beam import Windowing +from apache_beam import DoFn, ParDo, PTransform, Windowing, pvalue from apache_beam.transforms.window import GlobalWindows diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py index 7927219ab5ea..88afe8a2ee62 100644 --- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py @@ -24,8 +24,8 @@ import unittest import apache_beam as beam -from apache_beam.pipeline import Pipeline from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.pipeline import Pipeline from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py index 96e6a66caab4..b0a0159bce48 100644 --- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py @@ -18,10 +18,9 @@ """Wrapper of Beam runners that's built for running and verifying e2e tests.""" from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import TestOptions, GoogleCloudOptions +from apache_beam.options.pipeline_options import GoogleCloudOptions, TestOptions from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner - __all__ = ['TestDataflowRunner'] diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py index 97d1ee8bfb6b..0263904a7341 100644 --- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py +++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py @@ -21,18 +21,14 @@ import unittest from apache_beam import pvalue -from apache_beam.io import iobase -from apache_beam.io import Read +from apache_beam.io import Read, iobase from apache_beam.pipeline import Pipeline from apache_beam.pvalue import AsList from apache_beam.runners.direct import DirectRunner -from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor -from apache_beam.transforms import CoGroupByKey -from apache_beam.transforms import Create -from apache_beam.transforms import DoFn -from apache_beam.transforms import FlatMap -from apache_beam.transforms import Flatten -from apache_beam.transforms import ParDo +from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import \ + ConsumerTrackingPipelineVisitor +from apache_beam.transforms import (CoGroupByKey, Create, DoFn, FlatMap, + Flatten, ParDo) # Disable frequent lint warning due to pipe operator for chaining transforms. # pylint: disable=expression-not-assigned diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics.py b/sdks/python/apache_beam/runners/direct/direct_metrics.py index 9d234876035a..c8cf0344cbac 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics.py @@ -20,13 +20,11 @@ responding to queries of current metrics, but also of keeping the common state consistent. """ -from collections import defaultdict import threading +from collections import defaultdict -from apache_beam.metrics.cells import CounterAggregator -from apache_beam.metrics.cells import DistributionAggregator -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.cells import CounterAggregator, DistributionAggregator +from apache_beam.metrics.execution import MetricKey, MetricResult from apache_beam.metrics.metric import MetricResults diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py index 256b91f3e9af..b543f0dacc28 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py @@ -19,12 +19,9 @@ import hamcrest as hc +from apache_beam.metrics.cells import DistributionData, DistributionResult +from apache_beam.metrics.execution import MetricKey, MetricResult, MetricUpdates from apache_beam.metrics.metricbase import MetricName -from apache_beam.metrics.execution import MetricUpdates -from apache_beam.metrics.execution import MetricResult -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import DistributionResult from apache_beam.runners.direct.direct_metrics import DirectMetrics diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index 2deb7dace023..dff8689c5682 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -31,19 +31,14 @@ import apache_beam as beam from apache_beam import typehints from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.options.pipeline_options import DirectOptions, StandardOptions +from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.pvalue import PCollection from apache_beam.runners.direct.bundle_factory import BundleFactory -from apache_beam.runners.runner import PipelineResult -from apache_beam.runners.runner import PipelineRunner -from apache_beam.runners.runner import PipelineState -from apache_beam.runners.runner import PValueCache +from apache_beam.runners.runner import (PipelineResult, PipelineRunner, + PipelineState, PValueCache) +from apache_beam.transforms.core import _GroupAlsoByWindow, _GroupByKeyOnly from apache_beam.transforms.ptransform import PTransform -from apache_beam.transforms.core import _GroupAlsoByWindow -from apache_beam.transforms.core import _GroupByKeyOnly -from apache_beam.options.pipeline_options import DirectOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.value_provider import RuntimeValueProvider - __all__ = ['DirectRunner'] diff --git a/sdks/python/apache_beam/runners/direct/evaluation_context.py b/sdks/python/apache_beam/runners/direct/evaluation_context.py index 54c407c1c866..9913813f364a 100644 --- a/sdks/python/apache_beam/runners/direct/evaluation_context.py +++ b/sdks/python/apache_beam/runners/direct/evaluation_context.py @@ -22,11 +22,11 @@ import collections import threading -from apache_beam.transforms import sideinputs from apache_beam.runners.direct.clock import Clock -from apache_beam.runners.direct.watermark_manager import WatermarkManager -from apache_beam.runners.direct.executor import TransformExecutor from apache_beam.runners.direct.direct_metrics import DirectMetrics +from apache_beam.runners.direct.executor import TransformExecutor +from apache_beam.runners.direct.watermark_manager import WatermarkManager +from apache_beam.transforms import sideinputs from apache_beam.transforms.trigger import InMemoryUnmergedState from apache_beam.utils import counters diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py index 3e08b522d541..3509f4e341df 100644 --- a/sdks/python/apache_beam/runners/direct/executor.py +++ b/sdks/python/apache_beam/runners/direct/executor.py @@ -28,8 +28,8 @@ import traceback from weakref import WeakValueDictionary -from apache_beam.metrics.execution import MetricsContainer -from apache_beam.metrics.execution import ScopedMetricsContainer +from apache_beam.metrics.execution import (MetricsContainer, + ScopedMetricsContainer) from apache_beam.options.pipeline_options import DirectOptions diff --git a/sdks/python/apache_beam/runners/direct/helper_transforms.py b/sdks/python/apache_beam/runners/direct/helper_transforms.py index 374cd4ea69db..26b0701bd02b 100644 --- a/sdks/python/apache_beam/runners/direct/helper_transforms.py +++ b/sdks/python/apache_beam/runners/direct/helper_transforms.py @@ -20,8 +20,8 @@ import apache_beam as beam from apache_beam import typehints -from apache_beam.utils.windowed_value import WindowedValue from apache_beam.internal.util import ArgumentPlaceholder +from apache_beam.utils.windowed_value import WindowedValue class LiftedCombinePerKey(beam.PTransform): diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index cb2ace29f0ea..a7da152769aa 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -23,36 +23,29 @@ import random import time -from apache_beam import coders -from apache_beam import pvalue -from apache_beam.internal import pickler import apache_beam.io as io -from apache_beam.runners.common import DoFnRunner -from apache_beam.runners.common import DoFnState -from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly -from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow +from apache_beam import coders, pvalue +from apache_beam.internal import pickler +from apache_beam.options.pipeline_options import TypeOptions +from apache_beam.runners.common import DoFnRunner, DoFnState +from apache_beam.runners.dataflow.native_io.iobase import \ + _NativeWrite # pylint: disable=protected-access +from apache_beam.runners.direct.direct_runner import (_StreamingGroupAlsoByWindow, + _StreamingGroupByKeyOnly) +from apache_beam.runners.direct.util import KeyedWorkItem, TransformResult from apache_beam.runners.direct.watermark_manager import WatermarkManager -from apache_beam.runners.direct.util import KeyedWorkItem -from apache_beam.runners.direct.util import TransformResult -from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access -from apache_beam.testing.test_stream import TestStream -from apache_beam.testing.test_stream import ElementEvent -from apache_beam.testing.test_stream import WatermarkEvent -from apache_beam.testing.test_stream import ProcessingTimeEvent +from apache_beam.testing.test_stream import (ElementEvent, ProcessingTimeEvent, + TestStream, WatermarkEvent) from apache_beam.transforms import core -from apache_beam.transforms.window import GlobalWindows -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.trigger import create_trigger_driver -from apache_beam.transforms.trigger import _CombiningValueStateTag -from apache_beam.transforms.trigger import _ListStateTag -from apache_beam.transforms.trigger import TimeDomain -from apache_beam.typehints.typecheck import OutputCheckWrapperDoFn -from apache_beam.typehints.typecheck import TypeCheckError -from apache_beam.typehints.typecheck import TypeCheckWrapperDoFn +from apache_beam.transforms.trigger import (TimeDomain, _CombiningValueStateTag, + _ListStateTag, + create_trigger_driver) +from apache_beam.transforms.window import GlobalWindows, WindowedValue +from apache_beam.typehints.typecheck import (OutputCheckWrapperDoFn, + TypeCheckError, + TypeCheckWrapperDoFn) from apache_beam.utils import counters -from apache_beam.utils.timestamp import Timestamp -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.options.pipeline_options import TypeOptions +from apache_beam.utils.timestamp import MIN_TIMESTAMP, Timestamp class TransformEvaluatorRegistry(object): diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py index 935998d27de0..5ab83e94bf3d 100644 --- a/sdks/python/apache_beam/runners/direct/watermark_manager.py +++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py @@ -21,12 +21,10 @@ import threading -from apache_beam import pipeline -from apache_beam import pvalue +from apache_beam import pipeline, pvalue from apache_beam.runners.direct.util import TimerFiring -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils.timestamp import TIME_GRANULARITY +from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, + TIME_GRANULARITY) class WatermarkManager(object): diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py index 247ce1f0e195..98f3e68593de 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py @@ -27,9 +27,7 @@ from apache_beam.portability.api import beam_job_api_pb2 from apache_beam.runners.job import utils as job_utils from apache_beam.runners.job.manager import DockerRPCManager -from apache_beam.runners.runner import PipelineResult -from apache_beam.runners.runner import PipelineRunner - +from apache_beam.runners.runner import PipelineResult, PipelineRunner __all__ = ['PythonRPCDirectRunner'] diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py index bae25a4d07b1..4d0ef4e4cca3 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py @@ -19,14 +19,13 @@ """ import time import uuid - from concurrent import futures + import grpc -from apache_beam.portability.api import beam_job_api_pb2 -from apache_beam.portability.api import beam_job_api_pb2_grpc -from apache_beam.pipeline import Pipeline from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.pipeline import Pipeline +from apache_beam.portability.api import beam_job_api_pb2, beam_job_api_pb2_grpc from apache_beam.runners.runner import PipelineState _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/sdks/python/apache_beam/runners/job/utils.py b/sdks/python/apache_beam/runners/job/utils.py index 84c727fb4ebc..80b5e451f2d0 100644 --- a/sdks/python/apache_beam/runners/job/utils.py +++ b/sdks/python/apache_beam/runners/job/utils.py @@ -20,8 +20,7 @@ import json -from google.protobuf import json_format -from google.protobuf import struct_pb2 +from google.protobuf import json_format, struct_pb2 def dict_to_struct(dict_obj): diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index 42d7f5df14a8..a2436ce3ff08 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -21,11 +21,8 @@ """ -from apache_beam import pipeline -from apache_beam import pvalue -from apache_beam import coders -from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam import coders, pipeline, pvalue +from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 from apache_beam.transforms import core diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py index c9b3d9a5e831..820e4f022a4d 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py @@ -23,32 +23,26 @@ import logging import Queue as queue import threading - from concurrent import futures -from google.protobuf import wrappers_pb2 + import grpc +from google.protobuf import wrappers_pb2 import apache_beam as beam # pylint: disable=ungrouped-imports -from apache_beam.coders import registry -from apache_beam.coders import WindowedValueCoder -from apache_beam.coders.coder_impl import create_InputStream -from apache_beam.coders.coder_impl import create_OutputStream +from apache_beam.coders import WindowedValueCoder, registry +from apache_beam.coders.coder_impl import (create_InputStream, + create_OutputStream) from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 from apache_beam.runners import pipeline_context from apache_beam.runners.portability import maptask_executor_runner from apache_beam.runners.runner import PipelineState -from apache_beam.runners.worker import bundle_processor -from apache_beam.runners.worker import data_plane -from apache_beam.runners.worker import operation_specs -from apache_beam.runners.worker import sdk_worker +from apache_beam.runners.worker import (bundle_processor, data_plane, + operation_specs, sdk_worker) from apache_beam.transforms.window import GlobalWindows -from apache_beam.utils import proto_utils -from apache_beam.utils import urns - +from apache_beam.utils import proto_utils, urns # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py index ba219547e6cf..ef7660518bd5 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py @@ -19,10 +19,9 @@ import unittest import apache_beam as beam -from apache_beam.runners.portability import fn_api_runner -from apache_beam.runners.portability import maptask_executor_runner_test -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to +from apache_beam.runners.portability import (fn_api_runner, + maptask_executor_runner_test) +from apache_beam.testing.util import assert_that, equal_to class FnApiRunnerTest( diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py index ddfc4ccc43f1..c4c6c3813558 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py @@ -28,21 +28,21 @@ from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.options import pipeline_options from apache_beam.runners import DataflowRunner -from apache_beam.runners.dataflow.internal.dependency import _dependency_file_copy +from apache_beam.runners.dataflow.internal.dependency import \ + _dependency_file_copy from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.dataflow.native_io.iobase import NativeSource -from apache_beam.runners.runner import PipelineResult -from apache_beam.runners.runner import PipelineRunner -from apache_beam.runners.runner import PipelineState -from apache_beam.runners.worker import operation_specs -from apache_beam.runners.worker import operations +from apache_beam.runners.runner import (PipelineResult, PipelineRunner, + PipelineState) +from apache_beam.runners.worker import operation_specs, operations +from apache_beam.typehints import typehints +from apache_beam.utils import profiler +from apache_beam.utils.counters import CounterFactory + try: from apache_beam.runners.worker import statesampler except ImportError: from apache_beam.runners.worker import statesampler_fake as statesampler -from apache_beam.typehints import typehints -from apache_beam.utils import profiler -from apache_beam.utils.counters import CounterFactory # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py index b7ba15a502aa..17ba7788540f 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py @@ -21,18 +21,13 @@ import unittest import apache_beam as beam - from apache_beam.metrics import Metrics -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.metrics.execution import MetricKey, MetricsEnvironment from apache_beam.metrics.metricbase import MetricName - from apache_beam.pvalue import AsList -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import BeamAssertException -from apache_beam.testing.util import equal_to -from apache_beam.transforms.window import TimestampedValue from apache_beam.runners.portability import maptask_executor_runner +from apache_beam.testing.util import BeamAssertException, assert_that, equal_to +from apache_beam.transforms.window import TimestampedValue class MapTaskExecutorRunnerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py index 43ee27bc298f..4ac62237f1e4 100644 --- a/sdks/python/apache_beam/runners/runner.py +++ b/sdks/python/apache_beam/runners/runner.py @@ -25,7 +25,6 @@ import shutil import tempfile - __all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult'] diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py index fa80b1c9555c..1b3b1b4f69f3 100644 --- a/sdks/python/apache_beam/runners/runner_test.py +++ b/sdks/python/apache_beam/runners/runner_test.py @@ -28,17 +28,13 @@ import apache_beam as beam import apache_beam.transforms as ptransform -from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import DistributionResult -from apache_beam.metrics.execution import MetricKey -from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.cells import DistributionData, DistributionResult +from apache_beam.metrics.execution import MetricKey, MetricResult from apache_beam.metrics.metricbase import MetricName -from apache_beam.pipeline import Pipeline -from apache_beam.runners import DirectRunner -from apache_beam.runners import create_runner -from apache_beam.testing.util import assert_that -from apache_beam.testing.util import equal_to from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.pipeline import Pipeline +from apache_beam.runners import DirectRunner, create_runner +from apache_beam.testing.util import assert_that, equal_to class RunnerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py index 16c888c63c60..9b75b706726c 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py @@ -17,9 +17,7 @@ """SDK harness for executing Python Fns via the Fn API.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import base64 import collections @@ -29,19 +27,14 @@ from google.protobuf import wrappers_pb2 import apache_beam as beam -from apache_beam.coders import coder_impl -from apache_beam.coders import WindowedValueCoder +from apache_beam.coders import WindowedValueCoder, coder_impl from apache_beam.internal import pickler from apache_beam.io import iobase -from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.runners.dataflow.native_io import iobase as native_iobase +from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 from apache_beam.runners import pipeline_context -from apache_beam.runners.worker import operation_specs -from apache_beam.runners.worker import operations -from apache_beam.utils import counters -from apache_beam.utils import proto_utils -from apache_beam.utils import urns +from apache_beam.runners.dataflow.native_io import iobase as native_iobase +from apache_beam.runners.worker import operation_specs, operations +from apache_beam.utils import counters, proto_utils, urns # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py index e713041d7c83..e44f2d14276e 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane.py +++ b/sdks/python/apache_beam/runners/worker/data_plane.py @@ -17,9 +17,7 @@ """Implementation of DataChannels for communicating across the data plane.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import abc import collections @@ -27,9 +25,10 @@ import Queue as queue import threading +import grpc + from apache_beam.coders import coder_impl from apache_beam.portability.api import beam_fn_api_pb2 -import grpc # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index 360468a86874..e15aac9b8e54 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -17,16 +17,14 @@ """Tests for apache_beam.runners.worker.data_plane.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging import sys import threading import unittest - from concurrent import futures + import grpc from apache_beam.portability.api import beam_fn_api_pb2 diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py index b8f635210d2e..20bd49f16a6d 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler.py +++ b/sdks/python/apache_beam/runners/worker/log_handler.py @@ -21,9 +21,10 @@ import Queue as queue import threading -from apache_beam.portability.api import beam_fn_api_pb2 import grpc +from apache_beam.portability.api import beam_fn_api_pb2 + # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 2256bb5556f0..7edf66742b84 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -18,8 +18,8 @@ import logging import unittest - from concurrent import futures + import grpc from apache_beam.portability.api import beam_fn_api_pb2 diff --git a/sdks/python/apache_beam/runners/worker/opcounters.py b/sdks/python/apache_beam/runners/worker/opcounters.py index 2bb15fa7ee4b..f8f4b51b80b0 100644 --- a/sdks/python/apache_beam/runners/worker/opcounters.py +++ b/sdks/python/apache_beam/runners/worker/opcounters.py @@ -20,6 +20,7 @@ """Counters collect the progress of the Worker for reporting to the service.""" from __future__ import absolute_import + import math import random diff --git a/sdks/python/apache_beam/runners/worker/opcounters_test.py b/sdks/python/apache_beam/runners/worker/opcounters_test.py index 74561b81593e..008720f9493a 100644 --- a/sdks/python/apache_beam/runners/worker/opcounters_test.py +++ b/sdks/python/apache_beam/runners/worker/opcounters_test.py @@ -25,7 +25,6 @@ from apache_beam.transforms.window import GlobalWindows from apache_beam.utils.counters import CounterFactory - # Classes to test that we can handle a variety of objects. # These have to be at top level so the pickler can find them. diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py index c4f945bf2b9d..f99d0bda8074 100644 --- a/sdks/python/apache_beam/runners/worker/operations.py +++ b/sdks/python/apache_beam/runners/worker/operations.py @@ -26,20 +26,17 @@ from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.io import iobase -from apache_beam.metrics.execution import MetricsContainer -from apache_beam.metrics.execution import ScopedMetricsContainer +from apache_beam.metrics.execution import (MetricsContainer, + ScopedMetricsContainer) from apache_beam.runners import common from apache_beam.runners.common import Receiver from apache_beam.runners.dataflow.internal.names import PropertyNames -from apache_beam.runners.worker import logger -from apache_beam.runners.worker import opcounters -from apache_beam.runners.worker import operation_specs -from apache_beam.runners.worker import sideinputs -from apache_beam.transforms import combiners -from apache_beam.transforms import core +from apache_beam.runners.worker import (logger, opcounters, operation_specs, + sideinputs) from apache_beam.transforms import sideinputs as apache_sideinputs -from apache_beam.transforms.combiners import curry_combine_fn -from apache_beam.transforms.combiners import PhasedCombineFnExecutor +from apache_beam.transforms import combiners, core +from apache_beam.transforms.combiners import (PhasedCombineFnExecutor, + curry_combine_fn) from apache_beam.transforms.window import GlobalWindows from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index 6a236802b9a6..36fde7b6fd4b 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -17,9 +17,7 @@ """SDK harness for executing Python Fns via the Fn API.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging import Queue as queue @@ -27,8 +25,7 @@ import traceback from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.runners.worker import bundle_processor -from apache_beam.runners.worker import data_plane +from apache_beam.runners.worker import bundle_processor, data_plane class SdkHarness(object): diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index dc72a5ff4fae..407d589580ef 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -17,18 +17,15 @@ """Tests for apache_beam.runners.worker.sdk_worker.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import logging import unittest - from concurrent import futures + import grpc -from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 from apache_beam.runners.worker import sdk_worker diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py index 883343acdab6..c421e259398f 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py @@ -32,7 +32,6 @@ from apache_beam.testing import test_utils as utils from apache_beam.utils import retry - __all__ = [ 'PipelineStateMatcher', 'FileChecksumMatcher', diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py index 15e0a042008b..23cc00f1f1a6 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py @@ -25,10 +25,9 @@ from mock import Mock, patch from apache_beam.io.localfilesystem import LocalFileSystem -from apache_beam.runners.runner import PipelineResult -from apache_beam.runners.runner import PipelineState -from apache_beam.testing.test_utils import patch_retry +from apache_beam.runners.runner import PipelineResult, PipelineState from apache_beam.testing import pipeline_verifiers as verifiers +from apache_beam.testing.test_utils import patch_retry try: # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py index 83802423dbca..46eeb75183d7 100644 --- a/sdks/python/apache_beam/testing/test_pipeline.py +++ b/sdks/python/apache_beam/testing/test_pipeline.py @@ -20,12 +20,12 @@ import argparse import shlex +from nose.plugins.skip import SkipTest + from apache_beam.internal import pickler +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline from apache_beam.runners.runner import PipelineState -from apache_beam.options.pipeline_options import PipelineOptions -from nose.plugins.skip import SkipTest - __all__ = [ 'TestPipeline', diff --git a/sdks/python/apache_beam/testing/test_pipeline_test.py b/sdks/python/apache_beam/testing/test_pipeline_test.py index 747d64c7409f..c642c65a7b7d 100644 --- a/sdks/python/apache_beam/testing/test_pipeline_test.py +++ b/sdks/python/apache_beam/testing/test_pipeline_test.py @@ -20,12 +20,12 @@ import logging import unittest -from hamcrest.core.base_matcher import BaseMatcher from hamcrest.core.assert_that import assert_that as hc_assert_that +from hamcrest.core.base_matcher import BaseMatcher from apache_beam.internal import pickler -from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.testing.test_pipeline import TestPipeline # A simple matcher that is ued for testing extra options appending. diff --git a/sdks/python/apache_beam/testing/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py index 7989fb2eee94..1ee9915fb315 100644 --- a/sdks/python/apache_beam/testing/test_stream.py +++ b/sdks/python/apache_beam/testing/test_stream.py @@ -20,19 +20,14 @@ For internal use only; no backwards-compatibility guarantees. """ -from abc import ABCMeta -from abc import abstractmethod - -from apache_beam import coders -from apache_beam import core -from apache_beam import pvalue -from apache_beam.transforms import PTransform -from apache_beam.transforms import window +from abc import ABCMeta, abstractmethod + +from apache_beam import coders, core, pvalue +from apache_beam.transforms import PTransform, window from apache_beam.transforms.window import TimestampedValue from apache_beam.utils import timestamp from apache_beam.utils.windowed_value import WindowedValue - __all__ = [ 'Event', 'ElementEvent', diff --git a/sdks/python/apache_beam/testing/test_stream_test.py b/sdks/python/apache_beam/testing/test_stream_test.py index b7ca141f0598..caafe4f245d4 100644 --- a/sdks/python/apache_beam/testing/test_stream_test.py +++ b/sdks/python/apache_beam/testing/test_stream_test.py @@ -20,16 +20,13 @@ import unittest import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import (PipelineOptions, + StandardOptions) from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.test_stream import ElementEvent -from apache_beam.testing.test_stream import ProcessingTimeEvent -from apache_beam.testing.test_stream import TestStream -from apache_beam.testing.test_stream import WatermarkEvent +from apache_beam.testing.test_stream import (ElementEvent, ProcessingTimeEvent, + TestStream, WatermarkEvent) from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms.window import FixedWindows -from apache_beam.transforms.window import TimestampedValue +from apache_beam.transforms.window import FixedWindows, TimestampedValue from apache_beam.utils import timestamp from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index 26ca03d1375d..de7188795b67 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -22,12 +22,12 @@ import hashlib import imp + from mock import Mock, patch from apache_beam.io.filesystems import FileSystems from apache_beam.utils import retry - DEFAULT_HASHING_ALG = 'sha1' diff --git a/sdks/python/apache_beam/testing/test_utils_test.py b/sdks/python/apache_beam/testing/test_utils_test.py index bee0bd361e27..093e1f175d25 100644 --- a/sdks/python/apache_beam/testing/test_utils_test.py +++ b/sdks/python/apache_beam/testing/test_utils_test.py @@ -20,6 +20,7 @@ import logging import tempfile import unittest + from mock import patch from apache_beam.io.filesystem import BeamIOError diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py index 959f25f31873..49f9e9158009 100644 --- a/sdks/python/apache_beam/testing/util.py +++ b/sdks/python/apache_beam/testing/util.py @@ -24,14 +24,11 @@ from apache_beam import pvalue from apache_beam.transforms import window -from apache_beam.transforms.core import Create -from apache_beam.transforms.core import Map -from apache_beam.transforms.core import WindowInto -from apache_beam.transforms.util import CoGroupByKey +from apache_beam.transforms.core import Create, Map, WindowInto from apache_beam.transforms.ptransform import PTransform +from apache_beam.transforms.util import CoGroupByKey from apache_beam.utils.annotations import experimental - __all__ = [ 'assert_that', 'equal_to', diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py index 875306f80829..289a2a8103c3 100644 --- a/sdks/python/apache_beam/transforms/combiners.py +++ b/sdks/python/apache_beam/transforms/combiners.py @@ -22,20 +22,10 @@ import operator import random -from apache_beam.transforms import core -from apache_beam.transforms import cy_combiners -from apache_beam.transforms import ptransform +from apache_beam.transforms import core, cy_combiners, ptransform from apache_beam.transforms.display import DisplayDataItem -from apache_beam.typehints import Any -from apache_beam.typehints import Dict -from apache_beam.typehints import KV -from apache_beam.typehints import List -from apache_beam.typehints import Tuple -from apache_beam.typehints import TypeVariable -from apache_beam.typehints import Union -from apache_beam.typehints import with_input_types -from apache_beam.typehints import with_output_types - +from apache_beam.typehints import (KV, Any, Dict, List, Tuple, TypeVariable, + Union, with_input_types, with_output_types) __all__ = [ 'Count', diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index cd2b5956fef9..fb0241abbf62 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -22,12 +22,10 @@ import hamcrest as hc import apache_beam as beam -from apache_beam.testing.test_pipeline import TestPipeline import apache_beam.transforms.combiners as combine +from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms.core import CombineGlobally -from apache_beam.transforms.core import Create -from apache_beam.transforms.core import Map +from apache_beam.transforms.core import CombineGlobally, Create, Map from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher from apache_beam.transforms.ptransform import PTransform diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index d6f56d2d0cf0..81ba67db2c62 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -25,38 +25,24 @@ from google.protobuf import wrappers_pb2 -from apache_beam import pvalue -from apache_beam import typehints -from apache_beam import coders +from apache_beam import coders, pvalue, typehints from apache_beam.coders import typecoders -from apache_beam.internal import pickler -from apache_beam.internal import util +from apache_beam.internal import pickler, util +from apache_beam.options.pipeline_options import TypeOptions from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import ptransform -from apache_beam.transforms.display import DisplayDataItem -from apache_beam.transforms.display import HasDisplayData -from apache_beam.transforms.ptransform import PTransform -from apache_beam.transforms.ptransform import PTransformWithSideInputs -from apache_beam.transforms.window import MIN_TIMESTAMP -from apache_beam.transforms.window import TimestampCombiner -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.window import TimestampedValue -from apache_beam.transforms.window import GlobalWindows -from apache_beam.transforms.window import WindowFn -from apache_beam.typehints import Any -from apache_beam.typehints import Iterable -from apache_beam.typehints import KV -from apache_beam.typehints import trivial_inference -from apache_beam.typehints import Union -from apache_beam.typehints.decorators import get_type_hints -from apache_beam.typehints.decorators import TypeCheckError -from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.transforms.display import DisplayDataItem, HasDisplayData +from apache_beam.transforms.ptransform import (PTransform, + PTransformWithSideInputs) +from apache_beam.transforms.window import (MIN_TIMESTAMP, GlobalWindows, + TimestampCombiner, TimestampedValue, + WindowedValue, WindowFn) +from apache_beam.typehints import KV, Any, Iterable, Union, trivial_inference +from apache_beam.typehints.decorators import (TypeCheckError, WithTypeHints, + get_type_hints) from apache_beam.typehints.trivial_inference import element_type from apache_beam.typehints.typehints import is_consistent_with -from apache_beam.utils import proto_utils -from apache_beam.utils import urns -from apache_beam.options.pipeline_options import TypeOptions - +from apache_beam.utils import proto_utils, urns __all__ = [ 'DoFn', diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py index 55ad7f32982d..1ae3aaa705c7 100644 --- a/sdks/python/apache_beam/transforms/create_test.py +++ b/sdks/python/apache_beam/transforms/create_test.py @@ -18,10 +18,9 @@ """Unit tests for the Create and _CreateSource classes.""" import unittest -from apache_beam.io import source_test_utils - from apache_beam import Create from apache_beam.coders import FastPrimitivesCoder +from apache_beam.io import source_test_utils from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py index 88a1feef75e1..935f53c9cc76 100644 --- a/sdks/python/apache_beam/transforms/display.py +++ b/sdks/python/apache_beam/transforms/display.py @@ -39,10 +39,9 @@ from __future__ import absolute_import import calendar -from datetime import datetime, timedelta import inspect import json - +from datetime import datetime, timedelta __all__ = ['HasDisplayData', 'DisplayDataItem', 'DisplayData'] diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py index 15f178645bc3..b19b32c45bc4 100644 --- a/sdks/python/apache_beam/transforms/display_test.py +++ b/sdks/python/apache_beam/transforms/display_test.py @@ -19,17 +19,16 @@ from __future__ import absolute_import -from datetime import datetime import unittest +from datetime import datetime import hamcrest as hc from hamcrest.core.base_matcher import BaseMatcher import apache_beam as beam -from apache_beam.transforms.display import HasDisplayData -from apache_beam.transforms.display import DisplayData -from apache_beam.transforms.display import DisplayDataItem from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.transforms.display import (DisplayData, DisplayDataItem, + HasDisplayData) class DisplayDataItemMatcher(BaseMatcher): diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index f6e08ca9c0ce..27b9dd8b51e4 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -44,21 +44,15 @@ class and wrapper class that allows lambda functions to be used as from google.protobuf import wrappers_pb2 -from apache_beam import error -from apache_beam import pvalue -from apache_beam.internal import pickler -from apache_beam.internal import util -from apache_beam.transforms.display import HasDisplayData -from apache_beam.transforms.display import DisplayDataItem +from apache_beam import error, pvalue +from apache_beam.internal import pickler, util +from apache_beam.transforms.display import DisplayDataItem, HasDisplayData from apache_beam.typehints import typehints -from apache_beam.typehints.decorators import getcallargs_forhints -from apache_beam.typehints.decorators import TypeCheckError -from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.typehints.decorators import (TypeCheckError, WithTypeHints, + getcallargs_forhints) from apache_beam.typehints.trivial_inference import instance_to_type from apache_beam.typehints.typehints import validate_composite_type_param -from apache_beam.utils import proto_utils -from apache_beam.utils import urns - +from apache_beam.utils import proto_utils, urns __all__ = [ 'PTransform', diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index efc597877b29..fd0bbc8145b6 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -27,25 +27,23 @@ from nose.plugins.attrib import attr import apache_beam as beam +import apache_beam.pvalue as pvalue +import apache_beam.transforms.combiners as combine +import apache_beam.typehints as typehints +from apache_beam.io.iobase import Read from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.io.iobase import Read from apache_beam.options.pipeline_options import TypeOptions -import apache_beam.pvalue as pvalue from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to from apache_beam.transforms import window from apache_beam.transforms.core import _GroupByKeyOnly -import apache_beam.transforms.combiners as combine from apache_beam.transforms.display import DisplayData, DisplayDataItem from apache_beam.transforms.ptransform import PTransform -import apache_beam.typehints as typehints -from apache_beam.typehints import with_input_types -from apache_beam.typehints import with_output_types +from apache_beam.typehints import with_input_types, with_output_types from apache_beam.typehints.typehints_test import TypeHintTestCase from apache_beam.utils.windowed_value import WindowedValue - # Disable frequent lint warning due to pipe operator for chaining transforms. # pylint: disable=expression-not-assigned diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py index c0f91981d7d6..6af423f1bd1a 100644 --- a/sdks/python/apache_beam/transforms/timeutil.py +++ b/sdks/python/apache_beam/transforms/timeutil.py @@ -19,9 +19,7 @@ from __future__ import absolute_import -from abc import ABCMeta -from abc import abstractmethod - +from abc import ABCMeta, abstractmethod __all__ = [ 'TimeDomain', diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index c1fbfc54a3b8..fce2714ff2a6 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -20,24 +20,19 @@ Triggers control when in processing time windows get emitted. """ -from abc import ABCMeta -from abc import abstractmethod import collections import copy import itertools +from abc import ABCMeta, abstractmethod from apache_beam.coders import observable -from apache_beam.transforms import combiners -from apache_beam.transforms import core -from apache_beam.transforms.timeutil import TimeDomain -from apache_beam.transforms.window import GlobalWindow -from apache_beam.transforms.window import TimestampCombiner -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.window import WindowFn from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils.timestamp import TIME_GRANULARITY +from apache_beam.transforms import combiners, core +from apache_beam.transforms.timeutil import TimeDomain +from apache_beam.transforms.window import (GlobalWindow, TimestampCombiner, + WindowedValue, WindowFn) +from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, + TIME_GRANULARITY) # AfterCount is experimental. No backwards compatibility guarantees. diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py index 1ae1f0250320..be92212138bc 100644 --- a/sdks/python/apache_beam/transforms/trigger_test.py +++ b/sdks/python/apache_beam/transforms/trigger_test.py @@ -30,25 +30,16 @@ from apache_beam.testing.util import assert_that, equal_to from apache_beam.transforms import trigger from apache_beam.transforms.core import Windowing -from apache_beam.transforms.trigger import AccumulationMode -from apache_beam.transforms.trigger import AfterAll -from apache_beam.transforms.trigger import AfterCount -from apache_beam.transforms.trigger import AfterEach -from apache_beam.transforms.trigger import AfterAny -from apache_beam.transforms.trigger import AfterWatermark -from apache_beam.transforms.trigger import DefaultTrigger -from apache_beam.transforms.trigger import GeneralTriggerDriver -from apache_beam.transforms.trigger import InMemoryUnmergedState -from apache_beam.transforms.trigger import Repeatedly -from apache_beam.transforms.trigger import TriggerFn -from apache_beam.transforms.window import FixedWindows -from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms.window import MIN_TIMESTAMP -from apache_beam.transforms.window import TimestampCombiner -from apache_beam.transforms.window import Sessions -from apache_beam.transforms.window import TimestampedValue -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.window import WindowFn +from apache_beam.transforms.trigger import (AccumulationMode, AfterAll, + AfterAny, AfterCount, AfterEach, + AfterWatermark, DefaultTrigger, + GeneralTriggerDriver, + InMemoryUnmergedState, Repeatedly, + TriggerFn) +from apache_beam.transforms.window import (MIN_TIMESTAMP, FixedWindows, + IntervalWindow, Sessions, + TimestampCombiner, TimestampedValue, + WindowedValue, WindowFn) class CustomTimestampingFixedWindowsWindowFn(FixedWindows): diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index a7484acb2cc7..b83ed90ef500 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -20,13 +20,8 @@ from __future__ import absolute_import -from apache_beam.transforms.core import CombinePerKey -from apache_beam.transforms.core import Flatten -from apache_beam.transforms.core import GroupByKey -from apache_beam.transforms.core import Map -from apache_beam.transforms.ptransform import PTransform -from apache_beam.transforms.ptransform import ptransform_fn - +from apache_beam.transforms.core import CombinePerKey, Flatten, GroupByKey, Map +from apache_beam.transforms.ptransform import PTransform, ptransform_fn __all__ = [ 'CoGroupByKey', diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 458fb747eb22..c3c588c9b6c1 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -51,22 +51,17 @@ import abc -from google.protobuf import duration_pb2 -from google.protobuf import timestamp_pb2 +from google.protobuf import duration_pb2, timestamp_pb2 from apache_beam.coders import coders -from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.portability.api import standard_window_fns_pb2 +from apache_beam.portability.api import (beam_runner_api_pb2, + standard_window_fns_pb2) from apache_beam.transforms import timeutil -from apache_beam.utils import proto_utils -from apache_beam.utils import urns -from apache_beam.utils.timestamp import Duration -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils.timestamp import Timestamp +from apache_beam.utils import proto_utils, urns +from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, Duration, + Timestamp) from apache_beam.utils.windowed_value import WindowedValue - __all__ = [ 'TimestampCombiner', 'WindowFn', diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py index 977a364ad883..1cd344bdb466 100644 --- a/sdks/python/apache_beam/transforms/window_test.py +++ b/sdks/python/apache_beam/transforms/window_test.py @@ -22,28 +22,16 @@ from apache_beam.runners import pipeline_context from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms import CombinePerKey -from apache_beam.transforms import combiners -from apache_beam.transforms import core -from apache_beam.transforms import Create -from apache_beam.transforms import GroupByKey -from apache_beam.transforms import Map -from apache_beam.transforms import WindowInto +from apache_beam.transforms import (CombinePerKey, Create, GroupByKey, Map, + WindowInto, combiners, core) from apache_beam.transforms.core import Windowing -from apache_beam.transforms.trigger import AccumulationMode -from apache_beam.transforms.trigger import AfterCount -from apache_beam.transforms.window import FixedWindows -from apache_beam.transforms.window import GlobalWindow -from apache_beam.transforms.window import GlobalWindows -from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms.window import TimestampCombiner -from apache_beam.transforms.window import Sessions -from apache_beam.transforms.window import SlidingWindows -from apache_beam.transforms.window import TimestampedValue -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.window import WindowFn -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.transforms.trigger import AccumulationMode, AfterCount +from apache_beam.transforms.window import (FixedWindows, GlobalWindow, + GlobalWindows, IntervalWindow, + Sessions, SlidingWindows, + TimestampCombiner, TimestampedValue, + WindowedValue, WindowFn) +from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP def context(element, timestamp): diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py index 50f0debb0a70..dc4d333198f1 100644 --- a/sdks/python/apache_beam/transforms/write_ptransform_test.py +++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py @@ -20,7 +20,6 @@ import unittest import apache_beam as beam - from apache_beam.io import iobase from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, is_empty diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index 3f5b4c9c479b..c1604b9de175 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -86,13 +86,11 @@ def foo((a, b)): import inspect import types -from apache_beam.typehints import native_type_compatibility -from apache_beam.typehints import typehints -from apache_beam.typehints.typehints import check_constraint -from apache_beam.typehints.typehints import CompositeTypeHintError -from apache_beam.typehints.typehints import SimpleTypeHintError -from apache_beam.typehints.typehints import validate_composite_type_param - +from apache_beam.typehints import native_type_compatibility, typehints +from apache_beam.typehints.typehints import (CompositeTypeHintError, + SimpleTypeHintError, + check_constraint, + validate_composite_type_param) __all__ = [ 'with_input_types', diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility.py b/sdks/python/apache_beam/typehints/native_type_compatibility.py index 26c584e380d2..8a8e07ecb4be 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility.py @@ -19,6 +19,7 @@ import collections import typing + from apache_beam.typehints import typehints # Describes an entry in the type map in convert_to_beam_type. diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 0ff2b3b5e58d..81ca9014da9f 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -17,12 +17,10 @@ """Test for Beam type compatibility library.""" -import unittest - import typing +import unittest -from apache_beam.typehints import typehints -from apache_beam.typehints import native_type_compatibility +from apache_beam.typehints import native_type_compatibility, typehints _TestNamedTuple = typing.NamedTuple('_TestNamedTuple', [('age', int), ('name', bytes)]) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index 83f444c6c53b..8831eb226dce 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -28,8 +28,8 @@ """ import types -from trivial_inference import union, element_type, Const, BoundMethod import typehints +from trivial_inference import BoundMethod, Const, element_type, union from typehints import Any, Dict, Iterable, List, Tuple, Union diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index c7405963f161..3c5d54b5b8d3 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -26,8 +26,7 @@ import sys import types -from apache_beam.typehints import Any -from apache_beam.typehints import typehints +from apache_beam.typehints import Any, typehints class TypeInferenceError(ValueError): diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index e7f451da11a6..df21b16a5fec 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -18,9 +18,7 @@ """Tests for apache_beam.typehints.trivial_inference.""" import unittest - -from apache_beam.typehints import trivial_inference -from apache_beam.typehints import typehints +from apache_beam.typehints import trivial_inference, typehints global_int = 1 diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index 89a5f5c7e2cf..ac2cc15137b7 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -28,13 +28,12 @@ from apache_beam.pvalue import TaggedOutput from apache_beam.transforms.core import DoFn from apache_beam.transforms.window import WindowedValue -from apache_beam.typehints.decorators import _check_instance_type -from apache_beam.typehints.decorators import getcallargs_forhints -from apache_beam.typehints.decorators import GeneratorWrapper -from apache_beam.typehints.decorators import TypeCheckError -from apache_beam.typehints.typehints import check_constraint -from apache_beam.typehints.typehints import CompositeTypeHintError -from apache_beam.typehints.typehints import SimpleTypeHintError +from apache_beam.typehints.decorators import (GeneratorWrapper, TypeCheckError, + _check_instance_type, + getcallargs_forhints) +from apache_beam.typehints.typehints import (CompositeTypeHintError, + SimpleTypeHintError, + check_constraint) class AbstractDoFnWrapper(DoFn): diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 59d1e1c5714e..7d713296ca1c 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -17,13 +17,11 @@ """Unit tests for the type-hint objects and decorators.""" import inspect -import unittest - import typing +import unittest import apache_beam as beam -from apache_beam import pvalue -from apache_beam import typehints +from apache_beam import pvalue, typehints from apache_beam.options.pipeline_options import OptionsContext from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py index 98d399b82f3b..c19916fa3cc2 100644 --- a/sdks/python/apache_beam/typehints/typehints.py +++ b/sdks/python/apache_beam/typehints/typehints.py @@ -67,7 +67,6 @@ import copy import types - __all__ = [ 'Any', 'Union', diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index f1b92e096264..181686ffd7ae 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -20,20 +20,15 @@ import inspect import unittest - import apache_beam.typehints.typehints as typehints -from apache_beam.typehints import Any -from apache_beam.typehints import Tuple -from apache_beam.typehints import TypeCheckError -from apache_beam.typehints import Union -from apache_beam.typehints import with_input_types -from apache_beam.typehints import with_output_types -from apache_beam.typehints.decorators import _check_instance_type -from apache_beam.typehints.decorators import _interleave_type_check -from apache_beam.typehints.decorators import _positional_arg_hints -from apache_beam.typehints.decorators import get_type_hints -from apache_beam.typehints.decorators import getcallargs_forhints -from apache_beam.typehints.decorators import GeneratorWrapper +from apache_beam.typehints import (Any, Tuple, TypeCheckError, Union, + with_input_types, with_output_types) +from apache_beam.typehints.decorators import (GeneratorWrapper, + _check_instance_type, + _interleave_type_check, + _positional_arg_hints, + get_type_hints, + getcallargs_forhints) from apache_beam.typehints.typehints import is_consistent_with diff --git a/sdks/python/apache_beam/utils/annotations.py b/sdks/python/apache_beam/utils/annotations.py index 017dd6b81a4d..ec21093c1020 100644 --- a/sdks/python/apache_beam/utils/annotations.py +++ b/sdks/python/apache_beam/utils/annotations.py @@ -61,8 +61,7 @@ def exp_multiply(arg1, arg2): """ import warnings -from functools import partial -from functools import wraps +from functools import partial, wraps # Produce only the first occurrence of matching warnings regardless of # location per line of execution. Since the number of lines of execution diff --git a/sdks/python/apache_beam/utils/annotations_test.py b/sdks/python/apache_beam/utils/annotations_test.py index 32af8a991a79..94ed058b22d5 100644 --- a/sdks/python/apache_beam/utils/annotations_test.py +++ b/sdks/python/apache_beam/utils/annotations_test.py @@ -17,8 +17,8 @@ import unittest import warnings -from apache_beam.utils.annotations import deprecated -from apache_beam.utils.annotations import experimental + +from apache_beam.utils.annotations import deprecated, experimental class AnnotationTests(unittest.TestCase): diff --git a/sdks/python/apache_beam/utils/counters.py b/sdks/python/apache_beam/utils/counters.py index 08685aae2759..ae9743442590 100644 --- a/sdks/python/apache_beam/utils/counters.py +++ b/sdks/python/apache_beam/utils/counters.py @@ -23,12 +23,11 @@ For internal use only; no backwards-compatibility guarantees. """ -from collections import namedtuple import threading +from collections import namedtuple from apache_beam.transforms import cy_combiners - # Information identifying the IO being measured by a counter. IOTargetName = namedtuple('IOTargetName', ['side_input_step_name', 'side_input_index', diff --git a/sdks/python/apache_beam/utils/processes_test.py b/sdks/python/apache_beam/utils/processes_test.py index 920b621b1bbd..2dd45f44dc53 100644 --- a/sdks/python/apache_beam/utils/processes_test.py +++ b/sdks/python/apache_beam/utils/processes_test.py @@ -18,7 +18,6 @@ import unittest - import mock from apache_beam.utils import processes diff --git a/sdks/python/apache_beam/utils/proto_utils.py b/sdks/python/apache_beam/utils/proto_utils.py index d7693f3f7839..dce7cff7158a 100644 --- a/sdks/python/apache_beam/utils/proto_utils.py +++ b/sdks/python/apache_beam/utils/proto_utils.py @@ -17,8 +17,7 @@ """For internal use only; no backwards-compatibility guarantees.""" -from google.protobuf import any_pb2 -from google.protobuf import struct_pb2 +from google.protobuf import any_pb2, struct_pb2 def pack_Any(msg): diff --git a/sdks/python/apache_beam/utils/retry_test.py b/sdks/python/apache_beam/utils/retry_test.py index 1b03c835cd7f..3b372cdaec42 100644 --- a/sdks/python/apache_beam/utils/retry_test.py +++ b/sdks/python/apache_beam/utils/retry_test.py @@ -19,6 +19,8 @@ import unittest +from apache_beam.utils import retry + # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position # TODO(sourabhbajaj): Remove the GCP specific error code to a submodule @@ -29,7 +31,6 @@ # pylint: enable=wrong-import-order, wrong-import-position -from apache_beam.utils import retry class FakeClock(object): diff --git a/sdks/python/apache_beam/utils/timestamp_test.py b/sdks/python/apache_beam/utils/timestamp_test.py index 33229361b36f..04b19ed87a68 100644 --- a/sdks/python/apache_beam/utils/timestamp_test.py +++ b/sdks/python/apache_beam/utils/timestamp_test.py @@ -21,8 +21,7 @@ import unittest -from apache_beam.utils.timestamp import Duration -from apache_beam.utils.timestamp import Timestamp +from apache_beam.utils.timestamp import Duration, Timestamp class TimestampTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py index 18959be87503..7675d0548cfb 100644 --- a/sdks/python/apache_beam/utils/urns.py +++ b/sdks/python/apache_beam/utils/urns.py @@ -25,7 +25,6 @@ from apache_beam.internal import pickler from apache_beam.utils import proto_utils - PICKLED_WINDOW_FN = "beam:windowfn:pickled_python:v0.1" GLOBAL_WINDOWS_FN = "beam:windowfn:global_windows:v0.1" FIXED_WINDOWS_FN = "beam:windowfn:fixed_windows:v0.1" diff --git a/sdks/python/apache_beam/utils/windowed_value.py b/sdks/python/apache_beam/utils/windowed_value.py index be2785432a14..933c2c42d4d4 100644 --- a/sdks/python/apache_beam/utils/windowed_value.py +++ b/sdks/python/apache_beam/utils/windowed_value.py @@ -27,9 +27,7 @@ #cython: profile=True -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils.timestamp import Timestamp +from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP, Timestamp class WindowedValue(object): diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 2691be4ea42c..cbec255dd2f6 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -62,3 +62,7 @@ echo "Running pylint for module $MODULE:" pylint $MODULE --ignore-patterns="$FILES_TO_IGNORE" echo "Running pycodestyle for module $MODULE:" pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" +echo "Running isort for module $MODULE:" +pushd $MODULE +isort -w 80 -y -c +popd From 5de891093e73f92247be5f9c618478811ee05dbf Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 13:16:45 -0700 Subject: [PATCH 02/17] autopep8 whitespace fixes W391,W293,W291,E306,E305,E304,E303 --- .../python/apache_beam/coders/proto2_coder_test_messages_pb2.py | 2 -- sdks/python/apache_beam/examples/snippets/snippets.py | 2 -- sdks/python/apache_beam/io/gcp/datastore/v1/helper.py | 1 - sdks/python/apache_beam/pipeline_test.py | 1 - .../internal/clients/dataflow/dataflow_v1b3_messages.py | 1 - sdks/python/apache_beam/utils/retry_test.py | 2 -- 6 files changed, 9 deletions(-) diff --git a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py index 98667c8404c4..433d33f98841 100644 --- a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py +++ b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py @@ -32,8 +32,6 @@ _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor.FileDescriptor( name='apache_beam/coders/proto2_coder_test_messages.proto', package='proto2_coder_test_messages', diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index 64f21cf1d0b0..13e1476a22a4 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -658,8 +658,6 @@ def process(self, an_int): result.wait_until_finish() - - # Defining a new source. # [START model_custom_source_new_source] class CountingSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py index 84bf1eac7810..cfda48cfa398 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py @@ -45,7 +45,6 @@ # pylint: enable=wrong-import-order, wrong-import-position - def key_comparator(k1, k2): """A comparator for Datastore keys. diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 83ada1eae6d0..de0af6623440 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -40,7 +40,6 @@ # from nose.plugins.attrib import attr - class FakeSource(NativeSource): """Fake source returning a fixed list of values.""" diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py index d3fca37a7586..6f8e3e755ca3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py @@ -3067,7 +3067,6 @@ class ResourceUtilizationReportResponse(_messages.Message): """ - class RuntimeEnvironment(_messages.Message): """The environment values to set at runtime. diff --git a/sdks/python/apache_beam/utils/retry_test.py b/sdks/python/apache_beam/utils/retry_test.py index 3b372cdaec42..e5f07e88b420 100644 --- a/sdks/python/apache_beam/utils/retry_test.py +++ b/sdks/python/apache_beam/utils/retry_test.py @@ -31,8 +31,6 @@ # pylint: enable=wrong-import-order, wrong-import-position - - class FakeClock(object): """A fake clock object implementing sleep() and recording calls.""" From 5b1c62da008f7fa143994d316de392c4be706fac Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 13:38:59 -0700 Subject: [PATCH 03/17] Manual clean up of some isort issues --- sdks/python/apache_beam/io/avroio_test.py | 2 +- .../runners/dataflow/dataflow_runner_test.py | 4 ++-- .../runners/dataflow/native_io/iobase_test.py | 14 ++++---------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index 048c412230f8..c0c5b7a37116 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -23,9 +23,9 @@ import avro.datafile import avro.schema -import hamcrest as hc from avro.datafile import DataFileWriter from avro.io import DatumWriter +import hamcrest as hc import apache_beam as beam from apache_beam import Create diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index e77350c16831..ffbd7cb3ee28 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -30,8 +30,8 @@ from apache_beam.pvalue import PCollection from apache_beam.runners import (DataflowRunner, TestDataflowRunner, create_runner) -from apache_beam.runners.dataflow.dataflow_runner import (DataflowPipelineResult, - DataflowRuntimeException) +from apache_beam.runners.dataflow.dataflow_runner import \ + DataflowPipelineResult, DataflowRuntimeException from apache_beam.runners.dataflow.internal.clients import \ dataflow as dataflow_api from apache_beam.runners.runner import PipelineState diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index fdee2ee2c15f..99cbabc23fe1 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -21,16 +21,10 @@ import unittest from apache_beam import Create, error, pvalue -from apache_beam.runners.dataflow.native_io.iobase import (ConcatPosition, - DynamicSplitRequest, - DynamicSplitResultWithPosition, - NativeSink, - NativeSinkWriter, - NativeSource, - ReaderPosition, - ReaderProgress, - _dict_printable_fields, - _NativeWrite) +from apache_beam.runners.dataflow.native_io.iobase import \ + ConcatPosition, DynamicSplitRequest, DynamicSplitResultWithPosition, \ + NativeSink, NativeSinkWriter, NativeSource, ReaderPosition, ReaderProgress, \ + _dict_printable_fields, _NativeWrite from apache_beam.testing.test_pipeline import TestPipeline From 2442328bb0f5ef39b74b905df3d4b8a19c8f9d5a Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 13:45:22 -0700 Subject: [PATCH 04/17] re-runisort -p apache_beam -w 79 -y -c --- .../examples/complete/game/game_stats.py | 3 ++- .../examples/complete/game/hourly_team_score.py | 3 ++- .../examples/complete/game/leader_board.py | 3 ++- .../examples/cookbook/datastore_wordcount.py | 3 ++- sdks/python/apache_beam/io/avroio_test.py | 2 +- sdks/python/apache_beam/io/filebasedsink.py | 3 ++- sdks/python/apache_beam/io/filebasedsource.py | 3 ++- sdks/python/apache_beam/io/filebasedsource_test.py | 3 ++- .../io/gcp/datastore/v1/adaptive_throttler_test.py | 3 ++- .../apache_beam/io/gcp/datastore/v1/datastoreio.py | 3 ++- sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 ++- .../options/pipeline_options_validator.py | 6 +++--- sdks/python/apache_beam/pipeline.py | 3 ++- sdks/python/apache_beam/pipeline_test.py | 4 ++-- .../runners/dataflow/dataflow_runner.py | 4 ++-- .../runners/dataflow/dataflow_runner_test.py | 4 ++-- .../runners/dataflow/internal/dependency_test.py | 3 ++- .../runners/dataflow/native_io/iobase_test.py | 14 ++++++++++---- .../runners/dataflow/test_dataflow_runner.py | 3 ++- .../runners/direct/direct_metrics_test.py | 3 ++- .../runners/direct/transform_evaluator.py | 3 ++- sdks/python/apache_beam/transforms/window.py | 4 ++-- sdks/python/run_pylint.sh | 2 +- 23 files changed, 53 insertions(+), 32 deletions(-) diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py index 895e042aa830..6db07c73e771 100644 --- a/sdks/python/apache_beam/examples/complete/game/game_stats.py +++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py @@ -89,7 +89,8 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, SetupOptions, + PipelineOptions, + SetupOptions, StandardOptions) diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py index fbd7b5d0d057..2f988ab9f3e9 100644 --- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py +++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py @@ -76,7 +76,8 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, SetupOptions) + PipelineOptions, + SetupOptions) def str2timestamp(s, fmt='%Y-%m-%d-%H-%M'): diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py index 42c864e25fc0..6cda3613a6b2 100644 --- a/sdks/python/apache_beam/examples/complete/game/leader_board.py +++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py @@ -96,7 +96,8 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, SetupOptions, + PipelineOptions, + SetupOptions, StandardOptions) from apache_beam.transforms import trigger diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py index b76b4bfe3110..c8a828845c85 100644 --- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py +++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py @@ -77,7 +77,8 @@ from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, SetupOptions) + PipelineOptions, + SetupOptions) class WordExtractingDoFn(beam.DoFn): diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index c0c5b7a37116..048c412230f8 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -23,9 +23,9 @@ import avro.datafile import avro.schema +import hamcrest as hc from avro.datafile import DataFileWriter from avro.io import DatumWriter -import hamcrest as hc import apache_beam as beam from apache_beam import Create diff --git a/sdks/python/apache_beam/io/filebasedsink.py b/sdks/python/apache_beam/io/filebasedsink.py index fdad50efbce8..0284f4fb1c16 100644 --- a/sdks/python/apache_beam/io/filebasedsink.py +++ b/sdks/python/apache_beam/io/filebasedsink.py @@ -30,7 +30,8 @@ from apache_beam.io.filesystem import BeamIOError, CompressionTypes from apache_beam.io.filesystems import FileSystems from apache_beam.options.value_provider import (StaticValueProvider, - ValueProvider, check_accessible) + ValueProvider, + check_accessible) from apache_beam.transforms.display import DisplayDataItem DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN' diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py index 49498f5c41d8..6ab9e0c94698 100644 --- a/sdks/python/apache_beam/io/filebasedsource.py +++ b/sdks/python/apache_beam/io/filebasedsource.py @@ -33,7 +33,8 @@ from apache_beam.io.filesystems import FileSystems from apache_beam.io.range_trackers import OffsetRange from apache_beam.options.value_provider import (StaticValueProvider, - ValueProvider, check_accessible) + ValueProvider, + check_accessible) from apache_beam.transforms.core import (DoFn, FlatMap, GroupByKey, Map, ParDo, PTransform) from apache_beam.transforms.display import DisplayDataItem diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index 456d9daf1b52..09cdc69d872c 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -31,7 +31,8 @@ from apache_beam.io import filebasedsource, iobase, range_trackers # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource -from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource +from apache_beam.io.filebasedsource import \ + _SingleFileSource as SingleFileSource from apache_beam.io.filebasedsource import FileBasedSource from apache_beam.io.filesystem import CompressionTypes from apache_beam.options.value_provider import (RuntimeValueProvider, diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py index 1ac23930f65f..217698c6c540 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py @@ -19,7 +19,8 @@ from mock import patch -from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler +from apache_beam.io.gcp.datastore.v1.adaptive_throttler import \ + AdaptiveThrottler class AdaptiveThrottlerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py index c498933788d0..47a4788daad9 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py @@ -21,7 +21,8 @@ import time from apache_beam.io.gcp.datastore.v1 import helper, query_splitter, util -from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler +from apache_beam.io.gcp.datastore.v1.adaptive_throttler import \ + AdaptiveThrottler from apache_beam.metrics.metric import Metrics from apache_beam.transforms import (Create, DoFn, FlatMap, GroupByKey, Map, ParDo, PTransform) diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py index c49da0302179..9dcfa3959c50 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py @@ -23,7 +23,8 @@ import hamcrest as hc from apache_beam.io.gcp.pubsub import (ReadStringsFromPubSub, - WriteStringsToPubSub, _PubSubPayloadSink, + WriteStringsToPubSub, + _PubSubPayloadSink, _PubSubPayloadSource) from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData diff --git a/sdks/python/apache_beam/options/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py index df0b561a0474..72c0942460f9 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator.py @@ -24,9 +24,9 @@ from apache_beam.internal import pickler from apache_beam.options.pipeline_options import (DebugOptions, GoogleCloudOptions, - SetupOptions, StandardOptions, - TestOptions, TypeOptions, - WorkerOptions) + SetupOptions, + StandardOptions, TestOptions, + TypeOptions, WorkerOptions) class PipelineOptionsValidator(object): diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index d29026b56b5f..196da5b26e97 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -55,7 +55,8 @@ from apache_beam import pvalue from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import (PipelineOptions, SetupOptions, +from apache_beam.options.pipeline_options import (PipelineOptions, + SetupOptions, StandardOptions, TypeOptions) from apache_beam.options.pipeline_options_validator import \ PipelineOptionsValidator diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index de0af6623440..290b83243ea0 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -31,8 +31,8 @@ from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms import (CombineGlobally, Create, DoFn, FlatMap, Map, - ParDo, PTransform, WindowInto) +from apache_beam.transforms import (CombineGlobally, Create, DoFn, FlatMap, + Map, ParDo, PTransform, WindowInto) from apache_beam.transforms.window import SlidingWindows, TimestampedValue from apache_beam.utils.timestamp import MIN_TIMESTAMP diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 042c1f16c3a3..38fe25bf207a 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -32,8 +32,8 @@ from apache_beam import coders, error, pvalue from apache_beam.internal import pickler from apache_beam.internal.gcp import json_value -from apache_beam.options.pipeline_options import (SetupOptions, StandardOptions, - TestOptions) +from apache_beam.options.pipeline_options import (SetupOptions, + StandardOptions, TestOptions) from apache_beam.pvalue import AsSideInput from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics from apache_beam.runners.dataflow.internal import names diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index ffbd7cb3ee28..e77350c16831 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -30,8 +30,8 @@ from apache_beam.pvalue import PCollection from apache_beam.runners import (DataflowRunner, TestDataflowRunner, create_runner) -from apache_beam.runners.dataflow.dataflow_runner import \ - DataflowPipelineResult, DataflowRuntimeException +from apache_beam.runners.dataflow.dataflow_runner import (DataflowPipelineResult, + DataflowRuntimeException) from apache_beam.runners.dataflow.internal.clients import \ dataflow as dataflow_api from apache_beam.runners.runner import PipelineState diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py index 643de6af10b3..83fecb189109 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py @@ -25,7 +25,8 @@ from apache_beam.io.filesystems import FileSystems from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, SetupOptions) + PipelineOptions, + SetupOptions) from apache_beam.runners.dataflow.internal import dependency, names # Protect against environments where GCS library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index 99cbabc23fe1..fdee2ee2c15f 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -21,10 +21,16 @@ import unittest from apache_beam import Create, error, pvalue -from apache_beam.runners.dataflow.native_io.iobase import \ - ConcatPosition, DynamicSplitRequest, DynamicSplitResultWithPosition, \ - NativeSink, NativeSinkWriter, NativeSource, ReaderPosition, ReaderProgress, \ - _dict_printable_fields, _NativeWrite +from apache_beam.runners.dataflow.native_io.iobase import (ConcatPosition, + DynamicSplitRequest, + DynamicSplitResultWithPosition, + NativeSink, + NativeSinkWriter, + NativeSource, + ReaderPosition, + ReaderProgress, + _dict_printable_fields, + _NativeWrite) from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py index b0a0159bce48..6fd5dbf8ca75 100644 --- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py @@ -18,7 +18,8 @@ """Wrapper of Beam runners that's built for running and verifying e2e tests.""" from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import GoogleCloudOptions, TestOptions +from apache_beam.options.pipeline_options import (GoogleCloudOptions, + TestOptions) from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner __all__ = ['TestDataflowRunner'] diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py index b543f0dacc28..a9b3120d1dda 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py @@ -20,7 +20,8 @@ import hamcrest as hc from apache_beam.metrics.cells import DistributionData, DistributionResult -from apache_beam.metrics.execution import MetricKey, MetricResult, MetricUpdates +from apache_beam.metrics.execution import (MetricKey, MetricResult, + MetricUpdates) from apache_beam.metrics.metricbase import MetricName from apache_beam.runners.direct.direct_metrics import DirectMetrics diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index a7da152769aa..bf29f3932f56 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -37,7 +37,8 @@ from apache_beam.testing.test_stream import (ElementEvent, ProcessingTimeEvent, TestStream, WatermarkEvent) from apache_beam.transforms import core -from apache_beam.transforms.trigger import (TimeDomain, _CombiningValueStateTag, +from apache_beam.transforms.trigger import (TimeDomain, + _CombiningValueStateTag, _ListStateTag, create_trigger_driver) from apache_beam.transforms.window import GlobalWindows, WindowedValue diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index c3c588c9b6c1..4e54384d41e2 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -58,8 +58,8 @@ standard_window_fns_pb2) from apache_beam.transforms import timeutil from apache_beam.utils import proto_utils, urns -from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, Duration, - Timestamp) +from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, + Duration, Timestamp) from apache_beam.utils.windowed_value import WindowedValue __all__ = [ diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index cbec255dd2f6..a0b3a1073536 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -64,5 +64,5 @@ echo "Running pycodestyle for module $MODULE:" pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" echo "Running isort for module $MODULE:" pushd $MODULE -isort -w 80 -y -c +isort -p apache_beam -w 79 -y -c popd From 0b9c43d8afdbf8361239c2d46b6632b1cc8db36a Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 14:02:17 -0700 Subject: [PATCH 05/17] Re-do-cleanup & Manual fixes and skip the files that isort does annoying things to --- .../examples/cookbook/custom_ptransform.py | 2 +- sdks/python/apache_beam/io/avroio_test.py | 2 +- .../apache_beam/io/gcp/datastore/v1/helper.py | 9 ++++++--- .../apache_beam/io/gcp/datastore/v1/helper_test.py | 6 +++++- .../runners/dataflow/dataflow_runner_test.py | 4 ++-- .../runners/dataflow/native_io/iobase_test.py | 14 ++++---------- sdks/python/run_pylint.sh | 5 ++++- 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py index d274d019fa27..3b445d508c59 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py @@ -28,10 +28,10 @@ import apache_beam as beam from apache_beam.io import ReadFromText, WriteToText from apache_beam.options.pipeline_options import PipelineOptions - # pylint doesn't understand our pipeline syntax: # pylint:disable=expression-not-assigned + class Count1(beam.PTransform): """Count as a subclass of PTransform, with an apply method.""" diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index 048c412230f8..c0c5b7a37116 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -23,9 +23,9 @@ import avro.datafile import avro.schema -import hamcrest as hc from avro.datafile import DataFileWriter from avro.io import DatumWriter +import hamcrest as hc import apache_beam as beam from apache_beam import Create diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py index cfda48cfa398..008f0f57ed69 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py @@ -21,13 +21,11 @@ """ import errno +from socket import error as SocketError import logging import sys import time -from socket import error as SocketError -from apache_beam.internal.gcp import auth -from apache_beam.utils import retry # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position @@ -44,6 +42,11 @@ pass # pylint: enable=wrong-import-order, wrong-import-position +# pylint: disable=ungrouped-imports +from apache_beam.internal.gcp import auth +from apache_beam.utils import retry +# pylint: enable=ungrouped-imports + def key_comparator(k1, k2): """A comparator for Datastore keys. diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py index cde75a03f9bd..502a87354c32 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py @@ -24,9 +24,12 @@ from mock import MagicMock -from apache_beam.io.gcp.datastore.v1 import fake_datastore, helper +# pylint: disable=ungrouped-imports +from apache_beam.io.gcp.datastore.v1 import fake_datastore +from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.testing.test_utils import patch_retry + # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: @@ -40,6 +43,7 @@ except ImportError: datastore_helper = None # pylint: enable=wrong-import-order, wrong-import-position +# pylint: enable=ungrouped-imports @unittest.skipIf(datastore_helper is None, 'GCP dependencies are not installed') diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index e77350c16831..94e8ea5647e0 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -30,8 +30,8 @@ from apache_beam.pvalue import PCollection from apache_beam.runners import (DataflowRunner, TestDataflowRunner, create_runner) -from apache_beam.runners.dataflow.dataflow_runner import (DataflowPipelineResult, - DataflowRuntimeException) +from apache_beam.runners.dataflow.dataflow_runner import \ + DataflowPipelineResult, DataflowRuntimeException from apache_beam.runners.dataflow.internal.clients import \ dataflow as dataflow_api from apache_beam.runners.runner import PipelineState diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index fdee2ee2c15f..99cbabc23fe1 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -21,16 +21,10 @@ import unittest from apache_beam import Create, error, pvalue -from apache_beam.runners.dataflow.native_io.iobase import (ConcatPosition, - DynamicSplitRequest, - DynamicSplitResultWithPosition, - NativeSink, - NativeSinkWriter, - NativeSource, - ReaderPosition, - ReaderProgress, - _dict_printable_fields, - _NativeWrite) +from apache_beam.runners.dataflow.native_io.iobase import \ + ConcatPosition, DynamicSplitRequest, DynamicSplitResultWithPosition, \ + NativeSink, NativeSinkWriter, NativeSource, ReaderPosition, ReaderProgress, \ + _dict_printable_fields, _NativeWrite from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index a0b3a1073536..d4365701967c 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -64,5 +64,8 @@ echo "Running pycodestyle for module $MODULE:" pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" echo "Running isort for module $MODULE:" pushd $MODULE -isort -p apache_beam -w 79 -y -c +# Skip two files where isort is behaving weirdly +isort -p apache_beam -w 79 -y -c -ot -cs --skip avroio_test.py \ + --skip dataflow_runner_test.py --skip iobase_test.py \ + --skip helper_test.py --skip helper.py --skip custom_ptransform.py popd From f4b7a58b8cb69cea73565b9c17901b7800768c7b Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 17:56:39 -0700 Subject: [PATCH 06/17] Fix pylint bash to also skip doing isort check on the autogenreated files --- sdks/python/run_pylint.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index d4365701967c..d382c71716dc 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -63,9 +63,21 @@ pylint $MODULE --ignore-patterns="$FILES_TO_IGNORE" echo "Running pycodestyle for module $MODULE:" pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" echo "Running isort for module $MODULE:" +# Skip files where isort is behaving weirdly +ISORT_EXCLUDED=( + "avroio_test.py" + "dataflow_runner_test.py" + "iobase_test.py" + "helper_test.py" + "helper.py" + "custom_ptransform.py") +SKIP_PARAM="" +for file in "${ISORT_EXCLUDED[@]}"; do + SKIP_PARAM="$SKIP_PARAM --skip $file" +done +for file in "${EXCLUDED_GENERATED_FILES[@]}"; do + SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" +done pushd $MODULE -# Skip two files where isort is behaving weirdly -isort -p apache_beam -w 79 -y -c -ot -cs --skip avroio_test.py \ - --skip dataflow_runner_test.py --skip iobase_test.py \ - --skip helper_test.py --skip helper.py --skip custom_ptransform.py +isort -p apache_beam -w 79 -y -c -ot -cs ${SKIP_PARAM} popd From 7f8708f57faebae0402a6f86225ff4643ef399bc Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Aug 2017 22:27:08 -0700 Subject: [PATCH 07/17] Update list of isort exclusions --- sdks/python/run_pylint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index d382c71716dc..9b449aad49c3 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -70,7 +70,10 @@ ISORT_EXCLUDED=( "iobase_test.py" "helper_test.py" "helper.py" - "custom_ptransform.py") + "custom_ptransform.py" + "apiclient.py" + "datastore_wordcount.py" +) SKIP_PARAM="" for file in "${ISORT_EXCLUDED[@]}"; do SKIP_PARAM="$SKIP_PARAM --skip $file" From 73fd558b0b888e43ed81d05afb7f175d45150855 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Aug 2017 16:21:29 -0700 Subject: [PATCH 08/17] Switch to sl (or single line) mode for isort --- sdks/python/apache_beam/coders/coder_impl.py | 4 +- sdks/python/apache_beam/coders/coders.py | 3 +- .../apache_beam/coders/coders_test_common.py | 3 +- .../coders/standard_coders_test.py | 3 +- .../apache_beam/coders/typecoders_test.py | 3 +- .../examples/complete/autocomplete.py | 6 ++- .../examples/complete/autocomplete_test.py | 3 +- .../examples/complete/estimate_pi.py | 7 +++- .../examples/complete/estimate_pi_test.py | 3 +- .../examples/complete/game/game_stats.py | 11 ++--- .../examples/complete/game/game_stats_test.py | 3 +- .../complete/game/hourly_team_score.py | 9 +++-- .../complete/game/hourly_team_score_test.py | 3 +- .../examples/complete/game/leader_board.py | 11 ++--- .../complete/game/leader_board_test.py | 3 +- .../examples/complete/game/user_score_test.py | 3 +- .../apache_beam/examples/complete/tfidf.py | 6 ++- .../examples/complete/tfidf_test.py | 4 +- .../complete/top_wikipedia_sessions.py | 11 +++-- .../complete/top_wikipedia_sessions_test.py | 3 +- .../examples/cookbook/bigquery_side_input.py | 6 ++- .../cookbook/bigquery_side_input_test.py | 3 +- .../cookbook/bigquery_tornadoes_test.py | 3 +- .../apache_beam/examples/cookbook/coders.py | 6 ++- .../examples/cookbook/coders_test.py | 3 +- .../examples/cookbook/combiners_test.py | 3 +- .../examples/cookbook/custom_ptransform.py | 4 +- .../cookbook/custom_ptransform_test.py | 3 +- .../examples/cookbook/datastore_wordcount.py | 13 +++--- .../examples/cookbook/filters_test.py | 3 +- .../examples/cookbook/group_with_coder.py | 6 ++- .../examples/cookbook/mergecontacts.py | 9 +++-- .../cookbook/multiple_output_pardo.py | 6 ++- .../apache_beam/examples/snippets/snippets.py | 3 +- .../examples/snippets/snippets_test.py | 7 +++- .../examples/streaming_wordcount.py | 4 +- sdks/python/apache_beam/examples/wordcount.py | 6 ++- .../examples/wordcount_debugging.py | 9 +++-- .../apache_beam/examples/wordcount_it_test.py | 4 +- .../apache_beam/examples/wordcount_minimal.py | 6 ++- sdks/python/apache_beam/internal/gcp/auth.py | 3 +- .../internal/gcp/json_value_test.py | 7 ++-- .../apache_beam/internal/pickler_test.py | 3 +- sdks/python/apache_beam/internal/util_test.py | 6 +-- sdks/python/apache_beam/io/avroio.py | 7 +++- sdks/python/apache_beam/io/avroio_test.py | 10 +++-- .../apache_beam/io/concat_source_test.py | 7 +++- sdks/python/apache_beam/io/filebasedsink.py | 9 +++-- sdks/python/apache_beam/io/filebasedsource.py | 18 ++++++--- .../apache_beam/io/filebasedsource_test.py | 11 +++-- sdks/python/apache_beam/io/filesystem_test.py | 3 +- sdks/python/apache_beam/io/filesystems.py | 4 +- sdks/python/apache_beam/io/gcp/bigquery.py | 7 +++- .../apache_beam/io/gcp/bigquery_test.py | 5 ++- .../io/gcp/datastore/v1/datastoreio.py | 13 ++++-- .../io/gcp/datastore/v1/datastoreio_test.py | 17 ++++---- .../apache_beam/io/gcp/datastore/v1/helper.py | 8 ++-- .../io/gcp/datastore/v1/helper_test.py | 1 - .../gcp/datastore/v1/query_splitter_test.py | 6 ++- .../apache_beam/io/gcp/gcsfilesystem.py | 9 +++-- .../apache_beam/io/gcp/gcsfilesystem_test.py | 3 +- .../clients/bigquery/bigquery_v2_messages.py | 3 +- .../clients/storage/storage_v1_messages.py | 3 +- sdks/python/apache_beam/io/gcp/pubsub.py | 8 +++- sdks/python/apache_beam/io/gcp/pubsub_test.py | 8 ++-- .../io/gcp/tests/bigquery_matcher_test.py | 3 +- .../apache_beam/io/gcp/tests/utils_test.py | 3 +- sdks/python/apache_beam/io/iobase.py | 13 ++++-- sdks/python/apache_beam/io/localfilesystem.py | 9 +++-- sdks/python/apache_beam/io/sources_test.py | 6 ++- sdks/python/apache_beam/io/textio.py | 7 +++- sdks/python/apache_beam/io/textio_test.py | 13 ++++-- sdks/python/apache_beam/io/tfrecordio.py | 6 ++- sdks/python/apache_beam/io/tfrecordio_test.py | 11 +++-- sdks/python/apache_beam/metrics/cells.py | 3 +- sdks/python/apache_beam/metrics/cells_test.py | 6 ++- sdks/python/apache_beam/metrics/execution.py | 3 +- .../apache_beam/metrics/execution_test.py | 7 ++-- sdks/python/apache_beam/metrics/metric.py | 4 +- .../python/apache_beam/metrics/metric_test.py | 9 +++-- .../apache_beam/options/pipeline_options.py | 6 +-- .../options/pipeline_options_test.py | 4 +- .../options/pipeline_options_validator.py | 12 +++--- .../options/value_provider_test.py | 4 +- sdks/python/apache_beam/pipeline.py | 13 +++--- sdks/python/apache_beam/pipeline_test.py | 22 +++++++--- sdks/python/apache_beam/runners/common.py | 5 ++- .../runners/dataflow/dataflow_metrics.py | 6 ++- .../runners/dataflow/dataflow_metrics_test.py | 6 ++- .../runners/dataflow/dataflow_runner.py | 19 +++++---- .../runners/dataflow/dataflow_runner_test.py | 14 ++++--- .../runners/dataflow/internal/apiclient.py | 11 ++--- .../dataflow/dataflow_v1b3_messages.py | 3 +- .../runners/dataflow/internal/dependency.py | 4 +- .../dataflow/internal/dependency_test.py | 9 +++-- .../runners/dataflow/native_io/iobase_test.py | 18 +++++++-- .../dataflow/native_io/streaming_create.py | 6 ++- .../runners/dataflow/test_dataflow_runner.py | 4 +- ...consumer_tracking_pipeline_visitor_test.py | 11 +++-- .../runners/direct/direct_metrics.py | 6 ++- .../runners/direct/direct_metrics_test.py | 8 ++-- .../runners/direct/direct_runner.py | 12 ++++-- .../apache_beam/runners/direct/executor.py | 4 +- .../runners/direct/transform_evaluator.py | 40 +++++++++++-------- .../runners/direct/watermark_manager.py | 8 ++-- .../python_rpc_direct_runner.py | 3 +- .../experimental/python_rpc_direct/server.py | 3 +- sdks/python/apache_beam/runners/job/utils.py | 3 +- .../apache_beam/runners/pipeline_context.py | 7 +++- .../runners/portability/fn_api_runner.py | 19 +++++---- .../runners/portability/fn_api_runner_test.py | 7 ++-- .../portability/maptask_executor_runner.py | 8 ++-- .../maptask_executor_runner_test.py | 7 +++- .../python/apache_beam/runners/runner_test.py | 12 ++++-- .../runners/worker/bundle_processor.py | 17 +++++--- .../apache_beam/runners/worker/data_plane.py | 4 +- .../runners/worker/data_plane_test.py | 4 +- .../apache_beam/runners/worker/operations.py | 17 ++++---- .../apache_beam/runners/worker/sdk_worker.py | 7 +++- .../runners/worker/sdk_worker_test.py | 7 +++- .../testing/pipeline_verifiers_test.py | 6 ++- .../python/apache_beam/testing/test_stream.py | 12 ++++-- .../apache_beam/testing/test_stream_test.py | 16 +++++--- sdks/python/apache_beam/testing/test_utils.py | 3 +- sdks/python/apache_beam/testing/util.py | 4 +- sdks/python/apache_beam/testing/util_test.py | 4 +- .../apache_beam/transforms/combiners.py | 15 +++++-- .../apache_beam/transforms/combiners_test.py | 7 +++- sdks/python/apache_beam/transforms/core.py | 37 +++++++++++------ .../apache_beam/transforms/create_test.py | 3 +- sdks/python/apache_beam/transforms/display.py | 3 +- .../apache_beam/transforms/display_test.py | 5 ++- .../apache_beam/transforms/ptransform.py | 17 +++++--- .../apache_beam/transforms/ptransform_test.py | 9 +++-- .../apache_beam/transforms/sideinputs_test.py | 3 +- .../python/apache_beam/transforms/timeutil.py | 3 +- sdks/python/apache_beam/transforms/trigger.py | 17 +++++--- .../apache_beam/transforms/trigger_test.py | 32 ++++++++++----- sdks/python/apache_beam/transforms/util.py | 8 +++- sdks/python/apache_beam/transforms/window.py | 16 +++++--- .../apache_beam/transforms/window_test.py | 33 ++++++++++----- .../transforms/write_ptransform_test.py | 3 +- .../apache_beam/typehints/decorators.py | 11 ++--- .../native_type_compatibility_test.py | 3 +- sdks/python/apache_beam/typehints/opcodes.py | 12 +++++- .../typehints/trivial_inference.py | 3 +- .../typehints/trivial_inference_test.py | 3 +- .../python/apache_beam/typehints/typecheck.py | 13 +++--- .../typehints/typed_pipeline_test.py | 6 ++- .../apache_beam/typehints/typehints_test.py | 20 ++++++---- sdks/python/apache_beam/utils/annotations.py | 3 +- .../apache_beam/utils/annotations_test.py | 3 +- sdks/python/apache_beam/utils/proto_utils.py | 3 +- .../apache_beam/utils/timestamp_test.py | 3 +- .../apache_beam/utils/windowed_value.py | 4 +- sdks/python/run_pylint.sh | 2 +- 156 files changed, 793 insertions(+), 423 deletions(-) diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py index bea3d55caf5f..50702970694d 100644 --- a/sdks/python/apache_beam/coders/coder_impl.py +++ b/sdks/python/apache_beam/coders/coder_impl.py @@ -30,7 +30,9 @@ from apache_beam.coders import observable from apache_beam.utils import windowed_value -from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP, Timestamp +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Timestamp # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 70676f78b53f..3021da50b22d 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -27,7 +27,8 @@ from apache_beam.coders import coder_impl from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.utils import proto_utils, urns +from apache_beam.utils import proto_utils +from apache_beam.utils import urns # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py index 5547b05bb384..29ff2292c50a 100644 --- a/sdks/python/apache_beam/coders/coders_test_common.py +++ b/sdks/python/apache_beam/coders/coders_test_common.py @@ -29,7 +29,8 @@ from apache_beam.runners import pipeline_context from apache_beam.transforms import window from apache_beam.transforms.window import GlobalWindow -from apache_beam.utils import timestamp, windowed_value +from apache_beam.utils import timestamp +from apache_beam.utils import windowed_value from apache_beam.utils.timestamp import MIN_TIMESTAMP diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py index 0028a0bca4b5..ca4dffbcffe8 100644 --- a/sdks/python/apache_beam/coders/standard_coders_test.py +++ b/sdks/python/apache_beam/coders/standard_coders_test.py @@ -26,7 +26,8 @@ import yaml -from apache_beam.coders import coder_impl, coders +from apache_beam.coders import coder_impl +from apache_beam.coders import coders from apache_beam.transforms import window from apache_beam.transforms.window import IntervalWindow from apache_beam.utils import windowed_value diff --git a/sdks/python/apache_beam/coders/typecoders_test.py b/sdks/python/apache_beam/coders/typecoders_test.py index 552d185f88da..2b6aa7a51298 100644 --- a/sdks/python/apache_beam/coders/typecoders_test.py +++ b/sdks/python/apache_beam/coders/typecoders_test.py @@ -19,7 +19,8 @@ import unittest -from apache_beam.coders import coders, typecoders +from apache_beam.coders import coders +from apache_beam.coders import typecoders from apache_beam.internal import pickler from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py index 13c7e3b73d32..ab3397cfe335 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete.py @@ -24,8 +24,10 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions def run(argv=None): diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py index 050d35bb5d0e..e2c84d68d3d4 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py @@ -22,7 +22,8 @@ import apache_beam as beam from apache_beam.examples.complete import autocomplete from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class AutocompleteTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py index 7b05f385f33c..d0a5fb74f3fc 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py @@ -33,8 +33,11 @@ import apache_beam as beam from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions -from apache_beam.typehints import Any, Iterable, Tuple +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.typehints import Any +from apache_beam.typehints import Iterable +from apache_beam.typehints import Tuple @beam.typehints.with_output_types(Tuple[int, int, int]) diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py index 94399da32499..34633132e41c 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py @@ -22,7 +22,8 @@ from apache_beam.examples.complete import estimate_pi from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import BeamAssertException, assert_that +from apache_beam.testing.util import BeamAssertException +from apache_beam.testing.util import assert_that def in_between(lower, upper): diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py index 6db07c73e771..41813235e70e 100644 --- a/sdks/python/apache_beam/examples/complete/game/game_stats.py +++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py @@ -77,7 +77,8 @@ -------------------------------------------------------------------------------- """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import +from __future__ import print_function import argparse import csv @@ -88,10 +89,10 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, - SetupOptions, - StandardOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions def timestamp2str(t, fmt='%Y-%m-%d %H:%M:%S.000'): diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py index 249373d97be7..971f9dc9e299 100644 --- a/sdks/python/apache_beam/examples/complete/game/game_stats_test.py +++ b/sdks/python/apache_beam/examples/complete/game/game_stats_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.complete.game import game_stats from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class GameStatsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py index 2f988ab9f3e9..9dd8b05a5c5c 100644 --- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py +++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py @@ -64,7 +64,8 @@ --temp_location gs://$BUCKET/user_score/temp """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import +from __future__ import print_function import argparse import csv @@ -75,9 +76,9 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, - SetupOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions def str2timestamp(s, fmt='%Y-%m-%d-%H-%M'): diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py index 39fa72cb46d4..9c30127d7416 100644 --- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py +++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.complete.game import hourly_team_score from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class HourlyTeamScoreTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py index 6cda3613a6b2..2936bc96a31a 100644 --- a/sdks/python/apache_beam/examples/complete/game/leader_board.py +++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py @@ -84,7 +84,8 @@ -------------------------------------------------------------------------------- """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import +from __future__ import print_function import argparse import csv @@ -95,10 +96,10 @@ import apache_beam as beam from apache_beam.metrics.metric import Metrics -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, - SetupOptions, - StandardOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions from apache_beam.transforms import trigger diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py index 1a8445810f80..aece2649b898 100644 --- a/sdks/python/apache_beam/examples/complete/game/leader_board_test.py +++ b/sdks/python/apache_beam/examples/complete/game/leader_board_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.complete.game import leader_board from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class LeaderBoardTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/game/user_score_test.py b/sdks/python/apache_beam/examples/complete/game/user_score_test.py index e2880213bf3c..59903d985460 100644 --- a/sdks/python/apache_beam/examples/complete/game/user_score_test.py +++ b/sdks/python/apache_beam/examples/complete/game/user_score_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.complete.game import user_score from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class UserScoreTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py index 8f0c0040c49a..2132fbba1d8a 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf.py +++ b/sdks/python/apache_beam/examples/complete/tfidf.py @@ -29,8 +29,10 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.pvalue import AsSingleton diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py index 1e32ce0dbabe..518a47cf3baf 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf_test.py +++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py @@ -26,7 +26,9 @@ import apache_beam as beam from apache_beam.examples.complete import tfidf from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to, open_shards +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.testing.util import open_shards EXPECTED_RESULTS = set([ ('ghi', '1.txt', 0.3662040962227032), diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py index 0813bd01b521..dd827bc327ef 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py @@ -47,10 +47,13 @@ import apache_beam as beam from apache_beam import combiners -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions -from apache_beam.transforms.window import (FixedWindows, Sessions, - TimestampedValue) +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.transforms.window import FixedWindows +from apache_beam.transforms.window import Sessions +from apache_beam.transforms.window import TimestampedValue ONE_HOUR_IN_SECONDS = 3600 THIRTY_DAYS_IN_SECONDS = 30 * 24 * ONE_HOUR_IN_SECONDS diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py index bdf65be9f9c1..a0b368f60350 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.complete import top_wikipedia_sessions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class ComputeTopSessionsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py index 29735dc71c1d..e16ae7391b6e 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py @@ -33,8 +33,10 @@ import apache_beam as beam from apache_beam.io import WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions -from apache_beam.pvalue import AsList, AsSingleton +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.pvalue import AsList +from apache_beam.pvalue import AsSingleton def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py index 2be7abaf21ff..964b35b3f08f 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.cookbook import bigquery_side_input from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class BigQuerySideInputTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py index 2d12db5050f5..45dcabaf853c 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.cookbook import bigquery_tornadoes from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class BigQueryTornadoesTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py index b9e1a6896aa8..f97b0f2d9c9a 100644 --- a/sdks/python/apache_beam/examples/cookbook/coders.py +++ b/sdks/python/apache_beam/examples/cookbook/coders.py @@ -35,8 +35,10 @@ import logging import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions class JsonCoder(object): diff --git a/sdks/python/apache_beam/examples/cookbook/coders_test.py b/sdks/python/apache_beam/examples/cookbook/coders_test.py index 9bad505cea32..988d3c9d25e1 100644 --- a/sdks/python/apache_beam/examples/cookbook/coders_test.py +++ b/sdks/python/apache_beam/examples/cookbook/coders_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.cookbook import coders from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class CodersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/combiners_test.py b/sdks/python/apache_beam/examples/cookbook/combiners_test.py index 26c9f60b256b..ee1fb7765d47 100644 --- a/sdks/python/apache_beam/examples/cookbook/combiners_test.py +++ b/sdks/python/apache_beam/examples/cookbook/combiners_test.py @@ -28,7 +28,8 @@ import apache_beam as beam from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class CombinersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py index 3b445d508c59..db86003bff6a 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py @@ -26,8 +26,10 @@ import logging import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText from apache_beam.options.pipeline_options import PipelineOptions + # pylint doesn't understand our pipeline syntax: # pylint:disable=expression-not-assigned diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py index 88d83fe2a6ea..7aaccb4ac1c0 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.cookbook import custom_ptransform from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class CustomCountTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py index c8a828845c85..c42596f4aa82 100644 --- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py +++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py @@ -66,19 +66,20 @@ import re import uuid -from google.cloud.proto.datastore.v1 import entity_pb2, query_pb2 +from google.cloud.proto.datastore.v1 import entity_pb2 +from google.cloud.proto.datastore.v1 import query_pb2 from googledatastore import helper as datastore_helper from googledatastore import PropertyFilter import apache_beam as beam from apache_beam.io import ReadFromText -from apache_beam.io.gcp.datastore.v1.datastoreio import (ReadFromDatastore, - WriteToDatastore) +from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore +from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, - SetupOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions class WordExtractingDoFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/cookbook/filters_test.py b/sdks/python/apache_beam/examples/cookbook/filters_test.py index ff70caf7cba7..fd49f938742c 100644 --- a/sdks/python/apache_beam/examples/cookbook/filters_test.py +++ b/sdks/python/apache_beam/examples/cookbook/filters_test.py @@ -23,7 +23,8 @@ import apache_beam as beam from apache_beam.examples.cookbook import filters from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class FiltersTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py index 8ca22b3a8ee2..4c86f46a298b 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py @@ -33,8 +33,10 @@ import apache_beam as beam from apache_beam import coders -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.typehints import typehints from apache_beam.typehints.decorators import with_output_types diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py index 86e98a8d53ed..9acdd9073478 100644 --- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py +++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py @@ -36,9 +36,12 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to def run(argv=None, assert_results=None): diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py index aaa1e0363032..2316c6611c06 100644 --- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py +++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py @@ -57,8 +57,10 @@ import apache_beam as beam from apache_beam import pvalue -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions class SplitLinesToWordsFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index 13e1476a22a4..46696f430802 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -36,7 +36,8 @@ from apache_beam.metrics import Metrics from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.core import PTransform # Quiet some pylint warnings that happen because of the somewhat special diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index 3f6743ce8957..b0b3391f0026 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -26,7 +26,9 @@ import uuid import apache_beam as beam -from apache_beam import coders, pvalue, typehints +from apache_beam import coders +from apache_beam import pvalue +from apache_beam import typehints from apache_beam.coders.coders import ToStringCoder from apache_beam.examples.snippets import snippets from apache_beam.metrics import Metrics @@ -34,7 +36,8 @@ from apache_beam.options.pipeline_options import PipelineOptions # pylint: disable=expression-not-assigned from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.utils.windowed_value import WindowedValue # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py index d0effbf64bd1..94d4c701f677 100644 --- a/sdks/python/apache_beam/examples/streaming_wordcount.py +++ b/sdks/python/apache_beam/examples/streaming_wordcount.py @@ -28,8 +28,8 @@ import apache_beam as beam import apache_beam.transforms.window as window -from apache_beam.options.pipeline_options import (PipelineOptions, - StandardOptions) +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import StandardOptions def split_fn(lines): diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py index 1c4e2d728c8a..34dedb2b819a 100644 --- a/sdks/python/apache_beam/examples/wordcount.py +++ b/sdks/python/apache_beam/examples/wordcount.py @@ -24,10 +24,12 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions class WordExtractingDoFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py index cef76446f55e..c0ffd356364c 100644 --- a/sdks/python/apache_beam/examples/wordcount_debugging.py +++ b/sdks/python/apache_beam/examples/wordcount_debugging.py @@ -46,10 +46,13 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText from apache_beam.metrics import Metrics -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class FilterTextFn(beam.DoFn): diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index dc8e8482c80c..21ff3cebc6dc 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -25,8 +25,8 @@ from nose.plugins.attrib import attr from apache_beam.examples import wordcount -from apache_beam.testing.pipeline_verifiers import (FileChecksumMatcher, - PipelineStateMatcher) +from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher +from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.test_utils import delete_files diff --git a/sdks/python/apache_beam/examples/wordcount_minimal.py b/sdks/python/apache_beam/examples/wordcount_minimal.py index 794f170a840c..76b0a221df7a 100644 --- a/sdks/python/apache_beam/examples/wordcount_minimal.py +++ b/sdks/python/apache_beam/examples/wordcount_minimal.py @@ -51,8 +51,10 @@ import re import apache_beam as beam -from apache_beam.io import ReadFromText, WriteToText -from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions +from apache_beam.io import ReadFromText +from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions def run(argv=None): diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 5956b3ec6dbe..35676f1968b9 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -23,7 +23,8 @@ import os import urllib2 -from oauth2client.client import GoogleCredentials, OAuth2Credentials +from oauth2client.client import GoogleCredentials +from oauth2client.client import OAuth2Credentials from apache_beam.utils import retry diff --git a/sdks/python/apache_beam/internal/gcp/json_value_test.py b/sdks/python/apache_beam/internal/gcp/json_value_test.py index ab79e5e283eb..14223f11c786 100644 --- a/sdks/python/apache_beam/internal/gcp/json_value_test.py +++ b/sdks/python/apache_beam/internal/gcp/json_value_test.py @@ -19,9 +19,10 @@ import unittest -from apache_beam.internal.gcp.json_value import from_json_value, to_json_value -from apache_beam.options.value_provider import (RuntimeValueProvider, - StaticValueProvider) +from apache_beam.internal.gcp.json_value import from_json_value +from apache_beam.internal.gcp.json_value import to_json_value +from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/internal/pickler_test.py b/sdks/python/apache_beam/internal/pickler_test.py index d859b4ed62d5..05062d282da1 100644 --- a/sdks/python/apache_beam/internal/pickler_test.py +++ b/sdks/python/apache_beam/internal/pickler_test.py @@ -20,7 +20,8 @@ import unittest from apache_beam.internal import module_test -from apache_beam.internal.pickler import dumps, loads +from apache_beam.internal.pickler import dumps +from apache_beam.internal.pickler import loads class PicklerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/internal/util_test.py b/sdks/python/apache_beam/internal/util_test.py index 374307ac09bb..9a2e3977a708 100644 --- a/sdks/python/apache_beam/internal/util_test.py +++ b/sdks/python/apache_beam/internal/util_test.py @@ -19,9 +19,9 @@ import unittest -from apache_beam.internal.util import (ArgumentPlaceholder, - insert_values_in_args, - remove_objects_from_args) +from apache_beam.internal.util import ArgumentPlaceholder +from apache_beam.internal.util import insert_values_in_args +from apache_beam.internal.util import remove_objects_from_args class UtilTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py index e67c7df1390e..30fc8903283c 100644 --- a/sdks/python/apache_beam/io/avroio.py +++ b/sdks/python/apache_beam/io/avroio.py @@ -49,10 +49,13 @@ import avro from avro import io as avroio -from avro import datafile, schema +from avro import datafile +from avro import schema import apache_beam as beam -from apache_beam.io import filebasedsink, filebasedsource, iobase +from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource +from apache_beam.io import iobase from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.iobase import Read from apache_beam.transforms import PTransform diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index c0c5b7a37116..6c86635f3edd 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -23,17 +23,21 @@ import avro.datafile import avro.schema +import hamcrest as hc from avro.datafile import DataFileWriter from avro.io import DatumWriter -import hamcrest as hc import apache_beam as beam from apache_beam import Create -from apache_beam.io import avroio, filebasedsource, iobase, source_test_utils +from apache_beam.io import avroio +from apache_beam.io import filebasedsource +from apache_beam.io import iobase +from apache_beam.io import source_test_utils from apache_beam.io.avroio import _AvroSink as AvroSink from apache_beam.io.avroio import _AvroSource as AvroSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py index ec027e9f98df..0f7dd547e76e 100644 --- a/sdks/python/apache_beam/io/concat_source_test.py +++ b/sdks/python/apache_beam/io/concat_source_test.py @@ -21,10 +21,13 @@ import unittest import apache_beam as beam -from apache_beam.io import iobase, range_trackers, source_test_utils +from apache_beam.io import iobase +from apache_beam.io import range_trackers +from apache_beam.io import source_test_utils from apache_beam.io.concat_source import ConcatSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class RangeSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/io/filebasedsink.py b/sdks/python/apache_beam/io/filebasedsink.py index 0284f4fb1c16..ba1a49503260 100644 --- a/sdks/python/apache_beam/io/filebasedsink.py +++ b/sdks/python/apache_beam/io/filebasedsink.py @@ -27,11 +27,12 @@ from apache_beam.internal import util from apache_beam.io import iobase -from apache_beam.io.filesystem import BeamIOError, CompressionTypes +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems -from apache_beam.options.value_provider import (StaticValueProvider, - ValueProvider, - check_accessible) +from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import ValueProvider +from apache_beam.options.value_provider import check_accessible from apache_beam.transforms.display import DisplayDataItem DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN' diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py index 6ab9e0c94698..7b019edec424 100644 --- a/sdks/python/apache_beam/io/filebasedsource.py +++ b/sdks/python/apache_beam/io/filebasedsource.py @@ -28,15 +28,21 @@ import uuid from apache_beam.internal import pickler -from apache_beam.io import concat_source, iobase, range_trackers +from apache_beam.io import concat_source +from apache_beam.io import iobase +from apache_beam.io import range_trackers from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems from apache_beam.io.range_trackers import OffsetRange -from apache_beam.options.value_provider import (StaticValueProvider, - ValueProvider, - check_accessible) -from apache_beam.transforms.core import (DoFn, FlatMap, GroupByKey, Map, ParDo, - PTransform) +from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import ValueProvider +from apache_beam.options.value_provider import check_accessible +from apache_beam.transforms.core import DoFn +from apache_beam.transforms.core import FlatMap +from apache_beam.transforms.core import GroupByKey +from apache_beam.transforms.core import Map +from apache_beam.transforms.core import ParDo +from apache_beam.transforms.core import PTransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.trigger import DefaultTrigger diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index 09cdc69d872c..b263ec82265a 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -28,17 +28,20 @@ import hamcrest as hc import apache_beam as beam -from apache_beam.io import filebasedsource, iobase, range_trackers +from apache_beam.io import filebasedsource +from apache_beam.io import iobase +from apache_beam.io import range_trackers # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource from apache_beam.io.filebasedsource import \ _SingleFileSource as SingleFileSource from apache_beam.io.filebasedsource import FileBasedSource from apache_beam.io.filesystem import CompressionTypes -from apache_beam.options.value_provider import (RuntimeValueProvider, - StaticValueProvider) +from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py index 53cd4ef3451f..1c6cdd7a87ed 100644 --- a/sdks/python/apache_beam/io/filesystem_test.py +++ b/sdks/python/apache_beam/io/filesystem_test.py @@ -24,7 +24,8 @@ import unittest from StringIO import StringIO -from apache_beam.io.filesystem import CompressedFile, CompressionTypes +from apache_beam.io.filesystem import CompressedFile +from apache_beam.io.filesystem import CompressionTypes class TestCompressedFile(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py index 6ac54876cf2e..f9ce55396533 100644 --- a/sdks/python/apache_beam/io/filesystems.py +++ b/sdks/python/apache_beam/io/filesystems.py @@ -19,7 +19,9 @@ import re -from apache_beam.io.filesystem import BeamIOError, CompressionTypes, FileSystem +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystem import FileSystem # All filesystem implements should be added here # pylint: disable=wrong-import-position, unused-import from apache_beam.io.localfilesystem import LocalFileSystem diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 88c3fc36e1eb..40611ac2b6ec 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -112,11 +112,14 @@ from apache_beam import coders from apache_beam.internal.gcp import auth -from apache_beam.internal.gcp.json_value import from_json_value, to_json_value +from apache_beam.internal.gcp.json_value import from_json_value +from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.io.gcp.internal.clients import bigquery from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.runners.dataflow.native_io import iobase as dataflow_io -from apache_beam.transforms import DoFn, ParDo, PTransform +from apache_beam.transforms import DoFn +from apache_beam.transforms import ParDo +from apache_beam.transforms import PTransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.utils import retry diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index 5c33e3773977..849048164346 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -28,8 +28,9 @@ import apache_beam as beam from apache_beam.internal.gcp.json_value import to_json_value -from apache_beam.io.gcp.bigquery import (RowAsDictJsonCoder, TableRowJsonCoder, - parse_table_schema_from_json) +from apache_beam.io.gcp.bigquery import RowAsDictJsonCoder +from apache_beam.io.gcp.bigquery import TableRowJsonCoder +from apache_beam.io.gcp.bigquery import parse_table_schema_from_json from apache_beam.io.gcp.internal.clients import bigquery from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.transforms.display import DisplayData diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py index 47a4788daad9..4b9b73bea507 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py @@ -20,12 +20,19 @@ import logging import time -from apache_beam.io.gcp.datastore.v1 import helper, query_splitter, util +from apache_beam.io.gcp.datastore.v1 import helper +from apache_beam.io.gcp.datastore.v1 import query_splitter +from apache_beam.io.gcp.datastore.v1 import util from apache_beam.io.gcp.datastore.v1.adaptive_throttler import \ AdaptiveThrottler from apache_beam.metrics.metric import Metrics -from apache_beam.transforms import (Create, DoFn, FlatMap, GroupByKey, Map, - ParDo, PTransform) +from apache_beam.transforms import Create +from apache_beam.transforms import DoFn +from apache_beam.transforms import FlatMap +from apache_beam.transforms import GroupByKey +from apache_beam.transforms import Map +from apache_beam.transforms import ParDo +from apache_beam.transforms import PTransform from apache_beam.transforms.util import Values # Protect against environments where datastore library is not available. diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py index d0ba7eb8af28..fa7310f35ace 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py @@ -17,13 +17,16 @@ import unittest -from mock import MagicMock, call, patch - -from apache_beam.io.gcp.datastore.v1 import (fake_datastore, helper, - query_splitter) -from apache_beam.io.gcp.datastore.v1.datastoreio import (ReadFromDatastore, - WriteToDatastore, - _Mutate) +from mock import MagicMock +from mock import call +from mock import patch + +from apache_beam.io.gcp.datastore.v1 import fake_datastore +from apache_beam.io.gcp.datastore.v1 import helper +from apache_beam.io.gcp.datastore.v1 import query_splitter +from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore +from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore +from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py index 008f0f57ed69..01ced6a84a3d 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py @@ -21,11 +21,14 @@ """ import errno -from socket import error as SocketError import logging import sys import time +from socket import error as SocketError +# pylint: disable=ungrouped-imports +from apache_beam.internal.gcp import auth +from apache_beam.utils import retry # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position @@ -42,9 +45,6 @@ pass # pylint: enable=wrong-import-order, wrong-import-position -# pylint: disable=ungrouped-imports -from apache_beam.internal.gcp import auth -from apache_beam.utils import retry # pylint: enable=ungrouped-imports diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py index 502a87354c32..36cfb15e187c 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py @@ -29,7 +29,6 @@ from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.testing.test_utils import patch_retry - # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py index 80ca2277c740..52f25facd058 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter_test.py @@ -19,9 +19,11 @@ import unittest -from mock import MagicMock, call +from mock import MagicMock +from mock import call -from apache_beam.io.gcp.datastore.v1 import fake_datastore, query_splitter +from apache_beam.io.gcp.datastore.v1 import fake_datastore +from apache_beam.io.gcp.datastore.v1 import query_splitter # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py index e63e36ab35b3..ce8b5e6e4244 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py @@ -18,9 +18,12 @@ from __future__ import absolute_import -from apache_beam.io.filesystem import (BeamIOError, CompressedFile, - CompressionTypes, FileMetadata, - FileSystem, MatchResult) +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystem import CompressedFile +from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystem import FileMetadata +from apache_beam.io.filesystem import FileSystem +from apache_beam.io.filesystem import MatchResult from apache_beam.io.gcp import gcsio __all__ = ['GCSFileSystem'] diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py index 6d216fc4cb82..88a601f6d0fe 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py @@ -22,7 +22,8 @@ import mock -from apache_beam.io.filesystem import BeamIOError, FileMetadata +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystem import FileMetadata # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py index 4f3bae3d6abe..3e741cdbbd63 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py @@ -22,7 +22,8 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding, extra_types +from apitools.base.py import encoding +from apitools.base.py import extra_types package = 'bigquery' diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py index d1d226881adf..3c180a652bc5 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py @@ -23,7 +23,8 @@ from apitools.base.protorpclite import message_types as _message_types from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding, extra_types +from apitools.base.py import encoding +from apitools.base.py import extra_types package = 'storage' diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py index 130e145f2415..98aa884c71dc 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub.py +++ b/sdks/python/apache_beam/io/gcp/pubsub.py @@ -27,9 +27,13 @@ import re from apache_beam import coders -from apache_beam.io.iobase import Read, Write +from apache_beam.io.iobase import Read +from apache_beam.io.iobase import Write from apache_beam.runners.dataflow.native_io import iobase as dataflow_io -from apache_beam.transforms import Map, PTransform, core, window +from apache_beam.transforms import Map +from apache_beam.transforms import PTransform +from apache_beam.transforms import core +from apache_beam.transforms import window from apache_beam.transforms.display import DisplayDataItem __all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub'] diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py index 9dcfa3959c50..0c4ba02db87a 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py @@ -22,10 +22,10 @@ import hamcrest as hc -from apache_beam.io.gcp.pubsub import (ReadStringsFromPubSub, - WriteStringsToPubSub, - _PubSubPayloadSink, - _PubSubPayloadSource) +from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub +from apache_beam.io.gcp.pubsub import WriteStringsToPubSub +from apache_beam.io.gcp.pubsub import _PubSubPayloadSink +from apache_beam.io.gcp.pubsub import _PubSubPayloadSource from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py index 5b722856a7b9..a0977189e06b 100644 --- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py +++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py @@ -21,7 +21,8 @@ import unittest from hamcrest import assert_that as hc_assert_that -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.gcp.tests import bigquery_matcher as bq_verifier from apache_beam.testing.test_utils import patch_retry diff --git a/sdks/python/apache_beam/io/gcp/tests/utils_test.py b/sdks/python/apache_beam/io/gcp/tests/utils_test.py index 702c43b4955d..340aa6d2814f 100644 --- a/sdks/python/apache_beam/io/gcp/tests/utils_test.py +++ b/sdks/python/apache_beam/io/gcp/tests/utils_test.py @@ -20,7 +20,8 @@ import logging import unittest -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.gcp.tests import utils from apache_beam.testing.test_utils import patch_retry diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index d88eebcf6ca1..043666d3b8a0 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -34,11 +34,16 @@ import uuid from collections import namedtuple -from apache_beam import coders, pvalue +from apache_beam import coders +from apache_beam import pvalue from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.pvalue import AsIter, AsSingleton -from apache_beam.transforms import core, ptransform, window -from apache_beam.transforms.display import DisplayDataItem, HasDisplayData +from apache_beam.pvalue import AsIter +from apache_beam.pvalue import AsSingleton +from apache_beam.transforms import core +from apache_beam.transforms import ptransform +from apache_beam.transforms import window +from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData from apache_beam.utils import urns from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py index d784ba349e62..b08ac494846e 100644 --- a/sdks/python/apache_beam/io/localfilesystem.py +++ b/sdks/python/apache_beam/io/localfilesystem.py @@ -22,9 +22,12 @@ import os import shutil -from apache_beam.io.filesystem import (BeamIOError, CompressedFile, - CompressionTypes, FileMetadata, - FileSystem, MatchResult) +from apache_beam.io.filesystem import BeamIOError +from apache_beam.io.filesystem import CompressedFile +from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.filesystem import FileMetadata +from apache_beam.io.filesystem import FileSystem +from apache_beam.io.filesystem import MatchResult __all__ = ['LocalFileSystem'] diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py index 76df095c0c34..8f885e59fb0b 100644 --- a/sdks/python/apache_beam/io/sources_test.py +++ b/sdks/python/apache_beam/io/sources_test.py @@ -24,9 +24,11 @@ import apache_beam as beam from apache_beam import coders -from apache_beam.io import iobase, range_trackers +from apache_beam.io import iobase +from apache_beam.io import range_trackers from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class LineSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py index 98ca49554034..c25181db21a7 100644 --- a/sdks/python/apache_beam/io/textio.py +++ b/sdks/python/apache_beam/io/textio.py @@ -24,10 +24,13 @@ from functools import partial from apache_beam.coders import coders -from apache_beam.io import filebasedsink, filebasedsource, iobase +from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource +from apache_beam.io import iobase from apache_beam.io.filebasedsource import ReadAllFiles from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.iobase import Read, Write +from apache_beam.io.iobase import Read +from apache_beam.io.iobase import Write from apache_beam.transforms import PTransform from apache_beam.transforms.display import DisplayDataItem diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py index 9fe000bb3a1a..475848fcb356 100644 --- a/sdks/python/apache_beam/io/textio_test.py +++ b/sdks/python/apache_beam/io/textio_test.py @@ -29,15 +29,20 @@ import apache_beam as beam import apache_beam.io.source_test_utils as source_test_utils from apache_beam import coders -from apache_beam.io import ReadAllFromText, iobase -from apache_beam.io.filebasedsource_test import EOL, write_data, write_pattern +from apache_beam.io import ReadAllFromText +from apache_beam.io import iobase +from apache_beam.io.filebasedsource_test import EOL +from apache_beam.io.filebasedsource_test import write_data +from apache_beam.io.filebasedsource_test import write_pattern from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.textio import _TextSink as TextSink from apache_beam.io.textio import _TextSource as TextSource # Importing following private classes for testing. -from apache_beam.io.textio import ReadFromText, WriteToText +from apache_beam.io.textio import ReadFromText +from apache_beam.io.textio import WriteToText from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.core import Create diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py index 6119deeeff87..5af07164b669 100644 --- a/sdks/python/apache_beam/io/tfrecordio.py +++ b/sdks/python/apache_beam/io/tfrecordio.py @@ -24,9 +24,11 @@ import crcmod from apache_beam import coders -from apache_beam.io import filebasedsink, filebasedsource +from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.iobase import Read, Write +from apache_beam.io.iobase import Read +from apache_beam.io.iobase import Write from apache_beam.transforms import PTransform __all__ = ['ReadFromTFRecord', 'WriteToTFRecord'] diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py index e48b78de9ef2..f7a160a1ce41 100644 --- a/sdks/python/apache_beam/io/tfrecordio_test.py +++ b/sdks/python/apache_beam/io/tfrecordio_test.py @@ -32,11 +32,14 @@ import apache_beam as beam from apache_beam import coders from apache_beam.io.filesystem import CompressionTypes -from apache_beam.io.tfrecordio import (ReadFromTFRecord, WriteToTFRecord, - _TFRecordSink, _TFRecordSource, - _TFRecordUtil) +from apache_beam.io.tfrecordio import ReadFromTFRecord +from apache_beam.io.tfrecordio import WriteToTFRecord +from apache_beam.io.tfrecordio import _TFRecordSink +from apache_beam.io.tfrecordio import _TFRecordSource +from apache_beam.io.tfrecordio import _TFRecordUtil from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to try: import tensorflow as tf # pylint: disable=import-error diff --git a/sdks/python/apache_beam/metrics/cells.py b/sdks/python/apache_beam/metrics/cells.py index 5507cce0c70f..ba840f752b18 100644 --- a/sdks/python/apache_beam/metrics/cells.py +++ b/sdks/python/apache_beam/metrics/cells.py @@ -26,7 +26,8 @@ import threading -from apache_beam.metrics.metricbase import Counter, Distribution +from apache_beam.metrics.metricbase import Counter +from apache_beam.metrics.metricbase import Distribution __all__ = ['DistributionResult'] diff --git a/sdks/python/apache_beam/metrics/cells_test.py b/sdks/python/apache_beam/metrics/cells_test.py index ef3774498c14..c0664ab2bb0c 100644 --- a/sdks/python/apache_beam/metrics/cells_test.py +++ b/sdks/python/apache_beam/metrics/cells_test.py @@ -18,8 +18,10 @@ import threading import unittest -from apache_beam.metrics.cells import (CellCommitState, CounterCell, - DistributionCell, DistributionData) +from apache_beam.metrics.cells import CellCommitState +from apache_beam.metrics.cells import CounterCell +from apache_beam.metrics.cells import DistributionCell +from apache_beam.metrics.cells import DistributionData class TestCounterCell(unittest.TestCase): diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py index fe672241cf7a..1704b98d46a7 100644 --- a/sdks/python/apache_beam/metrics/execution.py +++ b/sdks/python/apache_beam/metrics/execution.py @@ -32,7 +32,8 @@ import threading from collections import defaultdict -from apache_beam.metrics.cells import CounterCell, DistributionCell +from apache_beam.metrics.cells import CounterCell +from apache_beam.metrics.cells import DistributionCell class MetricKey(object): diff --git a/sdks/python/apache_beam/metrics/execution_test.py b/sdks/python/apache_beam/metrics/execution_test.py index b25e0e5638f3..abf23e354a1c 100644 --- a/sdks/python/apache_beam/metrics/execution_test.py +++ b/sdks/python/apache_beam/metrics/execution_test.py @@ -18,9 +18,10 @@ import unittest from apache_beam.metrics.cells import CellCommitState -from apache_beam.metrics.execution import (MetricKey, MetricsContainer, - MetricsEnvironment, - ScopedMetricsContainer) +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricsContainer +from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.metrics.metric import Metrics from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py index 2283b64f8367..8fbf9804ac86 100644 --- a/sdks/python/apache_beam/metrics/metric.py +++ b/sdks/python/apache_beam/metrics/metric.py @@ -27,7 +27,9 @@ import inspect from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.metricbase import Counter, Distribution, MetricName +from apache_beam.metrics.metricbase import Counter +from apache_beam.metrics.metricbase import Distribution +from apache_beam.metrics.metricbase import MetricName __all__ = ['Metrics', 'MetricsFilter'] diff --git a/sdks/python/apache_beam/metrics/metric_test.py b/sdks/python/apache_beam/metrics/metric_test.py index e962074d8948..75c3aa09cbe5 100644 --- a/sdks/python/apache_beam/metrics/metric_test.py +++ b/sdks/python/apache_beam/metrics/metric_test.py @@ -18,9 +18,12 @@ import unittest from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.execution import (MetricKey, MetricsContainer, - MetricsEnvironment) -from apache_beam.metrics.metric import MetricResults, Metrics, MetricsFilter +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricsContainer +from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.metrics.metric import MetricResults +from apache_beam.metrics.metric import Metrics +from apache_beam.metrics.metric import MetricsFilter from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index 2285af1047da..3abcbf2edb24 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -19,9 +19,9 @@ import argparse -from apache_beam.options.value_provider import (RuntimeValueProvider, - StaticValueProvider, - ValueProvider) +from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import ValueProvider from apache_beam.transforms.display import HasDisplayData __all__ = [ diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index cc42ae9f7910..66c69bdfc70e 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -23,8 +23,8 @@ import hamcrest as hc from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.value_provider import (RuntimeValueProvider, - StaticValueProvider) +from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher diff --git a/sdks/python/apache_beam/options/pipeline_options_validator.py b/sdks/python/apache_beam/options/pipeline_options_validator.py index 72c0942460f9..24d2e55eadbe 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator.py @@ -22,11 +22,13 @@ import re from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import (DebugOptions, - GoogleCloudOptions, - SetupOptions, - StandardOptions, TestOptions, - TypeOptions, WorkerOptions) +from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TestOptions +from apache_beam.options.pipeline_options import TypeOptions +from apache_beam.options.pipeline_options import WorkerOptions class PipelineOptionsValidator(object): diff --git a/sdks/python/apache_beam/options/value_provider_test.py b/sdks/python/apache_beam/options/value_provider_test.py index 60d10a171359..17e9590d2a36 100644 --- a/sdks/python/apache_beam/options/value_provider_test.py +++ b/sdks/python/apache_beam/options/value_provider_test.py @@ -20,8 +20,8 @@ import unittest from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.value_provider import (RuntimeValueProvider, - StaticValueProvider) +from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider # TODO(BEAM-1319): Require unique names only within a test. diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 196da5b26e97..9daf7654dbc4 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -55,15 +55,18 @@ from apache_beam import pvalue from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import (PipelineOptions, - SetupOptions, - StandardOptions, TypeOptions) +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TypeOptions from apache_beam.options.pipeline_options_validator import \ PipelineOptionsValidator from apache_beam.pvalue import PCollection -from apache_beam.runners import PipelineRunner, create_runner +from apache_beam.runners import PipelineRunner +from apache_beam.runners import create_runner from apache_beam.transforms import ptransform -from apache_beam.typehints import TypeCheckError, typehints +from apache_beam.typehints import TypeCheckError +from apache_beam.typehints import typehints from apache_beam.utils import urns from apache_beam.utils.annotations import deprecated diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 290b83243ea0..0917c7846727 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -24,16 +24,26 @@ import apache_beam as beam from apache_beam.io import Read from apache_beam.metrics import Metrics -from apache_beam.pipeline import (Pipeline, PipelineOptions, PipelineVisitor, - PTransformOverride) +from apache_beam.pipeline import Pipeline +from apache_beam.pipeline import PipelineOptions +from apache_beam.pipeline import PipelineVisitor +from apache_beam.pipeline import PTransformOverride from apache_beam.pvalue import AsSingleton from apache_beam.runners import DirectRunner from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms import (CombineGlobally, Create, DoFn, FlatMap, - Map, ParDo, PTransform, WindowInto) -from apache_beam.transforms.window import SlidingWindows, TimestampedValue +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.transforms import CombineGlobally +from apache_beam.transforms import Create +from apache_beam.transforms import DoFn +from apache_beam.transforms import FlatMap +from apache_beam.transforms import Map +from apache_beam.transforms import ParDo +from apache_beam.transforms import PTransform +from apache_beam.transforms import WindowInto +from apache_beam.transforms.window import SlidingWindows +from apache_beam.transforms.window import TimestampedValue from apache_beam.utils.timestamp import MIN_TIMESTAMP # TODO(BEAM-1555): Test is failing on the service, with FakeSource. diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index 8f1ecb5695ec..66c033fc5c5f 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -29,8 +29,9 @@ from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.pvalue import TaggedOutput from apache_beam.transforms import core -from apache_beam.transforms.window import (GlobalWindow, TimestampedValue, - WindowFn) +from apache_beam.transforms.window import GlobalWindow +from apache_beam.transforms.window import TimestampedValue +from apache_beam.transforms.window import WindowFn from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py index ec224909d157..c7eb88ef578d 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py @@ -24,8 +24,10 @@ import numbers from collections import defaultdict -from apache_beam.metrics.cells import DistributionData, DistributionResult -from apache_beam.metrics.execution import MetricKey, MetricResult +from apache_beam.metrics.cells import DistributionData +from apache_beam.metrics.cells import DistributionResult +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult from apache_beam.metrics.metric import MetricResults from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py index c077ea393328..dd3cbe1156a4 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics_test.py @@ -23,8 +23,10 @@ import mock -from apache_beam.metrics.cells import DistributionData, DistributionResult -from apache_beam.metrics.execution import MetricKey, MetricResult +from apache_beam.metrics.cells import DistributionData +from apache_beam.metrics.cells import DistributionResult +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult from apache_beam.metrics.metricbase import MetricName from apache_beam.runners.dataflow import dataflow_metrics diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 38fe25bf207a..a67a712a334d 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -29,20 +29,25 @@ from collections import defaultdict import apache_beam as beam -from apache_beam import coders, error, pvalue +from apache_beam import coders +from apache_beam import error +from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.internal.gcp import json_value -from apache_beam.options.pipeline_options import (SetupOptions, - StandardOptions, TestOptions) +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TestOptions from apache_beam.pvalue import AsSideInput from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics from apache_beam.runners.dataflow.internal import names from apache_beam.runners.dataflow.internal.clients import \ dataflow as dataflow_api -from apache_beam.runners.dataflow.internal.names import (PropertyNames, - TransformNames) -from apache_beam.runners.runner import (PipelineResult, PipelineRunner, - PipelineState, PValueCache) +from apache_beam.runners.dataflow.internal.names import PropertyNames +from apache_beam.runners.dataflow.internal.names import TransformNames +from apache_beam.runners.runner import PipelineResult +from apache_beam.runners.runner import PipelineRunner +from apache_beam.runners.runner import PipelineState +from apache_beam.runners.runner import PValueCache from apache_beam.transforms.display import DisplayData from apache_beam.typehints import typehints from apache_beam.utils.plugin import BeamPlugin diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index 94e8ea5647e0..7f793f6cda35 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -26,18 +26,22 @@ import apache_beam as beam import apache_beam.transforms as ptransform from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.pipeline import AppliedPTransform, Pipeline +from apache_beam.pipeline import AppliedPTransform +from apache_beam.pipeline import Pipeline from apache_beam.pvalue import PCollection -from apache_beam.runners import (DataflowRunner, TestDataflowRunner, - create_runner) +from apache_beam.runners import DataflowRunner +from apache_beam.runners import TestDataflowRunner +from apache_beam.runners import create_runner +from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult from apache_beam.runners.dataflow.dataflow_runner import \ - DataflowPipelineResult, DataflowRuntimeException + DataflowRuntimeException from apache_beam.runners.dataflow.internal.clients import \ dataflow as dataflow_api from apache_beam.runners.runner import PipelineState from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms import window -from apache_beam.transforms.core import Windowing, _GroupByKeyOnly +from apache_beam.transforms.core import Windowing +from apache_beam.transforms.core import _GroupByKeyOnly from apache_beam.transforms.display import DisplayDataItem from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index a0c1be43f94e..9b661d26f9f1 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -29,16 +29,17 @@ from datetime import datetime from StringIO import StringIO -from apitools.base.py import encoding, exceptions +from apitools.base.py import encoding +from apitools.base.py import exceptions from apache_beam.internal.gcp.auth import get_service_credentials from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.io.filesystems import FileSystems from apache_beam.io.gcp.internal.clients import storage -from apache_beam.options.pipeline_options import (DebugOptions, - GoogleCloudOptions, - StandardOptions, - WorkerOptions) +from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.runners.dataflow.internal import dependency from apache_beam.runners.dataflow.internal.clients import dataflow from apache_beam.runners.dataflow.internal.dependency import \ diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py index 6f8e3e755ca3..b0d4e44816c1 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py @@ -23,7 +23,8 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages -from apitools.base.py import encoding, extra_types +from apitools.base.py import encoding +from apitools.base.py import extra_types package = 'dataflow' diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index d5e61241439b..f4131dc885c0 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -64,8 +64,8 @@ from apache_beam import version as beam_version from apache_beam.internal import pickler from apache_beam.io.filesystems import FileSystems -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - SetupOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.runners.dataflow.internal import names from apache_beam.utils import processes diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py index 83fecb189109..6d9b061c29a9 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py @@ -24,10 +24,11 @@ import unittest from apache_beam.io.filesystems import FileSystems -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - PipelineOptions, - SetupOptions) -from apache_beam.runners.dataflow.internal import dependency, names +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.dataflow.internal import dependency +from apache_beam.runners.dataflow.internal import names # Protect against environments where GCS library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index 99cbabc23fe1..4ebe806137e8 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -20,11 +20,21 @@ import unittest -from apache_beam import Create, error, pvalue +from apache_beam import Create +from apache_beam import error +from apache_beam import pvalue +from apache_beam.runners.dataflow.native_io.iobase import ConcatPosition +from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitRequest from apache_beam.runners.dataflow.native_io.iobase import \ - ConcatPosition, DynamicSplitRequest, DynamicSplitResultWithPosition, \ - NativeSink, NativeSinkWriter, NativeSource, ReaderPosition, ReaderProgress, \ - _dict_printable_fields, _NativeWrite + DynamicSplitResultWithPosition +from apache_beam.runners.dataflow.native_io.iobase import NativeSink +from apache_beam.runners.dataflow.native_io.iobase import NativeSinkWriter +from apache_beam.runners.dataflow.native_io.iobase import NativeSource +from apache_beam.runners.dataflow.native_io.iobase import ReaderPosition +from apache_beam.runners.dataflow.native_io.iobase import ReaderProgress +from apache_beam.runners.dataflow.native_io.iobase import \ + _dict_printable_fields +from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py index 929da81ed8a5..a54ee7767c5e 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py @@ -17,7 +17,11 @@ """Create transform for streaming.""" -from apache_beam import DoFn, ParDo, PTransform, Windowing, pvalue +from apache_beam import DoFn +from apache_beam import ParDo +from apache_beam import PTransform +from apache_beam import Windowing +from apache_beam import pvalue from apache_beam.transforms.window import GlobalWindows diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py index 6fd5dbf8ca75..e7c8d06d2f8f 100644 --- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py @@ -18,8 +18,8 @@ """Wrapper of Beam runners that's built for running and verifying e2e tests.""" from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import (GoogleCloudOptions, - TestOptions) +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import TestOptions from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner __all__ = ['TestDataflowRunner'] diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py index 0263904a7341..216ea3737183 100644 --- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py +++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py @@ -21,14 +21,19 @@ import unittest from apache_beam import pvalue -from apache_beam.io import Read, iobase +from apache_beam.io import Read +from apache_beam.io import iobase from apache_beam.pipeline import Pipeline from apache_beam.pvalue import AsList from apache_beam.runners.direct import DirectRunner from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import \ ConsumerTrackingPipelineVisitor -from apache_beam.transforms import (CoGroupByKey, Create, DoFn, FlatMap, - Flatten, ParDo) +from apache_beam.transforms import CoGroupByKey +from apache_beam.transforms import Create +from apache_beam.transforms import DoFn +from apache_beam.transforms import FlatMap +from apache_beam.transforms import Flatten +from apache_beam.transforms import ParDo # Disable frequent lint warning due to pipe operator for chaining transforms. # pylint: disable=expression-not-assigned diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics.py b/sdks/python/apache_beam/runners/direct/direct_metrics.py index c8cf0344cbac..aa35fb7ccbb1 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics.py @@ -23,8 +23,10 @@ import threading from collections import defaultdict -from apache_beam.metrics.cells import CounterAggregator, DistributionAggregator -from apache_beam.metrics.execution import MetricKey, MetricResult +from apache_beam.metrics.cells import CounterAggregator +from apache_beam.metrics.cells import DistributionAggregator +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult from apache_beam.metrics.metric import MetricResults diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py index a9b3120d1dda..f36178601ff8 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py @@ -19,9 +19,11 @@ import hamcrest as hc -from apache_beam.metrics.cells import DistributionData, DistributionResult -from apache_beam.metrics.execution import (MetricKey, MetricResult, - MetricUpdates) +from apache_beam.metrics.cells import DistributionData +from apache_beam.metrics.cells import DistributionResult +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.execution import MetricUpdates from apache_beam.metrics.metricbase import MetricName from apache_beam.runners.direct.direct_metrics import DirectMetrics diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index dff8689c5682..794a96be12ba 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -31,13 +31,17 @@ import apache_beam as beam from apache_beam import typehints from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.options.pipeline_options import DirectOptions, StandardOptions +from apache_beam.options.pipeline_options import DirectOptions +from apache_beam.options.pipeline_options import StandardOptions from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.pvalue import PCollection from apache_beam.runners.direct.bundle_factory import BundleFactory -from apache_beam.runners.runner import (PipelineResult, PipelineRunner, - PipelineState, PValueCache) -from apache_beam.transforms.core import _GroupAlsoByWindow, _GroupByKeyOnly +from apache_beam.runners.runner import PipelineResult +from apache_beam.runners.runner import PipelineRunner +from apache_beam.runners.runner import PipelineState +from apache_beam.runners.runner import PValueCache +from apache_beam.transforms.core import _GroupAlsoByWindow +from apache_beam.transforms.core import _GroupByKeyOnly from apache_beam.transforms.ptransform import PTransform __all__ = ['DirectRunner'] diff --git a/sdks/python/apache_beam/runners/direct/executor.py b/sdks/python/apache_beam/runners/direct/executor.py index 3509f4e341df..3e08b522d541 100644 --- a/sdks/python/apache_beam/runners/direct/executor.py +++ b/sdks/python/apache_beam/runners/direct/executor.py @@ -28,8 +28,8 @@ import traceback from weakref import WeakValueDictionary -from apache_beam.metrics.execution import (MetricsContainer, - ScopedMetricsContainer) +from apache_beam.metrics.execution import MetricsContainer +from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.options.pipeline_options import DirectOptions diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index bf29f3932f56..5cd3091da8b8 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -24,29 +24,37 @@ import time import apache_beam.io as io -from apache_beam import coders, pvalue +from apache_beam import coders +from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.options.pipeline_options import TypeOptions -from apache_beam.runners.common import DoFnRunner, DoFnState +from apache_beam.runners.common import DoFnRunner +from apache_beam.runners.common import DoFnState from apache_beam.runners.dataflow.native_io.iobase import \ _NativeWrite # pylint: disable=protected-access -from apache_beam.runners.direct.direct_runner import (_StreamingGroupAlsoByWindow, - _StreamingGroupByKeyOnly) -from apache_beam.runners.direct.util import KeyedWorkItem, TransformResult +from apache_beam.runners.direct.direct_runner import \ + _StreamingGroupAlsoByWindow +from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly +from apache_beam.runners.direct.util import KeyedWorkItem +from apache_beam.runners.direct.util import TransformResult from apache_beam.runners.direct.watermark_manager import WatermarkManager -from apache_beam.testing.test_stream import (ElementEvent, ProcessingTimeEvent, - TestStream, WatermarkEvent) +from apache_beam.testing.test_stream import ElementEvent +from apache_beam.testing.test_stream import ProcessingTimeEvent +from apache_beam.testing.test_stream import TestStream +from apache_beam.testing.test_stream import WatermarkEvent from apache_beam.transforms import core -from apache_beam.transforms.trigger import (TimeDomain, - _CombiningValueStateTag, - _ListStateTag, - create_trigger_driver) -from apache_beam.transforms.window import GlobalWindows, WindowedValue -from apache_beam.typehints.typecheck import (OutputCheckWrapperDoFn, - TypeCheckError, - TypeCheckWrapperDoFn) +from apache_beam.transforms.trigger import TimeDomain +from apache_beam.transforms.trigger import _CombiningValueStateTag +from apache_beam.transforms.trigger import _ListStateTag +from apache_beam.transforms.trigger import create_trigger_driver +from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import WindowedValue +from apache_beam.typehints.typecheck import OutputCheckWrapperDoFn +from apache_beam.typehints.typecheck import TypeCheckError +from apache_beam.typehints.typecheck import TypeCheckWrapperDoFn from apache_beam.utils import counters -from apache_beam.utils.timestamp import MIN_TIMESTAMP, Timestamp +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Timestamp class TransformEvaluatorRegistry(object): diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py index 5ab83e94bf3d..935998d27de0 100644 --- a/sdks/python/apache_beam/runners/direct/watermark_manager.py +++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py @@ -21,10 +21,12 @@ import threading -from apache_beam import pipeline, pvalue +from apache_beam import pipeline +from apache_beam import pvalue from apache_beam.runners.direct.util import TimerFiring -from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, - TIME_GRANULARITY) +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import TIME_GRANULARITY class WatermarkManager(object): diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py index 98f3e68593de..85e3f75be239 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py @@ -27,7 +27,8 @@ from apache_beam.portability.api import beam_job_api_pb2 from apache_beam.runners.job import utils as job_utils from apache_beam.runners.job.manager import DockerRPCManager -from apache_beam.runners.runner import PipelineResult, PipelineRunner +from apache_beam.runners.runner import PipelineResult +from apache_beam.runners.runner import PipelineRunner __all__ = ['PythonRPCDirectRunner'] diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py index 4d0ef4e4cca3..1d07e716f799 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py @@ -25,7 +25,8 @@ from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline -from apache_beam.portability.api import beam_job_api_pb2, beam_job_api_pb2_grpc +from apache_beam.portability.api import beam_job_api_pb2 +from apache_beam.portability.api import beam_job_api_pb2_grpc from apache_beam.runners.runner import PipelineState _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/sdks/python/apache_beam/runners/job/utils.py b/sdks/python/apache_beam/runners/job/utils.py index 80b5e451f2d0..84c727fb4ebc 100644 --- a/sdks/python/apache_beam/runners/job/utils.py +++ b/sdks/python/apache_beam/runners/job/utils.py @@ -20,7 +20,8 @@ import json -from google.protobuf import json_format, struct_pb2 +from google.protobuf import json_format +from google.protobuf import struct_pb2 def dict_to_struct(dict_obj): diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index a2436ce3ff08..35060999b723 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -21,8 +21,11 @@ """ -from apache_beam import coders, pipeline, pvalue -from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 +from apache_beam import coders +from apache_beam import pipeline +from apache_beam import pvalue +from apache_beam.portability.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import core diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py index 820e4f022a4d..32dc50acf660 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py @@ -29,20 +29,25 @@ from google.protobuf import wrappers_pb2 import apache_beam as beam # pylint: disable=ungrouped-imports -from apache_beam.coders import WindowedValueCoder, registry -from apache_beam.coders.coder_impl import (create_InputStream, - create_OutputStream) +from apache_beam.coders import WindowedValueCoder +from apache_beam.coders import registry +from apache_beam.coders.coder_impl import create_InputStream +from apache_beam.coders.coder_impl import create_OutputStream from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners import pipeline_context from apache_beam.runners.portability import maptask_executor_runner from apache_beam.runners.runner import PipelineState -from apache_beam.runners.worker import (bundle_processor, data_plane, - operation_specs, sdk_worker) +from apache_beam.runners.worker import bundle_processor +from apache_beam.runners.worker import data_plane +from apache_beam.runners.worker import operation_specs +from apache_beam.runners.worker import sdk_worker from apache_beam.transforms.window import GlobalWindows -from apache_beam.utils import proto_utils, urns +from apache_beam.utils import proto_utils +from apache_beam.utils import urns # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py index ef7660518bd5..ba219547e6cf 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py @@ -19,9 +19,10 @@ import unittest import apache_beam as beam -from apache_beam.runners.portability import (fn_api_runner, - maptask_executor_runner_test) -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.runners.portability import fn_api_runner +from apache_beam.runners.portability import maptask_executor_runner_test +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class FnApiRunnerTest( diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py index c4c6c3813558..05b021331b94 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py @@ -32,9 +32,11 @@ _dependency_file_copy from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.dataflow.native_io.iobase import NativeSource -from apache_beam.runners.runner import (PipelineResult, PipelineRunner, - PipelineState) -from apache_beam.runners.worker import operation_specs, operations +from apache_beam.runners.runner import PipelineResult +from apache_beam.runners.runner import PipelineRunner +from apache_beam.runners.runner import PipelineState +from apache_beam.runners.worker import operation_specs +from apache_beam.runners.worker import operations from apache_beam.typehints import typehints from apache_beam.utils import profiler from apache_beam.utils.counters import CounterFactory diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py index 17ba7788540f..519ab6e3d041 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py @@ -22,11 +22,14 @@ import apache_beam as beam from apache_beam.metrics import Metrics -from apache_beam.metrics.execution import MetricKey, MetricsEnvironment +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.metrics.metricbase import MetricName from apache_beam.pvalue import AsList from apache_beam.runners.portability import maptask_executor_runner -from apache_beam.testing.util import BeamAssertException, assert_that, equal_to +from apache_beam.testing.util import BeamAssertException +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.window import TimestampedValue diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py index 1b3b1b4f69f3..063c8a2bcecc 100644 --- a/sdks/python/apache_beam/runners/runner_test.py +++ b/sdks/python/apache_beam/runners/runner_test.py @@ -28,13 +28,17 @@ import apache_beam as beam import apache_beam.transforms as ptransform -from apache_beam.metrics.cells import DistributionData, DistributionResult -from apache_beam.metrics.execution import MetricKey, MetricResult +from apache_beam.metrics.cells import DistributionData +from apache_beam.metrics.cells import DistributionResult +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult from apache_beam.metrics.metricbase import MetricName from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline -from apache_beam.runners import DirectRunner, create_runner -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.runners import DirectRunner +from apache_beam.runners import create_runner +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class RunnerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py index 9b75b706726c..068aa0af2b04 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py @@ -17,7 +17,9 @@ """SDK harness for executing Python Fns via the Fn API.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import base64 import collections @@ -27,14 +29,19 @@ from google.protobuf import wrappers_pb2 import apache_beam as beam -from apache_beam.coders import WindowedValueCoder, coder_impl +from apache_beam.coders import WindowedValueCoder +from apache_beam.coders import coder_impl from apache_beam.internal import pickler from apache_beam.io import iobase -from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners import pipeline_context from apache_beam.runners.dataflow.native_io import iobase as native_iobase -from apache_beam.runners.worker import operation_specs, operations -from apache_beam.utils import counters, proto_utils, urns +from apache_beam.runners.worker import operation_specs +from apache_beam.runners.worker import operations +from apache_beam.utils import counters +from apache_beam.utils import proto_utils +from apache_beam.utils import urns # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py index e44f2d14276e..5c65a6d8ed90 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane.py +++ b/sdks/python/apache_beam/runners/worker/data_plane.py @@ -17,7 +17,9 @@ """Implementation of DataChannels for communicating across the data plane.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import abc import collections diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index e15aac9b8e54..db7ac0b4705e 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -17,7 +17,9 @@ """Tests for apache_beam.runners.worker.data_plane.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import sys diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py index f99d0bda8074..1b61f8e5f9be 100644 --- a/sdks/python/apache_beam/runners/worker/operations.py +++ b/sdks/python/apache_beam/runners/worker/operations.py @@ -26,17 +26,20 @@ from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.io import iobase -from apache_beam.metrics.execution import (MetricsContainer, - ScopedMetricsContainer) +from apache_beam.metrics.execution import MetricsContainer +from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.runners import common from apache_beam.runners.common import Receiver from apache_beam.runners.dataflow.internal.names import PropertyNames -from apache_beam.runners.worker import (logger, opcounters, operation_specs, - sideinputs) +from apache_beam.runners.worker import logger +from apache_beam.runners.worker import opcounters +from apache_beam.runners.worker import operation_specs +from apache_beam.runners.worker import sideinputs from apache_beam.transforms import sideinputs as apache_sideinputs -from apache_beam.transforms import combiners, core -from apache_beam.transforms.combiners import (PhasedCombineFnExecutor, - curry_combine_fn) +from apache_beam.transforms import combiners +from apache_beam.transforms import core +from apache_beam.transforms.combiners import PhasedCombineFnExecutor +from apache_beam.transforms.combiners import curry_combine_fn from apache_beam.transforms.window import GlobalWindows from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index 36fde7b6fd4b..6a236802b9a6 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -17,7 +17,9 @@ """SDK harness for executing Python Fns via the Fn API.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import Queue as queue @@ -25,7 +27,8 @@ import traceback from apache_beam.portability.api import beam_fn_api_pb2 -from apache_beam.runners.worker import bundle_processor, data_plane +from apache_beam.runners.worker import bundle_processor +from apache_beam.runners.worker import data_plane class SdkHarness(object): diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index 407d589580ef..d371747798ca 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -17,7 +17,9 @@ """Tests for apache_beam.runners.worker.sdk_worker.""" -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import logging import unittest @@ -25,7 +27,8 @@ import grpc -from apache_beam.portability.api import beam_fn_api_pb2, beam_runner_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.runners.worker import sdk_worker diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py index 23cc00f1f1a6..3b02431212de 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py @@ -22,10 +22,12 @@ import unittest from hamcrest import assert_that as hc_assert_that -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.localfilesystem import LocalFileSystem -from apache_beam.runners.runner import PipelineResult, PipelineState +from apache_beam.runners.runner import PipelineResult +from apache_beam.runners.runner import PipelineState from apache_beam.testing import pipeline_verifiers as verifiers from apache_beam.testing.test_utils import patch_retry diff --git a/sdks/python/apache_beam/testing/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py index 1ee9915fb315..8a63e7bd0561 100644 --- a/sdks/python/apache_beam/testing/test_stream.py +++ b/sdks/python/apache_beam/testing/test_stream.py @@ -20,10 +20,14 @@ For internal use only; no backwards-compatibility guarantees. """ -from abc import ABCMeta, abstractmethod - -from apache_beam import coders, core, pvalue -from apache_beam.transforms import PTransform, window +from abc import ABCMeta +from abc import abstractmethod + +from apache_beam import coders +from apache_beam import core +from apache_beam import pvalue +from apache_beam.transforms import PTransform +from apache_beam.transforms import window from apache_beam.transforms.window import TimestampedValue from apache_beam.utils import timestamp from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/testing/test_stream_test.py b/sdks/python/apache_beam/testing/test_stream_test.py index caafe4f245d4..0f6691f3d644 100644 --- a/sdks/python/apache_beam/testing/test_stream_test.py +++ b/sdks/python/apache_beam/testing/test_stream_test.py @@ -20,13 +20,17 @@ import unittest import apache_beam as beam -from apache_beam.options.pipeline_options import (PipelineOptions, - StandardOptions) +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import StandardOptions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.test_stream import (ElementEvent, ProcessingTimeEvent, - TestStream, WatermarkEvent) -from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms.window import FixedWindows, TimestampedValue +from apache_beam.testing.test_stream import ElementEvent +from apache_beam.testing.test_stream import ProcessingTimeEvent +from apache_beam.testing.test_stream import TestStream +from apache_beam.testing.test_stream import WatermarkEvent +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.transforms.window import FixedWindows +from apache_beam.transforms.window import TimestampedValue from apache_beam.utils import timestamp from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index de7188795b67..41a02cfba9c3 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -23,7 +23,8 @@ import hashlib import imp -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.filesystems import FileSystems from apache_beam.utils import retry diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py index 49f9e9158009..9bb18ccccb7f 100644 --- a/sdks/python/apache_beam/testing/util.py +++ b/sdks/python/apache_beam/testing/util.py @@ -24,7 +24,9 @@ from apache_beam import pvalue from apache_beam.transforms import window -from apache_beam.transforms.core import Create, Map, WindowInto +from apache_beam.transforms.core import Create +from apache_beam.transforms.core import Map +from apache_beam.transforms.core import WindowInto from apache_beam.transforms.ptransform import PTransform from apache_beam.transforms.util import CoGroupByKey from apache_beam.utils.annotations import experimental diff --git a/sdks/python/apache_beam/testing/util_test.py b/sdks/python/apache_beam/testing/util_test.py index 1acebb651c55..9d3869381b68 100644 --- a/sdks/python/apache_beam/testing/util_test.py +++ b/sdks/python/apache_beam/testing/util_test.py @@ -21,7 +21,9 @@ from apache_beam import Create from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to, is_empty +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.testing.util import is_empty class UtilTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py index 289a2a8103c3..60bf2d1bfbc4 100644 --- a/sdks/python/apache_beam/transforms/combiners.py +++ b/sdks/python/apache_beam/transforms/combiners.py @@ -22,10 +22,19 @@ import operator import random -from apache_beam.transforms import core, cy_combiners, ptransform +from apache_beam.transforms import core +from apache_beam.transforms import cy_combiners +from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayDataItem -from apache_beam.typehints import (KV, Any, Dict, List, Tuple, TypeVariable, - Union, with_input_types, with_output_types) +from apache_beam.typehints import KV +from apache_beam.typehints import Any +from apache_beam.typehints import Dict +from apache_beam.typehints import List +from apache_beam.typehints import Tuple +from apache_beam.typehints import TypeVariable +from apache_beam.typehints import Union +from apache_beam.typehints import with_input_types +from apache_beam.typehints import with_output_types __all__ = [ 'Count', diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index fb0241abbf62..8885d27c84d6 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -24,8 +24,11 @@ import apache_beam as beam import apache_beam.transforms.combiners as combine from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms.core import CombineGlobally, Create, Map +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.transforms.core import CombineGlobally +from apache_beam.transforms.core import Create +from apache_beam.transforms.core import Map from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher from apache_beam.transforms.ptransform import PTransform diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 81ba67db2c62..7f655a748384 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -25,24 +25,37 @@ from google.protobuf import wrappers_pb2 -from apache_beam import coders, pvalue, typehints +from apache_beam import coders +from apache_beam import pvalue +from apache_beam import typehints from apache_beam.coders import typecoders -from apache_beam.internal import pickler, util +from apache_beam.internal import pickler +from apache_beam.internal import util from apache_beam.options.pipeline_options import TypeOptions from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import ptransform -from apache_beam.transforms.display import DisplayDataItem, HasDisplayData -from apache_beam.transforms.ptransform import (PTransform, - PTransformWithSideInputs) -from apache_beam.transforms.window import (MIN_TIMESTAMP, GlobalWindows, - TimestampCombiner, TimestampedValue, - WindowedValue, WindowFn) -from apache_beam.typehints import KV, Any, Iterable, Union, trivial_inference -from apache_beam.typehints.decorators import (TypeCheckError, WithTypeHints, - get_type_hints) +from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData +from apache_beam.transforms.ptransform import PTransform +from apache_beam.transforms.ptransform import PTransformWithSideInputs +from apache_beam.transforms.window import MIN_TIMESTAMP +from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import TimestampCombiner +from apache_beam.transforms.window import TimestampedValue +from apache_beam.transforms.window import WindowedValue +from apache_beam.transforms.window import WindowFn +from apache_beam.typehints import KV +from apache_beam.typehints import Any +from apache_beam.typehints import Iterable +from apache_beam.typehints import Union +from apache_beam.typehints import trivial_inference +from apache_beam.typehints.decorators import TypeCheckError +from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.typehints.decorators import get_type_hints from apache_beam.typehints.trivial_inference import element_type from apache_beam.typehints.typehints import is_consistent_with -from apache_beam.utils import proto_utils, urns +from apache_beam.utils import proto_utils +from apache_beam.utils import urns __all__ = [ 'DoFn', diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py index 1ae3aaa705c7..e5863295ac17 100644 --- a/sdks/python/apache_beam/transforms/create_test.py +++ b/sdks/python/apache_beam/transforms/create_test.py @@ -22,7 +22,8 @@ from apache_beam.coders import FastPrimitivesCoder from apache_beam.io import source_test_utils from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class CreateTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py index 935f53c9cc76..cb7b53eb29aa 100644 --- a/sdks/python/apache_beam/transforms/display.py +++ b/sdks/python/apache_beam/transforms/display.py @@ -41,7 +41,8 @@ import calendar import inspect import json -from datetime import datetime, timedelta +from datetime import datetime +from datetime import timedelta __all__ = ['HasDisplayData', 'DisplayDataItem', 'DisplayData'] diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py index b19b32c45bc4..5c73cf39a92f 100644 --- a/sdks/python/apache_beam/transforms/display_test.py +++ b/sdks/python/apache_beam/transforms/display_test.py @@ -27,8 +27,9 @@ import apache_beam as beam from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.transforms.display import (DisplayData, DisplayDataItem, - HasDisplayData) +from apache_beam.transforms.display import DisplayData +from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData class DisplayDataItemMatcher(BaseMatcher): diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index 27b9dd8b51e4..eccaccd27065 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -44,15 +44,20 @@ class and wrapper class that allows lambda functions to be used as from google.protobuf import wrappers_pb2 -from apache_beam import error, pvalue -from apache_beam.internal import pickler, util -from apache_beam.transforms.display import DisplayDataItem, HasDisplayData +from apache_beam import error +from apache_beam import pvalue +from apache_beam.internal import pickler +from apache_beam.internal import util +from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData from apache_beam.typehints import typehints -from apache_beam.typehints.decorators import (TypeCheckError, WithTypeHints, - getcallargs_forhints) +from apache_beam.typehints.decorators import TypeCheckError +from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.trivial_inference import instance_to_type from apache_beam.typehints.typehints import validate_composite_type_param -from apache_beam.utils import proto_utils, urns +from apache_beam.utils import proto_utils +from apache_beam.utils import urns __all__ = [ 'PTransform', diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index fd0bbc8145b6..435270e27cdb 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -35,12 +35,15 @@ from apache_beam.metrics.metric import MetricsFilter from apache_beam.options.pipeline_options import TypeOptions from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import window from apache_beam.transforms.core import _GroupByKeyOnly -from apache_beam.transforms.display import DisplayData, DisplayDataItem +from apache_beam.transforms.display import DisplayData +from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.ptransform import PTransform -from apache_beam.typehints import with_input_types, with_output_types +from apache_beam.typehints import with_input_types +from apache_beam.typehints import with_output_types from apache_beam.typehints.typehints_test import TypeHintTestCase from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py index 650068141852..0a69c3b6050e 100644 --- a/sdks/python/apache_beam/transforms/sideinputs_test.py +++ b/sdks/python/apache_beam/transforms/sideinputs_test.py @@ -24,7 +24,8 @@ import apache_beam as beam from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import window diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py index 6af423f1bd1a..8d63d49baad0 100644 --- a/sdks/python/apache_beam/transforms/timeutil.py +++ b/sdks/python/apache_beam/transforms/timeutil.py @@ -19,7 +19,8 @@ from __future__ import absolute_import -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod __all__ = [ 'TimeDomain', diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index fce2714ff2a6..84d5be9b27bf 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -23,16 +23,21 @@ import collections import copy import itertools -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod from apache_beam.coders import observable from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.transforms import combiners, core +from apache_beam.transforms import combiners +from apache_beam.transforms import core from apache_beam.transforms.timeutil import TimeDomain -from apache_beam.transforms.window import (GlobalWindow, TimestampCombiner, - WindowedValue, WindowFn) -from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, - TIME_GRANULARITY) +from apache_beam.transforms.window import GlobalWindow +from apache_beam.transforms.window import TimestampCombiner +from apache_beam.transforms.window import WindowedValue +from apache_beam.transforms.window import WindowFn +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import TIME_GRANULARITY # AfterCount is experimental. No backwards compatibility guarantees. diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py index be92212138bc..b2fd761f7fd9 100644 --- a/sdks/python/apache_beam/transforms/trigger_test.py +++ b/sdks/python/apache_beam/transforms/trigger_test.py @@ -27,19 +27,29 @@ import apache_beam as beam from apache_beam.runners import pipeline_context from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import trigger from apache_beam.transforms.core import Windowing -from apache_beam.transforms.trigger import (AccumulationMode, AfterAll, - AfterAny, AfterCount, AfterEach, - AfterWatermark, DefaultTrigger, - GeneralTriggerDriver, - InMemoryUnmergedState, Repeatedly, - TriggerFn) -from apache_beam.transforms.window import (MIN_TIMESTAMP, FixedWindows, - IntervalWindow, Sessions, - TimestampCombiner, TimestampedValue, - WindowedValue, WindowFn) +from apache_beam.transforms.trigger import AccumulationMode +from apache_beam.transforms.trigger import AfterAll +from apache_beam.transforms.trigger import AfterAny +from apache_beam.transforms.trigger import AfterCount +from apache_beam.transforms.trigger import AfterEach +from apache_beam.transforms.trigger import AfterWatermark +from apache_beam.transforms.trigger import DefaultTrigger +from apache_beam.transforms.trigger import GeneralTriggerDriver +from apache_beam.transforms.trigger import InMemoryUnmergedState +from apache_beam.transforms.trigger import Repeatedly +from apache_beam.transforms.trigger import TriggerFn +from apache_beam.transforms.window import MIN_TIMESTAMP +from apache_beam.transforms.window import FixedWindows +from apache_beam.transforms.window import IntervalWindow +from apache_beam.transforms.window import Sessions +from apache_beam.transforms.window import TimestampCombiner +from apache_beam.transforms.window import TimestampedValue +from apache_beam.transforms.window import WindowedValue +from apache_beam.transforms.window import WindowFn class CustomTimestampingFixedWindowsWindowFn(FixedWindows): diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index b83ed90ef500..81b8c228ca27 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -20,8 +20,12 @@ from __future__ import absolute_import -from apache_beam.transforms.core import CombinePerKey, Flatten, GroupByKey, Map -from apache_beam.transforms.ptransform import PTransform, ptransform_fn +from apache_beam.transforms.core import CombinePerKey +from apache_beam.transforms.core import Flatten +from apache_beam.transforms.core import GroupByKey +from apache_beam.transforms.core import Map +from apache_beam.transforms.ptransform import PTransform +from apache_beam.transforms.ptransform import ptransform_fn __all__ = [ 'CoGroupByKey', diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 4e54384d41e2..a025019b3e72 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -51,15 +51,19 @@ import abc -from google.protobuf import duration_pb2, timestamp_pb2 +from google.protobuf import duration_pb2 +from google.protobuf import timestamp_pb2 from apache_beam.coders import coders -from apache_beam.portability.api import (beam_runner_api_pb2, - standard_window_fns_pb2) +from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.portability.api import standard_window_fns_pb2 from apache_beam.transforms import timeutil -from apache_beam.utils import proto_utils, urns -from apache_beam.utils.timestamp import (MAX_TIMESTAMP, MIN_TIMESTAMP, - Duration, Timestamp) +from apache_beam.utils import proto_utils +from apache_beam.utils import urns +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Duration +from apache_beam.utils.timestamp import Timestamp from apache_beam.utils.windowed_value import WindowedValue __all__ = [ diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py index 1cd344bdb466..71c0622dcee4 100644 --- a/sdks/python/apache_beam/transforms/window_test.py +++ b/sdks/python/apache_beam/transforms/window_test.py @@ -21,17 +21,30 @@ from apache_beam.runners import pipeline_context from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to -from apache_beam.transforms import (CombinePerKey, Create, GroupByKey, Map, - WindowInto, combiners, core) +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.transforms import CombinePerKey +from apache_beam.transforms import Create +from apache_beam.transforms import GroupByKey +from apache_beam.transforms import Map +from apache_beam.transforms import WindowInto +from apache_beam.transforms import combiners +from apache_beam.transforms import core from apache_beam.transforms.core import Windowing -from apache_beam.transforms.trigger import AccumulationMode, AfterCount -from apache_beam.transforms.window import (FixedWindows, GlobalWindow, - GlobalWindows, IntervalWindow, - Sessions, SlidingWindows, - TimestampCombiner, TimestampedValue, - WindowedValue, WindowFn) -from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP +from apache_beam.transforms.trigger import AccumulationMode +from apache_beam.transforms.trigger import AfterCount +from apache_beam.transforms.window import FixedWindows +from apache_beam.transforms.window import GlobalWindow +from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import IntervalWindow +from apache_beam.transforms.window import Sessions +from apache_beam.transforms.window import SlidingWindows +from apache_beam.transforms.window import TimestampCombiner +from apache_beam.transforms.window import TimestampedValue +from apache_beam.transforms.window import WindowedValue +from apache_beam.transforms.window import WindowFn +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP def context(element, timestamp): diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py index dc4d333198f1..c2a200516ee3 100644 --- a/sdks/python/apache_beam/transforms/write_ptransform_test.py +++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py @@ -22,7 +22,8 @@ import apache_beam as beam from apache_beam.io import iobase from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, is_empty +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import is_empty from apache_beam.transforms.ptransform import PTransform diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index c1604b9de175..694433ae7184 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -86,11 +86,12 @@ def foo((a, b)): import inspect import types -from apache_beam.typehints import native_type_compatibility, typehints -from apache_beam.typehints.typehints import (CompositeTypeHintError, - SimpleTypeHintError, - check_constraint, - validate_composite_type_param) +from apache_beam.typehints import native_type_compatibility +from apache_beam.typehints import typehints +from apache_beam.typehints.typehints import CompositeTypeHintError +from apache_beam.typehints.typehints import SimpleTypeHintError +from apache_beam.typehints.typehints import check_constraint +from apache_beam.typehints.typehints import validate_composite_type_param __all__ = [ 'with_input_types', diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 81ca9014da9f..4171507f3459 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -20,7 +20,8 @@ import typing import unittest -from apache_beam.typehints import native_type_compatibility, typehints +from apache_beam.typehints import native_type_compatibility +from apache_beam.typehints import typehints _TestNamedTuple = typing.NamedTuple('_TestNamedTuple', [('age', int), ('name', bytes)]) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index 8831eb226dce..923b848bf9bb 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -29,8 +29,16 @@ import types import typehints -from trivial_inference import BoundMethod, Const, element_type, union -from typehints import Any, Dict, Iterable, List, Tuple, Union +from trivial_inference import BoundMethod +from trivial_inference import Const +from trivial_inference import element_type +from trivial_inference import union +from typehints import Any +from typehints import Dict +from typehints import Iterable +from typehints import List +from typehints import Tuple +from typehints import Union def pop_one(state, unused_arg): diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index 3c5d54b5b8d3..c7405963f161 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -26,7 +26,8 @@ import sys import types -from apache_beam.typehints import Any, typehints +from apache_beam.typehints import Any +from apache_beam.typehints import typehints class TypeInferenceError(ValueError): diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index df21b16a5fec..38259c88c7b5 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -18,7 +18,8 @@ """Tests for apache_beam.typehints.trivial_inference.""" import unittest -from apache_beam.typehints import trivial_inference, typehints +from apache_beam.typehints import trivial_inference +from apache_beam.typehints import typehints global_int = 1 diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index ac2cc15137b7..c47e9bacb960 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -28,12 +28,13 @@ from apache_beam.pvalue import TaggedOutput from apache_beam.transforms.core import DoFn from apache_beam.transforms.window import WindowedValue -from apache_beam.typehints.decorators import (GeneratorWrapper, TypeCheckError, - _check_instance_type, - getcallargs_forhints) -from apache_beam.typehints.typehints import (CompositeTypeHintError, - SimpleTypeHintError, - check_constraint) +from apache_beam.typehints.decorators import GeneratorWrapper +from apache_beam.typehints.decorators import TypeCheckError +from apache_beam.typehints.decorators import _check_instance_type +from apache_beam.typehints.decorators import getcallargs_forhints +from apache_beam.typehints.typehints import CompositeTypeHintError +from apache_beam.typehints.typehints import SimpleTypeHintError +from apache_beam.typehints.typehints import check_constraint class AbstractDoFnWrapper(DoFn): diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 7d713296ca1c..1df11045ae72 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -21,10 +21,12 @@ import unittest import apache_beam as beam -from apache_beam import pvalue, typehints +from apache_beam import pvalue +from apache_beam import typehints from apache_beam.options.pipeline_options import OptionsContext from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.typehints import WithTypeHints # These test often construct a pipeline as value | PTransform to test side diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index 181686ffd7ae..af575f4aba42 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -21,14 +21,18 @@ import unittest import apache_beam.typehints.typehints as typehints -from apache_beam.typehints import (Any, Tuple, TypeCheckError, Union, - with_input_types, with_output_types) -from apache_beam.typehints.decorators import (GeneratorWrapper, - _check_instance_type, - _interleave_type_check, - _positional_arg_hints, - get_type_hints, - getcallargs_forhints) +from apache_beam.typehints import Any +from apache_beam.typehints import Tuple +from apache_beam.typehints import TypeCheckError +from apache_beam.typehints import Union +from apache_beam.typehints import with_input_types +from apache_beam.typehints import with_output_types +from apache_beam.typehints.decorators import GeneratorWrapper +from apache_beam.typehints.decorators import _check_instance_type +from apache_beam.typehints.decorators import _interleave_type_check +from apache_beam.typehints.decorators import _positional_arg_hints +from apache_beam.typehints.decorators import get_type_hints +from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.typehints import is_consistent_with diff --git a/sdks/python/apache_beam/utils/annotations.py b/sdks/python/apache_beam/utils/annotations.py index ec21093c1020..017dd6b81a4d 100644 --- a/sdks/python/apache_beam/utils/annotations.py +++ b/sdks/python/apache_beam/utils/annotations.py @@ -61,7 +61,8 @@ def exp_multiply(arg1, arg2): """ import warnings -from functools import partial, wraps +from functools import partial +from functools import wraps # Produce only the first occurrence of matching warnings regardless of # location per line of execution. Since the number of lines of execution diff --git a/sdks/python/apache_beam/utils/annotations_test.py b/sdks/python/apache_beam/utils/annotations_test.py index 94ed058b22d5..ddd1b9ff7571 100644 --- a/sdks/python/apache_beam/utils/annotations_test.py +++ b/sdks/python/apache_beam/utils/annotations_test.py @@ -18,7 +18,8 @@ import unittest import warnings -from apache_beam.utils.annotations import deprecated, experimental +from apache_beam.utils.annotations import deprecated +from apache_beam.utils.annotations import experimental class AnnotationTests(unittest.TestCase): diff --git a/sdks/python/apache_beam/utils/proto_utils.py b/sdks/python/apache_beam/utils/proto_utils.py index dce7cff7158a..d7693f3f7839 100644 --- a/sdks/python/apache_beam/utils/proto_utils.py +++ b/sdks/python/apache_beam/utils/proto_utils.py @@ -17,7 +17,8 @@ """For internal use only; no backwards-compatibility guarantees.""" -from google.protobuf import any_pb2, struct_pb2 +from google.protobuf import any_pb2 +from google.protobuf import struct_pb2 def pack_Any(msg): diff --git a/sdks/python/apache_beam/utils/timestamp_test.py b/sdks/python/apache_beam/utils/timestamp_test.py index 04b19ed87a68..33229361b36f 100644 --- a/sdks/python/apache_beam/utils/timestamp_test.py +++ b/sdks/python/apache_beam/utils/timestamp_test.py @@ -21,7 +21,8 @@ import unittest -from apache_beam.utils.timestamp import Duration, Timestamp +from apache_beam.utils.timestamp import Duration +from apache_beam.utils.timestamp import Timestamp class TimestampTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/utils/windowed_value.py b/sdks/python/apache_beam/utils/windowed_value.py index 933c2c42d4d4..be2785432a14 100644 --- a/sdks/python/apache_beam/utils/windowed_value.py +++ b/sdks/python/apache_beam/utils/windowed_value.py @@ -27,7 +27,9 @@ #cython: profile=True -from apache_beam.utils.timestamp import MAX_TIMESTAMP, MIN_TIMESTAMP, Timestamp +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Timestamp class WindowedValue(object): diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 9b449aad49c3..4a101e2339ff 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -82,5 +82,5 @@ for file in "${EXCLUDED_GENERATED_FILES[@]}"; do SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" done pushd $MODULE -isort -p apache_beam -w 79 -y -c -ot -cs ${SKIP_PARAM} +isort -p apache_beam -w 79 -y -c -ot -cs -sl ${SKIP_PARAM} popd From d580dfbc1739f28e90146317e1b6359bd1f702c5 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Aug 2017 16:24:46 -0700 Subject: [PATCH 09/17] Restore the comment that got cleaned up by automated tooling somehow --- sdks/python/apache_beam/coders/fast_coders_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/coders/fast_coders_test.py b/sdks/python/apache_beam/coders/fast_coders_test.py index d060117a147e..a13334a2c26f 100644 --- a/sdks/python/apache_beam/coders/fast_coders_test.py +++ b/sdks/python/apache_beam/coders/fast_coders_test.py @@ -20,6 +20,8 @@ import logging import unittest + +# Run all the standard coder test cases. from apache_beam.coders.coders_test_common import * From d020ee888b5969e04296a643944c6527bb3f0e7b Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Aug 2017 16:25:03 -0700 Subject: [PATCH 10/17] In single line mode isort and pylint agree a lot more :) --- sdks/python/apache_beam/io/avroio_test.py | 2 +- sdks/python/run_pylint.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index 6c86635f3edd..c64747eb7287 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -23,9 +23,9 @@ import avro.datafile import avro.schema -import hamcrest as hc from avro.datafile import DataFileWriter from avro.io import DatumWriter +import hamcrest as hc import apache_beam as beam from apache_beam import Create diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 4a101e2339ff..bef064dae755 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -66,13 +66,6 @@ echo "Running isort for module $MODULE:" # Skip files where isort is behaving weirdly ISORT_EXCLUDED=( "avroio_test.py" - "dataflow_runner_test.py" - "iobase_test.py" - "helper_test.py" - "helper.py" - "custom_ptransform.py" - "apiclient.py" - "datastore_wordcount.py" ) SKIP_PARAM="" for file in "${ISORT_EXCLUDED[@]}"; do From a82ae72949398cbcdcec5885b96d1c78ba19a0a1 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Aug 2017 16:28:57 -0700 Subject: [PATCH 11/17] Restoring the comment confuses isort, skip fast_coders_test.py --- sdks/python/run_pylint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index bef064dae755..53f382f0dda5 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -66,6 +66,7 @@ echo "Running isort for module $MODULE:" # Skip files where isort is behaving weirdly ISORT_EXCLUDED=( "avroio_test.py" + "fast_coders_test.py" ) SKIP_PARAM="" for file in "${ISORT_EXCLUDED[@]}"; do From 9323bafcb4732a103996ad5aa46ae3105c51f232 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Aug 2017 16:37:11 -0700 Subject: [PATCH 12/17] Remove unecessary pylint disable comment. --- sdks/python/apache_beam/examples/snippets/snippets_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index b0b3391f0026..ee1e50e460ef 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -34,7 +34,6 @@ from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter from apache_beam.options.pipeline_options import PipelineOptions -# pylint: disable=expression-not-assigned from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to From e1708bd558e9a613024fb728b43290381f660c00 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 31 Aug 2017 17:03:43 -0700 Subject: [PATCH 13/17] two more isort challanges --- sdks/python/run_pylint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 53f382f0dda5..4c84c62e52df 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -67,6 +67,8 @@ echo "Running isort for module $MODULE:" ISORT_EXCLUDED=( "avroio_test.py" "fast_coders_test.py" + "apiclient.py" + "datastore_wordcount.py" ) SKIP_PARAM="" for file in "${ISORT_EXCLUDED[@]}"; do From 031c13ecc6ab1f19219cf70479f91237a7246e5e Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 31 Aug 2017 19:14:51 -0700 Subject: [PATCH 14/17] Add slow_coders_test to ignore list since it keeps removing the comment --- sdks/python/apache_beam/coders/slow_coders_test.py | 2 ++ sdks/python/run_pylint.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/sdks/python/apache_beam/coders/slow_coders_test.py b/sdks/python/apache_beam/coders/slow_coders_test.py index 7032e081658a..97aa39ca094f 100644 --- a/sdks/python/apache_beam/coders/slow_coders_test.py +++ b/sdks/python/apache_beam/coders/slow_coders_test.py @@ -20,6 +20,8 @@ import logging import unittest + +# Run all the standard coder test cases. from apache_beam.coders.coders_test_common import * diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 4c84c62e52df..e42f918dd069 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -69,6 +69,7 @@ ISORT_EXCLUDED=( "fast_coders_test.py" "apiclient.py" "datastore_wordcount.py" + "slow_coders_test.py" ) SKIP_PARAM="" for file in "${ISORT_EXCLUDED[@]}"; do From d3a18c52c1546e29887d7187eae9f67e505424ec Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 31 Aug 2017 19:17:26 -0700 Subject: [PATCH 15/17] Allow 80 chars --- sdks/python/apache_beam/io/avroio_test.py | 4 ++-- sdks/python/run_pylint.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index c64747eb7287..8a344275a1a8 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -33,8 +33,8 @@ from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io import source_test_utils -from apache_beam.io.avroio import _AvroSink as AvroSink -from apache_beam.io.avroio import _AvroSource as AvroSource +from apache_beam.io.avroio import _AvroSink as AvroSink # For testing +from apache_beam.io.avroio import _AvroSource as AvroSource # For testing from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index e42f918dd069..d62b28fba978 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -79,5 +79,5 @@ for file in "${EXCLUDED_GENERATED_FILES[@]}"; do SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" done pushd $MODULE -isort -p apache_beam -w 79 -y -c -ot -cs -sl ${SKIP_PARAM} +isort -p apache_beam -w 80 -y -c -ot -cs -sl ${SKIP_PARAM} popd From 7ee5d4e023020a2820b697cb671b39dd3e357b90 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 31 Aug 2017 19:32:26 -0700 Subject: [PATCH 16/17] Allow 80 chars --- sdks/python/apache_beam/io/filebasedsource_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index b263ec82265a..0999510522f5 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -33,8 +33,7 @@ from apache_beam.io import range_trackers # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource -from apache_beam.io.filebasedsource import \ - _SingleFileSource as SingleFileSource +from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource from apache_beam.io.filebasedsource import FileBasedSource from apache_beam.io.filesystem import CompressionTypes from apache_beam.options.value_provider import RuntimeValueProvider From 2d79c8df457dddae1fc4348f12de3194c37abe42 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 1 Sep 2017 13:34:01 -0700 Subject: [PATCH 17/17] Switch isort long line to 120, add more isort exlcudes and fix the ordering a bit, manually remove single line isort continuations as asked to remove. --- .../io/gcp/datastore/v1/adaptive_throttler_test.py | 3 +-- sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py | 3 +-- .../io/gcp/internal/clients/bigquery/bigquery_v2_client.py | 3 +-- .../io/gcp/internal/clients/storage/storage_v1_client.py | 3 +-- .../apache_beam/options/pipeline_options_validator_test.py | 3 +-- sdks/python/apache_beam/pipeline.py | 3 +-- .../python/apache_beam/runners/dataflow/dataflow_runner.py | 3 +-- .../apache_beam/runners/dataflow/dataflow_runner_test.py | 6 ++---- .../apache_beam/runners/dataflow/internal/apiclient.py | 3 +-- .../internal/clients/dataflow/dataflow_v1b3_client.py | 3 +-- .../internal/clients/dataflow/message_matchers_test.py | 3 +-- .../apache_beam/runners/dataflow/native_io/iobase_test.py | 6 ++---- .../direct/consumer_tracking_pipeline_visitor_test.py | 3 +-- .../apache_beam/runners/direct/transform_evaluator.py | 6 ++---- .../runners/portability/maptask_executor_runner.py | 3 +-- sdks/python/run_pylint.sh | 7 ++++--- 16 files changed, 22 insertions(+), 39 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py index 217698c6c540..1ac23930f65f 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py @@ -19,8 +19,7 @@ from mock import patch -from apache_beam.io.gcp.datastore.v1.adaptive_throttler import \ - AdaptiveThrottler +from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler class AdaptiveThrottlerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py index 4b9b73bea507..078002c82d10 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py @@ -23,8 +23,7 @@ from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.io.gcp.datastore.v1 import query_splitter from apache_beam.io.gcp.datastore.v1 import util -from apache_beam.io.gcp.datastore.v1.adaptive_throttler import \ - AdaptiveThrottler +from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler from apache_beam.metrics.metric import Metrics from apache_beam.transforms import Create from apache_beam.transforms import DoFn diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py index b32fc5638f7d..201a1830b878 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py @@ -19,8 +19,7 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.io.gcp.internal.clients.bigquery import \ - bigquery_v2_messages as messages +from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_messages as messages class BigqueryV2(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py index 72b453dab19c..1b46d917f143 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py @@ -19,8 +19,7 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.io.gcp.internal.clients.storage import \ - storage_v1_messages as messages +from apache_beam.io.gcp.internal.clients.storage import storage_v1_messages as messages class StorageV1(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/options/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py index f7e576ec11f5..8ff66c78f17b 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py @@ -24,8 +24,7 @@ from apache_beam.internal import pickler from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options_validator import \ - PipelineOptionsValidator +from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator # Mock runners to use for validations. diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 9daf7654dbc4..1ebd099d37ec 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -59,8 +59,7 @@ from apache_beam.options.pipeline_options import SetupOptions from apache_beam.options.pipeline_options import StandardOptions from apache_beam.options.pipeline_options import TypeOptions -from apache_beam.options.pipeline_options_validator import \ - PipelineOptionsValidator +from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator from apache_beam.pvalue import PCollection from apache_beam.runners import PipelineRunner from apache_beam.runners import create_runner diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index a67a712a334d..f4cf459d904c 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -40,8 +40,7 @@ from apache_beam.pvalue import AsSideInput from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics from apache_beam.runners.dataflow.internal import names -from apache_beam.runners.dataflow.internal.clients import \ - dataflow as dataflow_api +from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.dataflow.internal.names import TransformNames from apache_beam.runners.runner import PipelineResult diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index 7f793f6cda35..e73a94fd8603 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -33,10 +33,8 @@ from apache_beam.runners import TestDataflowRunner from apache_beam.runners import create_runner from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult -from apache_beam.runners.dataflow.dataflow_runner import \ - DataflowRuntimeException -from apache_beam.runners.dataflow.internal.clients import \ - dataflow as dataflow_api +from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException +from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.runner import PipelineState from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms import window diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index 9b661d26f9f1..d010065237b3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -42,8 +42,7 @@ from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.runners.dataflow.internal import dependency from apache_beam.runners.dataflow.internal.clients import dataflow -from apache_beam.runners.dataflow.internal.dependency import \ - get_sdk_name_and_version +from apache_beam.runners.dataflow.internal.dependency import get_sdk_name_and_version from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.transforms import cy_combiners from apache_beam.transforms.display import DisplayData diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py index 977a4c03c038..61d02730ab54 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py @@ -19,8 +19,7 @@ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apache_beam.runners.dataflow.internal.clients.dataflow import \ - dataflow_v1b3_messages as messages +from apache_beam.runners.dataflow.internal.clients.dataflow import dataflow_v1b3_messages as messages class DataflowV1b3(base_api.BaseApiClient): diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py index 22b13f511efa..15bb9eff083f 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py @@ -20,8 +20,7 @@ import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow from apache_beam.internal.gcp.json_value import to_json_value -from apache_beam.runners.dataflow.internal.clients.dataflow import \ - message_matchers +from apache_beam.runners.dataflow.internal.clients.dataflow import message_matchers # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index 4ebe806137e8..01fd35f9cf95 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -25,15 +25,13 @@ from apache_beam import pvalue from apache_beam.runners.dataflow.native_io.iobase import ConcatPosition from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitRequest -from apache_beam.runners.dataflow.native_io.iobase import \ - DynamicSplitResultWithPosition +from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitResultWithPosition from apache_beam.runners.dataflow.native_io.iobase import NativeSink from apache_beam.runners.dataflow.native_io.iobase import NativeSinkWriter from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.runners.dataflow.native_io.iobase import ReaderPosition from apache_beam.runners.dataflow.native_io.iobase import ReaderProgress -from apache_beam.runners.dataflow.native_io.iobase import \ - _dict_printable_fields +from apache_beam.runners.dataflow.native_io.iobase import _dict_printable_fields from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py index 216ea3737183..4efaa27f0958 100644 --- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py +++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py @@ -26,8 +26,7 @@ from apache_beam.pipeline import Pipeline from apache_beam.pvalue import AsList from apache_beam.runners.direct import DirectRunner -from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import \ - ConsumerTrackingPipelineVisitor +from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor from apache_beam.transforms import CoGroupByKey from apache_beam.transforms import Create from apache_beam.transforms import DoFn diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index 5cd3091da8b8..8906ccb72e83 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -30,10 +30,8 @@ from apache_beam.options.pipeline_options import TypeOptions from apache_beam.runners.common import DoFnRunner from apache_beam.runners.common import DoFnState -from apache_beam.runners.dataflow.native_io.iobase import \ - _NativeWrite # pylint: disable=protected-access -from apache_beam.runners.direct.direct_runner import \ - _StreamingGroupAlsoByWindow +from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access +from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly from apache_beam.runners.direct.util import KeyedWorkItem from apache_beam.runners.direct.util import TransformResult diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py index 05b021331b94..a20ceef60fae 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py @@ -28,8 +28,7 @@ from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.options import pipeline_options from apache_beam.runners import DataflowRunner -from apache_beam.runners.dataflow.internal.dependency import \ - _dependency_file_copy +from apache_beam.runners.dataflow.internal.dependency import _dependency_file_copy from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.runners.runner import PipelineResult diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index d62b28fba978..df71e4402176 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -65,10 +65,11 @@ pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" echo "Running isort for module $MODULE:" # Skip files where isort is behaving weirdly ISORT_EXCLUDED=( - "avroio_test.py" - "fast_coders_test.py" "apiclient.py" + "avroio_test.py" "datastore_wordcount.py" + "iobase_test.py" + "fast_coders_test.py" "slow_coders_test.py" ) SKIP_PARAM="" @@ -79,5 +80,5 @@ for file in "${EXCLUDED_GENERATED_FILES[@]}"; do SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" done pushd $MODULE -isort -p apache_beam -w 80 -y -c -ot -cs -sl ${SKIP_PARAM} +isort -p apache_beam -w 120 -y -c -ot -cs -sl ${SKIP_PARAM} popd