diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py index 2670250c36b6..50702970694d 100644 --- a/sdks/python/apache_beam/coders/coder_impl.py +++ b/sdks/python/apache_beam/coders/coder_impl.py @@ -29,10 +29,10 @@ from types import NoneType from apache_beam.coders import observable -from apache_beam.utils.timestamp import Timestamp +from apache_beam.utils import windowed_value from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils import windowed_value +from apache_beam.utils.timestamp import Timestamp # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 10fb07b6a34f..3021da50b22d 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -22,12 +22,13 @@ import base64 import cPickle as pickle + import google.protobuf from apache_beam.coders import coder_impl from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.utils import urns from apache_beam.utils import proto_utils +from apache_beam.utils import urns # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: diff --git a/sdks/python/apache_beam/coders/coders_test.py b/sdks/python/apache_beam/coders/coders_test.py index c89e81028b8a..705de8920d52 100644 --- a/sdks/python/apache_beam/coders/coders_test.py +++ b/sdks/python/apache_beam/coders/coders_test.py @@ -20,8 +20,8 @@ import logging import unittest -from apache_beam.coders import coders from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.coders import coders from apache_beam.coders.typecoders import registry as coders_registry diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py index 8b0353d922e7..29ff2292c50a 100644 --- a/sdks/python/apache_beam/coders/coders_test_common.py +++ b/sdks/python/apache_beam/coders/coders_test_common.py @@ -23,16 +23,15 @@ import dill -from apache_beam.transforms.window import GlobalWindow -from apache_beam.utils.timestamp import MIN_TIMESTAMP import observable +from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.coders import coders from apache_beam.runners import pipeline_context from apache_beam.transforms import window +from apache_beam.transforms.window import GlobalWindow from apache_beam.utils import timestamp from apache_beam.utils import windowed_value - -from apache_beam.coders import coders -from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message +from apache_beam.utils.timestamp import MIN_TIMESTAMP # Defined out of line for picklability. diff --git a/sdks/python/apache_beam/coders/observable_test.py b/sdks/python/apache_beam/coders/observable_test.py index eaf1aec9e600..09ca3041c298 100644 --- a/sdks/python/apache_beam/coders/observable_test.py +++ b/sdks/python/apache_beam/coders/observable_test.py @@ -20,7 +20,6 @@ import logging import unittest - from apache_beam.coders import observable diff --git a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py index 16b1b4d6b4b3..433d33f98841 100644 --- a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py +++ b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py @@ -19,19 +19,19 @@ # source: sdks/java/core/src/main/proto/proto2_coder_test_messages.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor.FileDescriptor( name='apache_beam/coders/proto2_coder_test_messages.proto', package='proto2_coder_test_messages', diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py index 5f9845564477..ca4dffbcffe8 100644 --- a/sdks/python/apache_beam/coders/standard_coders_test.py +++ b/sdks/python/apache_beam/coders/standard_coders_test.py @@ -26,12 +26,12 @@ import yaml -from apache_beam.coders import coders from apache_beam.coders import coder_impl +from apache_beam.coders import coders +from apache_beam.transforms import window +from apache_beam.transforms.window import IntervalWindow from apache_beam.utils import windowed_value from apache_beam.utils.timestamp import Timestamp -from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms import window STANDARD_CODERS_YAML = os.path.join( os.path.dirname(__file__), '..', 'testing', 'data', 'standard_coders.yaml') diff --git a/sdks/python/apache_beam/coders/stream_test.py b/sdks/python/apache_beam/coders/stream_test.py index e6108b68aee9..15bc5eb9ba93 100644 --- a/sdks/python/apache_beam/coders/stream_test.py +++ b/sdks/python/apache_beam/coders/stream_test.py @@ -21,7 +21,6 @@ import math import unittest - from apache_beam.coders import slow_stream diff --git a/sdks/python/apache_beam/coders/typecoders.py b/sdks/python/apache_beam/coders/typecoders.py index 3894bb588172..797aee5815f5 100644 --- a/sdks/python/apache_beam/coders/typecoders.py +++ b/sdks/python/apache_beam/coders/typecoders.py @@ -70,7 +70,6 @@ def MakeXyzs(v): from apache_beam.coders import coders from apache_beam.typehints import typehints - __all__ = ['registry'] diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py index 7e3c4cd35a27..d0a5fb74f3fc 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py @@ -31,14 +31,13 @@ import logging import random - import apache_beam as beam from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.typehints import Any from apache_beam.typehints import Iterable from apache_beam.typehints import Tuple -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions @beam.typehints.with_output_types(Tuple[int, int, int]) diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py index f1cbb0a24d56..34633132e41c 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi_test.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi_test.py @@ -22,8 +22,8 @@ from apache_beam.examples.complete import estimate_pi from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that from apache_beam.testing.util import BeamAssertException +from apache_beam.testing.util import assert_that def in_between(lower, upper): diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py index 91c75aa57927..e4986271e940 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test.py @@ -23,7 +23,6 @@ import tempfile import unittest - from apache_beam.examples.complete.juliaset.juliaset import juliaset from apache_beam.testing.util import open_shards diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py index 0db5431e31c3..1d521bec2e4b 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py @@ -49,10 +49,8 @@ import logging - from apache_beam.examples.complete.juliaset.juliaset import juliaset - if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) juliaset.run() diff --git a/sdks/python/apache_beam/examples/complete/juliaset/setup.py b/sdks/python/apache_beam/examples/complete/juliaset/setup.py index 589e47c83059..2062e2a0cd81 100644 --- a/sdks/python/apache_beam/examples/complete/juliaset/setup.py +++ b/sdks/python/apache_beam/examples/complete/juliaset/setup.py @@ -25,8 +25,8 @@ when running the workflow for remote execution. """ -from distutils.command.build import build as _build import subprocess +from distutils.command.build import build as _build import setuptools diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py index a88ff827766c..2132fbba1d8a 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf.py +++ b/sdks/python/apache_beam/examples/complete/tfidf.py @@ -31,9 +31,9 @@ import apache_beam as beam from apache_beam.io import ReadFromText from apache_beam.io import WriteToText -from apache_beam.pvalue import AsSingleton from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.pvalue import AsSingleton def read_documents(pipeline, uris): diff --git a/sdks/python/apache_beam/examples/complete/tfidf_test.py b/sdks/python/apache_beam/examples/complete/tfidf_test.py index b6f88255887c..518a47cf3baf 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf_test.py +++ b/sdks/python/apache_beam/examples/complete/tfidf_test.py @@ -30,7 +30,6 @@ from apache_beam.testing.util import equal_to from apache_beam.testing.util import open_shards - EXPECTED_RESULTS = set([ ('ghi', '1.txt', 0.3662040962227032), ('abc', '1.txt', 0.0), diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py index 9a9ad7865c2c..dd827bc327ef 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py @@ -49,12 +49,11 @@ from apache_beam import combiners from apache_beam.io import ReadFromText from apache_beam.io import WriteToText +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions from apache_beam.transforms.window import FixedWindows from apache_beam.transforms.window import Sessions from apache_beam.transforms.window import TimestampedValue -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.pipeline_options import SetupOptions - ONE_HOUR_IN_SECONDS = 3600 THIRTY_DAYS_IN_SECONDS = 30 * 24 * ONE_HOUR_IN_SECONDS diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py index ced8a44af13c..a0b368f60350 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions_test.py @@ -20,7 +20,6 @@ import json import unittest - import apache_beam as beam from apache_beam.examples.complete import top_wikipedia_sessions from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py index 9911a6716bd8..e16ae7391b6e 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py @@ -32,12 +32,11 @@ from random import randrange import apache_beam as beam - from apache_beam.io import WriteToText -from apache_beam.pvalue import AsList -from apache_beam.pvalue import AsSingleton from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.pvalue import AsList +from apache_beam.pvalue import AsSingleton def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word): diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py index 05ee3c53f48f..961284908046 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py @@ -25,8 +25,8 @@ from nose.plugins.attrib import attr from apache_beam.examples.cookbook import bigquery_tornadoes -from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher from apache_beam.io.gcp.tests import utils +from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py index aee69d23ff65..db86003bff6a 100644 --- a/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py +++ b/sdks/python/apache_beam/examples/cookbook/custom_ptransform.py @@ -30,10 +30,10 @@ from apache_beam.io import WriteToText from apache_beam.options.pipeline_options import PipelineOptions - # pylint doesn't understand our pipeline syntax: # pylint:disable=expression-not-assigned + class Count1(beam.PTransform): """Count as a subclass of PTransform, with an apply method.""" diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py index 7161cff1c255..c42596f4aa82 100644 --- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py +++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount.py @@ -68,7 +68,8 @@ from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.proto.datastore.v1 import query_pb2 -from googledatastore import helper as datastore_helper, PropertyFilter +from googledatastore import helper as datastore_helper +from googledatastore import PropertyFilter import apache_beam as beam from apache_beam.io import ReadFromText diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py index 9c0d04b816ac..4c86f46a298b 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder.py @@ -35,10 +35,10 @@ from apache_beam import coders from apache_beam.io import ReadFromText from apache_beam.io import WriteToText -from apache_beam.typehints import typehints -from apache_beam.typehints.decorators import with_output_types from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.typehints import typehints +from apache_beam.typehints.decorators import with_output_types class Player(object): diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py index fb630ba465f7..ed38b5d8a4ed 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py @@ -24,7 +24,6 @@ from apache_beam.examples.cookbook import group_with_coder from apache_beam.testing.util import open_shards - # Patch group_with_coder.PlayerCoder.decode(). To test that the PlayerCoder was # used, we do not strip the prepended 'x:' string when decoding a Player object. group_with_coder.PlayerCoder.decode = lambda self, s: group_with_coder.Player(s) diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py index 27b81208fcd8..46696f430802 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets.py +++ b/sdks/python/apache_beam/examples/snippets/snippets.py @@ -31,10 +31,14 @@ """ import apache_beam as beam +from apache_beam.io import iobase +from apache_beam.io.range_trackers import OffsetRangeTracker from apache_beam.metrics import Metrics +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to +from apache_beam.transforms.core import PTransform # Quiet some pylint warnings that happen because of the somewhat special # format for the code snippets. @@ -655,13 +659,6 @@ def process(self, an_int): result.wait_until_finish() -import apache_beam as beam -from apache_beam.io import iobase -from apache_beam.io.range_trackers import OffsetRangeTracker -from apache_beam.transforms.core import PTransform -from apache_beam.options.pipeline_options import PipelineOptions - - # Defining a new source. # [START model_custom_source_new_source] class CountingSource(iobase.BoundedSource): diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index 9183d0dfea19..ee1e50e460ef 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -30,17 +30,15 @@ from apache_beam import pvalue from apache_beam import typehints from apache_beam.coders.coders import ToStringCoder -from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.examples.snippets import snippets from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to from apache_beam.utils.windowed_value import WindowedValue -# pylint: disable=expression-not-assigned -from apache_beam.testing.test_pipeline import TestPipeline - # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py index 7696d7789323..94d4c701f677 100644 --- a/sdks/python/apache_beam/examples/streaming_wordcount.py +++ b/sdks/python/apache_beam/examples/streaming_wordcount.py @@ -26,11 +26,10 @@ import argparse import logging - import apache_beam as beam +import apache_beam.transforms.window as window from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import StandardOptions -import apache_beam.transforms.window as window def split_fn(lines): diff --git a/sdks/python/apache_beam/examples/windowed_wordcount.py b/sdks/python/apache_beam/examples/windowed_wordcount.py index bd57847c67f9..508f18d00097 100644 --- a/sdks/python/apache_beam/examples/windowed_wordcount.py +++ b/sdks/python/apache_beam/examples/windowed_wordcount.py @@ -26,7 +26,6 @@ import argparse import logging - import apache_beam as beam import apache_beam.transforms.window as window diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index 8d2e73e28f6b..21ff3cebc6dc 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -25,10 +25,10 @@ from nose.plugins.attrib import attr from apache_beam.examples import wordcount -from apache_beam.testing.test_utils import delete_files from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.test_utils import delete_files class WordCountIT(unittest.TestCase): diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 9f32092f4166..35676f1968b9 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -28,7 +28,6 @@ from apache_beam.utils import retry - # When we are running in GCE, we can authenticate with VM credentials. is_running_in_gce = False diff --git a/sdks/python/apache_beam/internal/gcp/json_value.py b/sdks/python/apache_beam/internal/gcp/json_value.py index 167b173b1f84..7a5dc543eefa 100644 --- a/sdks/python/apache_beam/internal/gcp/json_value.py +++ b/sdks/python/apache_beam/internal/gcp/json_value.py @@ -27,7 +27,6 @@ from apache_beam.options.value_provider import ValueProvider - _MAXINT64 = (1 << 63) - 1 _MININT64 = - (1 << 63) diff --git a/sdks/python/apache_beam/internal/gcp/json_value_test.py b/sdks/python/apache_beam/internal/gcp/json_value_test.py index b1fd63f048e9..14223f11c786 100644 --- a/sdks/python/apache_beam/internal/gcp/json_value_test.py +++ b/sdks/python/apache_beam/internal/gcp/json_value_test.py @@ -21,9 +21,8 @@ from apache_beam.internal.gcp.json_value import from_json_value from apache_beam.internal.gcp.json_value import to_json_value -from apache_beam.options.value_provider import StaticValueProvider from apache_beam.options.value_provider import RuntimeValueProvider - +from apache_beam.options.value_provider import StaticValueProvider # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/internal/util.py b/sdks/python/apache_beam/internal/util.py index dbbeafcc1e75..3e943b0db1e3 100644 --- a/sdks/python/apache_beam/internal/util.py +++ b/sdks/python/apache_beam/internal/util.py @@ -21,9 +21,9 @@ """ import logging -from multiprocessing.pool import ThreadPool import threading import weakref +from multiprocessing.pool import ThreadPool class ArgumentPlaceholder(object): diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py index cb14c6542176..30fc8903283c 100644 --- a/sdks/python/apache_beam/io/avroio.py +++ b/sdks/python/apache_beam/io/avroio.py @@ -48,13 +48,13 @@ from functools import partial import avro -from avro import datafile from avro import io as avroio +from avro import datafile from avro import schema import apache_beam as beam -from apache_beam.io import filebasedsource from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.iobase import Read diff --git a/sdks/python/apache_beam/io/avroio_test.py b/sdks/python/apache_beam/io/avroio_test.py index 969f44050854..8a344275a1a8 100644 --- a/sdks/python/apache_beam/io/avroio_test.py +++ b/sdks/python/apache_beam/io/avroio_test.py @@ -21,29 +21,26 @@ import tempfile import unittest +import avro.datafile +import avro.schema +from avro.datafile import DataFileWriter +from avro.io import DatumWriter +import hamcrest as hc + import apache_beam as beam from apache_beam import Create -from apache_beam.io import iobase from apache_beam.io import avroio from apache_beam.io import filebasedsource +from apache_beam.io import iobase from apache_beam.io import source_test_utils +from apache_beam.io.avroio import _AvroSink as AvroSink # For testing +from apache_beam.io.avroio import _AvroSource as AvroSource # For testing from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -# Importing following private class for testing purposes. -from apache_beam.io.avroio import _AvroSource as AvroSource -from apache_beam.io.avroio import _AvroSink as AvroSink - -import avro.datafile -from avro.datafile import DataFileWriter -from avro.io import DatumWriter -import avro.schema -import hamcrest as hc - - # Import snappy optionally; some tests will be skipped when import fails. try: import snappy # pylint: disable=import-error diff --git a/sdks/python/apache_beam/io/concat_source_test.py b/sdks/python/apache_beam/io/concat_source_test.py index 4a8f5193dee2..0f7dd547e76e 100644 --- a/sdks/python/apache_beam/io/concat_source_test.py +++ b/sdks/python/apache_beam/io/concat_source_test.py @@ -21,7 +21,6 @@ import unittest import apache_beam as beam - from apache_beam.io import iobase from apache_beam.io import range_trackers from apache_beam.io import source_test_utils diff --git a/sdks/python/apache_beam/io/filebasedsink.py b/sdks/python/apache_beam/io/filebasedsink.py index eb99d0849017..ba1a49503260 100644 --- a/sdks/python/apache_beam/io/filebasedsink.py +++ b/sdks/python/apache_beam/io/filebasedsink.py @@ -30,10 +30,10 @@ from apache_beam.io.filesystem import BeamIOError from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems -from apache_beam.transforms.display import DisplayDataItem -from apache_beam.options.value_provider import ValueProvider from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import ValueProvider from apache_beam.options.value_provider import check_accessible +from apache_beam.transforms.display import DisplayDataItem DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN' diff --git a/sdks/python/apache_beam/io/filebasedsink_test.py b/sdks/python/apache_beam/io/filebasedsink_test.py index 7c8ddb4072ef..156251a09123 100644 --- a/sdks/python/apache_beam/io/filebasedsink_test.py +++ b/sdks/python/apache_beam/io/filebasedsink_test.py @@ -31,12 +31,11 @@ import apache_beam as beam from apache_beam.coders import coders from apache_beam.io import filebasedsink +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.options.value_provider import StaticValueProvider - # TODO: Refactor code so all io tests are using same library # TestCaseWithTempDirCleanup class. diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py index 649693076c5f..7b019edec424 100644 --- a/sdks/python/apache_beam/io/filebasedsource.py +++ b/sdks/python/apache_beam/io/filebasedsource.py @@ -27,12 +27,6 @@ """ import uuid -from apache_beam.transforms.core import DoFn -from apache_beam.transforms.core import ParDo -from apache_beam.transforms.core import GroupByKey -from apache_beam.transforms.core import PTransform -from apache_beam.transforms.core import FlatMap -from apache_beam.transforms.core import Map from apache_beam.internal import pickler from apache_beam.io import concat_source from apache_beam.io import iobase @@ -40,10 +34,16 @@ from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystems import FileSystems from apache_beam.io.range_trackers import OffsetRange -from apache_beam.transforms.display import DisplayDataItem -from apache_beam.options.value_provider import ValueProvider from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.options.value_provider import ValueProvider from apache_beam.options.value_provider import check_accessible +from apache_beam.transforms.core import DoFn +from apache_beam.transforms.core import FlatMap +from apache_beam.transforms.core import GroupByKey +from apache_beam.transforms.core import Map +from apache_beam.transforms.core import ParDo +from apache_beam.transforms.core import PTransform +from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.trigger import DefaultTrigger MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25 diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py b/sdks/python/apache_beam/io/filebasedsource_test.py index afb340d7608b..0999510522f5 100644 --- a/sdks/python/apache_beam/io/filebasedsource_test.py +++ b/sdks/python/apache_beam/io/filebasedsource_test.py @@ -20,8 +20,8 @@ import gzip import logging import math -import random import os +import random import tempfile import unittest @@ -31,15 +31,13 @@ from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io import range_trackers -from apache_beam.io.filesystem import CompressionTypes - # importing following private classes for testing from apache_beam.io.concat_source import ConcatSource from apache_beam.io.filebasedsource import _SingleFileSource as SingleFileSource - from apache_beam.io.filebasedsource import FileBasedSource -from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.io.filesystem import CompressionTypes from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py index 5804d0048af0..69049ae67a45 100644 --- a/sdks/python/apache_beam/io/filesystem.py +++ b/sdks/python/apache_beam/io/filesystem.py @@ -21,10 +21,10 @@ import abc import bz2 import cStringIO -import os -import zlib import logging +import os import time +import zlib from apache_beam.utils.plugin import BeamPlugin diff --git a/sdks/python/apache_beam/io/filesystem_test.py b/sdks/python/apache_beam/io/filesystem_test.py index 607393d3a555..1c6cdd7a87ed 100644 --- a/sdks/python/apache_beam/io/filesystem_test.py +++ b/sdks/python/apache_beam/io/filesystem_test.py @@ -20,11 +20,12 @@ import bz2 import gzip import os -import unittest import tempfile +import unittest from StringIO import StringIO -from apache_beam.io.filesystem import CompressedFile, CompressionTypes +from apache_beam.io.filesystem import CompressedFile +from apache_beam.io.filesystem import CompressionTypes class TestCompressedFile(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py index e03968628d59..f9ce55396533 100644 --- a/sdks/python/apache_beam/io/filesystems.py +++ b/sdks/python/apache_beam/io/filesystems.py @@ -22,7 +22,6 @@ from apache_beam.io.filesystem import BeamIOError from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.filesystem import FileSystem - # All filesystem implements should be added here # pylint: disable=wrong-import-position, unused-import from apache_beam.io.localfilesystem import LocalFileSystem diff --git a/sdks/python/apache_beam/io/filesystems_test.py b/sdks/python/apache_beam/io/filesystems_test.py index 9a6f013ff479..eaaa40f9f7eb 100644 --- a/sdks/python/apache_beam/io/filesystems_test.py +++ b/sdks/python/apache_beam/io/filesystems_test.py @@ -18,12 +18,12 @@ """Unit tests for LocalFileSystem.""" -import unittest - import filecmp import os import shutil import tempfile +import unittest + import mock from apache_beam.io import localfilesystem diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 33d67bf7c53a..40611ac2b6ec 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -114,14 +114,14 @@ from apache_beam.internal.gcp import auth from apache_beam.internal.gcp.json_value import from_json_value from apache_beam.internal.gcp.json_value import to_json_value +from apache_beam.io.gcp.internal.clients import bigquery +from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.runners.dataflow.native_io import iobase as dataflow_io from apache_beam.transforms import DoFn from apache_beam.transforms import ParDo from apache_beam.transforms import PTransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.utils import retry -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.io.gcp.internal.clients import bigquery # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py index bfd06ace6400..849048164346 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py @@ -27,14 +27,14 @@ import mock import apache_beam as beam +from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.io.gcp.bigquery import RowAsDictJsonCoder from apache_beam.io.gcp.bigquery import TableRowJsonCoder from apache_beam.io.gcp.bigquery import parse_table_schema_from_json from apache_beam.io.gcp.internal.clients import bigquery -from apache_beam.internal.gcp.json_value import to_json_value +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher -from apache_beam.options.pipeline_options import PipelineOptions # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py index 93b91ad7cfab..1ac23930f65f 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py @@ -16,6 +16,7 @@ # import unittest + from mock import patch from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py index 3cfba93d2e8c..078002c82d10 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py @@ -20,28 +20,29 @@ import logging import time -# Protect against environments where datastore library is not available. -# pylint: disable=wrong-import-order, wrong-import-position -try: - from google.cloud.proto.datastore.v1 import datastore_pb2 - from googledatastore import helper as datastore_helper -except ImportError: - pass -# pylint: enable=wrong-import-order, wrong-import-position - from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.io.gcp.datastore.v1 import query_splitter from apache_beam.io.gcp.datastore.v1 import util from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler +from apache_beam.metrics.metric import Metrics from apache_beam.transforms import Create from apache_beam.transforms import DoFn from apache_beam.transforms import FlatMap from apache_beam.transforms import GroupByKey from apache_beam.transforms import Map -from apache_beam.transforms import PTransform from apache_beam.transforms import ParDo +from apache_beam.transforms import PTransform from apache_beam.transforms.util import Values -from apache_beam.metrics.metric import Metrics + +# Protect against environments where datastore library is not available. +# pylint: disable=wrong-import-order, wrong-import-position +try: + from google.cloud.proto.datastore.v1 import datastore_pb2 + from googledatastore import helper as datastore_helper +except ImportError: + pass +# pylint: enable=wrong-import-order, wrong-import-position + __all__ = ['ReadFromDatastore', 'WriteToDatastore', 'DeleteFromDatastore'] diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py index 72c4c8cc0c13..fa7310f35ace 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py @@ -17,14 +17,16 @@ import unittest -from mock import MagicMock, call, patch +from mock import MagicMock +from mock import call +from mock import patch from apache_beam.io.gcp.datastore.v1 import fake_datastore from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.io.gcp.datastore.v1 import query_splitter -from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate from apache_beam.io.gcp.datastore.v1.datastoreio import ReadFromDatastore from apache_beam.io.gcp.datastore.v1.datastoreio import WriteToDatastore +from apache_beam.io.gcp.datastore.v1.datastoreio import _Mutate # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py index 5cde25540da4..01ced6a84a3d 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py @@ -21,10 +21,14 @@ """ import errno -from socket import error as SocketError import logging import sys import time +from socket import error as SocketError + +# pylint: disable=ungrouped-imports +from apache_beam.internal.gcp import auth +from apache_beam.utils import retry # Protect against environments where datastore library is not available. # pylint: disable=wrong-import-order, wrong-import-position @@ -41,8 +45,7 @@ pass # pylint: enable=wrong-import-order, wrong-import-position -from apache_beam.internal.gcp import auth -from apache_beam.utils import retry +# pylint: enable=ungrouped-imports def key_comparator(k1, k2): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py index a8b1bb11231f..36cfb15e187c 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper_test.py @@ -18,17 +18,17 @@ """Tests for datastore helper.""" import errno import random -from socket import error as SocketError import sys import unittest +from socket import error as SocketError from mock import MagicMock +# pylint: disable=ungrouped-imports from apache_beam.io.gcp.datastore.v1 import fake_datastore from apache_beam.io.gcp.datastore.v1 import helper from apache_beam.testing.test_utils import patch_retry - # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: @@ -42,6 +42,7 @@ except ImportError: datastore_helper = None # pylint: enable=wrong-import-order, wrong-import-position +# pylint: enable=ungrouped-imports @unittest.skipIf(datastore_helper is None, 'GCP dependencies are not installed') diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py index 923fc7dd4b68..88a601f6d0fe 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py @@ -21,6 +21,7 @@ import unittest import mock + from apache_beam.io.filesystem import BeamIOError from apache_beam.io.filesystem import FileMetadata diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index b18a2875aa0c..0db4ba5dc27c 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -31,6 +31,7 @@ import threading import time import traceback + import httplib2 from apache_beam.utils import retry diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py index 404542863f3c..3e741cdbbd63 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py @@ -25,7 +25,6 @@ from apitools.base.py import encoding from apitools.base.py import extra_types - package = 'bigquery' @@ -1906,5 +1905,3 @@ class ViewDefinition(_messages.Message): query = _messages.StringField(1) userDefinedFunctionResources = _messages.MessageField('UserDefinedFunctionResource', 2, repeated=True) - - diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py index dc9e5e692554..3c180a652bc5 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py @@ -26,7 +26,6 @@ from apitools.base.py import encoding from apitools.base.py import extra_types - package = 'storage' @@ -1916,5 +1915,3 @@ class ProjectionValueValuesEnum(_messages.Enum): prefix = _messages.StringField(6) projection = _messages.EnumField('ProjectionValueValuesEnum', 7) versions = _messages.BooleanField(8) - - diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py index 7d1f3553f4fa..98aa884c71dc 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub.py +++ b/sdks/python/apache_beam/io/gcp/pubsub.py @@ -30,13 +30,12 @@ from apache_beam.io.iobase import Read from apache_beam.io.iobase import Write from apache_beam.runners.dataflow.native_io import iobase as dataflow_io -from apache_beam.transforms import core -from apache_beam.transforms import PTransform from apache_beam.transforms import Map +from apache_beam.transforms import PTransform +from apache_beam.transforms import core from apache_beam.transforms import window from apache_beam.transforms.display import DisplayDataItem - __all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub'] diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py index 0dcc3c39ab5f..0c4ba02db87a 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py @@ -22,15 +22,14 @@ import hamcrest as hc -from apache_beam.io.gcp.pubsub import _PubSubPayloadSink -from apache_beam.io.gcp.pubsub import _PubSubPayloadSource from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub from apache_beam.io.gcp.pubsub import WriteStringsToPubSub +from apache_beam.io.gcp.pubsub import _PubSubPayloadSink +from apache_beam.io.gcp.pubsub import _PubSubPayloadSource from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display_test import DisplayDataItemMatcher - # Protect against environments where the PubSub library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py index 5b722856a7b9..a0977189e06b 100644 --- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py +++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher_test.py @@ -21,7 +21,8 @@ import unittest from hamcrest import assert_that as hc_assert_that -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.gcp.tests import bigquery_matcher as bq_verifier from apache_beam.testing.test_utils import patch_retry diff --git a/sdks/python/apache_beam/io/gcp/tests/utils_test.py b/sdks/python/apache_beam/io/gcp/tests/utils_test.py index 270750a25217..340aa6d2814f 100644 --- a/sdks/python/apache_beam/io/gcp/tests/utils_test.py +++ b/sdks/python/apache_beam/io/gcp/tests/utils_test.py @@ -19,7 +19,9 @@ import logging import unittest -from mock import Mock, patch + +from mock import Mock +from mock import patch from apache_beam.io.gcp.tests import utils from apache_beam.testing.test_utils import patch_retry diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index db75fe38baa9..043666d3b8a0 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -29,22 +29,21 @@ the sink. """ -from collections import namedtuple - import logging import random import uuid +from collections import namedtuple -from apache_beam import pvalue from apache_beam import coders +from apache_beam import pvalue from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.pvalue import AsIter from apache_beam.pvalue import AsSingleton from apache_beam.transforms import core from apache_beam.transforms import ptransform from apache_beam.transforms import window -from apache_beam.transforms.display import HasDisplayData from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData from apache_beam.utils import urns from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py b/sdks/python/apache_beam/io/localfilesystem_test.py index 04cf5b7c6f4d..8c34ecdf0d36 100644 --- a/sdks/python/apache_beam/io/localfilesystem_test.py +++ b/sdks/python/apache_beam/io/localfilesystem_test.py @@ -18,12 +18,12 @@ """Unit tests for LocalFileSystem.""" -import unittest - import filecmp import os import shutil import tempfile +import unittest + import mock from apache_beam.io import localfilesystem diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py index bea970850838..712049b99f1e 100644 --- a/sdks/python/apache_beam/io/source_test_utils.py +++ b/sdks/python/apache_beam/io/source_test_utils.py @@ -44,12 +44,12 @@ * apache_beam.io.avroio_test.py """ -from collections import namedtuple import logging import threading import weakref - +from collections import namedtuple from multiprocessing.pool import ThreadPool + from apache_beam.io import iobase __all__ = ['read_from_source', 'assert_sources_equal_reference_source', diff --git a/sdks/python/apache_beam/io/source_test_utils_test.py b/sdks/python/apache_beam/io/source_test_utils_test.py index 00522c9fd3f5..af2d4b87d6a7 100644 --- a/sdks/python/apache_beam/io/source_test_utils_test.py +++ b/sdks/python/apache_beam/io/source_test_utils_test.py @@ -19,8 +19,8 @@ import tempfile import unittest -from apache_beam.io.filebasedsource_test import LineSource import apache_beam.io.source_test_utils as source_test_utils +from apache_beam.io.filebasedsource_test import LineSource class SourceTestUtilsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/io/sources_test.py b/sdks/python/apache_beam/io/sources_test.py index 10d401b8d2a2..8f885e59fb0b 100644 --- a/sdks/python/apache_beam/io/sources_test.py +++ b/sdks/python/apache_beam/io/sources_test.py @@ -23,7 +23,6 @@ import unittest import apache_beam as beam - from apache_beam import coders from apache_beam.io import iobase from apache_beam.io import range_trackers diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py index 9708df7c753c..c25181db21a7 100644 --- a/sdks/python/apache_beam/io/textio.py +++ b/sdks/python/apache_beam/io/textio.py @@ -19,12 +19,13 @@ from __future__ import absolute_import -from functools import partial + import logging +from functools import partial from apache_beam.coders import coders -from apache_beam.io import filebasedsource from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource from apache_beam.io import iobase from apache_beam.io.filebasedsource import ReadAllFiles from apache_beam.io.filesystem import CompressionTypes diff --git a/sdks/python/apache_beam/io/textio_test.py b/sdks/python/apache_beam/io/textio_test.py index b29ca5a97137..475848fcb356 100644 --- a/sdks/python/apache_beam/io/textio_test.py +++ b/sdks/python/apache_beam/io/textio_test.py @@ -27,26 +27,22 @@ import unittest import apache_beam as beam -from apache_beam.io import iobase, ReadAllFromText import apache_beam.io.source_test_utils as source_test_utils - -# Importing following private classes for testing. -from apache_beam.io.textio import _TextSink as TextSink -from apache_beam.io.textio import _TextSource as TextSource - -from apache_beam.io.textio import ReadFromText -from apache_beam.io.textio import WriteToText - from apache_beam import coders +from apache_beam.io import ReadAllFromText +from apache_beam.io import iobase from apache_beam.io.filebasedsource_test import EOL from apache_beam.io.filebasedsource_test import write_data from apache_beam.io.filebasedsource_test import write_pattern from apache_beam.io.filesystem import CompressionTypes - +from apache_beam.io.textio import _TextSink as TextSink +from apache_beam.io.textio import _TextSource as TextSource +# Importing following private classes for testing. +from apache_beam.io.textio import ReadFromText +from apache_beam.io.textio import WriteToText from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to - from apache_beam.transforms.core import Create diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py index d7eb932cd655..5af07164b669 100644 --- a/sdks/python/apache_beam/io/tfrecordio.py +++ b/sdks/python/apache_beam/io/tfrecordio.py @@ -21,14 +21,15 @@ import logging import struct +import crcmod + from apache_beam import coders -from apache_beam.io import filebasedsource from apache_beam.io import filebasedsink +from apache_beam.io import filebasedsource from apache_beam.io.filesystem import CompressionTypes from apache_beam.io.iobase import Read from apache_beam.io.iobase import Write from apache_beam.transforms import PTransform -import crcmod __all__ = ['ReadFromTFRecord', 'WriteToTFRecord'] diff --git a/sdks/python/apache_beam/io/tfrecordio_test.py b/sdks/python/apache_beam/io/tfrecordio_test.py index 3c70ade3dbe3..f7a160a1ce41 100644 --- a/sdks/python/apache_beam/io/tfrecordio_test.py +++ b/sdks/python/apache_beam/io/tfrecordio_test.py @@ -27,19 +27,19 @@ import tempfile import unittest +import crcmod + import apache_beam as beam from apache_beam import coders from apache_beam.io.filesystem import CompressionTypes +from apache_beam.io.tfrecordio import ReadFromTFRecord +from apache_beam.io.tfrecordio import WriteToTFRecord from apache_beam.io.tfrecordio import _TFRecordSink from apache_beam.io.tfrecordio import _TFRecordSource from apache_beam.io.tfrecordio import _TFRecordUtil -from apache_beam.io.tfrecordio import ReadFromTFRecord -from apache_beam.io.tfrecordio import WriteToTFRecord from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to -import crcmod - try: import tensorflow as tf # pylint: disable=import-error diff --git a/sdks/python/apache_beam/metrics/cells_test.py b/sdks/python/apache_beam/metrics/cells_test.py index a4c8a43705fb..c0664ab2bb0c 100644 --- a/sdks/python/apache_beam/metrics/cells_test.py +++ b/sdks/python/apache_beam/metrics/cells_test.py @@ -18,10 +18,10 @@ import threading import unittest +from apache_beam.metrics.cells import CellCommitState from apache_beam.metrics.cells import CounterCell from apache_beam.metrics.cells import DistributionCell from apache_beam.metrics.cells import DistributionData -from apache_beam.metrics.cells import CellCommitState class TestCounterCell(unittest.TestCase): diff --git a/sdks/python/apache_beam/metrics/execution.py b/sdks/python/apache_beam/metrics/execution.py index 675e49c1234c..1704b98d46a7 100644 --- a/sdks/python/apache_beam/metrics/execution.py +++ b/sdks/python/apache_beam/metrics/execution.py @@ -29,10 +29,11 @@ - MetricsContainer - Holds the metrics of a single step and a single unit-of-commit (bundle). """ -from collections import defaultdict import threading +from collections import defaultdict -from apache_beam.metrics.cells import CounterCell, DistributionCell +from apache_beam.metrics.cells import CounterCell +from apache_beam.metrics.cells import DistributionCell class MetricKey(object): diff --git a/sdks/python/apache_beam/metrics/execution_test.py b/sdks/python/apache_beam/metrics/execution_test.py index 54569c10e99d..abf23e354a1c 100644 --- a/sdks/python/apache_beam/metrics/execution_test.py +++ b/sdks/python/apache_beam/metrics/execution_test.py @@ -18,10 +18,10 @@ import unittest from apache_beam.metrics.cells import CellCommitState +from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.execution import MetricsContainer -from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.metrics.metric import Metrics from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/metrics/metric.py b/sdks/python/apache_beam/metrics/metric.py index f99c0c415d56..8fbf9804ac86 100644 --- a/sdks/python/apache_beam/metrics/metric.py +++ b/sdks/python/apache_beam/metrics/metric.py @@ -27,7 +27,8 @@ import inspect from apache_beam.metrics.execution import MetricsEnvironment -from apache_beam.metrics.metricbase import Counter, Distribution +from apache_beam.metrics.metricbase import Counter +from apache_beam.metrics.metricbase import Distribution from apache_beam.metrics.metricbase import MetricName __all__ = ['Metrics', 'MetricsFilter'] diff --git a/sdks/python/apache_beam/metrics/metric_test.py b/sdks/python/apache_beam/metrics/metric_test.py index 56b74680a97d..75c3aa09cbe5 100644 --- a/sdks/python/apache_beam/metrics/metric_test.py +++ b/sdks/python/apache_beam/metrics/metric_test.py @@ -21,9 +21,9 @@ from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.execution import MetricsContainer from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.metrics.metric import MetricResults from apache_beam.metrics.metric import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.metrics.metric import MetricResults from apache_beam.metrics.metricbase import MetricName diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index db65b3c65837..3abcbf2edb24 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -19,11 +19,10 @@ import argparse -from apache_beam.transforms.display import HasDisplayData -from apache_beam.options.value_provider import StaticValueProvider from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider from apache_beam.options.value_provider import ValueProvider - +from apache_beam.transforms.display import HasDisplayData __all__ = [ 'PipelineOptions', diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index f4dd4d92b788..66c69bdfc70e 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -21,11 +21,12 @@ import unittest import hamcrest as hc -from apache_beam.transforms.display import DisplayData -from apache_beam.transforms.display_test import DisplayDataItemMatcher + from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.options.value_provider import StaticValueProvider from apache_beam.options.value_provider import RuntimeValueProvider +from apache_beam.options.value_provider import StaticValueProvider +from apache_beam.transforms.display import DisplayData +from apache_beam.transforms.display_test import DisplayDataItemMatcher class PipelineOptionsTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/options/pipeline_options_validator_test.py b/sdks/python/apache_beam/options/pipeline_options_validator_test.py index 97834cc4821b..8ff66c78f17b 100644 --- a/sdks/python/apache_beam/options/pipeline_options_validator_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_validator_test.py @@ -20,10 +20,11 @@ import logging import unittest +from hamcrest.core.base_matcher import BaseMatcher + from apache_beam.internal import pickler from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator -from hamcrest.core.base_matcher import BaseMatcher # Mock runners to use for validations. diff --git a/sdks/python/apache_beam/options/value_provider.py b/sdks/python/apache_beam/options/value_provider.py index f7630bd8d326..fe4614dc37da 100644 --- a/sdks/python/apache_beam/options/value_provider.py +++ b/sdks/python/apache_beam/options/value_provider.py @@ -23,7 +23,6 @@ from apache_beam import error - __all__ = [ 'ValueProvider', 'StaticValueProvider', diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 1ade6c067c5f..1ebd099d37ec 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -55,20 +55,19 @@ from apache_beam import pvalue from apache_beam.internal import pickler -from apache_beam.pvalue import PCollection -from apache_beam.runners import create_runner -from apache_beam.runners import PipelineRunner -from apache_beam.transforms import ptransform -from apache_beam.typehints import typehints -from apache_beam.typehints import TypeCheckError from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import SetupOptions from apache_beam.options.pipeline_options import StandardOptions from apache_beam.options.pipeline_options import TypeOptions from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator -from apache_beam.utils.annotations import deprecated +from apache_beam.pvalue import PCollection +from apache_beam.runners import PipelineRunner +from apache_beam.runners import create_runner +from apache_beam.transforms import ptransform +from apache_beam.typehints import TypeCheckError +from apache_beam.typehints import typehints from apache_beam.utils import urns - +from apache_beam.utils.annotations import deprecated __all__ = ['Pipeline'] diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index b3ac100780fe..0917c7846727 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -21,16 +21,13 @@ import platform import unittest -# TODO(BEAM-1555): Test is failing on the service, with FakeSource. -# from nose.plugins.attrib import attr - import apache_beam as beam from apache_beam.io import Read from apache_beam.metrics import Metrics from apache_beam.pipeline import Pipeline -from apache_beam.pipeline import PTransformOverride from apache_beam.pipeline import PipelineOptions from apache_beam.pipeline import PipelineVisitor +from apache_beam.pipeline import PTransformOverride from apache_beam.pvalue import AsSingleton from apache_beam.runners import DirectRunner from apache_beam.runners.dataflow.native_io.iobase import NativeSource @@ -39,9 +36,9 @@ from apache_beam.testing.util import equal_to from apache_beam.transforms import CombineGlobally from apache_beam.transforms import Create +from apache_beam.transforms import DoFn from apache_beam.transforms import FlatMap from apache_beam.transforms import Map -from apache_beam.transforms import DoFn from apache_beam.transforms import ParDo from apache_beam.transforms import PTransform from apache_beam.transforms import WindowInto @@ -49,6 +46,9 @@ from apache_beam.transforms.window import TimestampedValue from apache_beam.utils.timestamp import MIN_TIMESTAMP +# TODO(BEAM-1555): Test is failing on the service, with FakeSource. +# from nose.plugins.attrib import attr + class FakeSource(NativeSource): """Fake source returning a fixed list of values.""" diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 35686f7fa381..53a6121eef34 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -30,7 +30,6 @@ from apache_beam import typehints - __all__ = [ 'PCollection', 'TaggedOutput', diff --git a/sdks/python/apache_beam/runners/common.py b/sdks/python/apache_beam/runners/common.py index 84535692806b..66c033fc5c5f 100644 --- a/sdks/python/apache_beam/runners/common.py +++ b/sdks/python/apache_beam/runners/common.py @@ -29,9 +29,9 @@ from apache_beam.metrics.execution import ScopedMetricsContainer from apache_beam.pvalue import TaggedOutput from apache_beam.transforms import core +from apache_beam.transforms.window import GlobalWindow from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowFn -from apache_beam.transforms.window import GlobalWindow from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/common_test.py b/sdks/python/apache_beam/runners/common_test.py index 62a6955f6ce4..e0f628c71ee1 100644 --- a/sdks/python/apache_beam/runners/common_test.py +++ b/sdks/python/apache_beam/runners/common_test.py @@ -17,8 +17,8 @@ import unittest -from apache_beam.transforms.core import DoFn from apache_beam.runners.common import DoFnSignature +from apache_beam.transforms.core import DoFn class DoFnSignatureTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py index 24916fd49278..c7eb88ef578d 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_metrics.py @@ -21,8 +21,8 @@ service. """ -from collections import defaultdict import numbers +from collections import defaultdict from apache_beam.metrics.cells import DistributionData from apache_beam.metrics.cells import DistributionResult diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 56cc46c815a8..f4cf459d904c 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -21,37 +21,36 @@ to the Dataflow Service for remote execution by a worker. """ -from collections import defaultdict import logging import threading import time import traceback import urllib +from collections import defaultdict import apache_beam as beam -from apache_beam import error from apache_beam import coders +from apache_beam import error from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.internal.gcp import json_value +from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import TestOptions from apache_beam.pvalue import AsSideInput from apache_beam.runners.dataflow.dataflow_metrics import DataflowMetrics from apache_beam.runners.dataflow.internal import names from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.dataflow.internal.names import PropertyNames from apache_beam.runners.dataflow.internal.names import TransformNames -from apache_beam.runners.runner import PValueCache from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState +from apache_beam.runners.runner import PValueCache from apache_beam.transforms.display import DisplayData from apache_beam.typehints import typehints -from apache_beam.options.pipeline_options import SetupOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import TestOptions from apache_beam.utils.plugin import BeamPlugin - __all__ = ['DataflowRunner'] diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index 8e708e6fde29..e73a94fd8603 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -25,22 +25,22 @@ import apache_beam as beam import apache_beam.transforms as ptransform - from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.pipeline import Pipeline, AppliedPTransform +from apache_beam.pipeline import AppliedPTransform +from apache_beam.pipeline import Pipeline from apache_beam.pvalue import PCollection -from apache_beam.runners import create_runner from apache_beam.runners import DataflowRunner from apache_beam.runners import TestDataflowRunner +from apache_beam.runners import create_runner from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeException from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.runners.runner import PipelineState from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.transforms.display import DisplayDataItem -from apache_beam.transforms.core import _GroupByKeyOnly -from apache_beam.transforms.core import Windowing from apache_beam.transforms import window +from apache_beam.transforms.core import Windowing +from apache_beam.transforms.core import _GroupByKeyOnly +from apache_beam.transforms.display import DisplayDataItem from apache_beam.typehints import typehints # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py index a1f9301b19ba..d010065237b3 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py @@ -26,8 +26,8 @@ import os import re import time -from StringIO import StringIO from datetime import datetime +from StringIO import StringIO from apitools.base.py import encoding from apitools.base.py import exceptions @@ -36,6 +36,10 @@ from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.io.filesystems import FileSystems from apache_beam.io.gcp.internal.clients import storage +from apache_beam.options.pipeline_options import DebugOptions +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.pipeline_options import WorkerOptions from apache_beam.runners.dataflow.internal import dependency from apache_beam.runners.dataflow.internal.clients import dataflow from apache_beam.runners.dataflow.internal.dependency import get_sdk_name_and_version @@ -43,11 +47,6 @@ from apache_beam.transforms import cy_combiners from apache_beam.transforms.display import DisplayData from apache_beam.utils import retry -from apache_beam.options.pipeline_options import DebugOptions -from apache_beam.options.pipeline_options import GoogleCloudOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.pipeline_options import WorkerOptions - # Environment version information. It is passed to the service during a # a job submission and is used by the service to establish what features diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py index 407ffcf2ad72..e7cced7b4f3f 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py @@ -21,7 +21,6 @@ from apache_beam.metrics.cells import DistributionData from apache_beam.options.pipeline_options import PipelineOptions - from apache_beam.runners.dataflow.internal.clients import dataflow # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py index e81329fca80f..b0d4e44816c1 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py @@ -26,7 +26,6 @@ from apitools.base.py import encoding from apitools.base.py import extra_types - package = 'dataflow' @@ -3069,7 +3068,6 @@ class ResourceUtilizationReportResponse(_messages.Message): """ - class RuntimeEnvironment(_messages.Message): """The environment values to set at runtime. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py index 4dda47ad9e05..805473a8838c 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py @@ -17,7 +17,6 @@ from hamcrest.core.base_matcher import BaseMatcher - IGNORED = object() diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py index 3163c9b25125..15bb9eff083f 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py @@ -15,9 +15,10 @@ # limitations under the License. # import unittest + import hamcrest as hc -import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow +import apache_beam.runners.dataflow.internal.clients.dataflow as dataflow from apache_beam.internal.gcp.json_value import to_json_value from apache_beam.runners.dataflow.internal.clients.dataflow import message_matchers diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index f5dd2bd35732..f4131dc885c0 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -64,10 +64,10 @@ from apache_beam import version as beam_version from apache_beam.internal import pickler from apache_beam.io.filesystems import FileSystems -from apache_beam.runners.dataflow.internal import names -from apache_beam.utils import processes from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import SetupOptions +from apache_beam.runners.dataflow.internal import names +from apache_beam.utils import processes # All constants are for internal use only; no backwards-compatibility # guarantees. diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py index e555b69f15ad..6d9b061c29a9 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py @@ -24,12 +24,11 @@ import unittest from apache_beam.io.filesystems import FileSystems -from apache_beam.runners.dataflow.internal import dependency -from apache_beam.runners.dataflow.internal import names from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options import SetupOptions - +from apache_beam.runners.dataflow.internal import dependency +from apache_beam.runners.dataflow.internal import names # Protect against environments where GCS library is not available. # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py index 3d8c24f5651c..01fd35f9cf95 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/iobase_test.py @@ -23,18 +23,16 @@ from apache_beam import Create from apache_beam import error from apache_beam import pvalue -from apache_beam.runners.dataflow.native_io.iobase import ( - _dict_printable_fields, - _NativeWrite, - ConcatPosition, - DynamicSplitRequest, - DynamicSplitResultWithPosition, - NativeSink, - NativeSinkWriter, - NativeSource, - ReaderPosition, - ReaderProgress -) +from apache_beam.runners.dataflow.native_io.iobase import ConcatPosition +from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitRequest +from apache_beam.runners.dataflow.native_io.iobase import DynamicSplitResultWithPosition +from apache_beam.runners.dataflow.native_io.iobase import NativeSink +from apache_beam.runners.dataflow.native_io.iobase import NativeSinkWriter +from apache_beam.runners.dataflow.native_io.iobase import NativeSource +from apache_beam.runners.dataflow.native_io.iobase import ReaderPosition +from apache_beam.runners.dataflow.native_io.iobase import ReaderProgress +from apache_beam.runners.dataflow.native_io.iobase import _dict_printable_fields +from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite from apache_beam.testing.test_pipeline import TestPipeline diff --git a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py index 8c6c8d6d5299..a54ee7767c5e 100644 --- a/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py +++ b/sdks/python/apache_beam/runners/dataflow/native_io/streaming_create.py @@ -17,11 +17,11 @@ """Create transform for streaming.""" -from apache_beam import pvalue from apache_beam import DoFn from apache_beam import ParDo from apache_beam import PTransform from apache_beam import Windowing +from apache_beam import pvalue from apache_beam.transforms.window import GlobalWindows diff --git a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py index 7927219ab5ea..88afe8a2ee62 100644 --- a/sdks/python/apache_beam/runners/dataflow/template_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/template_runner_test.py @@ -24,8 +24,8 @@ import unittest import apache_beam as beam -from apache_beam.pipeline import Pipeline from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.pipeline import Pipeline from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner # Protect against environments where apitools library is not available. diff --git a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py index 96e6a66caab4..e7c8d06d2f8f 100644 --- a/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py @@ -18,10 +18,10 @@ """Wrapper of Beam runners that's built for running and verifying e2e tests.""" from apache_beam.internal import pickler -from apache_beam.options.pipeline_options import TestOptions, GoogleCloudOptions +from apache_beam.options.pipeline_options import GoogleCloudOptions +from apache_beam.options.pipeline_options import TestOptions from apache_beam.runners.dataflow.dataflow_runner import DataflowRunner - __all__ = ['TestDataflowRunner'] diff --git a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py index 97d1ee8bfb6b..4efaa27f0958 100644 --- a/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py +++ b/sdks/python/apache_beam/runners/direct/consumer_tracking_pipeline_visitor_test.py @@ -21,8 +21,8 @@ import unittest from apache_beam import pvalue -from apache_beam.io import iobase from apache_beam.io import Read +from apache_beam.io import iobase from apache_beam.pipeline import Pipeline from apache_beam.pvalue import AsList from apache_beam.runners.direct import DirectRunner diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics.py b/sdks/python/apache_beam/runners/direct/direct_metrics.py index 9d234876035a..aa35fb7ccbb1 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics.py @@ -20,8 +20,8 @@ responding to queries of current metrics, but also of keeping the common state consistent. """ -from collections import defaultdict import threading +from collections import defaultdict from apache_beam.metrics.cells import CounterAggregator from apache_beam.metrics.cells import DistributionAggregator diff --git a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py index 256b91f3e9af..f36178601ff8 100644 --- a/sdks/python/apache_beam/runners/direct/direct_metrics_test.py +++ b/sdks/python/apache_beam/runners/direct/direct_metrics_test.py @@ -19,12 +19,12 @@ import hamcrest as hc -from apache_beam.metrics.metricbase import MetricName -from apache_beam.metrics.execution import MetricUpdates -from apache_beam.metrics.execution import MetricResult -from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.cells import DistributionData from apache_beam.metrics.cells import DistributionResult +from apache_beam.metrics.execution import MetricKey +from apache_beam.metrics.execution import MetricResult +from apache_beam.metrics.execution import MetricUpdates +from apache_beam.metrics.metricbase import MetricName from apache_beam.runners.direct.direct_metrics import DirectMetrics diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py index 2deb7dace023..794a96be12ba 100644 --- a/sdks/python/apache_beam/runners/direct/direct_runner.py +++ b/sdks/python/apache_beam/runners/direct/direct_runner.py @@ -31,19 +31,18 @@ import apache_beam as beam from apache_beam import typehints from apache_beam.metrics.execution import MetricsEnvironment +from apache_beam.options.pipeline_options import DirectOptions +from apache_beam.options.pipeline_options import StandardOptions +from apache_beam.options.value_provider import RuntimeValueProvider from apache_beam.pvalue import PCollection from apache_beam.runners.direct.bundle_factory import BundleFactory from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState from apache_beam.runners.runner import PValueCache -from apache_beam.transforms.ptransform import PTransform from apache_beam.transforms.core import _GroupAlsoByWindow from apache_beam.transforms.core import _GroupByKeyOnly -from apache_beam.options.pipeline_options import DirectOptions -from apache_beam.options.pipeline_options import StandardOptions -from apache_beam.options.value_provider import RuntimeValueProvider - +from apache_beam.transforms.ptransform import PTransform __all__ = ['DirectRunner'] diff --git a/sdks/python/apache_beam/runners/direct/evaluation_context.py b/sdks/python/apache_beam/runners/direct/evaluation_context.py index 54c407c1c866..9913813f364a 100644 --- a/sdks/python/apache_beam/runners/direct/evaluation_context.py +++ b/sdks/python/apache_beam/runners/direct/evaluation_context.py @@ -22,11 +22,11 @@ import collections import threading -from apache_beam.transforms import sideinputs from apache_beam.runners.direct.clock import Clock -from apache_beam.runners.direct.watermark_manager import WatermarkManager -from apache_beam.runners.direct.executor import TransformExecutor from apache_beam.runners.direct.direct_metrics import DirectMetrics +from apache_beam.runners.direct.executor import TransformExecutor +from apache_beam.runners.direct.watermark_manager import WatermarkManager +from apache_beam.transforms import sideinputs from apache_beam.transforms.trigger import InMemoryUnmergedState from apache_beam.utils import counters diff --git a/sdks/python/apache_beam/runners/direct/helper_transforms.py b/sdks/python/apache_beam/runners/direct/helper_transforms.py index 374cd4ea69db..26b0701bd02b 100644 --- a/sdks/python/apache_beam/runners/direct/helper_transforms.py +++ b/sdks/python/apache_beam/runners/direct/helper_transforms.py @@ -20,8 +20,8 @@ import apache_beam as beam from apache_beam import typehints -from apache_beam.utils.windowed_value import WindowedValue from apache_beam.internal.util import ArgumentPlaceholder +from apache_beam.utils.windowed_value import WindowedValue class LiftedCombinePerKey(beam.PTransform): diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index cb2ace29f0ea..8906ccb72e83 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -23,36 +23,36 @@ import random import time +import apache_beam.io as io from apache_beam import coders from apache_beam import pvalue from apache_beam.internal import pickler -import apache_beam.io as io +from apache_beam.options.pipeline_options import TypeOptions from apache_beam.runners.common import DoFnRunner from apache_beam.runners.common import DoFnState -from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly +from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access from apache_beam.runners.direct.direct_runner import _StreamingGroupAlsoByWindow -from apache_beam.runners.direct.watermark_manager import WatermarkManager +from apache_beam.runners.direct.direct_runner import _StreamingGroupByKeyOnly from apache_beam.runners.direct.util import KeyedWorkItem from apache_beam.runners.direct.util import TransformResult -from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access -from apache_beam.testing.test_stream import TestStream +from apache_beam.runners.direct.watermark_manager import WatermarkManager from apache_beam.testing.test_stream import ElementEvent -from apache_beam.testing.test_stream import WatermarkEvent from apache_beam.testing.test_stream import ProcessingTimeEvent +from apache_beam.testing.test_stream import TestStream +from apache_beam.testing.test_stream import WatermarkEvent from apache_beam.transforms import core -from apache_beam.transforms.window import GlobalWindows -from apache_beam.transforms.window import WindowedValue -from apache_beam.transforms.trigger import create_trigger_driver +from apache_beam.transforms.trigger import TimeDomain from apache_beam.transforms.trigger import _CombiningValueStateTag from apache_beam.transforms.trigger import _ListStateTag -from apache_beam.transforms.trigger import TimeDomain +from apache_beam.transforms.trigger import create_trigger_driver +from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import WindowedValue from apache_beam.typehints.typecheck import OutputCheckWrapperDoFn from apache_beam.typehints.typecheck import TypeCheckError from apache_beam.typehints.typecheck import TypeCheckWrapperDoFn from apache_beam.utils import counters -from apache_beam.utils.timestamp import Timestamp from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.options.pipeline_options import TypeOptions +from apache_beam.utils.timestamp import Timestamp class TransformEvaluatorRegistry(object): diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py index 247ce1f0e195..85e3f75be239 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/python_rpc_direct_runner.py @@ -30,7 +30,6 @@ from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineRunner - __all__ = ['PythonRPCDirectRunner'] diff --git a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py index bae25a4d07b1..1d07e716f799 100644 --- a/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py +++ b/sdks/python/apache_beam/runners/experimental/python_rpc_direct/server.py @@ -19,14 +19,14 @@ """ import time import uuid - from concurrent import futures + import grpc +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.pipeline import Pipeline from apache_beam.portability.api import beam_job_api_pb2 from apache_beam.portability.api import beam_job_api_pb2_grpc -from apache_beam.pipeline import Pipeline -from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.runners.runner import PipelineState _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index 42d7f5df14a8..35060999b723 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -21,9 +21,9 @@ """ +from apache_beam import coders from apache_beam import pipeline from apache_beam import pvalue -from apache_beam import coders from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import core diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py index c9b3d9a5e831..32dc50acf660 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py @@ -23,14 +23,14 @@ import logging import Queue as queue import threading - from concurrent import futures -from google.protobuf import wrappers_pb2 + import grpc +from google.protobuf import wrappers_pb2 import apache_beam as beam # pylint: disable=ungrouped-imports -from apache_beam.coders import registry from apache_beam.coders import WindowedValueCoder +from apache_beam.coders import registry from apache_beam.coders.coder_impl import create_InputStream from apache_beam.coders.coder_impl import create_OutputStream from apache_beam.internal import pickler @@ -49,7 +49,6 @@ from apache_beam.utils import proto_utils from apache_beam.utils import urns - # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py index ddfc4ccc43f1..a20ceef60fae 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py @@ -36,13 +36,14 @@ from apache_beam.runners.runner import PipelineState from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import operations +from apache_beam.typehints import typehints +from apache_beam.utils import profiler +from apache_beam.utils.counters import CounterFactory + try: from apache_beam.runners.worker import statesampler except ImportError: from apache_beam.runners.worker import statesampler_fake as statesampler -from apache_beam.typehints import typehints -from apache_beam.utils import profiler -from apache_beam.utils.counters import CounterFactory # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py index b7ba15a502aa..519ab6e3d041 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner_test.py @@ -21,18 +21,16 @@ import unittest import apache_beam as beam - from apache_beam.metrics import Metrics from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.execution import MetricsEnvironment from apache_beam.metrics.metricbase import MetricName - from apache_beam.pvalue import AsList -from apache_beam.testing.util import assert_that +from apache_beam.runners.portability import maptask_executor_runner from apache_beam.testing.util import BeamAssertException +from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to from apache_beam.transforms.window import TimestampedValue -from apache_beam.runners.portability import maptask_executor_runner class MapTaskExecutorRunnerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/runner.py b/sdks/python/apache_beam/runners/runner.py index 43ee27bc298f..4ac62237f1e4 100644 --- a/sdks/python/apache_beam/runners/runner.py +++ b/sdks/python/apache_beam/runners/runner.py @@ -25,7 +25,6 @@ import shutil import tempfile - __all__ = ['PipelineRunner', 'PipelineState', 'PipelineResult'] diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py index fa80b1c9555c..063c8a2bcecc 100644 --- a/sdks/python/apache_beam/runners/runner_test.py +++ b/sdks/python/apache_beam/runners/runner_test.py @@ -33,12 +33,12 @@ from apache_beam.metrics.execution import MetricKey from apache_beam.metrics.execution import MetricResult from apache_beam.metrics.metricbase import MetricName +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline from apache_beam.runners import DirectRunner from apache_beam.runners import create_runner from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to -from apache_beam.options.pipeline_options import PipelineOptions class RunnerTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py index 16c888c63c60..068aa0af2b04 100644 --- a/sdks/python/apache_beam/runners/worker/bundle_processor.py +++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py @@ -29,14 +29,14 @@ from google.protobuf import wrappers_pb2 import apache_beam as beam -from apache_beam.coders import coder_impl from apache_beam.coders import WindowedValueCoder +from apache_beam.coders import coder_impl from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.portability.api import beam_runner_api_pb2 -from apache_beam.runners.dataflow.native_io import iobase as native_iobase from apache_beam.runners import pipeline_context +from apache_beam.runners.dataflow.native_io import iobase as native_iobase from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import operations from apache_beam.utils import counters diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py index e713041d7c83..5c65a6d8ed90 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane.py +++ b/sdks/python/apache_beam/runners/worker/data_plane.py @@ -27,9 +27,10 @@ import Queue as queue import threading +import grpc + from apache_beam.coders import coder_impl from apache_beam.portability.api import beam_fn_api_pb2 -import grpc # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index 360468a86874..db7ac0b4705e 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -25,8 +25,8 @@ import sys import threading import unittest - from concurrent import futures + import grpc from apache_beam.portability.api import beam_fn_api_pb2 diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py index b8f635210d2e..20bd49f16a6d 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler.py +++ b/sdks/python/apache_beam/runners/worker/log_handler.py @@ -21,9 +21,10 @@ import Queue as queue import threading -from apache_beam.portability.api import beam_fn_api_pb2 import grpc +from apache_beam.portability.api import beam_fn_api_pb2 + # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 2256bb5556f0..7edf66742b84 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -18,8 +18,8 @@ import logging import unittest - from concurrent import futures + import grpc from apache_beam.portability.api import beam_fn_api_pb2 diff --git a/sdks/python/apache_beam/runners/worker/opcounters.py b/sdks/python/apache_beam/runners/worker/opcounters.py index 2bb15fa7ee4b..f8f4b51b80b0 100644 --- a/sdks/python/apache_beam/runners/worker/opcounters.py +++ b/sdks/python/apache_beam/runners/worker/opcounters.py @@ -20,6 +20,7 @@ """Counters collect the progress of the Worker for reporting to the service.""" from __future__ import absolute_import + import math import random diff --git a/sdks/python/apache_beam/runners/worker/opcounters_test.py b/sdks/python/apache_beam/runners/worker/opcounters_test.py index 74561b81593e..008720f9493a 100644 --- a/sdks/python/apache_beam/runners/worker/opcounters_test.py +++ b/sdks/python/apache_beam/runners/worker/opcounters_test.py @@ -25,7 +25,6 @@ from apache_beam.transforms.window import GlobalWindows from apache_beam.utils.counters import CounterFactory - # Classes to test that we can handle a variety of objects. # These have to be at top level so the pickler can find them. diff --git a/sdks/python/apache_beam/runners/worker/operations.py b/sdks/python/apache_beam/runners/worker/operations.py index c4f945bf2b9d..1b61f8e5f9be 100644 --- a/sdks/python/apache_beam/runners/worker/operations.py +++ b/sdks/python/apache_beam/runners/worker/operations.py @@ -35,11 +35,11 @@ from apache_beam.runners.worker import opcounters from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import sideinputs +from apache_beam.transforms import sideinputs as apache_sideinputs from apache_beam.transforms import combiners from apache_beam.transforms import core -from apache_beam.transforms import sideinputs as apache_sideinputs -from apache_beam.transforms.combiners import curry_combine_fn from apache_beam.transforms.combiners import PhasedCombineFnExecutor +from apache_beam.transforms.combiners import curry_combine_fn from apache_beam.transforms.window import GlobalWindows from apache_beam.utils.windowed_value import WindowedValue diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index dc72a5ff4fae..d371747798ca 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -23,8 +23,8 @@ import logging import unittest - from concurrent import futures + import grpc from apache_beam.portability.api import beam_fn_api_pb2 diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers.py b/sdks/python/apache_beam/testing/pipeline_verifiers.py index 883343acdab6..c421e259398f 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers.py @@ -32,7 +32,6 @@ from apache_beam.testing import test_utils as utils from apache_beam.utils import retry - __all__ = [ 'PipelineStateMatcher', 'FileChecksumMatcher', diff --git a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py index 15e0a042008b..3b02431212de 100644 --- a/sdks/python/apache_beam/testing/pipeline_verifiers_test.py +++ b/sdks/python/apache_beam/testing/pipeline_verifiers_test.py @@ -22,13 +22,14 @@ import unittest from hamcrest import assert_that as hc_assert_that -from mock import Mock, patch +from mock import Mock +from mock import patch from apache_beam.io.localfilesystem import LocalFileSystem from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineState -from apache_beam.testing.test_utils import patch_retry from apache_beam.testing import pipeline_verifiers as verifiers +from apache_beam.testing.test_utils import patch_retry try: # pylint: disable=wrong-import-order, wrong-import-position diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py index 83802423dbca..46eeb75183d7 100644 --- a/sdks/python/apache_beam/testing/test_pipeline.py +++ b/sdks/python/apache_beam/testing/test_pipeline.py @@ -20,12 +20,12 @@ import argparse import shlex +from nose.plugins.skip import SkipTest + from apache_beam.internal import pickler +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.pipeline import Pipeline from apache_beam.runners.runner import PipelineState -from apache_beam.options.pipeline_options import PipelineOptions -from nose.plugins.skip import SkipTest - __all__ = [ 'TestPipeline', diff --git a/sdks/python/apache_beam/testing/test_pipeline_test.py b/sdks/python/apache_beam/testing/test_pipeline_test.py index 747d64c7409f..c642c65a7b7d 100644 --- a/sdks/python/apache_beam/testing/test_pipeline_test.py +++ b/sdks/python/apache_beam/testing/test_pipeline_test.py @@ -20,12 +20,12 @@ import logging import unittest -from hamcrest.core.base_matcher import BaseMatcher from hamcrest.core.assert_that import assert_that as hc_assert_that +from hamcrest.core.base_matcher import BaseMatcher from apache_beam.internal import pickler -from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.testing.test_pipeline import TestPipeline # A simple matcher that is ued for testing extra options appending. diff --git a/sdks/python/apache_beam/testing/test_stream.py b/sdks/python/apache_beam/testing/test_stream.py index 7989fb2eee94..8a63e7bd0561 100644 --- a/sdks/python/apache_beam/testing/test_stream.py +++ b/sdks/python/apache_beam/testing/test_stream.py @@ -32,7 +32,6 @@ from apache_beam.utils import timestamp from apache_beam.utils.windowed_value import WindowedValue - __all__ = [ 'Event', 'ElementEvent', diff --git a/sdks/python/apache_beam/testing/test_stream_test.py b/sdks/python/apache_beam/testing/test_stream_test.py index b7ca141f0598..0f6691f3d644 100644 --- a/sdks/python/apache_beam/testing/test_stream_test.py +++ b/sdks/python/apache_beam/testing/test_stream_test.py @@ -27,7 +27,8 @@ from apache_beam.testing.test_stream import ProcessingTimeEvent from apache_beam.testing.test_stream import TestStream from apache_beam.testing.test_stream import WatermarkEvent -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.window import FixedWindows from apache_beam.transforms.window import TimestampedValue from apache_beam.utils import timestamp diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index 26ca03d1375d..41a02cfba9c3 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -22,12 +22,13 @@ import hashlib import imp -from mock import Mock, patch + +from mock import Mock +from mock import patch from apache_beam.io.filesystems import FileSystems from apache_beam.utils import retry - DEFAULT_HASHING_ALG = 'sha1' diff --git a/sdks/python/apache_beam/testing/test_utils_test.py b/sdks/python/apache_beam/testing/test_utils_test.py index bee0bd361e27..093e1f175d25 100644 --- a/sdks/python/apache_beam/testing/test_utils_test.py +++ b/sdks/python/apache_beam/testing/test_utils_test.py @@ -20,6 +20,7 @@ import logging import tempfile import unittest + from mock import patch from apache_beam.io.filesystem import BeamIOError diff --git a/sdks/python/apache_beam/testing/util.py b/sdks/python/apache_beam/testing/util.py index 959f25f31873..9bb18ccccb7f 100644 --- a/sdks/python/apache_beam/testing/util.py +++ b/sdks/python/apache_beam/testing/util.py @@ -27,11 +27,10 @@ from apache_beam.transforms.core import Create from apache_beam.transforms.core import Map from apache_beam.transforms.core import WindowInto -from apache_beam.transforms.util import CoGroupByKey from apache_beam.transforms.ptransform import PTransform +from apache_beam.transforms.util import CoGroupByKey from apache_beam.utils.annotations import experimental - __all__ = [ 'assert_that', 'equal_to', diff --git a/sdks/python/apache_beam/testing/util_test.py b/sdks/python/apache_beam/testing/util_test.py index 1acebb651c55..9d3869381b68 100644 --- a/sdks/python/apache_beam/testing/util_test.py +++ b/sdks/python/apache_beam/testing/util_test.py @@ -21,7 +21,9 @@ from apache_beam import Create from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to, is_empty +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.testing.util import is_empty class UtilTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/transforms/combiners.py b/sdks/python/apache_beam/transforms/combiners.py index 875306f80829..60bf2d1bfbc4 100644 --- a/sdks/python/apache_beam/transforms/combiners.py +++ b/sdks/python/apache_beam/transforms/combiners.py @@ -26,9 +26,9 @@ from apache_beam.transforms import cy_combiners from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayDataItem +from apache_beam.typehints import KV from apache_beam.typehints import Any from apache_beam.typehints import Dict -from apache_beam.typehints import KV from apache_beam.typehints import List from apache_beam.typehints import Tuple from apache_beam.typehints import TypeVariable @@ -36,7 +36,6 @@ from apache_beam.typehints import with_input_types from apache_beam.typehints import with_output_types - __all__ = [ 'Count', 'Mean', diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py index cd2b5956fef9..8885d27c84d6 100644 --- a/sdks/python/apache_beam/transforms/combiners_test.py +++ b/sdks/python/apache_beam/transforms/combiners_test.py @@ -22,9 +22,10 @@ import hamcrest as hc import apache_beam as beam -from apache_beam.testing.test_pipeline import TestPipeline import apache_beam.transforms.combiners as combine -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms.core import CombineGlobally from apache_beam.transforms.core import Create from apache_beam.transforms.core import Map diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index d6f56d2d0cf0..7f655a748384 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -25,12 +25,13 @@ from google.protobuf import wrappers_pb2 +from apache_beam import coders from apache_beam import pvalue from apache_beam import typehints -from apache_beam import coders from apache_beam.coders import typecoders from apache_beam.internal import pickler from apache_beam.internal import util +from apache_beam.options.pipeline_options import TypeOptions from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayDataItem @@ -38,25 +39,23 @@ from apache_beam.transforms.ptransform import PTransform from apache_beam.transforms.ptransform import PTransformWithSideInputs from apache_beam.transforms.window import MIN_TIMESTAMP +from apache_beam.transforms.window import GlobalWindows from apache_beam.transforms.window import TimestampCombiner -from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import TimestampedValue -from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn +from apache_beam.typehints import KV from apache_beam.typehints import Any from apache_beam.typehints import Iterable -from apache_beam.typehints import KV -from apache_beam.typehints import trivial_inference from apache_beam.typehints import Union -from apache_beam.typehints.decorators import get_type_hints +from apache_beam.typehints import trivial_inference from apache_beam.typehints.decorators import TypeCheckError from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.typehints.decorators import get_type_hints from apache_beam.typehints.trivial_inference import element_type from apache_beam.typehints.typehints import is_consistent_with from apache_beam.utils import proto_utils from apache_beam.utils import urns -from apache_beam.options.pipeline_options import TypeOptions - __all__ = [ 'DoFn', diff --git a/sdks/python/apache_beam/transforms/create_test.py b/sdks/python/apache_beam/transforms/create_test.py index 55ad7f32982d..e5863295ac17 100644 --- a/sdks/python/apache_beam/transforms/create_test.py +++ b/sdks/python/apache_beam/transforms/create_test.py @@ -18,12 +18,12 @@ """Unit tests for the Create and _CreateSource classes.""" import unittest -from apache_beam.io import source_test_utils - from apache_beam import Create from apache_beam.coders import FastPrimitivesCoder +from apache_beam.io import source_test_utils from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to class CreateTest(unittest.TestCase): diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py index 88a1feef75e1..cb7b53eb29aa 100644 --- a/sdks/python/apache_beam/transforms/display.py +++ b/sdks/python/apache_beam/transforms/display.py @@ -39,10 +39,10 @@ from __future__ import absolute_import import calendar -from datetime import datetime, timedelta import inspect import json - +from datetime import datetime +from datetime import timedelta __all__ = ['HasDisplayData', 'DisplayDataItem', 'DisplayData'] diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py index 15f178645bc3..5c73cf39a92f 100644 --- a/sdks/python/apache_beam/transforms/display_test.py +++ b/sdks/python/apache_beam/transforms/display_test.py @@ -19,17 +19,17 @@ from __future__ import absolute_import -from datetime import datetime import unittest +from datetime import datetime import hamcrest as hc from hamcrest.core.base_matcher import BaseMatcher import apache_beam as beam -from apache_beam.transforms.display import HasDisplayData +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.transforms.display import DisplayData from apache_beam.transforms.display import DisplayDataItem -from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.transforms.display import HasDisplayData class DisplayDataItemMatcher(BaseMatcher): diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index f6e08ca9c0ce..eccaccd27065 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -48,18 +48,17 @@ class and wrapper class that allows lambda functions to be used as from apache_beam import pvalue from apache_beam.internal import pickler from apache_beam.internal import util -from apache_beam.transforms.display import HasDisplayData from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.display import HasDisplayData from apache_beam.typehints import typehints -from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.decorators import TypeCheckError from apache_beam.typehints.decorators import WithTypeHints +from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.trivial_inference import instance_to_type from apache_beam.typehints.typehints import validate_composite_type_param from apache_beam.utils import proto_utils from apache_beam.utils import urns - __all__ = [ 'PTransform', 'ptransform_fn', diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index efc597877b29..435270e27cdb 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -27,25 +27,26 @@ from nose.plugins.attrib import attr import apache_beam as beam +import apache_beam.pvalue as pvalue +import apache_beam.transforms.combiners as combine +import apache_beam.typehints as typehints +from apache_beam.io.iobase import Read from apache_beam.metrics import Metrics from apache_beam.metrics.metric import MetricsFilter -from apache_beam.io.iobase import Read from apache_beam.options.pipeline_options import TypeOptions -import apache_beam.pvalue as pvalue from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import window from apache_beam.transforms.core import _GroupByKeyOnly -import apache_beam.transforms.combiners as combine -from apache_beam.transforms.display import DisplayData, DisplayDataItem +from apache_beam.transforms.display import DisplayData +from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.ptransform import PTransform -import apache_beam.typehints as typehints from apache_beam.typehints import with_input_types from apache_beam.typehints import with_output_types from apache_beam.typehints.typehints_test import TypeHintTestCase from apache_beam.utils.windowed_value import WindowedValue - # Disable frequent lint warning due to pipe operator for chaining transforms. # pylint: disable=expression-not-assigned diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py index 650068141852..0a69c3b6050e 100644 --- a/sdks/python/apache_beam/transforms/sideinputs_test.py +++ b/sdks/python/apache_beam/transforms/sideinputs_test.py @@ -24,7 +24,8 @@ import apache_beam as beam from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import window diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py index c0f91981d7d6..8d63d49baad0 100644 --- a/sdks/python/apache_beam/transforms/timeutil.py +++ b/sdks/python/apache_beam/transforms/timeutil.py @@ -22,7 +22,6 @@ from abc import ABCMeta from abc import abstractmethod - __all__ = [ 'TimeDomain', ] diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index c1fbfc54a3b8..84d5be9b27bf 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -20,13 +20,14 @@ Triggers control when in processing time windows get emitted. """ -from abc import ABCMeta -from abc import abstractmethod import collections import copy import itertools +from abc import ABCMeta +from abc import abstractmethod from apache_beam.coders import observable +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import combiners from apache_beam.transforms import core from apache_beam.transforms.timeutil import TimeDomain @@ -34,7 +35,6 @@ from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn -from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP from apache_beam.utils.timestamp import TIME_GRANULARITY diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py index 1ae1f0250320..b2fd761f7fd9 100644 --- a/sdks/python/apache_beam/transforms/trigger_test.py +++ b/sdks/python/apache_beam/transforms/trigger_test.py @@ -27,25 +27,26 @@ import apache_beam as beam from apache_beam.runners import pipeline_context from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import trigger from apache_beam.transforms.core import Windowing from apache_beam.transforms.trigger import AccumulationMode from apache_beam.transforms.trigger import AfterAll +from apache_beam.transforms.trigger import AfterAny from apache_beam.transforms.trigger import AfterCount from apache_beam.transforms.trigger import AfterEach -from apache_beam.transforms.trigger import AfterAny from apache_beam.transforms.trigger import AfterWatermark from apache_beam.transforms.trigger import DefaultTrigger from apache_beam.transforms.trigger import GeneralTriggerDriver from apache_beam.transforms.trigger import InMemoryUnmergedState from apache_beam.transforms.trigger import Repeatedly from apache_beam.transforms.trigger import TriggerFn +from apache_beam.transforms.window import MIN_TIMESTAMP from apache_beam.transforms.window import FixedWindows from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms.window import MIN_TIMESTAMP -from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import Sessions +from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index a7484acb2cc7..81b8c228ca27 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -27,7 +27,6 @@ from apache_beam.transforms.ptransform import PTransform from apache_beam.transforms.ptransform import ptransform_fn - __all__ = [ 'CoGroupByKey', 'Keys', diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 458fb747eb22..a025019b3e72 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -60,13 +60,12 @@ from apache_beam.transforms import timeutil from apache_beam.utils import proto_utils from apache_beam.utils import urns -from apache_beam.utils.timestamp import Duration from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Duration from apache_beam.utils.timestamp import Timestamp from apache_beam.utils.windowed_value import WindowedValue - __all__ = [ 'TimestampCombiner', 'WindowFn', diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py index 977a364ad883..71c0622dcee4 100644 --- a/sdks/python/apache_beam/transforms/window_test.py +++ b/sdks/python/apache_beam/transforms/window_test.py @@ -21,14 +21,15 @@ from apache_beam.runners import pipeline_context from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.transforms import CombinePerKey -from apache_beam.transforms import combiners -from apache_beam.transforms import core from apache_beam.transforms import Create from apache_beam.transforms import GroupByKey from apache_beam.transforms import Map from apache_beam.transforms import WindowInto +from apache_beam.transforms import combiners +from apache_beam.transforms import core from apache_beam.transforms.core import Windowing from apache_beam.transforms.trigger import AccumulationMode from apache_beam.transforms.trigger import AfterCount @@ -36,9 +37,9 @@ from apache_beam.transforms.window import GlobalWindow from apache_beam.transforms.window import GlobalWindows from apache_beam.transforms.window import IntervalWindow -from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import Sessions from apache_beam.transforms.window import SlidingWindows +from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn diff --git a/sdks/python/apache_beam/transforms/write_ptransform_test.py b/sdks/python/apache_beam/transforms/write_ptransform_test.py index 50f0debb0a70..c2a200516ee3 100644 --- a/sdks/python/apache_beam/transforms/write_ptransform_test.py +++ b/sdks/python/apache_beam/transforms/write_ptransform_test.py @@ -20,10 +20,10 @@ import unittest import apache_beam as beam - from apache_beam.io import iobase from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, is_empty +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import is_empty from apache_beam.transforms.ptransform import PTransform diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index 3f5b4c9c479b..694433ae7184 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -88,12 +88,11 @@ def foo((a, b)): from apache_beam.typehints import native_type_compatibility from apache_beam.typehints import typehints -from apache_beam.typehints.typehints import check_constraint from apache_beam.typehints.typehints import CompositeTypeHintError from apache_beam.typehints.typehints import SimpleTypeHintError +from apache_beam.typehints.typehints import check_constraint from apache_beam.typehints.typehints import validate_composite_type_param - __all__ = [ 'with_input_types', 'with_output_types', diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility.py b/sdks/python/apache_beam/typehints/native_type_compatibility.py index 26c584e380d2..8a8e07ecb4be 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility.py @@ -19,6 +19,7 @@ import collections import typing + from apache_beam.typehints import typehints # Describes an entry in the type map in convert_to_beam_type. diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 0ff2b3b5e58d..4171507f3459 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -17,12 +17,11 @@ """Test for Beam type compatibility library.""" -import unittest - import typing +import unittest -from apache_beam.typehints import typehints from apache_beam.typehints import native_type_compatibility +from apache_beam.typehints import typehints _TestNamedTuple = typing.NamedTuple('_TestNamedTuple', [('age', int), ('name', bytes)]) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index 83f444c6c53b..923b848bf9bb 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -28,9 +28,17 @@ """ import types -from trivial_inference import union, element_type, Const, BoundMethod import typehints -from typehints import Any, Dict, Iterable, List, Tuple, Union +from trivial_inference import BoundMethod +from trivial_inference import Const +from trivial_inference import element_type +from trivial_inference import union +from typehints import Any +from typehints import Dict +from typehints import Iterable +from typehints import List +from typehints import Tuple +from typehints import Union def pop_one(state, unused_arg): diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index e7f451da11a6..38259c88c7b5 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -18,7 +18,6 @@ """Tests for apache_beam.typehints.trivial_inference.""" import unittest - from apache_beam.typehints import trivial_inference from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index 89a5f5c7e2cf..c47e9bacb960 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -28,13 +28,13 @@ from apache_beam.pvalue import TaggedOutput from apache_beam.transforms.core import DoFn from apache_beam.transforms.window import WindowedValue -from apache_beam.typehints.decorators import _check_instance_type -from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.decorators import GeneratorWrapper from apache_beam.typehints.decorators import TypeCheckError -from apache_beam.typehints.typehints import check_constraint +from apache_beam.typehints.decorators import _check_instance_type +from apache_beam.typehints.decorators import getcallargs_forhints from apache_beam.typehints.typehints import CompositeTypeHintError from apache_beam.typehints.typehints import SimpleTypeHintError +from apache_beam.typehints.typehints import check_constraint class AbstractDoFnWrapper(DoFn): diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 59d1e1c5714e..1df11045ae72 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -17,16 +17,16 @@ """Unit tests for the type-hint objects and decorators.""" import inspect -import unittest - import typing +import unittest import apache_beam as beam from apache_beam import pvalue from apache_beam import typehints from apache_beam.options.pipeline_options import OptionsContext from apache_beam.testing.test_pipeline import TestPipeline -from apache_beam.testing.util import assert_that, equal_to +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to from apache_beam.typehints import WithTypeHints # These test often construct a pipeline as value | PTransform to test side diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py index 98d399b82f3b..c19916fa3cc2 100644 --- a/sdks/python/apache_beam/typehints/typehints.py +++ b/sdks/python/apache_beam/typehints/typehints.py @@ -67,7 +67,6 @@ import copy import types - __all__ = [ 'Any', 'Union', diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index f1b92e096264..af575f4aba42 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -20,7 +20,6 @@ import inspect import unittest - import apache_beam.typehints.typehints as typehints from apache_beam.typehints import Any from apache_beam.typehints import Tuple @@ -28,12 +27,12 @@ from apache_beam.typehints import Union from apache_beam.typehints import with_input_types from apache_beam.typehints import with_output_types +from apache_beam.typehints.decorators import GeneratorWrapper from apache_beam.typehints.decorators import _check_instance_type from apache_beam.typehints.decorators import _interleave_type_check from apache_beam.typehints.decorators import _positional_arg_hints from apache_beam.typehints.decorators import get_type_hints from apache_beam.typehints.decorators import getcallargs_forhints -from apache_beam.typehints.decorators import GeneratorWrapper from apache_beam.typehints.typehints import is_consistent_with diff --git a/sdks/python/apache_beam/utils/annotations_test.py b/sdks/python/apache_beam/utils/annotations_test.py index 32af8a991a79..ddd1b9ff7571 100644 --- a/sdks/python/apache_beam/utils/annotations_test.py +++ b/sdks/python/apache_beam/utils/annotations_test.py @@ -17,6 +17,7 @@ import unittest import warnings + from apache_beam.utils.annotations import deprecated from apache_beam.utils.annotations import experimental diff --git a/sdks/python/apache_beam/utils/counters.py b/sdks/python/apache_beam/utils/counters.py index 08685aae2759..ae9743442590 100644 --- a/sdks/python/apache_beam/utils/counters.py +++ b/sdks/python/apache_beam/utils/counters.py @@ -23,12 +23,11 @@ For internal use only; no backwards-compatibility guarantees. """ -from collections import namedtuple import threading +from collections import namedtuple from apache_beam.transforms import cy_combiners - # Information identifying the IO being measured by a counter. IOTargetName = namedtuple('IOTargetName', ['side_input_step_name', 'side_input_index', diff --git a/sdks/python/apache_beam/utils/processes_test.py b/sdks/python/apache_beam/utils/processes_test.py index 920b621b1bbd..2dd45f44dc53 100644 --- a/sdks/python/apache_beam/utils/processes_test.py +++ b/sdks/python/apache_beam/utils/processes_test.py @@ -18,7 +18,6 @@ import unittest - import mock from apache_beam.utils import processes diff --git a/sdks/python/apache_beam/utils/retry_test.py b/sdks/python/apache_beam/utils/retry_test.py index 1b03c835cd7f..e5f07e88b420 100644 --- a/sdks/python/apache_beam/utils/retry_test.py +++ b/sdks/python/apache_beam/utils/retry_test.py @@ -19,6 +19,8 @@ import unittest +from apache_beam.utils import retry + # Protect against environments where apitools library is not available. # pylint: disable=wrong-import-order, wrong-import-position # TODO(sourabhbajaj): Remove the GCP specific error code to a submodule @@ -29,9 +31,6 @@ # pylint: enable=wrong-import-order, wrong-import-position -from apache_beam.utils import retry - - class FakeClock(object): """A fake clock object implementing sleep() and recording calls.""" diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py index 18959be87503..7675d0548cfb 100644 --- a/sdks/python/apache_beam/utils/urns.py +++ b/sdks/python/apache_beam/utils/urns.py @@ -25,7 +25,6 @@ from apache_beam.internal import pickler from apache_beam.utils import proto_utils - PICKLED_WINDOW_FN = "beam:windowfn:pickled_python:v0.1" GLOBAL_WINDOWS_FN = "beam:windowfn:global_windows:v0.1" FIXED_WINDOWS_FN = "beam:windowfn:fixed_windows:v0.1" diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 2691be4ea42c..df71e4402176 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -62,3 +62,23 @@ echo "Running pylint for module $MODULE:" pylint $MODULE --ignore-patterns="$FILES_TO_IGNORE" echo "Running pycodestyle for module $MODULE:" pycodestyle $MODULE --exclude="$FILES_TO_IGNORE" +echo "Running isort for module $MODULE:" +# Skip files where isort is behaving weirdly +ISORT_EXCLUDED=( + "apiclient.py" + "avroio_test.py" + "datastore_wordcount.py" + "iobase_test.py" + "fast_coders_test.py" + "slow_coders_test.py" +) +SKIP_PARAM="" +for file in "${ISORT_EXCLUDED[@]}"; do + SKIP_PARAM="$SKIP_PARAM --skip $file" +done +for file in "${EXCLUDED_GENERATED_FILES[@]}"; do + SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)" +done +pushd $MODULE +isort -p apache_beam -w 120 -y -c -ot -cs -sl ${SKIP_PARAM} +popd