Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/coders/coder_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@
from types import NoneType

from apache_beam.coders import observable
from apache_beam.utils.timestamp import Timestamp
from apache_beam.utils import windowed_value
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
from apache_beam.utils import windowed_value
from apache_beam.utils.timestamp import Timestamp

# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
try:
Expand Down
3 changes: 2 additions & 1 deletion sdks/python/apache_beam/coders/coders.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@

import base64
import cPickle as pickle

import google.protobuf

from apache_beam.coders import coder_impl
from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.utils import urns
from apache_beam.utils import proto_utils
from apache_beam.utils import urns

# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
try:
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/coders/coders_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
import logging
import unittest

from apache_beam.coders import coders
from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
from apache_beam.coders import coders
from apache_beam.coders.typecoders import registry as coders_registry


Expand Down
9 changes: 4 additions & 5 deletions sdks/python/apache_beam/coders/coders_test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,15 @@

import dill

from apache_beam.transforms.window import GlobalWindow
from apache_beam.utils.timestamp import MIN_TIMESTAMP
import observable
from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
from apache_beam.coders import coders
from apache_beam.runners import pipeline_context
from apache_beam.transforms import window
from apache_beam.transforms.window import GlobalWindow
from apache_beam.utils import timestamp
from apache_beam.utils import windowed_value

from apache_beam.coders import coders
from apache_beam.coders import proto2_coder_test_messages_pb2 as test_message
from apache_beam.utils.timestamp import MIN_TIMESTAMP


# Defined out of line for picklability.
Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/coders/observable_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import logging
import unittest


from apache_beam.coders import observable


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@
# source: sdks/java/core/src/main/proto/proto2_coder_test_messages.proto

import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))

from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2

_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()




DESCRIPTOR = _descriptor.FileDescriptor(
name='apache_beam/coders/proto2_coder_test_messages.proto',
package='proto2_coder_test_messages',
Expand Down
6 changes: 3 additions & 3 deletions sdks/python/apache_beam/coders/standard_coders_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@

import yaml

from apache_beam.coders import coders
from apache_beam.coders import coder_impl
from apache_beam.coders import coders
from apache_beam.transforms import window
from apache_beam.transforms.window import IntervalWindow
from apache_beam.utils import windowed_value
from apache_beam.utils.timestamp import Timestamp
from apache_beam.transforms.window import IntervalWindow
from apache_beam.transforms import window

STANDARD_CODERS_YAML = os.path.join(
os.path.dirname(__file__), '..', 'testing', 'data', 'standard_coders.yaml')
Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/coders/stream_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import math
import unittest


from apache_beam.coders import slow_stream


Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/coders/typecoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ def MakeXyzs(v):
from apache_beam.coders import coders
from apache_beam.typehints import typehints


__all__ = ['registry']


Expand Down
5 changes: 2 additions & 3 deletions sdks/python/apache_beam/examples/complete/estimate_pi.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,13 @@
import logging
import random


import apache_beam as beam
from apache_beam.io import WriteToText
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.typehints import Any
from apache_beam.typehints import Iterable
from apache_beam.typehints import Tuple
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions


@beam.typehints.with_output_types(Tuple[int, int, int])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@

from apache_beam.examples.complete import estimate_pi
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import BeamAssertException
from apache_beam.testing.util import assert_that


def in_between(lower, upper):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import tempfile
import unittest


from apache_beam.examples.complete.juliaset.juliaset import juliaset
from apache_beam.testing.util import open_shards

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,8 @@

import logging


from apache_beam.examples.complete.juliaset.juliaset import juliaset


if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
juliaset.run()
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
when running the workflow for remote execution.
"""

from distutils.command.build import build as _build
import subprocess
from distutils.command.build import build as _build

import setuptools

Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/examples/complete/tfidf.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import apache_beam as beam
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.pvalue import AsSingleton
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.pvalue import AsSingleton


def read_documents(pipeline, uris):
Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/examples/complete/tfidf_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
from apache_beam.testing.util import equal_to
from apache_beam.testing.util import open_shards


EXPECTED_RESULTS = set([
('ghi', '1.txt', 0.3662040962227032),
('abc', '1.txt', 0.0),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,11 @@
from apache_beam import combiners
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.window import Sessions
from apache_beam.transforms.window import TimestampedValue
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions


ONE_HOUR_IN_SECONDS = 3600
THIRTY_DAYS_IN_SECONDS = 30 * 24 * ONE_HOUR_IN_SECONDS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import json
import unittest


import apache_beam as beam
from apache_beam.examples.complete import top_wikipedia_sessions
from apache_beam.testing.test_pipeline import TestPipeline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,11 @@
from random import randrange

import apache_beam as beam

from apache_beam.io import WriteToText
from apache_beam.pvalue import AsList
from apache_beam.pvalue import AsSingleton
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.pvalue import AsList
from apache_beam.pvalue import AsSingleton


def create_groups(group_ids, corpus, word, ignore_corpus, ignore_word):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
from nose.plugins.attrib import attr

from apache_beam.examples.cookbook import bigquery_tornadoes
from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher
from apache_beam.io.gcp.tests import utils
from apache_beam.io.gcp.tests.bigquery_matcher import BigqueryMatcher
from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@
from apache_beam.io import WriteToText
from apache_beam.options.pipeline_options import PipelineOptions


# pylint doesn't understand our pipeline syntax:
# pylint:disable=expression-not-assigned


class Count1(beam.PTransform):
"""Count as a subclass of PTransform, with an apply method."""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@

from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.proto.datastore.v1 import query_pb2
from googledatastore import helper as datastore_helper, PropertyFilter
from googledatastore import helper as datastore_helper
from googledatastore import PropertyFilter

import apache_beam as beam
from apache_beam.io import ReadFromText
Expand Down
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/examples/cookbook/group_with_coder.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@
from apache_beam import coders
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.typehints import typehints
from apache_beam.typehints.decorators import with_output_types
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.typehints import typehints
from apache_beam.typehints.decorators import with_output_types


class Player(object):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from apache_beam.examples.cookbook import group_with_coder
from apache_beam.testing.util import open_shards


# Patch group_with_coder.PlayerCoder.decode(). To test that the PlayerCoder was
# used, we do not strip the prepended 'x:' string when decoding a Player object.
group_with_coder.PlayerCoder.decode = lambda self, s: group_with_coder.Player(s)
Expand Down
11 changes: 4 additions & 7 deletions sdks/python/apache_beam/examples/snippets/snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,14 @@
"""

import apache_beam as beam
from apache_beam.io import iobase
from apache_beam.io.range_trackers import OffsetRangeTracker
from apache_beam.metrics import Metrics
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms.core import PTransform

# Quiet some pylint warnings that happen because of the somewhat special
# format for the code snippets.
Expand Down Expand Up @@ -655,13 +659,6 @@ def process(self, an_int):
result.wait_until_finish()


import apache_beam as beam
from apache_beam.io import iobase
from apache_beam.io.range_trackers import OffsetRangeTracker
from apache_beam.transforms.core import PTransform
from apache_beam.options.pipeline_options import PipelineOptions


# Defining a new source.
# [START model_custom_source_new_source]
class CountingSource(iobase.BoundedSource):
Expand Down
6 changes: 2 additions & 4 deletions sdks/python/apache_beam/examples/snippets/snippets_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,15 @@
from apache_beam import pvalue
from apache_beam import typehints
from apache_beam.coders.coders import ToStringCoder
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.examples.snippets import snippets
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.utils.windowed_value import WindowedValue

# pylint: disable=expression-not-assigned
from apache_beam.testing.test_pipeline import TestPipeline

# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
try:
Expand Down
3 changes: 1 addition & 2 deletions sdks/python/apache_beam/examples/streaming_wordcount.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@
import argparse
import logging


import apache_beam as beam
import apache_beam.transforms.window as window
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import StandardOptions
import apache_beam.transforms.window as window


def split_fn(lines):
Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/examples/windowed_wordcount.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import argparse
import logging


import apache_beam as beam
import apache_beam.transforms.window as window

Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/examples/wordcount_it_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
from nose.plugins.attrib import attr

from apache_beam.examples import wordcount
from apache_beam.testing.test_utils import delete_files
from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher
from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.test_utils import delete_files


class WordCountIT(unittest.TestCase):
Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/internal/gcp/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@

from apache_beam.utils import retry


# When we are running in GCE, we can authenticate with VM credentials.
is_running_in_gce = False

Expand Down
1 change: 0 additions & 1 deletion sdks/python/apache_beam/internal/gcp/json_value.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@

from apache_beam.options.value_provider import ValueProvider


_MAXINT64 = (1 << 63) - 1
_MININT64 = - (1 << 63)

Expand Down
3 changes: 1 addition & 2 deletions sdks/python/apache_beam/internal/gcp/json_value_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@

from apache_beam.internal.gcp.json_value import from_json_value
from apache_beam.internal.gcp.json_value import to_json_value
from apache_beam.options.value_provider import StaticValueProvider
from apache_beam.options.value_provider import RuntimeValueProvider

from apache_beam.options.value_provider import StaticValueProvider

# Protect against environments where apitools library is not available.
# pylint: disable=wrong-import-order, wrong-import-position
Expand Down
Loading