From 427be241cd8981ad426c20ea0c7d1fd36d436aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 10:00:39 +0200 Subject: [PATCH 01/73] used exec-maven-plugin to map maven's lifecycle phases with python's setuptools --- sdks/pom.xml | 4 +- sdks/python/pom.xml | 91 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 sdks/python/pom.xml diff --git a/sdks/pom.xml b/sdks/pom.xml index aa027ebda022..cef1aa0170de 100644 --- a/sdks/pom.xml +++ b/sdks/pom.xml @@ -34,6 +34,7 @@ java + python @@ -53,8 +54,9 @@ + - \ No newline at end of file + diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml new file mode 100644 index 000000000000..1fa5dbe76b5a --- /dev/null +++ b/sdks/python/pom.xml @@ -0,0 +1,91 @@ + + + + + 4.0.0 + + + org.apache.beam + beam-sdks-parent + 0.2.0-incubating-SNAPSHOT + ../pom.xml + + + beam-sdks-python + + pom + + Apache Beam :: SDKs :: Python + + + + + org.codehaus.mojo + exec-maven-plugin + + python + ${basedir} + + + + setuptools-clean + clean + + exec + + + + setup.py + clean + + + + + setuptools-build + compile + + exec + + + + setup.py + build + --build-base + target/build + + + + + setuptools-test + test + + exec + + + + setup.py + test + + + + + + + + + From 23f42c31029b70214e62b50d341de65a08a96eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 10:22:15 +0200 Subject: [PATCH 02/73] synced version from maven --- sdks/python/apache_beam/version.py | 20 -------------------- sdks/python/setup.py | 12 +++++++----- 2 files changed, 7 insertions(+), 25 deletions(-) delete mode 100644 sdks/python/apache_beam/version.py diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py deleted file mode 100644 index c4c99f3c1605..000000000000 --- a/sdks/python/apache_beam/version.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Apache Beam SDK version information.""" - -__version__ = '0.3.0' diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 029226e117cd..22ac248df097 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -36,11 +36,13 @@ def get_version(): - global_names = {} - execfile(os.path.normpath('./apache_beam/version.py'), - global_names) - return global_names['__version__'] - + from lxml import etree + pom = etree.parse('pom.xml') + elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) + version = elements[0].text + version = version.replace("-SNAPSHOT", ".dev") + # TODO: PEP 440 and incubatign suffix + return version # Configure the required packages and scripts to install. REQUIRED_PACKAGES = [ From 061ef0a5914f658c0754a49ccfd2089f38008df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 10:32:12 +0200 Subject: [PATCH 03/73] synced version from maven --- sdks/python/apache_beam/utils/dependency.py | 4 ++-- sdks/python/apache_beam/utils/dependency_test.py | 4 ++-- sdks/python/apache_beam/version.py | 11 ++++++++++- sdks/python/setup.py | 8 +------- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 1c6ad9cf1d42..13d64e1a0036 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -64,7 +64,7 @@ from apache_beam.utils import processes from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import SetupOptions -from apache_beam.version import __version__ +from apache_beam.version import get_version # Standard file names used for staging files. @@ -337,7 +337,7 @@ def stage_job_resources( # will not stage a tarball. if setup_options.sdk_location == 'default': sdk_remote_location = '%s/v%s.tar.gz' % ( - PACKAGES_URL_PREFIX, __version__) + PACKAGES_URL_PREFIX, get_version()) else: sdk_remote_location = setup_options.sdk_location _stage_dataflow_sdk_tarball(sdk_remote_location, staged_path, temp_dir) diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index 4edcaa7b2c7f..e15b5d446a56 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -29,7 +29,7 @@ from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import PipelineOptions from apache_beam.utils.options import SetupOptions -from apache_beam.version import __version__ +from apache_beam.version import get_version class SetupTest(unittest.TestCase): @@ -246,7 +246,7 @@ def file_download(from_url, _): def test_sdk_location_default(self): staging_dir = tempfile.mkdtemp() expected_from_url = '%s/v%s.tar.gz' % ( - dependency.PACKAGES_URL_PREFIX, __version__) + dependency.PACKAGES_URL_PREFIX, get_version()) expected_from_path = self.override_file_download( expected_from_url, staging_dir) self.override_file_copy(expected_from_path, staging_dir) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index c4c99f3c1605..8f7edce924cb 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,4 +17,13 @@ """Apache Beam SDK version information.""" -__version__ = '0.3.0' +from lxml import etree + + +def get_version(): + pom = etree.parse('pom.xml') + elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) + version = elements[0].text + version = version.replace("-SNAPSHOT", ".dev") + # TODO: PEP 440 and incubatign suffix + return version \ No newline at end of file diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 029226e117cd..d34c12238a40 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -21,6 +21,7 @@ import platform import re import setuptools +from apache_beam.version import get_version # Currently all compiled modules are optional (for performance only). @@ -35,13 +36,6 @@ cythonize = lambda *args, **kwargs: [] -def get_version(): - global_names = {} - execfile(os.path.normpath('./apache_beam/version.py'), - global_names) - return global_names['__version__'] - - # Configure the required packages and scripts to install. REQUIRED_PACKAGES = [ 'avro>=1.7.7', From 2acaed2b87d6a5b0e3e2416c73b32e6eb18b9dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 10:59:00 +0200 Subject: [PATCH 04/73] mapped package phase --- sdks/python/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 1fa5dbe76b5a..110b719a014f 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -83,6 +83,21 @@ + + setuptools-sdist + package + + exec + + + + setup.py + sdist + --dist-dir + target + + + From 43bcc7a899049a54bff2612a9607a61f4ddbec31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 11:00:41 +0200 Subject: [PATCH 05/73] fixed author metadata --- sdks/python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a9cc329ffb60..6cca79b362eb 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -64,7 +64,8 @@ long_description='', url='https://beam.incubator.apache.org', download_url='TBD', - author='Apache Software Foundation', + author='Apache Beam (incubating)', + author_email='dev@beam.incubator.apache.org', packages=setuptools.find_packages(), ext_modules=cythonize([ '**/*.pyx', From 231ff5446b5aaf77671e66a6a8e30059699bc4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 27 Jun 2016 16:33:45 +0200 Subject: [PATCH 06/73] trying to get travis build working --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ec74fa4189cb..73db7c2ee0ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,8 @@ addons: apt: packages: - python2.7 + - python-setuptools + - python-dill matrix: include: From 1d815bb910c7d6c0ebe1b548e01c61a9a4f07bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 28 Jun 2016 08:43:20 +0200 Subject: [PATCH 07/73] BEAM-378: removed unnecessary packages previously introduced in ci environment --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 73db7c2ee0ff..ec74fa4189cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,6 @@ addons: apt: packages: - python2.7 - - python-setuptools - - python-dill matrix: include: From 97b52add3c16d72da522531b5ba7970a1595bbe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 28 Jun 2016 08:50:53 +0200 Subject: [PATCH 08/73] reformattted file according PEP 8 --- sdks/python/setup.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 6cca79b362eb..9c3e45ca05c7 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -17,23 +17,21 @@ """Apache Beam SDK setup configuration.""" -import os import platform -import re import setuptools from apache_beam.version import get_version # Currently all compiled modules are optional (for performance only). if platform.system() == 'Windows': - # Windows doesn't always provide int64_t. - cythonize = lambda *args, **kwargs: [] -else: - try: - # pylint: disable=g-statement-before-imports,g-import-not-at-top - from Cython.Build import cythonize - except ImportError: + # Windows doesn't always provide int64_t. cythonize = lambda *args, **kwargs: [] +else: + try: + # pylint: disable=g-statement-before-imports,g-import-not-at-top + from Cython.Build import cythonize + except ImportError: + cythonize = lambda *args, **kwargs: [] # Configure the required packages and scripts to install. @@ -54,8 +52,7 @@ 'protorpc>=0.9.1', 'python-gflags>=2.0', 'pyyaml>=3.10', - ] - +] setuptools.setup( name='apache-beam-sdk', @@ -85,7 +82,7 @@ 'Programming Language :: Python :: 2.7', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', - ], + ], license='Apache 2.0', keywords='apache beam', - ) +) From d7874fe1415fd380c2eb987dfda6d006bedc1c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 28 Jun 2016 10:12:06 +0200 Subject: [PATCH 09/73] BEAM-378: trying to get right the version on both build and runtime --- sdks/python/MANIFEST.in | 1 + sdks/python/apache_beam/__init__.py | 7 ++--- sdks/python/apache_beam/utils/dependency.py | 5 ++-- .../apache_beam/utils/dependency_test.py | 4 +-- sdks/python/apache_beam/version.py | 13 +++++--- sdks/python/pom.xml | 30 ++++++++++++++++--- sdks/python/setup.py | 10 ++++--- 7 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 sdks/python/MANIFEST.in diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in new file mode 100644 index 000000000000..2713aa6cb8ad --- /dev/null +++ b/sdks/python/MANIFEST.in @@ -0,0 +1 @@ +include pom.xml diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index c9bc736cb171..a66a3e643f01 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -65,17 +65,14 @@ import sys - if sys.version_info.major != 2: raise RuntimeError( - 'Dataflow SDK for Python is supported only on Python 2.7. ' + 'Apache Bean SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) - -import apache_beam.internal.pickler - from apache_beam import coders from apache_beam import io from apache_beam import typehints from apache_beam.pipeline import Pipeline from apache_beam.transforms import * +import apache_beam.internal.pickler diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 13d64e1a0036..984c745ceac5 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -57,14 +57,13 @@ import shutil import tempfile - from apache_beam import utils +from apache_beam.version import get_packaged_version from apache_beam.internal import pickler from apache_beam.utils import names from apache_beam.utils import processes from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import SetupOptions -from apache_beam.version import get_version # Standard file names used for staging files. @@ -337,7 +336,7 @@ def stage_job_resources( # will not stage a tarball. if setup_options.sdk_location == 'default': sdk_remote_location = '%s/v%s.tar.gz' % ( - PACKAGES_URL_PREFIX, get_version()) + PACKAGES_URL_PREFIX, get_packaged_version()) else: sdk_remote_location = setup_options.sdk_location _stage_dataflow_sdk_tarball(sdk_remote_location, staged_path, temp_dir) diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index e15b5d446a56..77d3c0cec816 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -24,12 +24,12 @@ import unittest from apache_beam import utils +from apache_beam.version import get_packaged_version from apache_beam.utils import dependency from apache_beam.utils import names from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import PipelineOptions from apache_beam.utils.options import SetupOptions -from apache_beam.version import get_version class SetupTest(unittest.TestCase): @@ -246,7 +246,7 @@ def file_download(from_url, _): def test_sdk_location_default(self): staging_dir = tempfile.mkdtemp() expected_from_url = '%s/v%s.tar.gz' % ( - dependency.PACKAGES_URL_PREFIX, get_version()) + dependency.PACKAGES_URL_PREFIX, get_packaged_version()) expected_from_path = self.override_file_download( expected_from_url, staging_dir) self.override_file_copy(expected_from_path, staging_dir) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index 8f7edce924cb..b5277f34a3e6 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,13 +17,18 @@ """Apache Beam SDK version information.""" +import pkg_resources from lxml import etree -def get_version(): +def get_version_from_pom(): pom = etree.parse('pom.xml') elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) version = elements[0].text - version = version.replace("-SNAPSHOT", ".dev") - # TODO: PEP 440 and incubatign suffix - return version \ No newline at end of file + version = version.replace("-SNAPSHOT", ".dev") # PEP 440 + # TODO: PEP 440 and incubating suffix + return version + + +def get_packaged_version(): + return pkg_resources.get_distribution("apache_beam").version diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 110b719a014f..bd116e21b882 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -32,16 +32,34 @@ Apache Beam :: SDKs :: Python + + true + + + + + ${basedir} + + org.codehaus.mojo exec-maven-plugin - - python - ${basedir} - + + process-resources + process-resources + + exec + + + echo + + ${project.version} + + + setuptools-clean clean @@ -49,6 +67,7 @@ exec + python setup.py clean @@ -62,6 +81,7 @@ exec + python setup.py build @@ -77,6 +97,7 @@ exec + python setup.py test @@ -90,6 +111,7 @@ exec + python setup.py sdist diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 9c3e45ca05c7..cb4a090955ad 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -19,7 +19,7 @@ import platform import setuptools -from apache_beam.version import get_version +from apache_beam.version import get_version_from_pom # Currently all compiled modules are optional (for performance only). @@ -33,6 +33,7 @@ except ImportError: cythonize = lambda *args, **kwargs: [] +version = get_version_from_pom() # Configure the required packages and scripts to install. REQUIRED_PACKAGES = [ @@ -52,11 +53,12 @@ 'protorpc>=0.9.1', 'python-gflags>=2.0', 'pyyaml>=3.10', + 'lxml' ] setuptools.setup( name='apache-beam-sdk', - version=get_version(), + version=version, description='Apache Beam SDK for Python', long_description='', url='https://beam.incubator.apache.org', @@ -71,7 +73,7 @@ 'apache_beam/utils/counters.py', ]), install_requires=REQUIRED_PACKAGES, - package_data={'': ['*.pyx', '*.pxd']}, + package_data={'apache_beam': ['*.pyx', '*.pxd', 'pom.xml']}, test_suite='nose.collector', zip_safe=False, # PyPI package information. @@ -83,6 +85,6 @@ 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', ], - license='Apache 2.0', + license='Apache License, 2.0', keywords='apache beam', ) From 12fedce3ffb745975d50a066fd7f9e8eada3381c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 28 Jun 2016 11:03:43 +0200 Subject: [PATCH 10/73] BEAM-378: changed completelly the strategy: version is only read on setup, and updated to be statically used as apache_beam.__version__ later on --- sdks/python/apache_beam/__init__.py | 2 ++ sdks/python/apache_beam/utils/dependency.py | 4 ++-- .../apache_beam/utils/dependency_test.py | 4 ++-- sdks/python/apache_beam/version.py | 4 ++-- sdks/python/pom.xml | 22 ----------------- sdks/python/setup.py | 24 +++++++++++++++++-- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index a66a3e643f01..d015c18c8e7d 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -70,6 +70,8 @@ 'Apache Bean SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) +__version__ = "0.2.0-incubating.dev" # FIXME: PEP 440 + from apache_beam import coders from apache_beam import io from apache_beam import typehints diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 984c745ceac5..d2c1b54f81ac 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -57,8 +57,8 @@ import shutil import tempfile +from apache_beam import __version__ from apache_beam import utils -from apache_beam.version import get_packaged_version from apache_beam.internal import pickler from apache_beam.utils import names from apache_beam.utils import processes @@ -336,7 +336,7 @@ def stage_job_resources( # will not stage a tarball. if setup_options.sdk_location == 'default': sdk_remote_location = '%s/v%s.tar.gz' % ( - PACKAGES_URL_PREFIX, get_packaged_version()) + PACKAGES_URL_PREFIX, __version__) else: sdk_remote_location = setup_options.sdk_location _stage_dataflow_sdk_tarball(sdk_remote_location, staged_path, temp_dir) diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index 77d3c0cec816..42ce6f28045e 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -23,8 +23,8 @@ import tempfile import unittest +from apache_beam import __version__ from apache_beam import utils -from apache_beam.version import get_packaged_version from apache_beam.utils import dependency from apache_beam.utils import names from apache_beam.utils.options import GoogleCloudOptions @@ -246,7 +246,7 @@ def file_download(from_url, _): def test_sdk_location_default(self): staging_dir = tempfile.mkdtemp() expected_from_url = '%s/v%s.tar.gz' % ( - dependency.PACKAGES_URL_PREFIX, get_packaged_version()) + dependency.PACKAGES_URL_PREFIX, __version__) expected_from_path = self.override_file_download( expected_from_url, staging_dir) self.override_file_copy(expected_from_path, staging_dir) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index b5277f34a3e6..d4c5df3b05b0 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -15,7 +15,7 @@ # limitations under the License. # -"""Apache Beam SDK version information.""" +"""Apache Beam SDK version utilities.""" import pkg_resources from lxml import etree @@ -30,5 +30,5 @@ def get_version_from_pom(): return version -def get_packaged_version(): +def get_distribution_version(): return pkg_resources.get_distribution("apache_beam").version diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index bd116e21b882..7f63ebe01a93 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -32,34 +32,12 @@ Apache Beam :: SDKs :: Python - - true - - - - - ${basedir} - - org.codehaus.mojo exec-maven-plugin - - process-resources - process-resources - - exec - - - echo - - ${project.version} - - - setuptools-clean clean diff --git a/sdks/python/setup.py b/sdks/python/setup.py index cb4a090955ad..ea009aa35243 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -19,10 +19,11 @@ import platform import setuptools +import re from apache_beam.version import get_version_from_pom -# Currently all compiled modules are optional (for performance only). +# Currently all compiled modules are optional (for performance only). if platform.system() == 'Windows': # Windows doesn't always provide int64_t. cythonize = lambda *args, **kwargs: [] @@ -33,7 +34,26 @@ except ImportError: cythonize = lambda *args, **kwargs: [] -version = get_version_from_pom() + +# Reads the actual version from pom.xml file, and synchronizes +# apache_beam.__version__ field for later usage. +def sync_version(): + version = get_version_from_pom() + init_path = "apache_beam/__init__.py" + regex = r'^__version__\s*=\s*".*"' + with open(init_path, "r") as f: + lines = f.readlines() + with open(init_path, "w") as f: + for line in lines: + if re.search(regex, line): + f.write(re.sub(regex, '__version__ = "%s"' % version, line)) + else: + f.write(line) + return version + + +version = sync_version() + # Configure the required packages and scripts to install. REQUIRED_PACKAGES = [ From 831d4b046b6a11c9bfc9b7e7ad972b3931087dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 29 Jun 2016 09:50:47 +0200 Subject: [PATCH 11/73] BEAM-378: fixed install_requires order --- sdks/python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index ea009aa35243..6791b98d103d 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -67,13 +67,13 @@ def sync_version(): # 'google-apitools-dataflow-v1b3>=0.4.20160217', # 'google-apitools-storage-v1', 'httplib2>=0.8', + 'lxml', 'mock>=1.0.1', 'nose>=1.0', 'oauth2client>=2.0.1', 'protorpc>=0.9.1', 'python-gflags>=2.0', - 'pyyaml>=3.10', - 'lxml' + 'pyyaml>=3.10' ] setuptools.setup( From d676108de9ec2d439ec75659c5716044def892fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 29 Jun 2016 09:55:49 +0200 Subject: [PATCH 12/73] BEAM-378: reverted PEP-8 change, since @silviulica pointed they currently use 2 spaces for indentation --- sdks/python/apache_beam/version.py | 14 +++++------ sdks/python/setup.py | 38 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index d4c5df3b05b0..7b5ff7c5b011 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -22,13 +22,13 @@ def get_version_from_pom(): - pom = etree.parse('pom.xml') - elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) - version = elements[0].text - version = version.replace("-SNAPSHOT", ".dev") # PEP 440 - # TODO: PEP 440 and incubating suffix - return version + pom = etree.parse('pom.xml') + elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) + version = elements[0].text + version = version.replace("-SNAPSHOT", ".dev") # PEP 440 + # TODO: PEP 440 and incubating suffix + return version def get_distribution_version(): - return pkg_resources.get_distribution("apache_beam").version + return pkg_resources.get_distribution("apache_beam").version diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 6791b98d103d..04191ed74040 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -25,31 +25,31 @@ # Currently all compiled modules are optional (for performance only). if platform.system() == 'Windows': - # Windows doesn't always provide int64_t. - cythonize = lambda *args, **kwargs: [] + # Windows doesn't always provide int64_t. + cythonize = lambda *args, **kwargs: [] else: - try: - # pylint: disable=g-statement-before-imports,g-import-not-at-top - from Cython.Build import cythonize - except ImportError: - cythonize = lambda *args, **kwargs: [] + try: + # pylint: disable=g-statement-before-imports,g-import-not-at-top + from Cython.Build import cythonize + except ImportError: + cythonize = lambda *args, **kwargs: [] # Reads the actual version from pom.xml file, and synchronizes # apache_beam.__version__ field for later usage. def sync_version(): - version = get_version_from_pom() - init_path = "apache_beam/__init__.py" - regex = r'^__version__\s*=\s*".*"' - with open(init_path, "r") as f: - lines = f.readlines() - with open(init_path, "w") as f: - for line in lines: - if re.search(regex, line): - f.write(re.sub(regex, '__version__ = "%s"' % version, line)) - else: - f.write(line) - return version + version = get_version_from_pom() + init_path = "apache_beam/__init__.py" + regex = r'^__version__\s*=\s*".*"' + with open(init_path, "r") as f: + lines = f.readlines() + with open(init_path, "w") as f: + for line in lines: + if re.search(regex, line): + f.write(re.sub(regex, '__version__ = "%s"' % version, line)) + else: + f.write(line) + return version version = sync_version() From 79b69fd59b0434ff612daa6f293f593f3bf01157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 29 Jun 2016 09:57:43 +0200 Subject: [PATCH 13/73] BEAM-378: added a comment about the default apache_beam.__version__ --- sdks/python/apache_beam/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index d015c18c8e7d..5c9b23b9b375 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -70,7 +70,7 @@ 'Apache Bean SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) -__version__ = "0.2.0-incubating.dev" # FIXME: PEP 440 +__version__ = "0.2.0.dev" # this is the default value, but it will be correctly updated by setup.py from apache_beam import coders from apache_beam import io From 18950476a4d88505d3c5d4ea4117035117238252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 29 Jun 2016 10:08:35 +0200 Subject: [PATCH 14/73] BEAM-378: switched to tox as target for 'mvn test' phase --- sdks/python/pom.xml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 7f63ebe01a93..2678840a9856 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -75,10 +75,12 @@ exec - python + tox - setup.py - test + -e + py27 + c + tox.ini From dcb5ea32e989a30b5319cd6cacf2cc817b5de821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 29 Jun 2016 10:22:06 +0200 Subject: [PATCH 15/73] BEAM-378: fixed tox cofig file argument --- sdks/python/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 2678840a9856..21a714628dc5 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -79,7 +79,7 @@ -e py27 - c + -c tox.ini From 137c4a0fb400c5636b55573efee704fbb260e6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 08:24:19 +0200 Subject: [PATCH 16/73] BEAM-378: fixed typo --- sdks/python/apache_beam/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index b93802f3b95b..426380bf6543 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -67,11 +67,11 @@ if sys.version_info.major != 2: raise RuntimeError( - 'Apache Bean SDK for Python is supported only on Python 2.7. ' + 'Apache Beam SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) -__version__ = "0.2.0.dev" # this is the default value, but it will be correctly updated by setup.py +__version__ = "0.2.0-incubating.dev" # this is the default value, but it will be correctly updated by setup.py from apache_beam import coders From a0a788ebaf7f96a5e2d1df50fc98339dfd6d238f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 08:34:41 +0200 Subject: [PATCH 17/73] BEAM-378: remove the depencency on apache_beam module from setup.py (now the implementation of parsing pom.xml is directly there) --- sdks/python/setup.py | 7 +++++-- sdks/python/tox.ini | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 04191ed74040..85ed0d6e7086 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -20,7 +20,7 @@ import platform import setuptools import re -from apache_beam.version import get_version_from_pom +from lxml import etree # Currently all compiled modules are optional (for performance only). @@ -38,7 +38,10 @@ # Reads the actual version from pom.xml file, and synchronizes # apache_beam.__version__ field for later usage. def sync_version(): - version = get_version_from_pom() + pom = etree.parse('pom.xml') + elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) + version = elements[0].text.replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix + init_path = "apache_beam/__init__.py" regex = r'^__version__\s*=\s*".*"' with open(init_path, "r") as f: diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 29674ed773da..ecb35916dcdb 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -30,6 +30,7 @@ exclude = windmill_pb2.py, windmill_service_pb2.py deps= pep8 pylint + lxml commands = python setup.py test {toxinidir}/run_pylint.sh From 65fde8d18ef1f708c766475cc8e18e16353a6e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 08:35:49 +0200 Subject: [PATCH 18/73] BEAM-378: clean-up --- sdks/python/apache_beam/version.py | 34 ------------------------------ sdks/python/setup.py | 1 - 2 files changed, 35 deletions(-) delete mode 100644 sdks/python/apache_beam/version.py diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py deleted file mode 100644 index 7b5ff7c5b011..000000000000 --- a/sdks/python/apache_beam/version.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Apache Beam SDK version utilities.""" - -import pkg_resources -from lxml import etree - - -def get_version_from_pom(): - pom = etree.parse('pom.xml') - elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) - version = elements[0].text - version = version.replace("-SNAPSHOT", ".dev") # PEP 440 - # TODO: PEP 440 and incubating suffix - return version - - -def get_distribution_version(): - return pkg_resources.get_distribution("apache_beam").version diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 85ed0d6e7086..e89597e5772d 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -70,7 +70,6 @@ def sync_version(): # 'google-apitools-dataflow-v1b3>=0.4.20160217', # 'google-apitools-storage-v1', 'httplib2>=0.8', - 'lxml', 'mock>=1.0.1', 'nose>=1.0', 'oauth2client>=2.0.1', From c9529646cb3e251f997cb2614dfa3efbda71bcee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 08:39:18 +0200 Subject: [PATCH 19/73] BEAM-378: fixed pylint warnings --- sdks/python/apache_beam/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 426380bf6543..bb67fa216cc7 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -63,17 +63,7 @@ """ -import sys - -if sys.version_info.major != 2: - raise RuntimeError( - 'Apache Beam SDK for Python is supported only on Python 2.7. ' - 'It is not supported on Python [%s].' % sys.version) - - -__version__ = "0.2.0-incubating.dev" # this is the default value, but it will be correctly updated by setup.py - - +from sys import version_info from apache_beam import coders from apache_beam import io from apache_beam import typehints @@ -81,3 +71,12 @@ from apache_beam.transforms import * import apache_beam.internal.pickler + +# this is the default value, but it will be correctly updated by setup.py +__version__ = "0.2.0-incubating.dev" + + +if version_info.major != 2: + raise RuntimeError( + 'Apache Beam SDK for Python is supported only on Python 2.7. ' + 'It is not supported on Python [%s].' % sys.version) From c5830942cfffa12372c3d44a4bb60266863cd046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 08:59:24 +0200 Subject: [PATCH 20/73] BEAM-378: removed deprecated version usage --- sdks/python/apache_beam/internal/apiclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py index 7dfb03528ff1..1b93c26dcba4 100644 --- a/sdks/python/apache_beam/internal/apiclient.py +++ b/sdks/python/apache_beam/internal/apiclient.py @@ -29,7 +29,7 @@ from apitools.base.py import exceptions from apache_beam import utils -from apache_beam import version +from apache_beam import __version__ from apache_beam.internal import pickler from apache_beam.internal.auth import get_service_credentials from apache_beam.internal.json_value import to_json_value @@ -190,7 +190,7 @@ def __init__(self, packages, options, environment_version): self.proto.userAgent = dataflow.Environment.UserAgentValue() self.local = 'localhost' in self.google_cloud_options.dataflow_endpoint - version_string = version.__version__ + version_string = __version__ self.proto.userAgent.additionalProperties.extend([ dataflow.Environment.UserAgentValue.AdditionalProperty( From b045a97d678e3102960af469c29c6362c0c904cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 1 Jul 2016 09:05:28 +0200 Subject: [PATCH 21/73] BEAM-378: brought back lxml as required to see if we get ci finally working --- sdks/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index e89597e5772d..85ed0d6e7086 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -70,6 +70,7 @@ def sync_version(): # 'google-apitools-dataflow-v1b3>=0.4.20160217', # 'google-apitools-storage-v1', 'httplib2>=0.8', + 'lxml', 'mock>=1.0.1', 'nose>=1.0', 'oauth2client>=2.0.1', From 27917a0f9e604e52183174fac2d82b427c2bda0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 6 Jul 2016 08:57:14 +0200 Subject: [PATCH 22/73] BEAM-378: replaced by regex pom.xml parsing to get rid of lxml dependency --- sdks/python/setup.py | 25 +++++++++++++++---------- sdks/python/tox.ini | 1 - 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 85ed0d6e7086..d59f00373032 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -17,10 +17,10 @@ """Apache Beam SDK setup configuration.""" + import platform import setuptools import re -from lxml import etree # Currently all compiled modules are optional (for performance only). @@ -35,14 +35,19 @@ cythonize = lambda *args, **kwargs: [] -# Reads the actual version from pom.xml file, and synchronizes -# apache_beam.__version__ field for later usage. -def sync_version(): - pom = etree.parse('pom.xml') - elements = pom.xpath(r'/pom:project/pom:parent/pom:version', namespaces={'pom':'http://maven.apache.org/POM/4.0.0'}) - version = elements[0].text.replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix +# Reads the actual version from pom.xml file, +def get_version_from_pom(): + with open('pom.xml', 'r') as f: + pom = f.read() + pom_version_regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' + pattern = re.compile(pom_version_regex) + search = pattern.search(pom) + return search.group(1).replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix + - init_path = "apache_beam/__init__.py" +# Synchronizes apache_beam.__version__ field for later usage +def sync_version(version): + init_path = 'apache_beam/__init__.py' regex = r'^__version__\s*=\s*".*"' with open(init_path, "r") as f: lines = f.readlines() @@ -55,7 +60,8 @@ def sync_version(): return version -version = sync_version() +version = get_version_from_pom() +sync_version(version) # Configure the required packages and scripts to install. @@ -70,7 +76,6 @@ def sync_version(): # 'google-apitools-dataflow-v1b3>=0.4.20160217', # 'google-apitools-storage-v1', 'httplib2>=0.8', - 'lxml', 'mock>=1.0.1', 'nose>=1.0', 'oauth2client>=2.0.1', diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index ecb35916dcdb..29674ed773da 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -30,7 +30,6 @@ exclude = windmill_pb2.py, windmill_service_pb2.py deps= pep8 pylint - lxml commands = python setup.py test {toxinidir}/run_pylint.sh From 456501e20586be966e3957014c9778d64ddf9b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 6 Jul 2016 09:28:01 +0200 Subject: [PATCH 23/73] BEAM-378: renamed variable --- sdks/python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index d59f00373032..908dec746cfa 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -39,8 +39,8 @@ def get_version_from_pom(): with open('pom.xml', 'r') as f: pom = f.read() - pom_version_regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' - pattern = re.compile(pom_version_regex) + regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' + pattern = re.compile(regex) search = pattern.search(pom) return search.group(1).replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix From 08502f7d4c5c5ed2152498b275e5d63da99f5db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 6 Jul 2016 11:31:54 +0200 Subject: [PATCH 24/73] BEAM-378: removed non-necessary anymore return statement --- sdks/python/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 908dec746cfa..ed9534080ca5 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -57,7 +57,6 @@ def sync_version(version): f.write(re.sub(regex, '__version__ = "%s"' % version, line)) else: f.write(line) - return version version = get_version_from_pom() From 3da6884ca82b36a8294fbb0207f249d5d2957c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 6 Jul 2016 12:03:37 +0200 Subject: [PATCH 25/73] BEAM-378: making rat happy --- pom.xml | 5 ++++- sdks/python/MANIFEST.in | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 49b77fa0ae97..7f0b8e3b2f9a 100644 --- a/pom.xml +++ b/pom.xml @@ -202,7 +202,7 @@ org.apache.rat apache-rat-plugin - 0.11 + 0.12 verify @@ -226,6 +226,9 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* + **/.tox/**/* + **/*.egg-info/** + **/.eggs/** diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in index 2713aa6cb8ad..f11a042585a5 100644 --- a/sdks/python/MANIFEST.in +++ b/sdks/python/MANIFEST.in @@ -1 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + include pom.xml From 3674690ea1d28b225d644b36e3eb71c4e1d1bc65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Thu, 7 Jul 2016 08:15:39 +0200 Subject: [PATCH 26/73] BEAM-378: fixed 2-spaces indentation --- sdks/python/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index ed9534080ca5..b5b8b9220f30 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -29,10 +29,10 @@ cythonize = lambda *args, **kwargs: [] else: try: - # pylint: disable=g-statement-before-imports,g-import-not-at-top - from Cython.Build import cythonize + # pylint: disable=g-statement-before-imports,g-import-not-at-top + from Cython.Build import cythonize except ImportError: - cythonize = lambda *args, **kwargs: [] + cythonize = lambda *args, **kwargs: [] # Reads the actual version from pom.xml file, From 93c31dfba439b1b241926308dabfb7b730e5a4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Thu, 7 Jul 2016 08:20:11 +0200 Subject: [PATCH 27/73] BEAM-378: added a comment in the manifest regarding why pom.xml is included in the distribution --- sdks/python/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in index f11a042585a5..baa2fdac6749 100644 --- a/sdks/python/MANIFEST.in +++ b/sdks/python/MANIFEST.in @@ -15,4 +15,5 @@ # limitations under the License. # +# This file is used from Python to sync versions include pom.xml From 72d128bfdd54575a0a6dbdb455d2693087dee26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Thu, 7 Jul 2016 15:04:04 +0200 Subject: [PATCH 28/73] BEAM-378: handled tox installation in process-test-resources maven phase --- sdks/python/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 21a714628dc5..043c73a7b51f 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -68,6 +68,21 @@ + + ssetup-test + process-test-resources + + exec + + + pip + + install + tox + --user + + + setuptools-test test From f0290bd9f6538b9f5f1afa8a5415c0f3b662d1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 09:18:35 +0200 Subject: [PATCH 29/73] BEAM-378: fixed execution id --- sdks/python/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 043c73a7b51f..5b70c8ac9e7e 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -69,7 +69,7 @@ - ssetup-test + setup-test-env process-test-resources exec From ebe4d0eef420c6feb8e3b1815d65d72a9e4a0487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 09:19:27 +0200 Subject: [PATCH 30/73] BEAM-378: now that it's clear versions are not going to be aligned, reverted to the old way to manage the version --- sdks/python/apache_beam/__init__.py | 5 +- sdks/python/apache_beam/internal/apiclient.py | 2 +- sdks/python/apache_beam/utils/dependency.py | 2 +- .../apache_beam/utils/dependency_test.py | 2 +- sdks/python/apache_beam/version.py | 46 +++++++++++++++++++ sdks/python/setup.py | 31 +++---------- 6 files changed, 57 insertions(+), 31 deletions(-) create mode 100644 sdks/python/apache_beam/version.py diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index bb67fa216cc7..994ab4182138 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -72,11 +72,8 @@ import apache_beam.internal.pickler -# this is the default value, but it will be correctly updated by setup.py -__version__ = "0.2.0-incubating.dev" - - if version_info.major != 2: raise RuntimeError( 'Apache Beam SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) + diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py index 1b93c26dcba4..4500730b3d9d 100644 --- a/sdks/python/apache_beam/internal/apiclient.py +++ b/sdks/python/apache_beam/internal/apiclient.py @@ -29,7 +29,7 @@ from apitools.base.py import exceptions from apache_beam import utils -from apache_beam import __version__ +from apache_beam.version import __version__ from apache_beam.internal import pickler from apache_beam.internal.auth import get_service_credentials from apache_beam.internal.json_value import to_json_value diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index d2c1b54f81ac..323ec7f5d9cb 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -57,7 +57,7 @@ import shutil import tempfile -from apache_beam import __version__ +from apache_beam.version import __version__ from apache_beam import utils from apache_beam.internal import pickler from apache_beam.utils import names diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index 11ce322677fb..ec4724c16ae9 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -23,7 +23,7 @@ import tempfile import unittest -from apache_beam import __version__ +from apache_beam.version import __version__ from apache_beam import utils from apache_beam.utils import dependency from apache_beam.utils import names diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py new file mode 100644 index 000000000000..853ae8beb2b9 --- /dev/null +++ b/sdks/python/apache_beam/version.py @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Beam SDK version information and utilities.""" + + +__version__ = '0.2.0-incubating.dev' # TODO: PEP 440 and incubating suffix + + +# Reads the actual version from pom.xml file, +def get_version_from_pom(): + with open('pom.xml', 'r') as f: + pom = f.read() + regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' + pattern = re.compile(regex) + search = pattern.search(pom) + return search.group(1).replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix + + +# Synchronizes apache_beam.__version__ field for later usage +def sync_version(version): + init_path = 'apache_beam/__init__.py' + regex = r'^__version__\s*=\s*".*"' + with open(init_path, "r") as f: + lines = f.readlines() + with open(init_path, "w") as f: + for line in lines: + if re.search(regex, line): + f.write(re.sub(regex, '__version__ = "%s"' % version, line)) + else: + f.write(line) + diff --git a/sdks/python/setup.py b/sdks/python/setup.py index b5b8b9220f30..30c6a5d6ff4c 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -18,6 +18,7 @@ """Apache Beam SDK setup configuration.""" +import os import platform import setuptools import re @@ -35,32 +36,14 @@ cythonize = lambda *args, **kwargs: [] -# Reads the actual version from pom.xml file, -def get_version_from_pom(): - with open('pom.xml', 'r') as f: - pom = f.read() - regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' - pattern = re.compile(regex) - search = pattern.search(pom) - return search.group(1).replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix +# Read the current version +def get_version(): + global_names = {} + execfile(os.path.normpath('./apache_beam/version.py'), global_names) + return global_names['__version__'] -# Synchronizes apache_beam.__version__ field for later usage -def sync_version(version): - init_path = 'apache_beam/__init__.py' - regex = r'^__version__\s*=\s*".*"' - with open(init_path, "r") as f: - lines = f.readlines() - with open(init_path, "w") as f: - for line in lines: - if re.search(regex, line): - f.write(re.sub(regex, '__version__ = "%s"' % version, line)) - else: - f.write(line) - - -version = get_version_from_pom() -sync_version(version) +version = get_version() # Configure the required packages and scripts to install. From aa8e9d135c1eb7595aa2440f432fe4cf7992f77b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 09:23:26 +0200 Subject: [PATCH 31/73] BEAM-378: fixed build path using the right maven property --- sdks/python/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 5b70c8ac9e7e..14fd3738cd92 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -64,7 +64,7 @@ setup.py build --build-base - target/build + ${project.build.directory}/build @@ -111,7 +111,7 @@ setup.py sdist --dist-dir - target + ${project.build.directory} From 49b1901782533c40799fb0170a95236e1f0af1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 09:26:26 +0200 Subject: [PATCH 32/73] BEAM-378: kept the ',' at the end of the list according to @dhalperi comment --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 30c6a5d6ff4c..bbcc6674b3d3 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -63,7 +63,7 @@ def get_version(): 'oauth2client>=2.0.1', 'protorpc>=0.9.1', 'python-gflags>=2.0', - 'pyyaml>=3.10' + 'pyyaml>=3.10', ] setuptools.setup( From 0a193fa3693bcfe1dfa49b7cd6cec349d7fe41a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 09:37:15 +0200 Subject: [PATCH 33/73] BEAM-378: trying to use virtualenv to run tox from maven everywhere --- sdks/python/pom.xml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 14fd3738cd92..cb8550870ea5 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -69,17 +69,29 @@ - setup-test-env + setup-test-virtualenv process-test-resources exec - pip + virtualenv + + ${project.build.directory}/virtualenv + + + + + setup-test-tox + process-test-resources + + exec + + + ${project.build.directory}/virtualenv/bin/pip install tox - --user @@ -90,7 +102,7 @@ exec - tox + ${project.build.directory}/virtualenv/bin/tox -e py27 From 62646b58eaec6d14096d64a886c89358f7a863dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 10:17:03 +0200 Subject: [PATCH 34/73] BEAM-378: making pylint happy again --- sdks/python/apache_beam/version.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index 853ae8beb2b9..3e2f1ae592d6 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -18,17 +18,30 @@ """Apache Beam SDK version information and utilities.""" +import re + + __version__ = '0.2.0-incubating.dev' # TODO: PEP 440 and incubating suffix +# The following utilities are legacy code from the Maven integration; +# see BEAM-378 for further details. + + # Reads the actual version from pom.xml file, def get_version_from_pom(): with open('pom.xml', 'r') as f: pom = f.read() - regex = '.*\s*[a-z\.]+\s*[a-z\-]+\s*([0-9a-zA-Z\.\-]+).*' - pattern = re.compile(regex) + regex = (r'.*\s*' + r'[a-z\.]+\s*' + r'[a-z\-]+\s*' + r'([0-9a-zA-Z\.\-]+).*') + pattern = re.compile(str(regex)) search = pattern.search(pom) - return search.group(1).replace("-SNAPSHOT", ".dev") # TODO: PEP 440 and incubating suffix + version = search.group(1) + version = version.replace("-SNAPSHOT", ".dev") + # TODO: PEP 440 and incubating suffix + return version # Synchronizes apache_beam.__version__ field for later usage @@ -43,4 +56,3 @@ def sync_version(version): f.write(re.sub(regex, '__version__ = "%s"' % version, line)) else: f.write(line) - From 71b59bf98d1e2b3bf1ec460007e5a4e8b1ebda2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 10:33:11 +0200 Subject: [PATCH 35/73] BEAM-378: remove unused import to make pylint happy again --- sdks/python/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index e58423188068..311482e499bb 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -19,7 +19,6 @@ import os import platform -import re import setuptools From 7f3c4cbf98528aa1adf9e76dbecf3004a3d45857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Fri, 8 Jul 2016 10:35:45 +0200 Subject: [PATCH 36/73] BEAM-378: removed trailing newlines to make pylint happy again --- sdks/python/apache_beam/__init__.py | 1 - sdks/python/apache_beam/utils/dependency.py | 1 - 2 files changed, 2 deletions(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 994ab4182138..37f8746db597 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -76,4 +76,3 @@ raise RuntimeError( 'Apache Beam SDK for Python is supported only on Python 2.7. ' 'It is not supported on Python [%s].' % sys.version) - diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 8221e7ecc812..177dc09cf02c 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -459,4 +459,3 @@ def _download_pypi_sdk_package(temp_dir): 'Failed to download a source distribution for the running SDK. Expected ' 'either %s or %s to be found in the download folder.' % ( zip_expected, tgz_expected)) - From 36b4d809e7e9c0e52c17e5d9632d9d3d84267d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 12:34:13 +0200 Subject: [PATCH 37/73] BEAM-378: as @aaltay suggested, changed the strategy to use tox under a custom /home/wikier/tmp/beam/sdks/python/target/python --- sdks/python/pom.xml | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index cb8550870ea5..08d50fedc986 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -32,6 +32,13 @@ Apache Beam :: SDKs :: Python + + python2 + pip2 + ${project.build.directory}/build + ${project.build.directory}/python + + @@ -45,7 +52,7 @@ exec - python + ${python.interpreter.bin} setup.py clean @@ -59,25 +66,25 @@ exec - python + ${python.interpreter.bin} setup.py build --build-base - ${project.build.directory}/build + ${python.build.base} - setup-test-virtualenv + setup-test-create-python-base process-test-resources exec - virtualenv + mkdir - ${project.build.directory}/virtualenv + ${python.user.base} @@ -88,11 +95,17 @@ exec - ${project.build.directory}/virtualenv/bin/pip + ${python.pip.bin} install + --user + --upgrade + --ignore-installed tox + + ${python.user.base} + @@ -102,13 +115,16 @@ exec - ${project.build.directory}/virtualenv/bin/tox + ${python.user.base}/bin/tox -e py27 -c tox.ini + + ${python.user.base} + @@ -118,13 +134,16 @@ exec - python + ${python.interpreter.bin} setup.py sdist --dist-dir ${project.build.directory} + + ${python.user.base} + From 2d19489a08b5d6f69959dd0f7abe749469b568f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 12:34:13 +0200 Subject: [PATCH 38/73] BEAM-378: as @aaltay suggested, changed the strategy to use tox under a custom PYTHONUSERBASE env variable --- sdks/python/pom.xml | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index cb8550870ea5..08d50fedc986 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -32,6 +32,13 @@ Apache Beam :: SDKs :: Python + + python2 + pip2 + ${project.build.directory}/build + ${project.build.directory}/python + + @@ -45,7 +52,7 @@ exec - python + ${python.interpreter.bin} setup.py clean @@ -59,25 +66,25 @@ exec - python + ${python.interpreter.bin} setup.py build --build-base - ${project.build.directory}/build + ${python.build.base} - setup-test-virtualenv + setup-test-create-python-base process-test-resources exec - virtualenv + mkdir - ${project.build.directory}/virtualenv + ${python.user.base} @@ -88,11 +95,17 @@ exec - ${project.build.directory}/virtualenv/bin/pip + ${python.pip.bin} install + --user + --upgrade + --ignore-installed tox + + ${python.user.base} + @@ -102,13 +115,16 @@ exec - ${project.build.directory}/virtualenv/bin/tox + ${python.user.base}/bin/tox -e py27 -c tox.ini + + ${python.user.base} + @@ -118,13 +134,16 @@ exec - python + ${python.interpreter.bin} setup.py sdist --dist-dir ${project.build.directory} + + ${python.user.base} + From a38a3a8c55f2f7f7b181e0a5cebf08657c0604d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 14:04:02 +0200 Subject: [PATCH 39/73] BEAM-378: trying to skip goal exec:java with the old maven version we have in jenkins --- sdks/python/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 08d50fedc986..747a93586a31 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -45,6 +45,14 @@ org.codehaus.mojo exec-maven-plugin + + + java + + + true + + setuptools-clean clean From 684cfb2c14de9af905b5b5a631c9e3b377e8a22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 14:35:37 +0200 Subject: [PATCH 40/73] BEAM-368: another try to skip exec:java goal on jenkins --- sdks/python/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 747a93586a31..3d4e34b5f8a7 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -44,6 +44,9 @@ org.codehaus.mojo exec-maven-plugin + + exec + From db7dd39b5ac8c0ceae0c8b1b8b964efb00ca9a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 14:49:52 +0200 Subject: [PATCH 41/73] BEAM-368: yet another try to get rig of exec:java goal on jenkins --- sdks/python/pom.xml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 3d4e34b5f8a7..f86d32f051d6 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -45,17 +45,10 @@ org.codehaus.mojo exec-maven-plugin - exec + + org.apache.beam.FakeMain - - - java - - - true - - setuptools-clean clean From f1f354f846adcb04eda0b0ab62d26b74cd1caf0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 11 Jul 2016 14:58:19 +0200 Subject: [PATCH 42/73] BEAM-368: trying to know the maven version is running at jenkins --- sdks/python/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index f86d32f051d6..e805cb4dbc1c 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -39,15 +39,15 @@ ${project.build.directory}/python + + 3.3.0 + + org.codehaus.mojo exec-maven-plugin - - - org.apache.beam.FakeMain - setuptools-clean From c7be1d36f429bf73d7c9b3f47080df5ed287e035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 12 Jul 2016 11:15:37 +0200 Subject: [PATCH 43/73] BEAM-378: aligned tox directories with maven build --- pom.xml | 3 --- sdks/python/pom.xml | 4 ---- sdks/python/tox.ini | 3 +++ 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 7f0b8e3b2f9a..cb41ae4a025c 100644 --- a/pom.xml +++ b/pom.xml @@ -226,9 +226,6 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* - **/.tox/**/* - **/*.egg-info/** - **/.eggs/** diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index e805cb4dbc1c..08d50fedc986 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -39,10 +39,6 @@ ${project.build.directory}/python - - 3.3.0 - - diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 29674ed773da..b753b7c80c68 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,6 +17,9 @@ [tox] envlist = py27 +toxworkdir={toxinidir}/target/tox +distdir={toxinidir}/target/dist +distshare={toxinidir}/target/distshare [pep8] # Disable all errors and warnings except for the ones related to blank lines. From 6e616d83c848d914bd0ea0ccaa37a0953c2cef76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 19 Jul 2016 12:03:19 +0200 Subject: [PATCH 44/73] BEAM-378: trying not to inherit exec:java execution in the python sdk --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index cb41ae4a025c..5da063e2a253 100644 --- a/pom.xml +++ b/pom.xml @@ -754,6 +754,7 @@ + false false From 69fe354b01103f706441faaaca7396fc5301d48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 19 Jul 2016 14:55:25 +0200 Subject: [PATCH 45/73] Revert "BEAM-378: trying not to inherit exec:java execution in the python sdk" This reverts commit 6e616d83c848d914bd0ea0ccaa37a0953c2cef76. --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5da063e2a253..cb41ae4a025c 100644 --- a/pom.xml +++ b/pom.xml @@ -754,7 +754,6 @@ - false false From ec578d2597165a8ed7ca045be0b018cbd7c08eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 08:23:17 +0200 Subject: [PATCH 46/73] BEAM-378: another try to skip the exec:java goal in the python sdk --- sdks/python/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 08d50fedc986..ca2f2ff53f2d 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -45,6 +45,15 @@ org.codehaus.mojo exec-maven-plugin + + verify + + java + + + true + + setuptools-clean clean From 78779bb57400af96c12807f65673de78a5f1cdea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 08:47:20 +0200 Subject: [PATCH 47/73] BEAM-379: preventing warning with mkdir --- sdks/python/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index ca2f2ff53f2d..6fbc6679d0fd 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -93,6 +93,7 @@ mkdir + -p ${python.user.base} From c7093e7c48d637ef1c2603ca32e194c9e59590d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 08:53:53 +0200 Subject: [PATCH 48/73] BEAM-378: removed unused imports --- sdks/python/apache_beam/internal/apiclient.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py index 09475acadfb2..d0cb430fff83 100644 --- a/sdks/python/apache_beam/internal/apiclient.py +++ b/sdks/python/apache_beam/internal/apiclient.py @@ -30,14 +30,12 @@ from apache_beam import utils from apache_beam import version -from apache_beam.internal import pickler from apache_beam.internal.auth import get_service_credentials from apache_beam.internal.json_value import to_json_value from apache_beam.transforms import cy_combiners from apache_beam.utils import dependency from apache_beam.utils import retry from apache_beam.utils.dependency import get_required_container_version -from apache_beam.utils.dependency import get_sdk_name_and_version from apache_beam.utils.names import PropertyNames from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import StandardOptions From 2620bfbd0942e88cb1552a8a8d3c013be6b8b73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 09:15:40 +0200 Subject: [PATCH 49/73] BEAM-378: trying to use the execution id to skip exec:maven in a submodule --- pom.xml | 1 + sdks/python/pom.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index cb41ae4a025c..a132d1679da4 100644 --- a/pom.xml +++ b/pom.xml @@ -747,6 +747,7 @@ 1.4.0 + beam-exec-java verify java diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 6fbc6679d0fd..2a2cb4e4f277 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -46,6 +46,7 @@ exec-maven-plugin + beam-exec-java verify java From 23003ac3f70969d4cc57751c3a410ad3f1eb5ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 09:22:18 +0200 Subject: [PATCH 50/73] BEAM-378: moved exec:java execution to a non-existent phase to actually get skipped the goal in the python sdk --- sdks/python/pom.xml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 2a2cb4e4f277..886f2d70c65a 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -47,13 +47,8 @@ beam-exec-java - verify - - java - - - true - + + dont-execute setuptools-clean From a92320704a0f496722711e93caa6f9e2eb0c5352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2016 09:50:47 +0200 Subject: [PATCH 51/73] BEAM-378: fied egg_base to make rat happy again --- sdks/python/setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/setup.cfg b/sdks/python/setup.cfg index 547a74b5cb10..fcfe003a7d36 100644 --- a/sdks/python/setup.cfg +++ b/sdks/python/setup.cfg @@ -26,3 +26,5 @@ verbosity=2 # fast_coders_test and typecoders_test. exclude=fast_coders_test|typecoders_test +[egg_info] +egg_base = target From 1d9378a74498850ec89b67b4fc620808421a71dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:09:58 +0200 Subject: [PATCH 52/73] BEAM-378: fixed wrong merge on setup.py --- sdks/python/setup.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 12d650b39376..1cfdea7c78b2 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -53,27 +53,12 @@ def get_version(): cythonize = lambda *args, **kwargs: [] -def get_version(): - global_names = {} - execfile(os.path.normpath('./apache_beam/version.py'), - global_names) - return global_names['__version__'] - - -# Configure the required packages and scripts to install. REQUIRED_PACKAGES = [ 'avro>=1.7.7', 'dill>=0.2.5', 'google-apitools>=0.5.2', - # TODO(silviuc): Reenable api client package dependencies when we can - # update the packages to the latest version without affecting previous - # SDK releases. - # 'google-apitools-bigquery-v2', - # 'google-apitools-dataflow-v1b3>=0.4.20160217', - # 'google-apitools-storage-v1', 'httplib2>=0.8', 'mock>=1.0.1', - 'nose>=1.0', 'oauth2client>=2.0.1', 'protorpc>=0.9.1', 'python-gflags>=2.0', @@ -113,6 +98,6 @@ def get_version(): 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', ], - license='Apache License, Version 2.0', + license='Apache License, 2.0', keywords=PACKAGE_KEYWORDS, ) From fd7ddf01aedddcc51263044b5ad2d2e5a5f22e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:12:04 +0200 Subject: [PATCH 53/73] BEAM-378: fixed wrong merge on apiclient.py --- sdks/python/apache_beam/internal/apiclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py index d0cb430fff83..bc4a4e0832eb 100644 --- a/sdks/python/apache_beam/internal/apiclient.py +++ b/sdks/python/apache_beam/internal/apiclient.py @@ -29,13 +29,13 @@ from apitools.base.py import exceptions from apache_beam import utils -from apache_beam import version from apache_beam.internal.auth import get_service_credentials from apache_beam.internal.json_value import to_json_value from apache_beam.transforms import cy_combiners from apache_beam.utils import dependency from apache_beam.utils import retry from apache_beam.utils.dependency import get_required_container_version +from apache_beam.utils.dependency import get_sdk_name_and_version from apache_beam.utils.names import PropertyNames from apache_beam.utils.options import GoogleCloudOptions from apache_beam.utils.options import StandardOptions @@ -117,7 +117,7 @@ def __init__(self, packages, options, environment_version): self.proto.userAgent = dataflow.Environment.UserAgentValue() self.local = 'localhost' in self.google_cloud_options.dataflow_endpoint - version_string = version.__version__ + sdk_name, version_string = get_sdk_name_and_version() self.proto.userAgent.additionalProperties.extend([ dataflow.Environment.UserAgentValue.AdditionalProperty( From e044d8d5c5540cbb6590b6bca4811de455b5b46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:14:29 +0200 Subject: [PATCH 54/73] BEAM-378: removed non-used import --- sdks/python/apache_beam/utils/dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 4e5701fc31aa..aacaf903740c 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -58,7 +58,7 @@ import shutil import tempfile -from apache_beam.version import __version__ + from apache_beam import utils from apache_beam import version as beam_version from apache_beam.internal import pickler From cb82ec1e571b2f5c4086aa17fe1f5e15bf48dcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:16:05 +0200 Subject: [PATCH 55/73] BEAM-378: reverted to reference __init__.py --- sdks/python/apache_beam/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sdks/python/apache_beam/__init__.py b/sdks/python/apache_beam/__init__.py index 37f8746db597..eed251b83e4e 100644 --- a/sdks/python/apache_beam/__init__.py +++ b/sdks/python/apache_beam/__init__.py @@ -63,16 +63,20 @@ """ -from sys import version_info +import sys + + +if sys.version_info.major != 2: + raise RuntimeError( + 'Dataflow SDK for Python is supported only on Python 2.7. ' + 'It is not supported on Python [%s].' % sys.version) + +# pylint: disable=wrong-import-position +import apache_beam.internal.pickler + from apache_beam import coders from apache_beam import io from apache_beam import typehints from apache_beam.pipeline import Pipeline from apache_beam.transforms import * -import apache_beam.internal.pickler - - -if version_info.major != 2: - raise RuntimeError( - 'Apache Beam SDK for Python is supported only on Python 2.7. ' - 'It is not supported on Python [%s].' % sys.version) +# pylint: enable=wrong-import-position From cd9432ffa64b77d9e89d7b065745b9054aa7f7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 10:45:10 +0200 Subject: [PATCH 56/73] BEAM-378: moved nose setup requirements from setuptools to tox, so we get rid of custom .eggs in unmanageable locations that was causing issues with rat --- sdks/python/setup.py | 1 - sdks/python/tox.ini | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 1cfdea7c78b2..5b42dcac3e01 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -85,7 +85,6 @@ def get_version(): 'apache_beam/utils/counters.py', 'apache_beam/utils/windowed_value.py', ]), - setup_requires=['nose>=1.0'], install_requires=REQUIRED_PACKAGES, test_suite='nose.collector', zip_safe=False, diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index cba9626a0593..d510601888dc 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -33,6 +33,7 @@ exclude = windmill_pb2.py, windmill_service_pb2.py deps= pep8 pylint + nose commands = python apache_beam/examples/complete/autocomplete_test.py python setup.py test From 44222078a02a5e6a456a72da25bf47a38af8617b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 2 Aug 2016 11:41:53 +0200 Subject: [PATCH 57/73] BEAM-378: trying to fix the lateral issue on beam-runners-flink_2.10-examples' exec:exec --- runners/flink/examples/pom.xml | 36 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml index 1fba81cfec99..3bfe8c4530ba 100644 --- a/runners/flink/examples/pom.xml +++ b/runners/flink/examples/pom.xml @@ -114,17 +114,31 @@ org.codehaus.mojo exec-maven-plugin - - java - - -classpath - - ${clazz} - --input=${input} - --output=${output} - --parallelism=${parallelism} - - + + + beam-exec-java + + dont-execute + + + beam-runners-flink_2.10-examples-exec + verify + + exec + + + java + + -classpath + + ${clazz} + --input=${input} + --output=${output} + --parallelism=${parallelism} + + + + From a0ecce5b0a2f1bbfbf791634c10c013512eafcfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 3 Aug 2016 07:55:50 +0200 Subject: [PATCH 58/73] Revert "BEAM-378: trying to fix the lateral issue on beam-runners-flink_2.10-examples' exec:exec" This reverts commit 44222078a02a5e6a456a72da25bf47a38af8617b. --- runners/flink/examples/pom.xml | 36 +++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml index 3bfe8c4530ba..1fba81cfec99 100644 --- a/runners/flink/examples/pom.xml +++ b/runners/flink/examples/pom.xml @@ -114,31 +114,17 @@ org.codehaus.mojo exec-maven-plugin - - - beam-exec-java - - dont-execute - - - beam-runners-flink_2.10-examples-exec - verify - - exec - - - java - - -classpath - - ${clazz} - --input=${input} - --output=${output} - --parallelism=${parallelism} - - - - + + java + + -classpath + + ${clazz} + --input=${input} + --output=${output} + --parallelism=${parallelism} + + From d2328929aca1e006d2aafd7670c56c4894653ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 3 Aug 2016 07:58:53 +0200 Subject: [PATCH 59/73] BEAM-378: clean-up --- sdks/python/apache_beam/utils/dependency_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index 33e1863f653e..ca318067c6d9 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -23,7 +23,6 @@ import tempfile import unittest -from apache_beam.version import __version__ from apache_beam import utils from apache_beam.utils import dependency from apache_beam.utils import names From 51b13f588d017a9f5a166f403e3dffddfc157643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 3 Aug 2016 08:01:20 +0200 Subject: [PATCH 60/73] BEAM-378: upgraded dev version according the new release --- sdks/python/apache_beam/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index 3e2f1ae592d6..60d963482b2d 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -21,7 +21,7 @@ import re -__version__ = '0.2.0-incubating.dev' # TODO: PEP 440 and incubating suffix +__version__ = '0.3.0-incubating.dev' # TODO: PEP 440 and incubating suffix # The following utilities are legacy code from the Maven integration; From f0d79f735ac0e8a015578acc5414bb7653257d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 3 Aug 2016 08:02:34 +0200 Subject: [PATCH 61/73] BEAM-378: right license label --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 5b42dcac3e01..a937c0c822dc 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -97,6 +97,6 @@ def get_version(): 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', ], - license='Apache License, 2.0', + license='Apache License, Version 2.0', keywords=PACKAGE_KEYWORDS, ) From e9fc5f0df5aa0baca51c6182da49a1acad37db2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 8 Aug 2016 18:22:55 +0200 Subject: [PATCH 62/73] BEAM-378: fixed parent version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 14d49a6558b1..95a8aec6a3e0 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ http://beam.incubator.apache.org 2016 - 0.2.0-incubating-SNAPSHOT + 0.3.0-incubating-SNAPSHOT From 403c4252a33e574b29a97b205e3e1636494252c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Mon, 8 Aug 2016 18:59:42 +0200 Subject: [PATCH 63/73] BEAm-378: recovered exec:java execution id for being able to skip it later on --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 95a8aec6a3e0..f56bc7916769 100644 --- a/pom.xml +++ b/pom.xml @@ -839,7 +839,8 @@ exec-maven-plugin 1.4.0 - + + beam-exec-java verify java From cf0bb30910e9ce447f16d856a84f7b596c6f6af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Tue, 9 Aug 2016 08:47:08 +0200 Subject: [PATCH 64/73] fixed exec-maven-plugin configuration on beam-runners-flink_2.10-examples --- runners/flink/README.md | 2 +- runners/flink/examples/pom.xml | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/runners/flink/README.md b/runners/flink/README.md index aeb16922a96b..f778c03c4518 100644 --- a/runners/flink/README.md +++ b/runners/flink/README.md @@ -169,7 +169,7 @@ The contents of the root `pom.xml` should be slightly changed aftewards (explana org.apache.beam beam-runners-flink_2.10 - 0.2.0-incubating-SNAPSHOT + 0.3.0-incubating-SNAPSHOT diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml index 9f705db88dbe..5b010e7e4485 100644 --- a/runners/flink/examples/pom.xml +++ b/runners/flink/examples/pom.xml @@ -127,15 +127,29 @@ org.codehaus.mojo exec-maven-plugin - - java - - --runner=org.apache.beam.runners.flink.FlinkRunner - --parallelism=${parallelism} - --input=${input} - --output=${output} - - + + + beam-exec-java + + dont-execute + + + beam-runners-flink-example-exec-java + verify + + exec + + + java + + --runner=org.apache.beam.runners.flink.FlinkRunner + --parallelism=${parallelism} + --input=${input} + --output=${output} + + + + From 4c602536478d816542b8e4a2666ca5a0f5079004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 08:37:50 +0200 Subject: [PATCH 65/73] BEAM-378: applied the trick by @aaltay to skip inheritance in exec-maven-plugin --- pom.xml | 3 +-- sdks/python/pom.xml | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index f56bc7916769..95a8aec6a3e0 100644 --- a/pom.xml +++ b/pom.xml @@ -839,8 +839,7 @@ exec-maven-plugin 1.4.0 - - beam-exec-java + verify java diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index ccb39ab041d9..79d20bf0e057 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -40,16 +40,25 @@ + + + + org.codehaus.mojo + exec-maven-plugin + + + + none + + + + + org.codehaus.mojo exec-maven-plugin - - beam-exec-java - - dont-execute - setuptools-clean clean From f7c8a13205f1c9b5c8d9fa4e8758ce7c6baa178e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 08:39:10 +0200 Subject: [PATCH 66/73] Revert "fixed exec-maven-plugin configuration on beam-runners-flink_2.10-examples" This reverts commit cf0bb30910e9ce447f16d856a84f7b596c6f6af9. --- runners/flink/README.md | 2 +- runners/flink/examples/pom.xml | 32 +++++++++----------------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/runners/flink/README.md b/runners/flink/README.md index f778c03c4518..aeb16922a96b 100644 --- a/runners/flink/README.md +++ b/runners/flink/README.md @@ -169,7 +169,7 @@ The contents of the root `pom.xml` should be slightly changed aftewards (explana org.apache.beam beam-runners-flink_2.10 - 0.3.0-incubating-SNAPSHOT + 0.2.0-incubating-SNAPSHOT diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml index 5b010e7e4485..9f705db88dbe 100644 --- a/runners/flink/examples/pom.xml +++ b/runners/flink/examples/pom.xml @@ -127,29 +127,15 @@ org.codehaus.mojo exec-maven-plugin - - - beam-exec-java - - dont-execute - - - beam-runners-flink-example-exec-java - verify - - exec - - - java - - --runner=org.apache.beam.runners.flink.FlinkRunner - --parallelism=${parallelism} - --input=${input} - --output=${output} - - - - + + java + + --runner=org.apache.beam.runners.flink.FlinkRunner + --parallelism=${parallelism} + --input=${input} + --output=${output} + + From 909889da8c2085a3b7edf92342f282d94c72944f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 08:57:13 +0200 Subject: [PATCH 67/73] Revert "BEAM-378: moved nose setup requirements from setuptools to tox, so we get rid of custom .eggs in unmanageable locations that was causing issues with rat" This reverts commit cd9432ffa64b77d9e89d7b065745b9054aa7f7d7. --- sdks/python/setup.py | 1 + sdks/python/tox.ini | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a937c0c822dc..58e677e9bbe4 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -85,6 +85,7 @@ def get_version(): 'apache_beam/utils/counters.py', 'apache_beam/utils/windowed_value.py', ]), + setup_requires=['nose>=1.0'], install_requires=REQUIRED_PACKAGES, test_suite='nose.collector', zip_safe=False, diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index d510601888dc..cba9626a0593 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -33,7 +33,6 @@ exclude = windmill_pb2.py, windmill_service_pb2.py deps= pep8 pylint - nose commands = python apache_beam/examples/complete/autocomplete_test.py python setup.py test From 5014ed19ec16b5c94d2eb9f35094cae1ad9ef97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 09:15:31 +0200 Subject: [PATCH 68/73] BEAM-378: excluded nose egg now that we are back managing its setup_requires with setuptools --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 95a8aec6a3e0..bbd8ae9473f4 100644 --- a/pom.xml +++ b/pom.xml @@ -817,6 +817,7 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* + **/nose-*/**/* **/.checkstyle From 82a8515b0422531eff348840640ea72406e1193d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 09:43:36 +0200 Subject: [PATCH 69/73] BEAM-378: actually upgraded to apache-rat-plugin 0.12 --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bbd8ae9473f4..8af849a5f980 100644 --- a/pom.xml +++ b/pom.xml @@ -212,7 +212,6 @@ org.apache.rat apache-rat-plugin - 0.12 verify @@ -801,7 +800,7 @@ org.apache.rat apache-rat-plugin - 0.11 + 0.12 ${project.build.directory}/${project.build.finalName}.rat false From 35c6e2c51d0dbb227fd1e834276b23f032d3dc37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 09:48:40 +0200 Subject: [PATCH 70/73] BEAM-378: more fine grained rat exclusion for nose egg --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8af849a5f980..5da5a33f873b 100644 --- a/pom.xml +++ b/pom.xml @@ -235,6 +235,7 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* + **/.eggs/nose-*.egg/**/* @@ -816,7 +817,7 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* - **/nose-*/**/* + **/.eggs/nose-*.egg/**/* **/.checkstyle From 7020e73aa07e5f30486f563c4bfa1dda0ac5a8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2016 09:56:57 +0200 Subject: [PATCH 71/73] BEAM-378: pr clean-up --- pom.xml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/pom.xml b/pom.xml index 5da5a33f873b..2236eb7ec31b 100644 --- a/pom.xml +++ b/pom.xml @@ -90,10 +90,6 @@ - - 3.0.3 - - UTF-8 @@ -220,24 +216,6 @@ - - ${project.build.directory}/${project.build.finalName}.rat - false - true - - - **/target/**/* - **/dependency-reduced-pom.xml - .github/**/* - **/*.iml - **/package-list - **/user.avsc - **/test/resources/**/*.txt - **/test/**/.placeholder - .repository/**/* - **/.eggs/nose-*.egg/**/* - - @@ -977,16 +955,6 @@ - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - true - - - org.codehaus.mojo build-helper-maven-plugin From 86b3419f073e7d5af20e22177fcc9f3c12ad3ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Thu, 11 Aug 2016 08:01:08 +0200 Subject: [PATCH 72/73] BEAM-378: more flexible rat exclusion for nose's egg --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2236eb7ec31b..52df50f724b0 100644 --- a/pom.xml +++ b/pom.xml @@ -795,7 +795,7 @@ **/test/resources/**/*.txt **/test/**/.placeholder .repository/**/* - **/.eggs/nose-*.egg/**/* + **/nose-*.egg/**/* **/.checkstyle From bbd48b3614fd8b1955baceef152a5ca6c294e85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Fern=C3=A1ndez?= Date: Thu, 11 Aug 2016 08:50:36 +0200 Subject: [PATCH 73/73] BEAM-378: pr clean-up --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 52df50f724b0..afe24ee37c64 100644 --- a/pom.xml +++ b/pom.xml @@ -120,7 +120,6 @@ 2.7.2 3.0.1 2.4 - 3.0.1 4.11 1.9.5 4.1.1.Final