diff --git a/.github/workflows/pushtest.yml b/.github/workflows/pushtest.yml index 01ba741..76356be 100644 --- a/.github/workflows/pushtest.yml +++ b/.github/workflows/pushtest.yml @@ -26,7 +26,34 @@ jobs: pytest --cov=imagine --cov-report term-missing tests/ - name: Lint with pycodestyle run: | - pycodestyle imagine.py tests/ + pycodestyle imagine/ tests/ + - name: Build the latest binary + run: | + python setup.py sdist bdist_wheel + - name: Install the built wheel and test usage (UNIX) + if: matrix.operating-system == 'ubuntu-latest' || matrix.operating-system == 'macOS-latest' + run: | + pip install --ignore-installed dist/imageinary-*-py3-none-any.whl + imagine --help + - name: Install the built wheel and test usage (Windows) + if: matrix.operating-system == 'windows-latest' || matrix.operating-system == 'windows-2016' + run: | + pip install --ignore-installed --find-links=dist imageinary + imagine --help + - name: Install the built wheel with all dependencies (UNIX) + if: matrix.operating-system == 'ubuntu-latest' || matrix.operating-system == 'macOS-latest' + run: | + pip install imageinary['all'] + imagine --help + pip freeze | grep "tensorflow" + pip freeze | grep "mxnet" + - name: Install the built wheel with all dependencies (Windows) + if: matrix.operating-system == 'windows-latest' || matrix.operating-system == 'windows-2016' + run: | + pip install imageinary['all'] + imagine --help + pip freeze | grep "tensorflow" + pip freeze | grep "mxnet" test_tf1: name: Test code for TensorFlow 1.x @@ -51,4 +78,31 @@ jobs: pytest --cov=imagine --cov-report term-missing tests/ - name: Lint with pycodestyle run: | - pycodestyle imagine.py tests/ + pycodestyle imagine/ tests/ + - name: Build the latest binary + run: | + python setup.py sdist bdist_wheel + - name: Install the built wheel and test usage (UNIX) + if: matrix.operating-system == 'ubuntu-latest' || matrix.operating-system == 'macOS-latest' + run: | + pip install --ignore-installed dist/imageinary-*-py3-none-any.whl + imagine --help + - name: Install the built wheel and test usage (Windows) + if: matrix.operating-system == 'windows-latest' || matrix.operating-system == 'windows-2016' + run: | + pip install --ignore-installed --find-links=dist imageinary + imagine --help + - name: Install the built wheel with all dependencies (UNIX) + if: matrix.operating-system == 'ubuntu-latest' || matrix.operating-system == 'macOS-latest' + run: | + pip install tensorflow==1.14.0 imageinary['all'] + imagine --help + pip freeze | grep "tensorflow" + pip freeze | grep "mxnet" + - name: Install the built wheel with all dependencies (Windows) + if: matrix.operating-system == 'windows-latest' || matrix.operating-system == 'windows-2016' + run: | + pip install tensorflow==1.14.0 imageinary['all'] + imagine --help + pip freeze | grep "tensorflow" + pip freeze | grep "mxnet" diff --git a/.gitignore b/.gitignore index 62f7142..f7a9199 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *vscode* *env* *coverage* +build/* +dist/* +*egg* diff --git a/README.md b/README.md index ee71d29..d57b4e1 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,60 @@ Once finished running the code, you can leave the virtual environment with: deactivate ``` +## Building from source +Imageinary includes a `setup.py` file which makes it easy to build and install +a binary locally. To build the package, run the following: + +```bash +python3 setup.py sdist bdist_wheel +``` + +This will generate a new package for Imageinary in the `dist/` directory which +can be installed via `pip`. + +```bash +$ ls dist/ +imageinary-1.0.0-py3-none-any.whl imageinary-1.0.0.tar.gz +``` + +## Installing +Once the package is built, it can be installed locally with `pip` using a few +different options depending on your needs. + +### Minimal Install +The minimal install supports standard image types, such as JPG, PNG, and BMP +and only installs the dependencies necessary for those tools. + +```bash +pip install imageinary +``` + +### TFRecord Support +To add support for TFRecords in addition to the standard image types, TensorFlow +needs to be included as a dependency. This can be done by running the following +which installs TensorFlow alongside all other dependencies: + +```bash +pip install imageinary['tfrecord'] +``` + +### RecordIO Support +RecordIO files are supported using MXNet, which can be included as a dependency +using the following: + +```bash +pip install imageinary['mxnet'] +``` + +### Complete Install +If desired, all dependencies can be installed to support standard images, +TFRecords, and RecordIO files without installing extra packages later. Run the +following to install all dependencies: + +```bash +pip install imageinary['all'] +``` + ## Running Imageinary supports many different image types which can be specified while running the application. @@ -60,7 +114,7 @@ A basic run to create 1000 4K JPEGs, and display the size of the first file and all files in the target directory path (not including subdirectories): ```bash -python3 imagine.py create-images \ +imagine create-images \ --path /mnt/nvme/test_dir \ --name random_image_ \ --width 3840 \ @@ -82,7 +136,7 @@ TFRecords can also be easily generated using the application. This command expects images to be pre-loaded to be used as the basis for the TFRecord files. ```bash -python3 imagine.py create-tfrecords \ +imagine create-tfrecords \ --source_path /mnt/nvme/test_dir \ --dest_path /mnt/nvme/tf_record_dir \ --name random_tfrecord_ \ @@ -97,7 +151,7 @@ TFRecords based on those images. The TFRecords will be saved to Similarly, RecordIO files can be generated with a single command: ```bash -python3 imagine.py create-recordio \ +imagine create-recordio \ --source_path /mnt/nvme/test_dir \ --dest_path /mnt/nvme/record_files \ --name random_recordio_ \ diff --git a/imagine/__init__.py b/imagine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/imagine.py b/imagine/imagine.py similarity index 91% rename from imagine.py rename to imagine/imagine.py index cff745a..2543239 100755 --- a/imagine.py +++ b/imagine/imagine.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +18,21 @@ import click from PIL import Image from multiprocessing.pool import Pool -from mxnet.recordio import IRHeader, MXIndexedRecordIO, pack +try: + from mxnet.recordio import IRHeader, MXIndexedRecordIO, pack +except ImportError: + IRHeader = None from time import perf_counter from math import ceil -from tensorflow.io import TFRecordWriter -from tensorflow.train import BytesList, Example, Feature, Features, Int64List +try: + from tensorflow.io import TFRecordWriter + from tensorflow.train import (BytesList, + Example, + Feature, + Features, + Int64List) +except ImportError: + TFRecordWriter = None SUPPORTED_IMAGE_FORMATS = {"jpg": "jpg", "jpeg": "jpg", "bmp": "bmp", @@ -111,6 +122,9 @@ def create_recordio(source_path, dest_path, name, img_per_file): source_path, img_per_file, name)) + if not IRHeader: + raise ImportError('MXNet not found! Please install MXNet dependency ' + 'using "pip install imageinary[\'mxnet\']".') image_files = [] source_path = os.path.abspath(source_path) dest_path = os.path.abspath(dest_path) @@ -153,6 +167,10 @@ def create_tfrecords(source_path, dest_path, name, img_per_file): source_path, img_per_file, name)) + if not TFRecordWriter: + raise ImportError('TensorFlow not found! Please install TensorFlow ' + 'dependency using "pip install ' + 'imageinary[\'tfrecord\']".') check_directory_exists(source_path) try_create_directory(dest_path) combined_path = os.path.join(dest_path, name) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..07cb6fa --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from setuptools import find_packages, setup + +with open('README.md', 'r') as f: + long_description = f.read() + +with open('VERSION', 'r') as f: + version = f.read().strip() + +extras = { + 'tfrecord': ['tensorflow >= 1.14.0,!=2.0.x,!=2.1.x,!=2.2.0'], + 'mxnet': ['mxnet >= 1.6.0'] +} + +extras['all'] = [item for group in extras.values() for item in group] + +setup( + name='imageinary', + version=version, + description='''A reproducible mechanism which is used to generate large +image datasets at various resolutions. The tool supports multiple image types, +including JPEGs, PNGs, BMPs, RecordIO, and TFRecord files''', + long_description=long_description, + packages=find_packages(include=['imagine'], exclude=['tests']), + license='Apache v2.0', + python_requires='>=3.6', + entry_points={ + 'console_scripts': ['imagine=imagine.imagine:main'] + }, + install_requires=[ + 'click >= 7.1.2', + 'numpy >= 1.18.0', + 'Pillow >= 7.1.2' + ], + extras_require=extras +) diff --git a/tests/functional/test_jpg_creation.py b/tests/functional/test_jpg_creation.py index 57cc19e..63938d3 100644 --- a/tests/functional/test_jpg_creation.py +++ b/tests/functional/test_jpg_creation.py @@ -16,7 +16,7 @@ import os from click.testing import CliRunner from glob import glob -from imagine import main +from imagine.imagine import main from PIL import Image diff --git a/tests/functional/test_png_creation.py b/tests/functional/test_png_creation.py index 31a820b..b96409b 100644 --- a/tests/functional/test_png_creation.py +++ b/tests/functional/test_png_creation.py @@ -16,7 +16,7 @@ import os from click.testing import CliRunner from glob import glob -from imagine import main +from imagine.imagine import main from PIL import Image diff --git a/tests/functional/test_recordio.py b/tests/functional/test_recordio.py index 454f882..b6aa12f 100644 --- a/tests/functional/test_recordio.py +++ b/tests/functional/test_recordio.py @@ -16,7 +16,7 @@ import os from click.testing import CliRunner from glob import glob -from imagine import main +from imagine.imagine import main from PIL import Image diff --git a/tests/functional/test_tfrecord.py b/tests/functional/test_tfrecord.py index 6c87827..706f95a 100644 --- a/tests/functional/test_tfrecord.py +++ b/tests/functional/test_tfrecord.py @@ -16,7 +16,7 @@ import os from click.testing import CliRunner from glob import glob -from imagine import main +from imagine.imagine import main from PIL import Image diff --git a/tests/unit/test_units.py b/tests/unit/test_units.py index 77557ee..5702784 100644 --- a/tests/unit/test_units.py +++ b/tests/unit/test_units.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import imagine import pytest import os +from imagine import imagine class TestUnits: