diff --git a/README.md b/README.md new file mode 100644 index 0000000..e1bba5d --- /dev/null +++ b/README.md @@ -0,0 +1,272 @@ +[![json2python-models](/etc/logo.png)](https://github.com/bogdandm/json2python-models) + +[![PyPI version](https://badge.fury.io/py/json2python-models.svg)](https://badge.fury.io/py/json2python-models) +[![Build Status](https://travis-ci.org/bogdandm/json2python-models.svg?branch=master)](https://travis-ci.org/bogdandm/json2python-models) +[![Coverage Status](https://coveralls.io/repos/github/bogdandm/json2python-models/badge.svg?branch=master)](https://coveralls.io/github/bogdandm/json2python-models?branch=master) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/11e13f2b81d7450eb0bca4b941d16d81)](https://www.codacy.com/app/bogdandm/json2python-models?utm_source=github.com&utm_medium=referral&utm_content=bogdandm/json2python-models&utm_campaign=Badge_Grade) + +![Example](/etc/convert.png) + +json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes +(dataclasses, [attrs](https://github.com/python-attrs/attrs)) from JSON dataset. + +## Features + +* Full **`typing` module** support +* **Types merging** - if some field contains data of different types this will be represent as `Union` type +* Fields and models **names** generation (unicode support included) +* Similar **models generalization** +* Handling **recursive data** structures (i.e family tree) +* Detecting **string literals** (i.e. datetime or just stringify numbers) + and providing decorators to easily convert into Python representation +* Generation models as **tree** (nested models) or **list** +* Specifying when dictionaries should be processed as **`dict` type** (by default every dict is considered as some model) +* **CLI** tool + +## Table of Contents + +* [Features](#features) +* [Table of Contents](#table-of-contents) +* [Example](#example) +* [Installation](#installation) +* [Usage](#usage) + * [CLI](#cli) + * [Low level API]() +* [Tests](#tests) + * [Test examples](#test-examples) +* [Built With](#built-with) +* [Contributing](#contributing) +* [License](#license) + +## Example + +``` +driver_standings.json +[ + { + "season": "2019", + "round": "3", + "DriverStandings": [ + { + "position": "1", + "positionText": "1", + "points": "68", + "wins": "2", + "Driver": { + "driverId": "hamilton", + "permanentNumber": "44", + "code": "HAM", + "url": "http://en.wikipedia.org/wiki/Lewis_Hamilton", + "givenName": "Lewis", + "familyName": "Hamilton", + "dateOfBirth": "1985-01-07", + "nationality": "British" + }, + "Constructors": [ + { + "constructorId": "mercedes", + "url": "http://en.wikipedia.org/wiki/Mercedes-Benz_in_Formula_One", + "name": "Mercedes", + "nationality": "German" + } + ] + }, + ... + ] + } +] +``` + +``` +json2models -f attrs -l DriverStandings driver_standings.json +``` + +```python +import attr +from json_to_models.dynamic_typing import IntString, IsoDateString +from typing import List + + +@attr.s +class DriverStandings: + @attr.s + class DriverStanding: + @attr.s + class Driver: + driver_id: str = attr.ib() + permanent_number: IntString = attr.ib(converter=IntString) + code: str = attr.ib() + url: str = attr.ib() + given_name: str = attr.ib() + family_name: str = attr.ib() + date_of_birth: IsoDateString = attr.ib(converter=IsoDateString) + nationality: str = attr.ib() + + @attr.s + class Constructor: + constructor_id: str = attr.ib() + url: str = attr.ib() + name: str = attr.ib() + nationality: str = attr.ib() + + position: IntString = attr.ib(converter=IntString) + position_text: IntString = attr.ib(converter=IntString) + points: IntString = attr.ib(converter=IntString) + wins: IntString = attr.ib(converter=IntString) + driver: 'Driver' = attr.ib() + constructors: List['Constructor'] = attr.ib() + + season: IntString = attr.ib(converter=IntString) + round: IntString = attr.ib(converter=IntString) + driver_standings: List['DriverStanding'] = attr.ib() +``` + +## Installation + +| **Be ware**: this project supports only `python3.7` and higher. | +| --- | + +To install it, use `pip`: + +`pip install json2python-models` + +Or you can build it from source: + +``` +git clone https://github.com/bogdandm/json2python-models.git +cd json2python-models +python setup.py install +``` + +## Usage + +### CLI + +For regular usage CLI tool is the best option. After you install this package you could use it as `json2models ` +or `python -m json_to_models `. I.e.: +``` +json2models -m Car car_*.json -f attrs > car.py +``` + +Arguments: +* `-h`, `--help` - Show help message and exit + +* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`, `**` or `?` patterns symbols are supported. + * **Format**: `-m [ ...]` + * **Example**: `-m Car audi.json reno.json` or `-m Car audi.json -m Car reno.json` (results will be the same) + +* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). + If this file contains dict with nested list than you can pass `` to lookup. + Deep lookups are supported by dot-separated path. If no lookup needed pass `-` as ``. + * **Format**: `-l ` + * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json` + * **Note**: Models names under this arguments should be unique. + +* `-f`, `--framework` - Model framework for which python code is generated. + `base` (default) mean no framework so code will be generated without any decorators and additional meta-data. + * **Format**: `-f {base,attrs,dataclasses,custom}` + * **Example**: `-f attrs` + * **Default**: `-f base` + +* `-s , --structure` - Models composition style. + * **Format**: `-s {nested, flat}` + * **Example**: `-s flat` + * **Default**: `-s nested` + +* `--datetime` - Enable datetime/date/time strings parsing. + * **Default**: disabled + * **Warning**: This can lead to 6-7 times slowdown on large datasets. Be sure that you really need this option. + +* `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`). + * **Default**: disabled + +* `--merge` - Merge policy settings. Possible values are: + * **Format**: `--merge MERGE_POLICY [MERGE_POLICY ...]` + * **Possible values** (MERGE_POLICY): + * `percent[_]` - two models had a certain percentage of matched field names. + Custom value could be i.e. `percent_95`. + * `number[_]` - two models had a certain number of matched field names. + * `exact` - two models should have exact same field names to merge. + * **Example**: `--merge percent_95 number_20` - merge if 95% of fields are matched or 20 of fields are matched + * **Default**: `--merge percent_70 number_10` + +* `--dict-keys-regex`, `--dkr` - List of regular expressions (Python syntax). + If all keys of some dict are match one of the pattern then + this dict will be marked as dict field but not nested model. + * **Format**: `--dkr RegEx [RegEx ...]` + * **Example**: `--dkr node_\d+ \d+_\d+_\d+` + * **Note**: `^` and `$` (string borders) tokens will be added automatically but you + have escape to other special characters manually. + * **Optional** + +* `--dict-keys-fields`, `--dkf` - List of model fields names that will be marked as dict fields + * **Format**: `--dkf FIELD_NAME [FIELD_NAME ...]` + * **Example**: `--dkf "dict_data" "mapping"` + * **Optional** + +* `--code-generator` - Absolute import path to `GenericModelCodeGenerator` subclass. + * **Format**: `--code-generator CODE_GENERATOR` + * **Example**: `-f mypackage.mymodule.DjangoModelsGenerator` + * **Note**: Is ignored without `-f custom` but is required with it. + +* `--code-generator-kwargs` - List of GenericModelCodeGenerator subclass arguments (for `__init__` method, + see docs of specific subclass). + Each argument should be in following format: `argument_name=value` or `"argument_name=value with space"`. + Boolean values should be passed in JS style: `true` or `false` + * **Format**: `--code-generator-kwargs [NAME=VALUE [NAME=VALUE ...]]` + * **Example**: `--code-generator-kwargs kwarg1=true kwarg2=10 "kwarg3=It is string with spaces"` + * **Optional** + +One of model arguments (`-m` or `-l`) is required. + +### Low level API + +> Coming soon (Wiki) + +## Tests + +To run tests you should clone project and run `setup.py` script: + +``` +git clone https://github.com/bogdandm/json2python-models.git +cd json2python-models +python setup.py test -a '' +``` + +Also I would recommend you to install `pytest-sugar` for pretty printing test results + +### Test examples + +You can find out some examples of usage of this project at [testing_tools/real_apis/...](/testing_tools/real_apis) + +Each file contains functions to download data from some online API (references included at the top of file) and +`main` function that generates and prints code. Some examples may print debug data before actual code. +Downloaded data will be saved at `testing_tools/real_apis//.json` + +## Built With + +* [python-dateutil](https://github.com/dateutil/dateutil) - Datetime parsing +* [inflection](https://github.com/jpvanhal/inflection) - String transformations +* [Unidecode](https://pypi.org/project/Unidecode/) - Unicode to ASCII conversion +* [Jinja2](https://github.com/pallets/jinja) - Code templates +* [ordered-set](https://github.com/LuminosoInsight/ordered-set) is used in models merging algorithm + +Test tools: +* [pytest](https://github.com/pytest-dev/pytest) - Test framework +* [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) - Parallel execution of test suites +* [pytest-sugar](https://github.com/Frozenball/pytest-sugar) - Test results pretty printing +* [requests](https://github.com/kennethreitz/requests) - Test data download + +## Contributing + +Feel free to open pull requests with new features or bug fixes. Just follow few rules: + +1. Always use some code formatter ([black](https://github.com/ambv/black) or PyCharm built-in) +2. Keep code coverage above 95-98% +3. All existing tests should be passed (including test examples from `testing_tools/real_apis`) +4. Use `typing` module +5. Fix [codacy](https://app.codacy.com/project/bogdandm/json2python-models/dashboard) issues from your PR + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details diff --git a/etc/convert.png b/etc/convert.png new file mode 100644 index 0000000..2912ad7 Binary files /dev/null and b/etc/convert.png differ diff --git a/etc/logo.png b/etc/logo.png new file mode 100644 index 0000000..c05be9d Binary files /dev/null and b/etc/logo.png differ diff --git a/json_to_models/__init__.py b/json_to_models/__init__.py index bf770bf..4ce5e3f 100644 --- a/json_to_models/__init__.py +++ b/json_to_models/__init__.py @@ -1,4 +1,4 @@ from pkg_resources import parse_version -__version__ = "0.1b2" +__version__ = "0.1.0" VERSION = parse_version(__version__) diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index 400ccf5..559b87f 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -8,6 +8,7 @@ class StringSerializable(BaseType): """ Mixin for classes which are used to (de-)serialize some values in a string form """ + @classmethod def to_internal_value(cls, value: str) -> 'StringSerializable': """ @@ -62,6 +63,7 @@ def add(self, replace_types: Iterable[T_StringSerializable] = (), cls: type = No :param cls: StringSerializable class :return: decorator """ + def decorator(cls): self.types.append(cls) for t in replace_types: diff --git a/setup.py b/setup.py index 18c5e88..ffec23e 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,10 @@ with open('requirements.txt') as f: required = f.read().splitlines() -URL = "https://github.com/bogdandm/json2python-models" +with open('README.md') as f: + long_description = f.read() +URL = "https://github.com/bogdandm/json2python-models" CPU_N = multiprocessing.cpu_count() @@ -38,6 +40,8 @@ def run_tests(self): author="bogdandm (Bogdan Kalashnikov)", author_email="bogdan.dm1995@yandex.ru", description="Python models (attrs, dataclasses or custom) generator from JSON data with typing module support", + long_description=long_description, + long_description_content_type='text/markdown', license="MIT", packages=find_packages(exclude=['test', 'testing_tools']), entry_points={ @@ -46,5 +50,5 @@ def run_tests(self): install_requires=required, cmdclass={"test": PyTest}, tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs"], - data_files=[('', ['pytest.ini', '.coveragerc', 'LICENSE'])] + data_files=[('', ['pytest.ini', '.coveragerc', 'LICENSE', 'README.md', 'CHANGELOG.md'])] ) diff --git a/test/test_dynamic_typing/test_string_datetime.py b/test/test_dynamic_typing/test_string_datetime.py index de71aff..257960e 100644 --- a/test/test_dynamic_typing/test_string_datetime.py +++ b/test/test_dynamic_typing/test_string_datetime.py @@ -2,9 +2,8 @@ import pytest -from json_to_models.dynamic_typing import ( - FloatString, IntString, IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes -) +from json_to_models.dynamic_typing import (BooleanString, FloatString, IntString, IsoDateString, IsoDatetimeString, + IsoTimeString, register_datetime_classes) from json_to_models.generator import MetadataGenerator register_datetime_classes() @@ -21,6 +20,11 @@ FloatString, id="default_check_float" ), + pytest.param( + "true", + BooleanString, + id="bool" + ), pytest.param( "2018-12-31", IsoDateString, @@ -46,6 +50,11 @@ def test_detect_type(models_generator: MetadataGenerator, value, expected): test_parse_data = [ + pytest.param( + "true", + BooleanString(True), + id="bool" + ), pytest.param( "2018-12-31", IsoDateString(2018, 12, 31), diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index e31e15d..71d550c 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -11,7 +11,6 @@ from json_to_models.models.dataclasses import DataclassModelCodeGenerator from json_to_models.models.structure import compose_models from json_to_models.registry import ModelRegistry -from json_to_models.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta from testing_tools.real_apis import dump_response @@ -58,8 +57,8 @@ def main(): print("=" * 20, end='') structure = compose_models(reg.models_map) - print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) - print("=" * 20) + # print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) + # print("=" * 20) print(generate_code(structure, DataclassModelCodeGenerator, class_generator_kwargs={"post_init_converters": True})) diff --git a/testing_tools/real_apis/openlibrary.py b/testing_tools/real_apis/openlibrary.py index 3287130..37e6625 100644 --- a/testing_tools/real_apis/openlibrary.py +++ b/testing_tools/real_apis/openlibrary.py @@ -4,8 +4,10 @@ import requests from json_to_models.generator import MetadataGenerator +from json_to_models.models import compose_models +from json_to_models.models.attr import AttrsModelCodeGenerator +from json_to_models.models.base import generate_code from json_to_models.registry import ModelRegistry -from testing_tools.pprint_meta_data import pretty_format_meta from testing_tools.real_apis import dump_response session = requests.Session() @@ -44,16 +46,16 @@ def main(): gen = MetadataGenerator() reg = ModelRegistry() - for data in (search_result, books): - reg.process_meta_data(gen.generate(*data)) + reg.process_meta_data(gen.generate(*search_result), model_name="Search") + reg.process_meta_data(gen.generate(*books), model_name="Book") reg.merge_models(generator=gen) - print("\n" + "=" * 20, end='') + print("\n" + "=" * 20) for model in reg.models: model.generate_name() - for model in reg.models: - print(pretty_format_meta(model)) - print("\n" + "=" * 20, end='') + + structure = compose_models(reg.models_map) + print(generate_code(structure, AttrsModelCodeGenerator)) if __name__ == '__main__': diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index b3b53be..8ded980 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -1,5 +1,5 @@ """ -Path of Exile API http://www.pathofexile.com/developer/docs/api-resource-public-stash-tabs +Path of Exile Stash API http://www.pathofexile.com/developer/docs/api-resource-public-stash-tabs """ from datetime import datetime @@ -10,7 +10,6 @@ from json_to_models.models.base import generate_code from json_to_models.models.structure import compose_models_flat from json_to_models.registry import ModelRegistry -from json_to_models.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta from testing_tools.real_apis import dump_response @@ -34,12 +33,13 @@ def main(): reg.merge_models(generator=gen) reg.generate_names() + print("Meta tree:") print(pretty_format_meta(next(iter(reg.models)))) print("\n" + "=" * 20, end='') structure = compose_models_flat(reg.models_map) - print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) - print("=" * 20) + # print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) + # print("=" * 20) print(generate_code(structure, AttrsModelCodeGenerator)) print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds") diff --git a/testing_tools/real_apis/randomapis.py b/testing_tools/real_apis/randomapis.py index d942db8..7d22ba6 100644 --- a/testing_tools/real_apis/randomapis.py +++ b/testing_tools/real_apis/randomapis.py @@ -39,11 +39,19 @@ def main(): gen = MetadataGenerator() reg = ModelRegistry() - for data in ([chroniclingamerica_data], [launchlibrary_data], university_domains_data): - fields = gen.generate(*data) - reg.process_meta_data(fields) + + fields = gen.generate(chroniclingamerica_data) + reg.process_meta_data(fields, model_name="CHRONICLING") + + fields = gen.generate(launchlibrary_data) + reg.process_meta_data(fields, model_name="LaunchLibrary") + + fields = gen.generate(*university_domains_data) + reg.process_meta_data(fields, model_name="Universities") + reg.merge_models(generator=gen) reg.generate_names() + structure = compose_models(reg.models_map) print(generate_code(structure, AttrsModelCodeGenerator))