From 8f589dfbca352a7089a7c441d493f97cbb956a76 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 10:55:14 -0700 Subject: [PATCH 1/7] fix docstring and docs --- .../azure-schemaregistry-avroserializer/CHANGELOG.md | 2 +- .../azure-schemaregistry-avroserializer/README.md | 2 +- sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md | 2 +- sdk/schemaregistry/azure-schemaregistry/README.md | 2 ++ .../azure/schemaregistry/_common/_schema.py | 2 +- .../schemaregistry/aio/_schema_registry_client_async.py | 6 +++--- ...ry_code_async.py => sample_code_schemaregistry_async.py} | 0 ...chema_registry_code.py => sample_code_schemaregistry.py} | 0 8 files changed, 9 insertions(+), 7 deletions(-) rename sdk/schemaregistry/azure-schemaregistry/samples/async_samples/{sample_schema_registry_code_async.py => sample_code_schemaregistry_async.py} (100%) rename sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/{sample_schema_registry_code.py => sample_code_schemaregistry.py} (100%) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md index d132099f47a4..05d6cbecb23b 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0b1 (2020-09-08) +## 1.0.0b1 (2020-09-09) Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry Avro Serializer. diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index c4ef1006c343..cb5b1632b380 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -3,7 +3,7 @@ Azure Schema Registry Avro Serializer provides the ability to serialize and deserialize data according to the given avro schema. It is integrated with Azure Schema Registry SDK and will automatically register and get schema. -[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_docs] | [Samples][sr_avro_samples] | [Changelog][change_log] +[Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_avro_samples] | [Changelog][change_log] ## Getting started diff --git a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md index 22deb5fe313e..22b4ce02c481 100644 --- a/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +++ b/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0b1 (2020-09-08) +## 1.0.0b1 (2020-09-09) Version 1.0.0b1 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Schema Registry. diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index 8ef6b8790650..c8f3dcd7300d 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -39,6 +39,8 @@ schema_registry_client = SchemaRegistryClient(endpoint, credential) - Schema: Schema is the organization or structure for data. +- SchemaRegistryClient: `SchemaRegistryClient ` provides the API for storing and retrieving schemas in schema registry. + ## Examples The following sections provide several code snippets covering some of the most common Schema Registry tasks, including: diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py index 6864db3ae5e3..cf34225f61c4 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_common/_schema.py @@ -48,7 +48,7 @@ class SchemaProperties(object): :end-before: [END print_schema_properties] :language: python :dedent: 4 - :caption: SchemaId object. + :caption: SchemaProperties object. """ def __init__( diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 6c3835a94d98..29fb6a2c1df5 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -97,7 +97,7 @@ async def register_schema( .. admonition:: Example: - .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry.py + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry_async.py :start-after: [START register_schema_async] :end-before: [END register_schema_async] :language: python @@ -133,7 +133,7 @@ async def get_schema( .. admonition:: Example: - .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry.py + .. literalinclude:: ../samples/async_samples/sample_code_schemaregistry_async.py :start-after: [START get_schema_async] :end-before: [END get_schema_async] :language: python @@ -173,7 +173,7 @@ async def get_schema_id( :end-before: [END get_schema_id_async] :language: python :dedent: 4 - :caption:Get schema id. + :caption: Get schema by id. """ try: diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py b/sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_code_schemaregistry_async.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_schema_registry_code_async.py rename to sdk/schemaregistry/azure-schemaregistry/samples/async_samples/sample_code_schemaregistry_async.py diff --git a/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py b/sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_code_schemaregistry.py similarity index 100% rename from sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_schema_registry_code.py rename to sdk/schemaregistry/azure-schemaregistry/samples/sync_samples/sample_code_schemaregistry.py From bcd67b9775cf65938e8f33a5d6dfa3d950a1cfa2 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 11:42:28 -0700 Subject: [PATCH 2/7] update codeowner and ci config --- .github/CODEOWNERS | 3 +++ sdk/schemaregistry/ci.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1822a5b802b5..5c2685e37bdc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -67,6 +67,9 @@ # PRLabel: %Recovery Services /sdk/recoveryservices/ @DheerendraRathor +# PRLabel: %Schema Registry +/sdk/schemaregistry/ @yunhaoling + # PRLabel: %Search /sdk/search/ @xiangyan99 @rakshith91 diff --git a/sdk/schemaregistry/ci.yml b/sdk/schemaregistry/ci.yml index aaa6f14460ea..07ac07773862 100644 --- a/sdk/schemaregistry/ci.yml +++ b/sdk/schemaregistry/ci.yml @@ -30,3 +30,5 @@ extends: Artifacts: - name: azure_schemaregistry safeName: azureschemaregistry + - name: azure_schemaregistry_avroserializer + safeName: azureschemaregistryavroserializer \ No newline at end of file From 037123ca6a110360eef4eecd35562afc3efe3046 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 14:43:42 -0700 Subject: [PATCH 3/7] update init in serializer --- .../azure/schemaregistry/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py index c36aaed14908..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/__init__.py @@ -23,3 +23,4 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore From f1e251166e4ef18c25824df1663bfecf8ff6478f Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 15:41:21 -0700 Subject: [PATCH 4/7] update readme --- .../README.md | 40 ++++++++++++++++--- .../azure-schemaregistry/README.md | 10 +++-- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md index cb5b1632b380..0973eb1ad23d 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/README.md @@ -1,7 +1,8 @@ # Azure Schema Registry Avro Serializer client library for Python -Azure Schema Registry Avro Serializer provides the ability to serialize and deserialize data according -to the given avro schema. It is integrated with Azure Schema Registry SDK and will automatically register and get schema. +Azure Schema Registry is a schema repository service hosted by Azure Event Hubs, providing schema storage, versioning, +and management. This package provides an Avro serializer capable of serializing and deserializing payloads containing +Schema Registry schema identifiers and Avro-encoded data. [Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_avro_samples] | [Changelog][change_log] @@ -18,7 +19,7 @@ pip install azure-schemaregistry-avroserializer azure-identity ### Prerequisites: To use this package, you must have: * Azure subscription - [Create a free account][azure_sub] -* Azure Schema Registry +* [Azure Schema Registry][schemaregistry_service] * Python 2.7, 3.5 or later - [Install Python][python] ### Authenticate the client @@ -40,7 +41,34 @@ serializer = SchemaRegistryAvroSerializer(schema_registry_client, schema_group) ## Key concepts -- Avro: Apache Avro™ is a data serialization system. +### SchemaRegistryAvroSerializer + +Provides API to serialize to and deserialize from Avro Binary Encoding plus a +header with schema ID. Uses [SchemaRegistryClient][schemaregistry_client] to get schema IDs from schema content or vice versa. + +### Message format + +The same format is used by schema registry serializers across Azure SDK languages. + +Messages are encoded as follows: + +- 4 bytes: Format Indicator + + - Currently always zero to indicate format below. + +- 32 bytes: Schema ID + + - UTF-8 hexadecimal representation of GUID. + - 32 hex digits, no hyphens. + - Same format and byte order as string from Schema Registry service. + +- Remaining bytes: Avro payload (in general, format-specific payload) + + - Avro Binary Encoding + - NOT Avro Object Container File, which includes the schema and defeats the + purpose of this serialzer to move the schema out of the message payload and + into the schema registry. + ## Examples @@ -171,4 +199,6 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio [sr_avro_samples]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/samples [api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry-avroserializer/latest/index.html [source_code]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer -[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md \ No newline at end of file +[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry-avroserializer/CHANGELOG.md +[schemaregistry_client]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry +[schemaregistry_service]: https://aka.ms/schemaregistry \ No newline at end of file diff --git a/sdk/schemaregistry/azure-schemaregistry/README.md b/sdk/schemaregistry/azure-schemaregistry/README.md index c8f3dcd7300d..1f632899cada 100644 --- a/sdk/schemaregistry/azure-schemaregistry/README.md +++ b/sdk/schemaregistry/azure-schemaregistry/README.md @@ -1,7 +1,8 @@ # Azure Schema Registry client library for Python -Azure Schema Registry is a service that provides the ability to store and retrieve different types of schemas such as -Avro, Json, etc. +Azure Schema Registry is a schema repository service hosted by Azure Event Hubs, providing schema storage, versioning, +and management. The registry is leveraged by serializers to reduce payload size while describing payload structure with +schema identifiers rather than full schemas. [Source code][source_code] | [Package (PyPi)][pypi] | [API reference documentation][api_reference] | [Samples][sr_samples] | [Changelog][change_log] @@ -18,7 +19,7 @@ pip install azure-schemaregistry azure-identity ### Prerequisites: To use this package, you must have: * Azure subscription - [Create a free account][azure_sub] -* Azure Schema Registry +* [Azure Schema Registry][schemaregistry_service] * Python 2.7, 3.5 or later - [Install Python][python] ### Authenticate the client @@ -198,4 +199,5 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio [sr_samples]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/samples [api_reference]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-schemaregistry/latest/index.html [source_code]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry -[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md \ No newline at end of file +[change_log]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/schemaregistry/azure-schemaregistry/CHANGELOG.md +[schemaregistry_service]: https://aka.ms/schemaregistry \ No newline at end of file From 61c6ceaf5f74b2b08a3c2e8a7afab4c52eddff10 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 15:41:49 -0700 Subject: [PATCH 5/7] update sr dependecy in avro serializer --- sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py | 2 +- shared_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index c97916e14530..dbbb37794d4c 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -65,7 +65,7 @@ zip_safe=False, packages=find_packages(exclude=exclude_packages), install_requires=[ - 'azure-schemaregistry<2.0.0,>=1.0.0b1', + 'azure-schemaregistry==1.0.0b1', 'avro<2.0.0,>=1.10.0' ] ) diff --git a/shared_requirements.txt b/shared_requirements.txt index 2e397e351331..bdc116ab9ca6 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -83,7 +83,7 @@ azure-mgmt-trafficmanager~=0.50.0 azure-mgmt-web~=0.35.0 azure-nspkg azure-keyvault-nspkg -azure-schemaregistry<2.0.0,>=1.0.0b1 +azure-schemaregistry==1.0.0b1 azure-search-nspkg azure-security-nspkg azure-synapse-nspkg From 8617d8f562b7b3501aff232931e1f44778648344 Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 15:43:04 -0700 Subject: [PATCH 6/7] update module __init__.py --- .../azure/schemaregistry/serializer/__init__.py | 1 + .../azure-schemaregistry/azure/schemaregistry/__init__.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py index c36aaed14908..80f86cb969ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/__init__.py @@ -23,3 +23,4 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py index 6c68f01b1092..b0b43faf4258 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/__init__.py @@ -23,8 +23,6 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore - from ._version import VERSION __version__ = VERSION From c0c44e178ee4177023120853a31b1016311db38f Mon Sep 17 00:00:00 2001 From: Yunhao Ling Date: Tue, 8 Sep 2020 16:09:36 -0700 Subject: [PATCH 7/7] revert dependcy to see if it helps doc generetion --- sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py | 2 +- shared_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py index dbbb37794d4c..c97916e14530 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/setup.py @@ -65,7 +65,7 @@ zip_safe=False, packages=find_packages(exclude=exclude_packages), install_requires=[ - 'azure-schemaregistry==1.0.0b1', + 'azure-schemaregistry<2.0.0,>=1.0.0b1', 'avro<2.0.0,>=1.10.0' ] ) diff --git a/shared_requirements.txt b/shared_requirements.txt index bdc116ab9ca6..2e397e351331 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -83,7 +83,7 @@ azure-mgmt-trafficmanager~=0.50.0 azure-mgmt-web~=0.35.0 azure-nspkg azure-keyvault-nspkg -azure-schemaregistry==1.0.0b1 +azure-schemaregistry<2.0.0,>=1.0.0b1 azure-search-nspkg azure-security-nspkg azure-synapse-nspkg